From 53dedc7ccd83f1b0b54d754743527b5029e5ed74 Mon Sep 17 00:00:00 2001 From: May Liu Date: Wed, 10 Dec 2025 10:53:41 -0800 Subject: [PATCH 1/5] Support override_condition in save_as_table for targeted delete-insert --- CHANGELOG.md | 2 + .../snowpark/_internal/analyzer/analyzer.py | 6 + .../_internal/analyzer/snowflake_plan.py | 65 ++++++-- .../_internal/analyzer/snowflake_plan_node.py | 2 + src/snowflake/snowpark/dataframe_writer.py | 37 ++++- tests/integ/test_dataframe.py | 148 ++++++++++++++++++ 6 files changed, 250 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bdf0b3a85..b0a22bc2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ #### New Features +- Added support for targeted delete-insert via the `override_condition` parameter in `DataFrameWriter.save_as_table` + #### Bug Fixes #### Improvements diff --git a/src/snowflake/snowpark/_internal/analyzer/analyzer.py b/src/snowflake/snowpark/_internal/analyzer/analyzer.py index fe939a5b14..25c4807e15 100644 --- a/src/snowflake/snowpark/_internal/analyzer/analyzer.py +++ b/src/snowflake/snowpark/_internal/analyzer/analyzer.py @@ -1228,6 +1228,12 @@ def do_resolve_with_resolved_children( child_attributes=resolved_child.attributes, iceberg_config=iceberg_config, table_exists=logical_plan.table_exists, + override_condition=self.analyze( + logical_plan.override_condition, + df_aliased_col_name_to_real_col_name, + ) + if logical_plan.override_condition + else None, ) if isinstance(logical_plan, Limit): diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py index a3d40995a7..f52f6a1487 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py @@ -1267,6 +1267,7 @@ def save_as_table( child_attributes: Optional[List[Attribute]], iceberg_config: Optional[dict] = None, table_exists: Optional[bool] = None, + override_condition: Optional[str] = None, ) -> SnowflakePlan: """Returns a SnowflakePlan to materialize the child plan into a table. @@ -1417,19 +1418,65 @@ def get_create_and_insert_plan(child: SnowflakePlan, replace, error): referenced_ctes=child.referenced_ctes, ) + def get_override_delete_insert_plan(child: SnowflakePlan): + """Build a plan for targeted delete + insert with transaction. + + Deletes rows matching the override_condition condition, then inserts + all rows from the source DataFrame. Wrapped in a transaction for atomicity. + """ + child = self.add_result_scan_if_not_select(child) + + return SnowflakePlan( + [ + *child.queries[:-1], + Query("BEGIN TRANSACTION"), + Query( + delete_statement( + table_name=full_table_name, + condition=override_condition, + source_data=None, + ), + params=child.queries[-1].params, + is_ddl_on_temp_object=is_temp_table_type, + ), + Query( + insert_into_statement( + table_name=full_table_name, + child=child.queries[-1].sql, + column_names=column_names, + ), + params=child.queries[-1].params, + is_ddl_on_temp_object=is_temp_table_type, + ), + Query("COMMIT"), + ], + schema_query=None, + post_actions=child.post_actions, + expr_to_alias={}, + source_plan=source_plan, + api_calls=child.api_calls, + session=self.session, + referenced_ctes=child.referenced_ctes, + ) + if mode == SaveMode.APPEND: assert table_exists is not None if table_exists: - return self.build( - lambda x: insert_into_statement( - table_name=full_table_name, - child=x, - column_names=column_names, - ), - child, - source_plan, - ) + if override_condition is not None: + return get_override_delete_insert_plan(child) + else: + # Normal append without override_condition + return self.build( + lambda x: insert_into_statement( + table_name=full_table_name, + child=x, + column_names=column_names, + ), + child, + source_plan, + ) else: + # Table doesn't exist, just create and insert (override_condition is no-op) return get_create_and_insert_plan(child, replace=False, error=False) elif mode == SaveMode.TRUNCATE: diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py index ff43b07cb2..1e6d9032d9 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py @@ -243,6 +243,7 @@ def __init__( copy_grants: bool = False, iceberg_config: Optional[dict] = None, table_exists: Optional[bool] = None, + override_condition: Optional[Expression] = None, ) -> None: super().__init__() @@ -267,6 +268,7 @@ def __init__( # whether the table already exists in the database # determines the compiled SQL for APPEND and TRUNCATE mode self.table_exists = table_exists + self.override_condition = override_condition @property def individual_node_complexity(self) -> Dict[PlanNodeCategory, int]: diff --git a/src/snowflake/snowpark/dataframe_writer.py b/src/snowflake/snowpark/dataframe_writer.py index 4f272da557..26e61711e8 100644 --- a/src/snowflake/snowpark/dataframe_writer.py +++ b/src/snowflake/snowpark/dataframe_writer.py @@ -48,7 +48,7 @@ warning, ) from snowflake.snowpark.async_job import AsyncJob, _AsyncResultType -from snowflake.snowpark.column import Column, _to_col_if_str +from snowflake.snowpark.column import Column, _to_col_if_str, _to_col_if_sql_expr from snowflake.snowpark.exceptions import SnowparkClientException from snowflake.snowpark.functions import sql_expr from snowflake.snowpark.mock._connection import MockServerConnection @@ -256,6 +256,7 @@ def save_as_table( Dict[str, Union[str, Iterable[ColumnOrSqlExpr]]] ] = None, table_exists: Optional[bool] = None, + override_condition: Optional[ColumnOrSqlExpr] = None, _emit_ast: bool = True, **kwargs: Optional[Dict[str, Any]], ) -> Optional[AsyncJob]: @@ -331,6 +332,9 @@ def save_as_table( table_exists: Optional parameter to specify if the table is known to exist or not. Set to ``True`` if table exists, ``False`` if it doesn't, or ``None`` (default) for automatic detection. Primarily useful for "append" and "truncate" modes to avoid running query for automatic detection. + override_condition: Specifies the override condition to perform atomic targeted delete-insert. + Can only be used when ``mode`` is "append" and the table exists. Rows matching the + condition are deleted from the target table, then all rows from the DataFrame are inserted. Example 1:: @@ -364,6 +368,21 @@ def save_as_table( ... "partition_by": ["a", bucket(3, col("b"))], ... } >>> df.write.mode("overwrite").save_as_table("my_table", iceberg_config=iceberg_config) # doctest: +SKIP + + Example 3:: + + Using override_condition for targeted delete and insert: + + >>> from snowflake.snowpark.functions import col + >>> df = session.create_dataframe([[1, "a"], [2, "b"], [3, "c"]], schema=["id", "val"]) + >>> df.write.mode("overwrite").save_as_table("my_table", table_type="temporary") + >>> session.table("my_table").order_by("id").collect() + [Row(ID=1, VAL='a'), Row(ID=2, VAL='b'), Row(ID=3, VAL='c')] + + >>> new_df = session.create_dataframe([[2, "updated2"], [5, "updated5"]], schema=["id", "val"]) + >>> new_df.write.mode("append").save_as_table("my_table", override_condition="id = 1 or val = 'b'") + >>> session.table("my_table").order_by("id").collect() + [Row(ID=2, VAL='updated2'), Row(ID=3, VAL='c'), Row(ID=5, VAL='updated5')] """ statement_params = track_data_source_statement_params( @@ -486,6 +505,21 @@ def save_as_table( f"Unsupported table type. Expected table types: {SUPPORTED_TABLE_TYPES}" ) + # override_condition must be used with APPEND mode + if override_condition is not None and save_mode != SaveMode.APPEND: + raise ValueError( + f"'override_condition' is only supported with mode='append'. " + f"Got mode='{save_mode.value}'." + ) + + override_condition_expr = ( + _to_col_if_sql_expr( + override_condition, "DataFrameWriter.save_as_table" + )._expression + if override_condition is not None + else None + ) + session = self._dataframe._session if ( table_exists is None @@ -518,6 +552,7 @@ def save_as_table( copy_grants, iceberg_config, table_exists, + override_condition_expr, ) snowflake_plan = session._analyzer.resolve(create_table_logic_plan) result = session._conn.execute( diff --git a/tests/integ/test_dataframe.py b/tests/integ/test_dataframe.py index 30e1805e5c..239a715cb0 100644 --- a/tests/integ/test_dataframe.py +++ b/tests/integ/test_dataframe.py @@ -4566,6 +4566,154 @@ def test_write_table_with_clustering_keys_and_comment( Utils.drop_table(session, table_name3) +@pytest.mark.xfail( + "config.getoption('local_testing_mode', default=False)", + reason="override_condition is a SQL feature", + run=False, +) +def test_write_table_with_override_condition(session): + """Test override_condition parameter for targeted delete + insert.""" + table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) + try: + # Setup and verify initial data + initial_df = session.create_dataframe( + [[1, "a"], [2, "b"], [3, "c"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + initial_df.write.mode("overwrite").save_as_table(table_name) + result = session.table(table_name).order_by("id").collect() + assert result == [ + Row(ID=1, VAL="a"), + Row(ID=2, VAL="b"), + Row(ID=3, VAL="c"), + ] + + # Test 1: override_condition with SQL string expr + new_df1 = session.create_dataframe( + [[2, "updated2"], [5, "new5"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df1.write.mode("append").save_as_table( + table_name, override_condition="id = 1 or val = 'b'" + ) + result = session.table(table_name).order_by("id").collect() + # id=1 and id=2 (val='b') deleted, new rows inserted + assert result == [ + Row(ID=2, VAL="updated2"), + Row(ID=3, VAL="c"), + Row(ID=5, VAL="new5"), + ] + + # Test 2: override_condition with Column expr + new_df2 = session.create_dataframe( + [[2, "replaced2"], [4, "new4"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df2.write.mode("append").save_as_table( + table_name, override_condition=col("id") == 2 + ) + result = session.table(table_name).order_by("id").collect() + # id=2 deleted, new rows inserted + assert result == [ + Row(ID=2, VAL="replaced2"), + Row(ID=3, VAL="c"), + Row(ID=4, VAL="new4"), + Row(ID=5, VAL="new5"), + ] + + # Test 3: override_condition with multiple Column expr + new_df3 = session.create_dataframe( + [[6, "new6"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df3.write.mode("append").save_as_table( + table_name, override_condition=(col("id") > 4) | (col("val") == "c") + ) + result = session.table(table_name).order_by("id").collect() + # id=3 (val='c') and id=5 (id > 4) deleted, id=4 remains (4 is not > 4), new row inserted + assert result == [ + Row(ID=2, VAL="replaced2"), + Row(ID=4, VAL="new4"), + Row(ID=6, VAL="new6"), + ] + + # Test 4: override_condition that matches all rows + new_df4 = session.create_dataframe( + [[10, "new"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df4.write.mode("append").save_as_table( + table_name, override_condition="id > 0" + ) + result = session.table(table_name).collect() + assert result == [Row(ID=10, VAL="new")] + + # Test 5: override_condition that matches no rows + new_df5 = session.create_dataframe( + [[20, "another"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df5.write.mode("append").save_as_table( + table_name, override_condition="id = 999" + ) + result = session.table(table_name).order_by("id").collect() + assert result == [ + Row(ID=10, VAL="new"), + Row(ID=20, VAL="another"), + ] + + finally: + Utils.drop_table(session, table_name) + + +@pytest.mark.xfail( + "config.getoption('local_testing_mode', default=False)", + reason="override_condition is a SQL feature", + run=False, +) +@pytest.mark.parametrize( + "invalid_mode", ["overwrite", "truncate", "errorifexists", "ignore"] +) +def test_write_table_with_override_condition_edge_cases(session, invalid_mode): + """Test override_condition edge cases: table not exists, and invalid modes.""" + table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) + try: + # Edge case 1: Table doesn't exist - override_condition is no-op + df = session.create_dataframe( + [[1, "a"], [2, "b"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + df.write.mode("append").save_as_table(table_name, override_condition="id = 999") + result = session.table(table_name).order_by("id").collect() + assert result == [Row(ID=1, VAL="a"), Row(ID=2, VAL="b")] + + # Edge case 2: Invalid mode raises ValueError + with pytest.raises( + ValueError, + match="'override_condition' is only supported with mode='append'", + ): + df.write.mode(invalid_mode).save_as_table( + table_name, override_condition="id = 1" + ) + + finally: + Utils.drop_table(session, table_name) + + @pytest.mark.xfail( "config.getoption('local_testing_mode', default=False)", reason="Clustering is a SQL feature", From 557d26068dea9fb143f0277d97dfc1ae874d03bf Mon Sep 17 00:00:00 2001 From: May Liu Date: Wed, 10 Dec 2025 16:55:52 -0800 Subject: [PATCH 2/5] rename param to overwrite_condition; can work with both Append and Overwrite modes --- CHANGELOG.md | 2 +- .../snowpark/_internal/analyzer/analyzer.py | 6 +- .../_internal/analyzer/snowflake_plan.py | 23 ++++--- .../_internal/analyzer/snowflake_plan_node.py | 4 +- src/snowflake/snowpark/dataframe_writer.py | 41 ++++++------ tests/integ/test_dataframe.py | 64 ++++++++++++------- 6 files changed, 83 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0a22bc2d6..7c02958856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ #### New Features -- Added support for targeted delete-insert via the `override_condition` parameter in `DataFrameWriter.save_as_table` +- Added support for targeted delete-insert via the `overwrite_condition` parameter in `DataFrameWriter.save_as_table` #### Bug Fixes diff --git a/src/snowflake/snowpark/_internal/analyzer/analyzer.py b/src/snowflake/snowpark/_internal/analyzer/analyzer.py index 25c4807e15..39d21f2e02 100644 --- a/src/snowflake/snowpark/_internal/analyzer/analyzer.py +++ b/src/snowflake/snowpark/_internal/analyzer/analyzer.py @@ -1228,11 +1228,11 @@ def do_resolve_with_resolved_children( child_attributes=resolved_child.attributes, iceberg_config=iceberg_config, table_exists=logical_plan.table_exists, - override_condition=self.analyze( - logical_plan.override_condition, + overwrite_condition=self.analyze( + logical_plan.overwrite_condition, df_aliased_col_name_to_real_col_name, ) - if logical_plan.override_condition + if logical_plan.overwrite_condition else None, ) diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py index f52f6a1487..ee87724c5e 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py @@ -1267,7 +1267,7 @@ def save_as_table( child_attributes: Optional[List[Attribute]], iceberg_config: Optional[dict] = None, table_exists: Optional[bool] = None, - override_condition: Optional[str] = None, + overwrite_condition: Optional[str] = None, ) -> SnowflakePlan: """Returns a SnowflakePlan to materialize the child plan into a table. @@ -1418,10 +1418,10 @@ def get_create_and_insert_plan(child: SnowflakePlan, replace, error): referenced_ctes=child.referenced_ctes, ) - def get_override_delete_insert_plan(child: SnowflakePlan): + def get_overwrite_delete_insert_plan(child: SnowflakePlan): """Build a plan for targeted delete + insert with transaction. - Deletes rows matching the override_condition condition, then inserts + Deletes rows matching the overwrite_condition condition, then inserts all rows from the source DataFrame. Wrapped in a transaction for atomicity. """ child = self.add_result_scan_if_not_select(child) @@ -1433,7 +1433,7 @@ def get_override_delete_insert_plan(child: SnowflakePlan): Query( delete_statement( table_name=full_table_name, - condition=override_condition, + condition=overwrite_condition, source_data=None, ), params=child.queries[-1].params, @@ -1462,10 +1462,10 @@ def get_override_delete_insert_plan(child: SnowflakePlan): if mode == SaveMode.APPEND: assert table_exists is not None if table_exists: - if override_condition is not None: - return get_override_delete_insert_plan(child) + if overwrite_condition is not None: + return get_overwrite_delete_insert_plan(child) else: - # Normal append without override_condition + # Normal append without overwrite_condition return self.build( lambda x: insert_into_statement( table_name=full_table_name, @@ -1476,7 +1476,7 @@ def get_override_delete_insert_plan(child: SnowflakePlan): source_plan, ) else: - # Table doesn't exist, just create and insert (override_condition is no-op) + # Table doesn't exist, just create and insert (overwrite_condition is no-op) return get_create_and_insert_plan(child, replace=False, error=False) elif mode == SaveMode.TRUNCATE: @@ -1493,7 +1493,12 @@ def get_override_delete_insert_plan(child: SnowflakePlan): return get_create_table_as_select_plan(child, replace=True, error=True) elif mode == SaveMode.OVERWRITE: - return get_create_table_as_select_plan(child, replace=True, error=True) + if overwrite_condition is not None and table_exists: + # Selective overwrite: delete matching rows, then insert + return get_overwrite_delete_insert_plan(child) + else: + # Default overwrite: drop and recreate table + return get_create_table_as_select_plan(child, replace=True, error=True) elif mode == SaveMode.IGNORE: return get_create_table_as_select_plan(child, replace=False, error=False) diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py index 1e6d9032d9..0d2169e9eb 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py @@ -243,7 +243,7 @@ def __init__( copy_grants: bool = False, iceberg_config: Optional[dict] = None, table_exists: Optional[bool] = None, - override_condition: Optional[Expression] = None, + overwrite_condition: Optional[Expression] = None, ) -> None: super().__init__() @@ -268,7 +268,7 @@ def __init__( # whether the table already exists in the database # determines the compiled SQL for APPEND and TRUNCATE mode self.table_exists = table_exists - self.override_condition = override_condition + self.overwrite_condition = overwrite_condition @property def individual_node_complexity(self) -> Dict[PlanNodeCategory, int]: diff --git a/src/snowflake/snowpark/dataframe_writer.py b/src/snowflake/snowpark/dataframe_writer.py index 26e61711e8..8bc1ee1df2 100644 --- a/src/snowflake/snowpark/dataframe_writer.py +++ b/src/snowflake/snowpark/dataframe_writer.py @@ -256,7 +256,7 @@ def save_as_table( Dict[str, Union[str, Iterable[ColumnOrSqlExpr]]] ] = None, table_exists: Optional[bool] = None, - override_condition: Optional[ColumnOrSqlExpr] = None, + overwrite_condition: Optional[ColumnOrSqlExpr] = None, _emit_ast: bool = True, **kwargs: Optional[Dict[str, Any]], ) -> Optional[AsyncJob]: @@ -331,9 +331,9 @@ def save_as_table( * iceberg_version: Overrides the version of iceberg to use. Defaults to 2 when unset. table_exists: Optional parameter to specify if the table is known to exist or not. Set to ``True`` if table exists, ``False`` if it doesn't, or ``None`` (default) for automatic detection. - Primarily useful for "append" and "truncate" modes to avoid running query for automatic detection. - override_condition: Specifies the override condition to perform atomic targeted delete-insert. - Can only be used when ``mode`` is "append" and the table exists. Rows matching the + Primarily useful for "append", "truncate", and "overwrite" with overwrite_condition modes to avoid running query for automatic detection. + overwrite_condition: Specifies the overwrite condition to perform atomic targeted delete-insert. + Can be used when ``mode`` is "append" or "overwrite" when the table exists. Rows matching the condition are deleted from the target table, then all rows from the DataFrame are inserted. @@ -371,7 +371,7 @@ def save_as_table( Example 3:: - Using override_condition for targeted delete and insert: + Using overwrite_condition for targeted delete and insert: >>> from snowflake.snowpark.functions import col >>> df = session.create_dataframe([[1, "a"], [2, "b"], [3, "c"]], schema=["id", "val"]) @@ -380,7 +380,7 @@ def save_as_table( [Row(ID=1, VAL='a'), Row(ID=2, VAL='b'), Row(ID=3, VAL='c')] >>> new_df = session.create_dataframe([[2, "updated2"], [5, "updated5"]], schema=["id", "val"]) - >>> new_df.write.mode("append").save_as_table("my_table", override_condition="id = 1 or val = 'b'") + >>> new_df.write.mode("append").save_as_table("my_table", overwrite_condition="id = 1 or val = 'b'") >>> session.table("my_table").order_by("id").collect() [Row(ID=2, VAL='updated2'), Row(ID=3, VAL='c'), Row(ID=5, VAL='updated5')] """ @@ -505,33 +505,36 @@ def save_as_table( f"Unsupported table type. Expected table types: {SUPPORTED_TABLE_TYPES}" ) - # override_condition must be used with APPEND mode - if override_condition is not None and save_mode != SaveMode.APPEND: + # overwrite_condition must be used with APPEND or OVERWRITE mode + if overwrite_condition is not None and save_mode not in ( + SaveMode.APPEND, + SaveMode.OVERWRITE, + ): raise ValueError( - f"'override_condition' is only supported with mode='append'. " + f"'overwrite_condition' is only supported with mode='append' or mode='overwrite'. " f"Got mode='{save_mode.value}'." ) - override_condition_expr = ( + overwrite_condition_expr = ( _to_col_if_sql_expr( - override_condition, "DataFrameWriter.save_as_table" + overwrite_condition, "DataFrameWriter.save_as_table" )._expression - if override_condition is not None + if overwrite_condition is not None else None ) session = self._dataframe._session + needs_table_exists_check = save_mode in [ + SaveMode.APPEND, + SaveMode.TRUNCATE, + ] or (save_mode == SaveMode.OVERWRITE and overwrite_condition is not None) if ( table_exists is None and not isinstance(session._conn, MockServerConnection) - and save_mode - in [ - SaveMode.APPEND, - SaveMode.TRUNCATE, - ] + and needs_table_exists_check ): # whether the table already exists in the database - # determines the compiled SQL for APPEND and TRUNCATE mode + # determines the compiled SQL for APPEND, TRUNCATE, and OVERWRITE with overwrite_condition # if the table does not exist, we need to create it first; # if the table exists, we can skip the creation step and insert data directly table_exists = session._table_exists(table_name) @@ -552,7 +555,7 @@ def save_as_table( copy_grants, iceberg_config, table_exists, - override_condition_expr, + overwrite_condition_expr, ) snowflake_plan = session._analyzer.resolve(create_table_logic_plan) result = session._conn.execute( diff --git a/tests/integ/test_dataframe.py b/tests/integ/test_dataframe.py index 239a715cb0..229dc04bde 100644 --- a/tests/integ/test_dataframe.py +++ b/tests/integ/test_dataframe.py @@ -4568,11 +4568,11 @@ def test_write_table_with_clustering_keys_and_comment( @pytest.mark.xfail( "config.getoption('local_testing_mode', default=False)", - reason="override_condition is a SQL feature", + reason="overwrite_condition is a SQL feature", run=False, ) -def test_write_table_with_override_condition(session): - """Test override_condition parameter for targeted delete + insert.""" +def test_write_table_with_overwrite_condition(session): + """Test overwrite_condition parameter for targeted delete + insert.""" table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) try: # Setup and verify initial data @@ -4590,7 +4590,7 @@ def test_write_table_with_override_condition(session): Row(ID=3, VAL="c"), ] - # Test 1: override_condition with SQL string expr + # Test 1: overwrite_condition with SQL string expr new_df1 = session.create_dataframe( [[2, "updated2"], [5, "new5"]], schema=StructType( @@ -4598,7 +4598,7 @@ def test_write_table_with_override_condition(session): ), ) new_df1.write.mode("append").save_as_table( - table_name, override_condition="id = 1 or val = 'b'" + table_name, overwrite_condition="id = 1 or val = 'b'" ) result = session.table(table_name).order_by("id").collect() # id=1 and id=2 (val='b') deleted, new rows inserted @@ -4608,7 +4608,7 @@ def test_write_table_with_override_condition(session): Row(ID=5, VAL="new5"), ] - # Test 2: override_condition with Column expr + # Test 2: overwrite_condition with Column expr new_df2 = session.create_dataframe( [[2, "replaced2"], [4, "new4"]], schema=StructType( @@ -4616,7 +4616,7 @@ def test_write_table_with_override_condition(session): ), ) new_df2.write.mode("append").save_as_table( - table_name, override_condition=col("id") == 2 + table_name, overwrite_condition=col("id") == 2 ) result = session.table(table_name).order_by("id").collect() # id=2 deleted, new rows inserted @@ -4627,7 +4627,7 @@ def test_write_table_with_override_condition(session): Row(ID=5, VAL="new5"), ] - # Test 3: override_condition with multiple Column expr + # Test 3: overwrite_condition with multiple Column expr new_df3 = session.create_dataframe( [[6, "new6"]], schema=StructType( @@ -4635,7 +4635,7 @@ def test_write_table_with_override_condition(session): ), ) new_df3.write.mode("append").save_as_table( - table_name, override_condition=(col("id") > 4) | (col("val") == "c") + table_name, overwrite_condition=(col("id") > 4) | (col("val") == "c") ) result = session.table(table_name).order_by("id").collect() # id=3 (val='c') and id=5 (id > 4) deleted, id=4 remains (4 is not > 4), new row inserted @@ -4645,7 +4645,7 @@ def test_write_table_with_override_condition(session): Row(ID=6, VAL="new6"), ] - # Test 4: override_condition that matches all rows + # Test 4: overwrite_condition that matches all rows new_df4 = session.create_dataframe( [[10, "new"]], schema=StructType( @@ -4653,12 +4653,12 @@ def test_write_table_with_override_condition(session): ), ) new_df4.write.mode("append").save_as_table( - table_name, override_condition="id > 0" + table_name, overwrite_condition="id > 0" ) result = session.table(table_name).collect() assert result == [Row(ID=10, VAL="new")] - # Test 5: override_condition that matches no rows + # Test 5: overwrite_condition that matches no rows new_df5 = session.create_dataframe( [[20, "another"]], schema=StructType( @@ -4666,7 +4666,7 @@ def test_write_table_with_override_condition(session): ), ) new_df5.write.mode("append").save_as_table( - table_name, override_condition="id = 999" + table_name, overwrite_condition="id = 999" ) result = session.table(table_name).order_by("id").collect() assert result == [ @@ -4674,40 +4674,58 @@ def test_write_table_with_override_condition(session): Row(ID=20, VAL="another"), ] + # Test 6: overwrite_condition with mode="overwrite" (selective overwrite) + new_df6 = session.create_dataframe( + [[10, "replaced10"], [30, "new30"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df6.write.mode("overwrite").save_as_table( + table_name, overwrite_condition=col("id") == 10 + ) + result = session.table(table_name).order_by("id").collect() + # id=10 deleted, new rows inserted, id=20 preserved + assert result == [ + Row(ID=10, VAL="replaced10"), + Row(ID=20, VAL="another"), + Row(ID=30, VAL="new30"), + ] + finally: Utils.drop_table(session, table_name) @pytest.mark.xfail( "config.getoption('local_testing_mode', default=False)", - reason="override_condition is a SQL feature", + reason="overwrite_condition is a SQL feature", run=False, ) -@pytest.mark.parametrize( - "invalid_mode", ["overwrite", "truncate", "errorifexists", "ignore"] -) -def test_write_table_with_override_condition_edge_cases(session, invalid_mode): - """Test override_condition edge cases: table not exists, and invalid modes.""" +@pytest.mark.parametrize("invalid_mode", ["truncate", "errorifexists", "ignore"]) +def test_write_table_with_overwrite_condition_edge_cases(session, invalid_mode): + """Test overwrite_condition edge cases: table not exists, and invalid modes.""" table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) try: - # Edge case 1: Table doesn't exist - override_condition is no-op + # Edge case 1: Table doesn't exist - overwrite_condition is no-op df = session.create_dataframe( [[1, "a"], [2, "b"]], schema=StructType( [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - df.write.mode("append").save_as_table(table_name, override_condition="id = 999") + df.write.mode("append").save_as_table( + table_name, overwrite_condition="id = 999" + ) result = session.table(table_name).order_by("id").collect() assert result == [Row(ID=1, VAL="a"), Row(ID=2, VAL="b")] # Edge case 2: Invalid mode raises ValueError with pytest.raises( ValueError, - match="'override_condition' is only supported with mode='append'", + match="'overwrite_condition' is only supported with mode='append' or mode='overwrite'", ): df.write.mode(invalid_mode).save_as_table( - table_name, override_condition="id = 1" + table_name, overwrite_condition="id = 1" ) finally: From 7a463f90a4c71d0b01347b485f61331ffc9f0f26 Mon Sep 17 00:00:00 2001 From: May Liu Date: Wed, 10 Dec 2025 18:42:01 -0800 Subject: [PATCH 3/5] Add AST support --- .../snowpark/_internal/proto/ast.proto | 32 +- src/snowflake/snowpark/dataframe_writer.py | 8 + tests/ast/data/DataFrame.write.test | 1402 ++++++++++++----- 3 files changed, 1007 insertions(+), 435 deletions(-) diff --git a/src/snowflake/snowpark/_internal/proto/ast.proto b/src/snowflake/snowpark/_internal/proto/ast.proto index 9d32c71d7c..b3f792d4c4 100644 --- a/src/snowflake/snowpark/_internal/proto/ast.proto +++ b/src/snowflake/snowpark/_internal/proto/ast.proto @@ -1,5 +1,5 @@ // N.B. This file is generated by `//ir-dsl-c`. DO NOT EDIT! -// Generated from `{git@github.com:snowflakedb/snowflake.git}/Snowpark/ast`. +// Generated from `{git@github.com:snowflake-eng/snowflake.git}/Snowpark/ast`. syntax = "proto3"; @@ -987,7 +987,7 @@ message DataframeCollect { repeated Tuple_String_String statement_params = 7; } -// dataframe-io.ir:165 +// dataframe-io.ir:167 message DataframeCopyIntoTable { repeated Tuple_String_Expr copy_options = 1; Expr df = 2; @@ -1011,7 +1011,7 @@ message DataframeCount { repeated Tuple_String_String statement_params = 4; } -// dataframe-io.ir:148 +// dataframe-io.ir:150 message DataframeCreateOrReplaceDynamicTable { repeated Expr clustering_keys = 1; google.protobuf.StringValue comment = 2; @@ -1030,7 +1030,7 @@ message DataframeCreateOrReplaceDynamicTable { string warehouse = 15; } -// dataframe-io.ir:139 +// dataframe-io.ir:141 message DataframeCreateOrReplaceView { google.protobuf.StringValue comment = 1; bool copy_grants = 2; @@ -2685,7 +2685,7 @@ message WindowSpecRowsBetween { WindowSpecExpr wnd = 4; } -// dataframe-io.ir:116 +// dataframe-io.ir:118 message WriteCopyIntoLocation { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2704,7 +2704,7 @@ message WriteCopyIntoLocation { Expr writer = 15; } -// dataframe-io.ir:123 +// dataframe-io.ir:125 message WriteCsv { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2731,7 +2731,7 @@ message WriteFile { } } -// dataframe-io.ir:129 +// dataframe-io.ir:131 message WriteInsertInto { bool overwrite = 1; SrcPosition src = 2; @@ -2739,7 +2739,7 @@ message WriteInsertInto { Expr writer = 4; } -// dataframe-io.ir:125 +// dataframe-io.ir:127 message WriteJson { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2773,7 +2773,7 @@ message WritePandas { string table_type = 13; } -// dataframe-io.ir:127 +// dataframe-io.ir:129 message WriteParquet { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2790,7 +2790,7 @@ message WriteParquet { Expr writer = 13; } -// dataframe-io.ir:121 +// dataframe-io.ir:123 message WriteSave { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2821,9 +2821,11 @@ message WriteTable { repeated Tuple_String_Expr iceberg_config = 10; google.protobuf.Int64Value max_data_extension_time = 11; SaveMode mode = 12; - SrcPosition src = 13; - repeated Tuple_String_String statement_params = 14; - NameRef table_name = 15; - string table_type = 16; - Expr writer = 17; + Expr overwrite_condition = 13; + SrcPosition src = 14; + repeated Tuple_String_String statement_params = 15; + google.protobuf.BoolValue table_exists = 16; + NameRef table_name = 17; + string table_type = 18; + Expr writer = 19; } diff --git a/src/snowflake/snowpark/dataframe_writer.py b/src/snowflake/snowpark/dataframe_writer.py index 8bc1ee1df2..ed1d14c76b 100644 --- a/src/snowflake/snowpark/dataframe_writer.py +++ b/src/snowflake/snowpark/dataframe_writer.py @@ -411,6 +411,8 @@ def save_as_table( # change_tracking: Optional[bool] = None, # copy_grants: bool = False, # iceberg_config: Optional[dict] = None, + # table_exists: Optional[bool] = None, + # overwrite_condition: Optional[ColumnOrSqlExpr] = None, build_table_name(expr.table_name, table_name) @@ -452,6 +454,12 @@ def save_as_table( t = expr.iceberg_config.add() t._1 = k build_expr_from_python_val(t._2, v) + if table_exists is not None: + expr.table_exists.value = table_exists + if overwrite_condition is not None: + build_expr_from_snowpark_column_or_sql_str( + expr.overwrite_condition, overwrite_condition + ) self._dataframe._session._ast_batch.eval(stmt) diff --git a/tests/ast/data/DataFrame.write.test b/tests/ast/data/DataFrame.write.test index 4a67d868fc..c65bc3c398 100644 --- a/tests/ast/data/DataFrame.write.test +++ b/tests/ast/data/DataFrame.write.test @@ -24,6 +24,14 @@ df.write.mode("overwrite").save_as_table("iceberg_table_target_size", iceberg_co df.write.mode("overwrite").save_as_table("iceberg_table_full", iceberg_config={"external_volume": "example_volume", "partition_by": [bucket(10, "A"), year("date_col")], "target_file_size": "AUTO", "catalog": "my_catalog"}) +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition=col("A")==10) + +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition="B = 100") + +df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition=(col("A") > 4) | (col("B") == "c")) + +df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition="A = 1 or B = 'b'") + df.write.mode("truncate").save_as_table("test_destination", column_order="name", create_temp_table=False, table_type="transient", clustering_keys=['STR', col('num1')], comment="test", block=True, statement_params={"k":"v"}) df.write.partition_by("value").mode("overwrite").save_as_table("saved_table", table_type="temporary") @@ -130,6 +138,22 @@ df.write.mode("overwrite").save_as_table("iceberg_table_full", iceberg_config={" df = session.table("table1") +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition=col("A") == 10) + +df = session.table("table1") + +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition="B = 100") + +df = session.table("table1") + +df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition=(col("A") > 4) | (col("B") == "c")) + +df = session.table("table1") + +df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition="A = 1 or B = 'b'") + +df = session.table("table1") + df.write.mode("truncate").save_as_table("test_destination", column_order="name", table_type="transient", clustering_keys=["STR", col("num1")], statement_params={"k": "v"}, comment="test") df = session.table("table1") @@ -1849,6 +1873,544 @@ body { uid: 1 } } +body { + bind { + expr { + write_table { + block: true + column_order: "index" + overwrite_condition { + eq { + lhs { + apply_expr { + fn { + builtin_fn { + name { + name { + name_flat { + name: "col" + } + } + } + } + } + pos_args { + string_val { + src { + end_column: 116 + end_line: 49 + file: 2 + start_column: 108 + start_line: 49 + } + v: "A" + } + } + src { + end_column: 116 + end_line: 49 + file: 2 + start_column: 108 + start_line: 49 + } + } + } + rhs { + int64_val { + src { + end_column: 120 + end_line: 49 + file: 2 + start_column: 108 + start_line: 49 + } + v: 10 + } + } + src { + end_column: 120 + end_line: 49 + file: 2 + start_column: 108 + start_line: 49 + } + } + } + src { + end_column: 121 + end_line: 49 + file: 2 + start_column: 8 + start_line: 49 + } + table_name { + name { + name_flat { + name: "saved_table" + } + } + } + table_type: "temporary" + writer { + dataframe_writer { + df { + dataframe_ref { + id: 1 + } + } + save_mode { + save_mode_overwrite: true + } + src { + end_column: 16 + end_line: 29 + file: 2 + start_column: 8 + start_line: 29 + } + } + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + } + uid: 12 + } +} +body { + eval { + bind_id: 12 + } +} +body { + bind { + expr { + table { + name { + name { + name_flat { + name: "table1" + } + } + } + src { + end_column: 41 + end_line: 27 + file: 2 + start_column: 13 + start_line: 27 + } + variant { + session_table: true + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + value: "df" + } + uid: 1 + } +} +body { + bind { + expr { + write_table { + block: true + column_order: "index" + overwrite_condition { + sql_expr { + sql: "B = 100" + src { + end_column: 118 + end_line: 51 + file: 2 + start_column: 8 + start_line: 51 + } + } + } + src { + end_column: 118 + end_line: 51 + file: 2 + start_column: 8 + start_line: 51 + } + table_name { + name { + name_flat { + name: "saved_table" + } + } + } + table_type: "temporary" + writer { + dataframe_writer { + df { + dataframe_ref { + id: 1 + } + } + save_mode { + save_mode_overwrite: true + } + src { + end_column: 16 + end_line: 29 + file: 2 + start_column: 8 + start_line: 29 + } + } + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + } + uid: 13 + } +} +body { + eval { + bind_id: 13 + } +} +body { + bind { + expr { + table { + name { + name { + name_flat { + name: "table1" + } + } + } + src { + end_column: 41 + end_line: 27 + file: 2 + start_column: 13 + start_line: 27 + } + variant { + session_table: true + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + value: "df" + } + uid: 1 + } +} +body { + bind { + expr { + write_table { + block: true + column_order: "index" + overwrite_condition { + or { + lhs { + gt { + lhs { + apply_expr { + fn { + builtin_fn { + name { + name { + name_flat { + name: "col" + } + } + } + } + } + pos_args { + string_val { + src { + end_column: 114 + end_line: 53 + file: 2 + start_column: 106 + start_line: 53 + } + v: "A" + } + } + src { + end_column: 114 + end_line: 53 + file: 2 + start_column: 106 + start_line: 53 + } + } + } + rhs { + int64_val { + src { + end_column: 118 + end_line: 53 + file: 2 + start_column: 106 + start_line: 53 + } + v: 4 + } + } + src { + end_column: 118 + end_line: 53 + file: 2 + start_column: 106 + start_line: 53 + } + } + } + rhs { + eq { + lhs { + apply_expr { + fn { + builtin_fn { + name { + name { + name_flat { + name: "col" + } + } + } + } + } + pos_args { + string_val { + src { + end_column: 131 + end_line: 53 + file: 2 + start_column: 123 + start_line: 53 + } + v: "B" + } + } + src { + end_column: 131 + end_line: 53 + file: 2 + start_column: 123 + start_line: 53 + } + } + } + rhs { + string_val { + src { + end_column: 138 + end_line: 53 + file: 2 + start_column: 123 + start_line: 53 + } + v: "c" + } + } + src { + end_column: 138 + end_line: 53 + file: 2 + start_column: 123 + start_line: 53 + } + } + } + src { + end_column: 139 + end_line: 53 + file: 2 + start_column: 105 + start_line: 53 + } + } + } + src { + end_column: 140 + end_line: 53 + file: 2 + start_column: 8 + start_line: 53 + } + table_name { + name { + name_flat { + name: "saved_table" + } + } + } + table_type: "temporary" + writer { + dataframe_writer { + df { + dataframe_ref { + id: 1 + } + } + save_mode { + save_mode_append: true + } + src { + end_column: 16 + end_line: 29 + file: 2 + start_column: 8 + start_line: 29 + } + } + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + } + uid: 14 + } +} +body { + eval { + bind_id: 14 + } +} +body { + bind { + expr { + table { + name { + name { + name_flat { + name: "table1" + } + } + } + src { + end_column: 41 + end_line: 27 + file: 2 + start_column: 13 + start_line: 27 + } + variant { + session_table: true + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + value: "df" + } + uid: 1 + } +} +body { + bind { + expr { + write_table { + block: true + column_order: "index" + overwrite_condition { + sql_expr { + sql: "A = 1 or B = \'b\'" + src { + end_column: 124 + end_line: 55 + file: 2 + start_column: 8 + start_line: 55 + } + } + } + src { + end_column: 124 + end_line: 55 + file: 2 + start_column: 8 + start_line: 55 + } + table_name { + name { + name_flat { + name: "saved_table" + } + } + } + table_type: "temporary" + writer { + dataframe_writer { + df { + dataframe_ref { + id: 1 + } + } + save_mode { + save_mode_append: true + } + src { + end_column: 16 + end_line: 29 + file: 2 + start_column: 8 + start_line: 29 + } + } + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + } + uid: 15 + } +} +body { + eval { + bind_id: 15 + } +} +body { + bind { + expr { + table { + name { + name { + name_flat { + name: "table1" + } + } + } + src { + end_column: 41 + end_line: 27 + file: 2 + start_column: 13 + start_line: 27 + } + variant { + session_table: true + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + value: "df" + } + uid: 1 + } +} body { bind { expr { @@ -1858,10 +2420,10 @@ body { string_val { src { end_column: 231 - end_line: 49 + end_line: 57 file: 2 start_column: 8 - start_line: 49 + start_line: 57 } v: "STR" } @@ -1883,20 +2445,20 @@ body { string_val { src { end_column: 173 - end_line: 49 + end_line: 57 file: 2 start_column: 162 - start_line: 49 + start_line: 57 } v: "num1" } } src { end_column: 173 - end_line: 49 + end_line: 57 file: 2 start_column: 162 - start_line: 49 + start_line: 57 } } } @@ -1906,10 +2468,10 @@ body { } src { end_column: 231 - end_line: 49 + end_line: 57 file: 2 start_column: 8 - start_line: 49 + start_line: 57 } statement_params { _1: "k" @@ -1947,12 +2509,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 12 + uid: 16 } } body { eval { - bind_id: 12 + bind_id: 16 } } body { @@ -1993,10 +2555,10 @@ body { column_order: "index" src { end_column: 109 - end_line: 51 + end_line: 59 file: 2 start_column: 8 - start_line: 51 + start_line: 59 } table_name { name { @@ -2018,10 +2580,10 @@ body { sql: "value" src { end_column: 38 - end_line: 51 + end_line: 59 file: 2 start_column: 8 - start_line: 51 + start_line: 59 } } } @@ -2042,12 +2604,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 13 + uid: 17 } } body { eval { - bind_id: 13 + bind_id: 17 } } body { @@ -2088,10 +2650,10 @@ body { column_order: "index" src { end_column: 114 - end_line: 53 + end_line: 61 file: 2 start_column: 8 - start_line: 53 + start_line: 61 } table_name { name { @@ -2125,20 +2687,20 @@ body { string_val { src { end_column: 42 - end_line: 53 + end_line: 61 file: 2 start_column: 30 - start_line: 53 + start_line: 61 } v: "value" } } src { end_column: 42 - end_line: 53 + end_line: 61 file: 2 start_column: 30 - start_line: 53 + start_line: 61 } } } @@ -2159,12 +2721,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 14 + uid: 18 } } body { eval { - bind_id: 14 + bind_id: 18 } } body { @@ -2205,10 +2767,10 @@ body { column_order: "index" src { end_column: 151 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } table_name { name { @@ -2231,10 +2793,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -2257,20 +2819,20 @@ body { string_val { src { end_column: 79 - end_line: 55 + end_line: 63 file: 2 start_column: 67 - start_line: 55 + start_line: 63 } v: "value" } } src { end_column: 79 - end_line: 55 + end_line: 63 file: 2 start_column: 67 - start_line: 55 + start_line: 63 } } } @@ -2291,12 +2853,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 15 + uid: 19 } } body { eval { - bind_id: 15 + bind_id: 19 } } body { @@ -2337,10 +2899,10 @@ body { column_order: "index" src { end_column: 239 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } table_name { name { @@ -2363,10 +2925,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -2378,10 +2940,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -2393,10 +2955,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -2408,10 +2970,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -2423,10 +2985,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -2449,20 +3011,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -2483,12 +3045,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 16 + uid: 20 } } body { eval { - bind_id: 16 + bind_id: 20 } } body { @@ -2498,10 +3060,10 @@ body { query: "create temp stage if not exists test_stage" src { end_column: 88 - end_line: 59 + end_line: 67 file: 2 start_column: 31 - start_line: 59 + start_line: 67 } } } @@ -2509,7 +3071,7 @@ body { symbol { value: "stage_created_result" } - uid: 17 + uid: 21 } } body { @@ -2520,27 +3082,27 @@ body { case_sensitive: true df { dataframe_ref { - id: 17 + id: 21 } } src { end_column: 98 - end_line: 59 + end_line: 67 file: 2 start_column: 31 - start_line: 59 + start_line: 67 } } } first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 18 + uid: 22 } } body { eval { - bind_id: 18 + bind_id: 22 } } body { @@ -2581,10 +3143,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 72 - end_line: 61 + end_line: 69 file: 2 start_column: 8 - start_line: 61 + start_line: 69 } writer { dataframe_writer { @@ -2599,10 +3161,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -2614,10 +3176,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -2629,10 +3191,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -2644,10 +3206,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -2659,10 +3221,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -2685,20 +3247,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -2719,12 +3281,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 19 + uid: 23 } } body { eval { - bind_id: 19 + bind_id: 23 } } body { @@ -2768,10 +3330,10 @@ body { bool_val { src { end_column: 123 - end_line: 65 + end_line: 73 file: 2 start_column: 8 - start_line: 65 + start_line: 73 } v: true } @@ -2783,10 +3345,10 @@ body { bool_val { src { end_column: 123 - end_line: 65 + end_line: 73 file: 2 start_column: 8 - start_line: 65 + start_line: 73 } v: true } @@ -2799,10 +3361,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 123 - end_line: 65 + end_line: 73 file: 2 start_column: 8 - start_line: 65 + start_line: 73 } writer { dataframe_writer { @@ -2817,10 +3379,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -2832,10 +3394,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -2847,10 +3409,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -2862,10 +3424,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -2877,10 +3439,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -2903,20 +3465,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -2937,12 +3499,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 20 + uid: 24 } } body { eval { - bind_id: 20 + bind_id: 24 } } body { @@ -2985,10 +3547,10 @@ body { bool_val { src { end_column: 181 - end_line: 67 + end_line: 75 file: 2 start_column: 8 - start_line: 67 + start_line: 75 } v: true } @@ -3000,10 +3562,10 @@ body { bool_val { src { end_column: 181 - end_line: 67 + end_line: 75 file: 2 start_column: 8 - start_line: 67 + start_line: 75 } } } @@ -3018,10 +3580,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 181 - end_line: 67 + end_line: 75 file: 2 start_column: 8 - start_line: 67 + start_line: 75 } writer { dataframe_writer { @@ -3036,10 +3598,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -3051,10 +3613,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -3066,10 +3628,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -3081,10 +3643,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -3096,10 +3658,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -3122,20 +3684,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -3156,12 +3718,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 21 + uid: 25 } } body { eval { - bind_id: 21 + bind_id: 25 } } body { @@ -3204,10 +3766,10 @@ body { bool_val { src { end_column: 207 - end_line: 69 + end_line: 77 file: 2 start_column: 8 - start_line: 69 + start_line: 77 } v: true } @@ -3219,10 +3781,10 @@ body { bool_val { src { end_column: 207 - end_line: 69 + end_line: 77 file: 2 start_column: 8 - start_line: 69 + start_line: 77 } } } @@ -3241,10 +3803,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 207 - end_line: 69 + end_line: 77 file: 2 start_column: 8 - start_line: 69 + start_line: 77 } writer { dataframe_writer { @@ -3259,10 +3821,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -3274,10 +3836,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -3289,10 +3851,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -3304,10 +3866,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -3319,10 +3881,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -3345,20 +3907,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -3379,12 +3941,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 22 + uid: 26 } } body { eval { - bind_id: 22 + bind_id: 26 } } body { @@ -3428,10 +3990,10 @@ body { bool_val { src { end_column: 125 - end_line: 73 + end_line: 81 file: 2 start_column: 8 - start_line: 73 + start_line: 81 } v: true } @@ -3443,10 +4005,10 @@ body { bool_val { src { end_column: 125 - end_line: 73 + end_line: 81 file: 2 start_column: 8 - start_line: 73 + start_line: 81 } v: true } @@ -3460,10 +4022,10 @@ body { location: "@test_stage/test.csv" src { end_column: 125 - end_line: 73 + end_line: 81 file: 2 start_column: 8 - start_line: 73 + start_line: 81 } writer { dataframe_writer { @@ -3478,10 +4040,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -3493,10 +4055,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -3508,10 +4070,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -3523,10 +4085,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -3538,10 +4100,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -3564,20 +4126,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -3598,12 +4160,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 23 + uid: 27 } } body { eval { - bind_id: 23 + bind_id: 27 } } body { @@ -3648,10 +4210,10 @@ body { location: "@test_stage/test.csv" src { end_column: 139 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } writer { dataframe_writer { @@ -3669,10 +4231,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -3684,10 +4246,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -3699,10 +4261,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -3714,10 +4276,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -3729,10 +4291,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -3744,10 +4306,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -3770,20 +4332,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -3804,12 +4366,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 24 + uid: 28 } } body { eval { - bind_id: 24 + bind_id: 28 } } body { @@ -3853,10 +4415,10 @@ body { bool_val { src { end_column: 114 - end_line: 79 + end_line: 87 file: 2 start_column: 8 - start_line: 79 + start_line: 87 } v: true } @@ -3868,10 +4430,10 @@ body { bool_val { src { end_column: 114 - end_line: 79 + end_line: 87 file: 2 start_column: 8 - start_line: 79 + start_line: 87 } v: true } @@ -3884,10 +4446,10 @@ body { location: "@test_stage/test.json" src { end_column: 114 - end_line: 79 + end_line: 87 file: 2 start_column: 8 - start_line: 79 + start_line: 87 } writer { dataframe_writer { @@ -3905,10 +4467,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -3920,10 +4482,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -3935,10 +4497,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -3950,10 +4512,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -3965,10 +4527,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -3980,10 +4542,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -4006,20 +4568,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -4040,12 +4602,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 25 + uid: 29 } } body { eval { - bind_id: 25 + bind_id: 29 } } body { @@ -4090,10 +4652,10 @@ body { location: "@test_stage/test.json" src { end_column: 117 - end_line: 81 + end_line: 89 file: 2 start_column: 8 - start_line: 81 + start_line: 89 } writer { dataframe_writer { @@ -4111,10 +4673,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -4126,10 +4688,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -4141,10 +4703,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -4156,10 +4718,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -4171,10 +4733,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -4186,10 +4748,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -4212,20 +4774,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -4246,12 +4808,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 26 + uid: 30 } } body { eval { - bind_id: 26 + bind_id: 30 } } body { @@ -4295,10 +4857,10 @@ body { bool_val { src { end_column: 130 - end_line: 85 + end_line: 93 file: 2 start_column: 8 - start_line: 85 + start_line: 93 } v: true } @@ -4310,10 +4872,10 @@ body { bool_val { src { end_column: 130 - end_line: 85 + end_line: 93 file: 2 start_column: 8 - start_line: 85 + start_line: 93 } v: true } @@ -4326,10 +4888,10 @@ body { location: "@test_stage/test.parquet" src { end_column: 130 - end_line: 85 + end_line: 93 file: 2 start_column: 8 - start_line: 85 + start_line: 93 } writer { dataframe_writer { @@ -4347,10 +4909,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -4362,10 +4924,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -4377,10 +4939,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -4392,10 +4954,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -4407,10 +4969,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -4422,10 +4984,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -4448,20 +5010,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -4482,12 +5044,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 27 + uid: 31 } } body { eval { - bind_id: 27 + bind_id: 31 } } body { @@ -4532,10 +5094,10 @@ body { location: "@test_stage/test.parquet" src { end_column: 120 - end_line: 87 + end_line: 95 file: 2 start_column: 8 - start_line: 87 + start_line: 95 } writer { dataframe_writer { @@ -4553,10 +5115,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -4568,10 +5130,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -4583,10 +5145,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -4598,10 +5160,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -4613,10 +5175,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -4628,10 +5190,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -4654,20 +5216,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -4688,12 +5250,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 28 + uid: 32 } } body { eval { - bind_id: 28 + bind_id: 32 } } body { @@ -4733,10 +5295,10 @@ body { overwrite: true src { end_column: 59 - end_line: 89 + end_line: 97 file: 2 start_column: 8 - start_line: 89 + start_line: 97 } table_name { name { @@ -4761,10 +5323,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -4776,10 +5338,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -4791,10 +5353,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -4806,10 +5368,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -4821,10 +5383,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -4836,10 +5398,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -4862,20 +5424,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -4896,12 +5458,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 29 + uid: 33 } } body { eval { - bind_id: 29 + bind_id: 33 } } body { @@ -4940,10 +5502,10 @@ body { write_insert_into { src { end_column: 60 - end_line: 91 + end_line: 99 file: 2 start_column: 8 - start_line: 91 + start_line: 99 } table_name { name { @@ -4968,10 +5530,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 63 file: 2 start_column: 8 - start_line: 55 + start_line: 63 } v: "abcd" } @@ -4983,10 +5545,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "abcd" } @@ -4998,10 +5560,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "pqrs" } @@ -5013,10 +5575,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: true } @@ -5028,10 +5590,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 65 file: 2 start_column: 8 - start_line: 57 + start_line: 65 } v: "append" } @@ -5043,10 +5605,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 83 file: 2 start_column: 8 - start_line: 75 + start_line: 83 } v: true } @@ -5069,20 +5631,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 65 file: 2 start_column: 155 - start_line: 57 + start_line: 65 } } } @@ -5103,12 +5665,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 30 + uid: 34 } } body { eval { - bind_id: 30 + bind_id: 34 } } body { @@ -5124,10 +5686,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5138,7 +5700,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5149,10 +5711,10 @@ body { location: "s3://testbucket" src { end_column: 94 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } storage_integration { value: "test_integration" @@ -5161,15 +5723,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5178,12 +5740,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 32 + uid: 36 } } body { eval { - bind_id: 32 + bind_id: 36 } } body { @@ -5199,10 +5761,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5213,7 +5775,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5224,10 +5786,10 @@ body { location: "azure://testcontainer" src { end_column: 131 - end_line: 97 + end_line: 105 file: 2 start_column: 8 - start_line: 97 + start_line: 105 } storage_integration { value: "test_integration" @@ -5239,15 +5801,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5256,12 +5818,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 33 + uid: 37 } } body { eval { - bind_id: 33 + bind_id: 37 } } body { @@ -5277,10 +5839,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5291,7 +5853,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5302,24 +5864,24 @@ body { location: "gcp://teststorage" src { end_column: 87 - end_line: 99 + end_line: 107 file: 2 start_column: 8 - start_line: 99 + start_line: 107 } writer { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5328,12 +5890,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 34 + uid: 38 } } body { eval { - bind_id: 34 + bind_id: 38 } } body { @@ -5349,10 +5911,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5363,7 +5925,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5377,10 +5939,10 @@ body { string_val { src { end_column: 174 - end_line: 101 + end_line: 109 file: 2 start_column: 8 - start_line: 101 + start_line: 109 } v: "awsid" } @@ -5392,10 +5954,10 @@ body { string_val { src { end_column: 174 - end_line: 101 + end_line: 109 file: 2 start_column: 8 - start_line: 101 + start_line: 109 } v: "!23@" } @@ -5407,10 +5969,10 @@ body { null_val { src { end_column: 174 - end_line: 101 + end_line: 109 file: 2 start_column: 8 - start_line: 101 + start_line: 109 } } } @@ -5421,10 +5983,10 @@ body { string_val { src { end_column: 174 - end_line: 101 + end_line: 109 file: 2 start_column: 8 - start_line: 101 + start_line: 109 } v: "master key" } @@ -5433,24 +5995,24 @@ body { location: "gcp://teststorage" src { end_column: 174 - end_line: 101 + end_line: 109 file: 2 start_column: 8 - start_line: 101 + start_line: 109 } writer { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5459,12 +6021,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 35 + uid: 39 } } body { eval { - bind_id: 35 + bind_id: 39 } } body { @@ -5480,10 +6042,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5494,7 +6056,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5505,10 +6067,10 @@ body { location: "azure://testcontainer" src { end_column: 122 - end_line: 103 + end_line: 111 file: 2 start_column: 8 - start_line: 103 + start_line: 111 } storage_integration { value: "test_integration" @@ -5520,15 +6082,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5537,12 +6099,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 36 + uid: 40 } } body { eval { - bind_id: 36 + bind_id: 40 } } body { @@ -5558,10 +6120,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5572,7 +6134,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5586,10 +6148,10 @@ body { string_val { src { end_column: 158 - end_line: 105 + end_line: 113 file: 2 start_column: 8 - start_line: 105 + start_line: 113 } v: "awsid" } @@ -5601,10 +6163,10 @@ body { string_val { src { end_column: 158 - end_line: 105 + end_line: 113 file: 2 start_column: 8 - start_line: 105 + start_line: 113 } v: "!23@" } @@ -5616,10 +6178,10 @@ body { null_val { src { end_column: 158 - end_line: 105 + end_line: 113 file: 2 start_column: 8 - start_line: 105 + start_line: 113 } } } @@ -5630,10 +6192,10 @@ body { string_val { src { end_column: 158 - end_line: 105 + end_line: 113 file: 2 start_column: 8 - start_line: 105 + start_line: 113 } v: "master key" } @@ -5642,24 +6204,24 @@ body { location: "s3://testbucket" src { end_column: 158 - end_line: 105 + end_line: 113 file: 2 start_column: 8 - start_line: 105 + start_line: 113 } writer { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5668,12 +6230,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 37 + uid: 41 } } body { eval { - bind_id: 37 + bind_id: 41 } } body { @@ -5689,10 +6251,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5703,7 +6265,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5714,10 +6276,10 @@ body { location: "azure://testcontainer" src { end_column: 149 - end_line: 107 + end_line: 115 file: 2 start_column: 8 - start_line: 107 + start_line: 115 } storage_integration { value: "test_integration" @@ -5729,15 +6291,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5746,12 +6308,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 38 + uid: 42 } } body { eval { - bind_id: 38 + bind_id: 42 } } body { @@ -5767,10 +6329,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 101 file: 2 start_column: 13 - start_line: 93 + start_line: 101 } variant { session_table: true @@ -5781,7 +6343,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 35 } } body { @@ -5795,10 +6357,10 @@ body { string_val { src { end_column: 237 - end_line: 109 + end_line: 117 file: 2 start_column: 8 - start_line: 109 + start_line: 117 } v: "awsid" } @@ -5810,10 +6372,10 @@ body { string_val { src { end_column: 237 - end_line: 109 + end_line: 117 file: 2 start_column: 8 - start_line: 109 + start_line: 117 } v: "!23@" } @@ -5825,10 +6387,10 @@ body { null_val { src { end_column: 237 - end_line: 109 + end_line: 117 file: 2 start_column: 8 - start_line: 109 + start_line: 117 } } } @@ -5839,10 +6401,10 @@ body { string_val { src { end_column: 237 - end_line: 109 + end_line: 117 file: 2 start_column: 8 - start_line: 109 + start_line: 117 } v: "master key" } @@ -5854,19 +6416,19 @@ body { sql: "value" src { end_column: 237 - end_line: 109 + end_line: 117 file: 2 start_column: 8 - start_line: 109 + start_line: 117 } } } src { end_column: 237 - end_line: 109 + end_line: 117 file: 2 start_column: 8 - start_line: 109 + start_line: 117 } storage_integration { value: "test_integration" @@ -5875,7 +6437,7 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 35 } } format { @@ -5883,10 +6445,10 @@ body { } src { end_column: 16 - end_line: 95 + end_line: 103 file: 2 start_column: 8 - start_line: 95 + start_line: 103 } } } @@ -5895,12 +6457,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 39 + uid: 43 } } body { eval { - bind_id: 39 + bind_id: 43 } } client_ast_version: 1 From eaf42926975780299a7524a4fef2b97082df3a43 Mon Sep 17 00:00:00 2001 From: May Liu Date: Thu, 11 Dec 2025 11:18:59 -0800 Subject: [PATCH 4/5] overwrite_condition should only work with overwrite mode, reverting support for append mode --- .../_internal/analyzer/snowflake_plan.py | 23 +- src/snowflake/snowpark/dataframe_writer.py | 13 +- tests/ast/data/DataFrame.write.test | 1203 +++++++---------- tests/integ/test_dataframe.py | 36 +- 4 files changed, 505 insertions(+), 770 deletions(-) diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py index ee87724c5e..b979cd38c9 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py @@ -1462,21 +1462,16 @@ def get_overwrite_delete_insert_plan(child: SnowflakePlan): if mode == SaveMode.APPEND: assert table_exists is not None if table_exists: - if overwrite_condition is not None: - return get_overwrite_delete_insert_plan(child) - else: - # Normal append without overwrite_condition - return self.build( - lambda x: insert_into_statement( - table_name=full_table_name, - child=x, - column_names=column_names, - ), - child, - source_plan, - ) + return self.build( + lambda x: insert_into_statement( + table_name=full_table_name, + child=x, + column_names=column_names, + ), + child, + source_plan, + ) else: - # Table doesn't exist, just create and insert (overwrite_condition is no-op) return get_create_and_insert_plan(child, replace=False, error=False) elif mode == SaveMode.TRUNCATE: diff --git a/src/snowflake/snowpark/dataframe_writer.py b/src/snowflake/snowpark/dataframe_writer.py index ed1d14c76b..9c297a65f4 100644 --- a/src/snowflake/snowpark/dataframe_writer.py +++ b/src/snowflake/snowpark/dataframe_writer.py @@ -333,7 +333,7 @@ def save_as_table( Set to ``True`` if table exists, ``False`` if it doesn't, or ``None`` (default) for automatic detection. Primarily useful for "append", "truncate", and "overwrite" with overwrite_condition modes to avoid running query for automatic detection. overwrite_condition: Specifies the overwrite condition to perform atomic targeted delete-insert. - Can be used when ``mode`` is "append" or "overwrite" when the table exists. Rows matching the + Can only be used when ``mode`` is "overwrite" and the table exists. Rows matching the condition are deleted from the target table, then all rows from the DataFrame are inserted. @@ -380,7 +380,7 @@ def save_as_table( [Row(ID=1, VAL='a'), Row(ID=2, VAL='b'), Row(ID=3, VAL='c')] >>> new_df = session.create_dataframe([[2, "updated2"], [5, "updated5"]], schema=["id", "val"]) - >>> new_df.write.mode("append").save_as_table("my_table", overwrite_condition="id = 1 or val = 'b'") + >>> new_df.write.mode("overwrite").save_as_table("my_table", overwrite_condition="id = 1 or val = 'b'") >>> session.table("my_table").order_by("id").collect() [Row(ID=2, VAL='updated2'), Row(ID=3, VAL='c'), Row(ID=5, VAL='updated5')] """ @@ -513,13 +513,10 @@ def save_as_table( f"Unsupported table type. Expected table types: {SUPPORTED_TABLE_TYPES}" ) - # overwrite_condition must be used with APPEND or OVERWRITE mode - if overwrite_condition is not None and save_mode not in ( - SaveMode.APPEND, - SaveMode.OVERWRITE, - ): + # overwrite_condition must be used with OVERWRITE mode only + if overwrite_condition is not None and save_mode != SaveMode.OVERWRITE: raise ValueError( - f"'overwrite_condition' is only supported with mode='append' or mode='overwrite'. " + f"'overwrite_condition' is only supported with mode='overwrite'. " f"Got mode='{save_mode.value}'." ) diff --git a/tests/ast/data/DataFrame.write.test b/tests/ast/data/DataFrame.write.test index c65bc3c398..6590c3f162 100644 --- a/tests/ast/data/DataFrame.write.test +++ b/tests/ast/data/DataFrame.write.test @@ -24,13 +24,9 @@ df.write.mode("overwrite").save_as_table("iceberg_table_target_size", iceberg_co df.write.mode("overwrite").save_as_table("iceberg_table_full", iceberg_config={"external_volume": "example_volume", "partition_by": [bucket(10, "A"), year("date_col")], "target_file_size": "AUTO", "catalog": "my_catalog"}) -df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition=col("A")==10) +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition=(col("A") > 4) | (col("B") == "c")) -df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition="B = 100") - -df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition=(col("A") > 4) | (col("B") == "c")) - -df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition="A = 1 or B = 'b'") +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition="A = 1 or B = 'b'") df.write.mode("truncate").save_as_table("test_destination", column_order="name", create_temp_table=False, table_type="transient", clustering_keys=['STR', col('num1')], comment="test", block=True, statement_params={"k":"v"}) @@ -138,19 +134,11 @@ df.write.mode("overwrite").save_as_table("iceberg_table_full", iceberg_config={" df = session.table("table1") -df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition=col("A") == 10) - -df = session.table("table1") - -df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", overwrite_condition="B = 100") - -df = session.table("table1") - -df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition=(col("A") > 4) | (col("B") == "c")) +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition=(col("A") > 4) | (col("B") == "c")) df = session.table("table1") -df.write.mode("append").save_as_table("saved_table", table_type="temporary", overwrite_condition="A = 1 or B = 'b'") +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition="A = 1 or B = 'b'") df = session.table("table1") @@ -1873,241 +1861,6 @@ body { uid: 1 } } -body { - bind { - expr { - write_table { - block: true - column_order: "index" - overwrite_condition { - eq { - lhs { - apply_expr { - fn { - builtin_fn { - name { - name { - name_flat { - name: "col" - } - } - } - } - } - pos_args { - string_val { - src { - end_column: 116 - end_line: 49 - file: 2 - start_column: 108 - start_line: 49 - } - v: "A" - } - } - src { - end_column: 116 - end_line: 49 - file: 2 - start_column: 108 - start_line: 49 - } - } - } - rhs { - int64_val { - src { - end_column: 120 - end_line: 49 - file: 2 - start_column: 108 - start_line: 49 - } - v: 10 - } - } - src { - end_column: 120 - end_line: 49 - file: 2 - start_column: 108 - start_line: 49 - } - } - } - src { - end_column: 121 - end_line: 49 - file: 2 - start_column: 8 - start_line: 49 - } - table_name { - name { - name_flat { - name: "saved_table" - } - } - } - table_type: "temporary" - writer { - dataframe_writer { - df { - dataframe_ref { - id: 1 - } - } - save_mode { - save_mode_overwrite: true - } - src { - end_column: 16 - end_line: 29 - file: 2 - start_column: 8 - start_line: 29 - } - } - } - } - } - first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" - symbol { - } - uid: 12 - } -} -body { - eval { - bind_id: 12 - } -} -body { - bind { - expr { - table { - name { - name { - name_flat { - name: "table1" - } - } - } - src { - end_column: 41 - end_line: 27 - file: 2 - start_column: 13 - start_line: 27 - } - variant { - session_table: true - } - } - } - first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" - symbol { - value: "df" - } - uid: 1 - } -} -body { - bind { - expr { - write_table { - block: true - column_order: "index" - overwrite_condition { - sql_expr { - sql: "B = 100" - src { - end_column: 118 - end_line: 51 - file: 2 - start_column: 8 - start_line: 51 - } - } - } - src { - end_column: 118 - end_line: 51 - file: 2 - start_column: 8 - start_line: 51 - } - table_name { - name { - name_flat { - name: "saved_table" - } - } - } - table_type: "temporary" - writer { - dataframe_writer { - df { - dataframe_ref { - id: 1 - } - } - save_mode { - save_mode_overwrite: true - } - src { - end_column: 16 - end_line: 29 - file: 2 - start_column: 8 - start_line: 29 - } - } - } - } - } - first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" - symbol { - } - uid: 13 - } -} -body { - eval { - bind_id: 13 - } -} -body { - bind { - expr { - table { - name { - name { - name_flat { - name: "table1" - } - } - } - src { - end_column: 41 - end_line: 27 - file: 2 - start_column: 13 - start_line: 27 - } - variant { - session_table: true - } - } - } - first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" - symbol { - value: "df" - } - uid: 1 - } -} body { bind { expr { @@ -2134,42 +1887,42 @@ body { pos_args { string_val { src { - end_column: 114 - end_line: 53 + end_column: 136 + end_line: 49 file: 2 - start_column: 106 - start_line: 53 + start_column: 128 + start_line: 49 } v: "A" } } src { - end_column: 114 - end_line: 53 + end_column: 136 + end_line: 49 file: 2 - start_column: 106 - start_line: 53 + start_column: 128 + start_line: 49 } } } rhs { int64_val { src { - end_column: 118 - end_line: 53 + end_column: 140 + end_line: 49 file: 2 - start_column: 106 - start_line: 53 + start_column: 128 + start_line: 49 } v: 4 } } src { - end_column: 118 - end_line: 53 + end_column: 140 + end_line: 49 file: 2 - start_column: 106 - start_line: 53 + start_column: 128 + start_line: 49 } } } @@ -2191,60 +1944,63 @@ body { pos_args { string_val { src { - end_column: 131 - end_line: 53 + end_column: 153 + end_line: 49 file: 2 - start_column: 123 - start_line: 53 + start_column: 145 + start_line: 49 } v: "B" } } src { - end_column: 131 - end_line: 53 + end_column: 153 + end_line: 49 file: 2 - start_column: 123 - start_line: 53 + start_column: 145 + start_line: 49 } } } rhs { string_val { src { - end_column: 138 - end_line: 53 + end_column: 160 + end_line: 49 file: 2 - start_column: 123 - start_line: 53 + start_column: 145 + start_line: 49 } v: "c" } } src { - end_column: 138 - end_line: 53 + end_column: 160 + end_line: 49 file: 2 - start_column: 123 - start_line: 53 + start_column: 145 + start_line: 49 } } } src { - end_column: 139 - end_line: 53 + end_column: 161 + end_line: 49 file: 2 - start_column: 105 - start_line: 53 + start_column: 127 + start_line: 49 } } } src { - end_column: 140 - end_line: 53 + end_column: 162 + end_line: 49 file: 2 start_column: 8 - start_line: 53 + start_line: 49 + } + table_exists { + value: true } table_name { name { @@ -2262,7 +2018,7 @@ body { } } save_mode { - save_mode_append: true + save_mode_overwrite: true } src { end_column: 16 @@ -2278,12 +2034,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 14 + uid: 12 } } body { eval { - bind_id: 14 + bind_id: 12 } } body { @@ -2326,20 +2082,23 @@ body { sql_expr { sql: "A = 1 or B = \'b\'" src { - end_column: 124 - end_line: 55 + end_column: 146 + end_line: 51 file: 2 start_column: 8 - start_line: 55 + start_line: 51 } } } src { - end_column: 124 - end_line: 55 + end_column: 146 + end_line: 51 file: 2 start_column: 8 - start_line: 55 + start_line: 51 + } + table_exists { + value: true } table_name { name { @@ -2357,7 +2116,7 @@ body { } } save_mode { - save_mode_append: true + save_mode_overwrite: true } src { end_column: 16 @@ -2373,12 +2132,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 15 + uid: 13 } } body { eval { - bind_id: 15 + bind_id: 13 } } body { @@ -2420,10 +2179,10 @@ body { string_val { src { end_column: 231 - end_line: 57 + end_line: 53 file: 2 start_column: 8 - start_line: 57 + start_line: 53 } v: "STR" } @@ -2445,20 +2204,20 @@ body { string_val { src { end_column: 173 - end_line: 57 + end_line: 53 file: 2 start_column: 162 - start_line: 57 + start_line: 53 } v: "num1" } } src { end_column: 173 - end_line: 57 + end_line: 53 file: 2 start_column: 162 - start_line: 57 + start_line: 53 } } } @@ -2468,10 +2227,10 @@ body { } src { end_column: 231 - end_line: 57 + end_line: 53 file: 2 start_column: 8 - start_line: 57 + start_line: 53 } statement_params { _1: "k" @@ -2509,12 +2268,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 16 + uid: 14 } } body { eval { - bind_id: 16 + bind_id: 14 } } body { @@ -2555,10 +2314,10 @@ body { column_order: "index" src { end_column: 109 - end_line: 59 + end_line: 55 file: 2 start_column: 8 - start_line: 59 + start_line: 55 } table_name { name { @@ -2580,10 +2339,10 @@ body { sql: "value" src { end_column: 38 - end_line: 59 + end_line: 55 file: 2 start_column: 8 - start_line: 59 + start_line: 55 } } } @@ -2604,12 +2363,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 17 + uid: 15 } } body { eval { - bind_id: 17 + bind_id: 15 } } body { @@ -2650,10 +2409,10 @@ body { column_order: "index" src { end_column: 114 - end_line: 61 + end_line: 57 file: 2 start_column: 8 - start_line: 61 + start_line: 57 } table_name { name { @@ -2687,20 +2446,20 @@ body { string_val { src { end_column: 42 - end_line: 61 + end_line: 57 file: 2 start_column: 30 - start_line: 61 + start_line: 57 } v: "value" } } src { end_column: 42 - end_line: 61 + end_line: 57 file: 2 start_column: 30 - start_line: 61 + start_line: 57 } } } @@ -2721,12 +2480,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 18 + uid: 16 } } body { eval { - bind_id: 18 + bind_id: 16 } } body { @@ -2767,10 +2526,10 @@ body { column_order: "index" src { end_column: 151 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } table_name { name { @@ -2793,10 +2552,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -2819,20 +2578,20 @@ body { string_val { src { end_column: 79 - end_line: 63 + end_line: 59 file: 2 start_column: 67 - start_line: 63 + start_line: 59 } v: "value" } } src { end_column: 79 - end_line: 63 + end_line: 59 file: 2 start_column: 67 - start_line: 63 + start_line: 59 } } } @@ -2853,12 +2612,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 19 + uid: 17 } } body { eval { - bind_id: 19 + bind_id: 17 } } body { @@ -2899,10 +2658,10 @@ body { column_order: "index" src { end_column: 239 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } table_name { name { @@ -2925,10 +2684,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -2940,10 +2699,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -2955,10 +2714,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -2970,10 +2729,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -2985,10 +2744,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -3011,20 +2770,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -3045,12 +2804,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 20 + uid: 18 } } body { eval { - bind_id: 20 + bind_id: 18 } } body { @@ -3060,10 +2819,10 @@ body { query: "create temp stage if not exists test_stage" src { end_column: 88 - end_line: 67 + end_line: 63 file: 2 start_column: 31 - start_line: 67 + start_line: 63 } } } @@ -3071,7 +2830,7 @@ body { symbol { value: "stage_created_result" } - uid: 21 + uid: 19 } } body { @@ -3082,27 +2841,27 @@ body { case_sensitive: true df { dataframe_ref { - id: 21 + id: 19 } } src { end_column: 98 - end_line: 67 + end_line: 63 file: 2 start_column: 31 - start_line: 67 + start_line: 63 } } } first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 22 + uid: 20 } } body { eval { - bind_id: 22 + bind_id: 20 } } body { @@ -3143,10 +2902,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 72 - end_line: 69 + end_line: 65 file: 2 start_column: 8 - start_line: 69 + start_line: 65 } writer { dataframe_writer { @@ -3161,10 +2920,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -3176,10 +2935,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -3191,10 +2950,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -3206,10 +2965,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -3221,10 +2980,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -3247,20 +3006,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -3281,12 +3040,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 23 + uid: 21 } } body { eval { - bind_id: 23 + bind_id: 21 } } body { @@ -3330,10 +3089,10 @@ body { bool_val { src { end_column: 123 - end_line: 73 + end_line: 69 file: 2 start_column: 8 - start_line: 73 + start_line: 69 } v: true } @@ -3345,10 +3104,10 @@ body { bool_val { src { end_column: 123 - end_line: 73 + end_line: 69 file: 2 start_column: 8 - start_line: 73 + start_line: 69 } v: true } @@ -3361,10 +3120,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 123 - end_line: 73 + end_line: 69 file: 2 start_column: 8 - start_line: 73 + start_line: 69 } writer { dataframe_writer { @@ -3379,10 +3138,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -3394,10 +3153,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -3409,10 +3168,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -3424,10 +3183,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -3439,10 +3198,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -3465,20 +3224,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -3499,12 +3258,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 24 + uid: 22 } } body { eval { - bind_id: 24 + bind_id: 22 } } body { @@ -3547,10 +3306,10 @@ body { bool_val { src { end_column: 181 - end_line: 75 + end_line: 71 file: 2 start_column: 8 - start_line: 75 + start_line: 71 } v: true } @@ -3562,10 +3321,10 @@ body { bool_val { src { end_column: 181 - end_line: 75 + end_line: 71 file: 2 start_column: 8 - start_line: 75 + start_line: 71 } } } @@ -3580,10 +3339,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 181 - end_line: 75 + end_line: 71 file: 2 start_column: 8 - start_line: 75 + start_line: 71 } writer { dataframe_writer { @@ -3598,10 +3357,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -3613,10 +3372,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -3628,10 +3387,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -3643,10 +3402,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -3658,10 +3417,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -3684,20 +3443,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -3718,12 +3477,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 25 + uid: 23 } } body { eval { - bind_id: 25 + bind_id: 23 } } body { @@ -3766,10 +3525,10 @@ body { bool_val { src { end_column: 207 - end_line: 77 + end_line: 73 file: 2 start_column: 8 - start_line: 77 + start_line: 73 } v: true } @@ -3781,10 +3540,10 @@ body { bool_val { src { end_column: 207 - end_line: 77 + end_line: 73 file: 2 start_column: 8 - start_line: 77 + start_line: 73 } } } @@ -3803,10 +3562,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 207 - end_line: 77 + end_line: 73 file: 2 start_column: 8 - start_line: 77 + start_line: 73 } writer { dataframe_writer { @@ -3821,10 +3580,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -3836,10 +3595,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -3851,10 +3610,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -3866,10 +3625,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -3881,10 +3640,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -3907,20 +3666,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -3941,12 +3700,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 26 + uid: 24 } } body { eval { - bind_id: 26 + bind_id: 24 } } body { @@ -3990,10 +3749,10 @@ body { bool_val { src { end_column: 125 - end_line: 81 + end_line: 77 file: 2 start_column: 8 - start_line: 81 + start_line: 77 } v: true } @@ -4005,10 +3764,10 @@ body { bool_val { src { end_column: 125 - end_line: 81 + end_line: 77 file: 2 start_column: 8 - start_line: 81 + start_line: 77 } v: true } @@ -4022,10 +3781,10 @@ body { location: "@test_stage/test.csv" src { end_column: 125 - end_line: 81 + end_line: 77 file: 2 start_column: 8 - start_line: 81 + start_line: 77 } writer { dataframe_writer { @@ -4040,10 +3799,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -4055,10 +3814,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -4070,10 +3829,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -4085,10 +3844,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -4100,10 +3859,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -4126,20 +3885,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -4160,12 +3919,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 27 + uid: 25 } } body { eval { - bind_id: 27 + bind_id: 25 } } body { @@ -4210,10 +3969,10 @@ body { location: "@test_stage/test.csv" src { end_column: 139 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } writer { dataframe_writer { @@ -4231,10 +3990,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -4246,10 +4005,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -4261,10 +4020,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -4276,10 +4035,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -4291,10 +4050,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -4306,10 +4065,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -4332,20 +4091,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -4366,12 +4125,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 28 + uid: 26 } } body { eval { - bind_id: 28 + bind_id: 26 } } body { @@ -4415,10 +4174,10 @@ body { bool_val { src { end_column: 114 - end_line: 87 + end_line: 83 file: 2 start_column: 8 - start_line: 87 + start_line: 83 } v: true } @@ -4430,10 +4189,10 @@ body { bool_val { src { end_column: 114 - end_line: 87 + end_line: 83 file: 2 start_column: 8 - start_line: 87 + start_line: 83 } v: true } @@ -4446,10 +4205,10 @@ body { location: "@test_stage/test.json" src { end_column: 114 - end_line: 87 + end_line: 83 file: 2 start_column: 8 - start_line: 87 + start_line: 83 } writer { dataframe_writer { @@ -4467,10 +4226,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -4482,10 +4241,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -4497,10 +4256,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -4512,10 +4271,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -4527,10 +4286,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -4542,10 +4301,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -4568,20 +4327,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -4602,12 +4361,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 29 + uid: 27 } } body { eval { - bind_id: 29 + bind_id: 27 } } body { @@ -4652,10 +4411,10 @@ body { location: "@test_stage/test.json" src { end_column: 117 - end_line: 89 + end_line: 85 file: 2 start_column: 8 - start_line: 89 + start_line: 85 } writer { dataframe_writer { @@ -4673,10 +4432,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -4688,10 +4447,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -4703,10 +4462,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -4718,10 +4477,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -4733,10 +4492,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -4748,10 +4507,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -4774,20 +4533,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -4808,12 +4567,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 30 + uid: 28 } } body { eval { - bind_id: 30 + bind_id: 28 } } body { @@ -4857,10 +4616,10 @@ body { bool_val { src { end_column: 130 - end_line: 93 + end_line: 89 file: 2 start_column: 8 - start_line: 93 + start_line: 89 } v: true } @@ -4872,10 +4631,10 @@ body { bool_val { src { end_column: 130 - end_line: 93 + end_line: 89 file: 2 start_column: 8 - start_line: 93 + start_line: 89 } v: true } @@ -4888,10 +4647,10 @@ body { location: "@test_stage/test.parquet" src { end_column: 130 - end_line: 93 + end_line: 89 file: 2 start_column: 8 - start_line: 93 + start_line: 89 } writer { dataframe_writer { @@ -4909,10 +4668,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -4924,10 +4683,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -4939,10 +4698,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -4954,10 +4713,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -4969,10 +4728,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -4984,10 +4743,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -5010,20 +4769,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -5044,12 +4803,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 31 + uid: 29 } } body { eval { - bind_id: 31 + bind_id: 29 } } body { @@ -5094,10 +4853,10 @@ body { location: "@test_stage/test.parquet" src { end_column: 120 - end_line: 95 + end_line: 91 file: 2 start_column: 8 - start_line: 95 + start_line: 91 } writer { dataframe_writer { @@ -5115,10 +4874,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -5130,10 +4889,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -5145,10 +4904,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -5160,10 +4919,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -5175,10 +4934,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -5190,10 +4949,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -5216,20 +4975,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -5250,12 +5009,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 32 + uid: 30 } } body { eval { - bind_id: 32 + bind_id: 30 } } body { @@ -5295,10 +5054,10 @@ body { overwrite: true src { end_column: 59 - end_line: 97 + end_line: 93 file: 2 start_column: 8 - start_line: 97 + start_line: 93 } table_name { name { @@ -5323,10 +5082,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -5338,10 +5097,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -5353,10 +5112,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -5368,10 +5127,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -5383,10 +5142,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -5398,10 +5157,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -5424,20 +5183,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -5458,12 +5217,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 33 + uid: 31 } } body { eval { - bind_id: 33 + bind_id: 31 } } body { @@ -5502,10 +5261,10 @@ body { write_insert_into { src { end_column: 60 - end_line: 99 + end_line: 95 file: 2 start_column: 8 - start_line: 99 + start_line: 95 } table_name { name { @@ -5530,10 +5289,10 @@ body { string_val { src { end_column: 53 - end_line: 63 + end_line: 59 file: 2 start_column: 8 - start_line: 63 + start_line: 59 } v: "abcd" } @@ -5545,10 +5304,10 @@ body { string_val { src { end_column: 53 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "abcd" } @@ -5560,10 +5319,10 @@ body { string_val { src { end_column: 90 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "pqrs" } @@ -5575,10 +5334,10 @@ body { bool_val { src { end_column: 116 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: true } @@ -5590,10 +5349,10 @@ body { string_val { src { end_column: 141 - end_line: 65 + end_line: 61 file: 2 start_column: 8 - start_line: 65 + start_line: 61 } v: "append" } @@ -5605,10 +5364,10 @@ body { bool_val { src { end_column: 53 - end_line: 83 + end_line: 79 file: 2 start_column: 8 - start_line: 83 + start_line: 79 } v: true } @@ -5631,20 +5390,20 @@ body { string_val { src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 65 + end_line: 61 file: 2 start_column: 155 - start_line: 65 + start_line: 61 } } } @@ -5665,12 +5424,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 34 + uid: 32 } } body { eval { - bind_id: 34 + bind_id: 32 } } body { @@ -5686,10 +5445,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -5700,7 +5459,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -5711,10 +5470,10 @@ body { location: "s3://testbucket" src { end_column: 94 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } storage_integration { value: "test_integration" @@ -5723,15 +5482,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -5740,12 +5499,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 36 + uid: 34 } } body { eval { - bind_id: 36 + bind_id: 34 } } body { @@ -5761,10 +5520,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -5775,7 +5534,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -5786,10 +5545,10 @@ body { location: "azure://testcontainer" src { end_column: 131 - end_line: 105 + end_line: 101 file: 2 start_column: 8 - start_line: 105 + start_line: 101 } storage_integration { value: "test_integration" @@ -5801,15 +5560,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -5818,12 +5577,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 37 + uid: 35 } } body { eval { - bind_id: 37 + bind_id: 35 } } body { @@ -5839,10 +5598,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -5853,7 +5612,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -5864,24 +5623,24 @@ body { location: "gcp://teststorage" src { end_column: 87 - end_line: 107 + end_line: 103 file: 2 start_column: 8 - start_line: 107 + start_line: 103 } writer { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -5890,12 +5649,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 38 + uid: 36 } } body { eval { - bind_id: 38 + bind_id: 36 } } body { @@ -5911,10 +5670,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -5925,7 +5684,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -5939,10 +5698,10 @@ body { string_val { src { end_column: 174 - end_line: 109 + end_line: 105 file: 2 start_column: 8 - start_line: 109 + start_line: 105 } v: "awsid" } @@ -5954,10 +5713,10 @@ body { string_val { src { end_column: 174 - end_line: 109 + end_line: 105 file: 2 start_column: 8 - start_line: 109 + start_line: 105 } v: "!23@" } @@ -5969,10 +5728,10 @@ body { null_val { src { end_column: 174 - end_line: 109 + end_line: 105 file: 2 start_column: 8 - start_line: 109 + start_line: 105 } } } @@ -5983,10 +5742,10 @@ body { string_val { src { end_column: 174 - end_line: 109 + end_line: 105 file: 2 start_column: 8 - start_line: 109 + start_line: 105 } v: "master key" } @@ -5995,24 +5754,24 @@ body { location: "gcp://teststorage" src { end_column: 174 - end_line: 109 + end_line: 105 file: 2 start_column: 8 - start_line: 109 + start_line: 105 } writer { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -6021,12 +5780,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 39 + uid: 37 } } body { eval { - bind_id: 39 + bind_id: 37 } } body { @@ -6042,10 +5801,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -6056,7 +5815,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -6067,10 +5826,10 @@ body { location: "azure://testcontainer" src { end_column: 122 - end_line: 111 + end_line: 107 file: 2 start_column: 8 - start_line: 111 + start_line: 107 } storage_integration { value: "test_integration" @@ -6082,15 +5841,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -6099,12 +5858,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 40 + uid: 38 } } body { eval { - bind_id: 40 + bind_id: 38 } } body { @@ -6120,10 +5879,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -6134,7 +5893,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -6148,10 +5907,10 @@ body { string_val { src { end_column: 158 - end_line: 113 + end_line: 109 file: 2 start_column: 8 - start_line: 113 + start_line: 109 } v: "awsid" } @@ -6163,10 +5922,10 @@ body { string_val { src { end_column: 158 - end_line: 113 + end_line: 109 file: 2 start_column: 8 - start_line: 113 + start_line: 109 } v: "!23@" } @@ -6178,10 +5937,10 @@ body { null_val { src { end_column: 158 - end_line: 113 + end_line: 109 file: 2 start_column: 8 - start_line: 113 + start_line: 109 } } } @@ -6192,10 +5951,10 @@ body { string_val { src { end_column: 158 - end_line: 113 + end_line: 109 file: 2 start_column: 8 - start_line: 113 + start_line: 109 } v: "master key" } @@ -6204,24 +5963,24 @@ body { location: "s3://testbucket" src { end_column: 158 - end_line: 113 + end_line: 109 file: 2 start_column: 8 - start_line: 113 + start_line: 109 } writer { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -6230,12 +5989,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 41 + uid: 39 } } body { eval { - bind_id: 41 + bind_id: 39 } } body { @@ -6251,10 +6010,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -6265,7 +6024,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -6276,10 +6035,10 @@ body { location: "azure://testcontainer" src { end_column: 149 - end_line: 115 + end_line: 111 file: 2 start_column: 8 - start_line: 115 + start_line: 111 } storage_integration { value: "test_integration" @@ -6291,15 +6050,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -6308,12 +6067,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 42 + uid: 40 } } body { eval { - bind_id: 42 + bind_id: 40 } } body { @@ -6329,10 +6088,10 @@ body { } src { end_column: 41 - end_line: 101 + end_line: 97 file: 2 start_column: 13 - start_line: 101 + start_line: 97 } variant { session_table: true @@ -6343,7 +6102,7 @@ body { symbol { value: "df" } - uid: 35 + uid: 33 } } body { @@ -6357,10 +6116,10 @@ body { string_val { src { end_column: 237 - end_line: 117 + end_line: 113 file: 2 start_column: 8 - start_line: 117 + start_line: 113 } v: "awsid" } @@ -6372,10 +6131,10 @@ body { string_val { src { end_column: 237 - end_line: 117 + end_line: 113 file: 2 start_column: 8 - start_line: 117 + start_line: 113 } v: "!23@" } @@ -6387,10 +6146,10 @@ body { null_val { src { end_column: 237 - end_line: 117 + end_line: 113 file: 2 start_column: 8 - start_line: 117 + start_line: 113 } } } @@ -6401,10 +6160,10 @@ body { string_val { src { end_column: 237 - end_line: 117 + end_line: 113 file: 2 start_column: 8 - start_line: 117 + start_line: 113 } v: "master key" } @@ -6416,19 +6175,19 @@ body { sql: "value" src { end_column: 237 - end_line: 117 + end_line: 113 file: 2 start_column: 8 - start_line: 117 + start_line: 113 } } } src { end_column: 237 - end_line: 117 + end_line: 113 file: 2 start_column: 8 - start_line: 117 + start_line: 113 } storage_integration { value: "test_integration" @@ -6437,7 +6196,7 @@ body { dataframe_writer { df { dataframe_ref { - id: 35 + id: 33 } } format { @@ -6445,10 +6204,10 @@ body { } src { end_column: 16 - end_line: 103 + end_line: 99 file: 2 start_column: 8 - start_line: 103 + start_line: 99 } } } @@ -6457,12 +6216,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 43 + uid: 41 } } body { eval { - bind_id: 43 + bind_id: 41 } } client_ast_version: 1 diff --git a/tests/integ/test_dataframe.py b/tests/integ/test_dataframe.py index 229dc04bde..c2ec824c27 100644 --- a/tests/integ/test_dataframe.py +++ b/tests/integ/test_dataframe.py @@ -4597,7 +4597,7 @@ def test_write_table_with_overwrite_condition(session): [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - new_df1.write.mode("append").save_as_table( + new_df1.write.mode("overwrite").save_as_table( table_name, overwrite_condition="id = 1 or val = 'b'" ) result = session.table(table_name).order_by("id").collect() @@ -4615,7 +4615,7 @@ def test_write_table_with_overwrite_condition(session): [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - new_df2.write.mode("append").save_as_table( + new_df2.write.mode("overwrite").save_as_table( table_name, overwrite_condition=col("id") == 2 ) result = session.table(table_name).order_by("id").collect() @@ -4634,7 +4634,7 @@ def test_write_table_with_overwrite_condition(session): [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - new_df3.write.mode("append").save_as_table( + new_df3.write.mode("overwrite").save_as_table( table_name, overwrite_condition=(col("id") > 4) | (col("val") == "c") ) result = session.table(table_name).order_by("id").collect() @@ -4652,7 +4652,7 @@ def test_write_table_with_overwrite_condition(session): [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - new_df4.write.mode("append").save_as_table( + new_df4.write.mode("overwrite").save_as_table( table_name, overwrite_condition="id > 0" ) result = session.table(table_name).collect() @@ -4665,7 +4665,7 @@ def test_write_table_with_overwrite_condition(session): [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - new_df5.write.mode("append").save_as_table( + new_df5.write.mode("overwrite").save_as_table( table_name, overwrite_condition="id = 999" ) result = session.table(table_name).order_by("id").collect() @@ -4674,24 +4674,6 @@ def test_write_table_with_overwrite_condition(session): Row(ID=20, VAL="another"), ] - # Test 6: overwrite_condition with mode="overwrite" (selective overwrite) - new_df6 = session.create_dataframe( - [[10, "replaced10"], [30, "new30"]], - schema=StructType( - [StructField("id", IntegerType()), StructField("val", StringType())] - ), - ) - new_df6.write.mode("overwrite").save_as_table( - table_name, overwrite_condition=col("id") == 10 - ) - result = session.table(table_name).order_by("id").collect() - # id=10 deleted, new rows inserted, id=20 preserved - assert result == [ - Row(ID=10, VAL="replaced10"), - Row(ID=20, VAL="another"), - Row(ID=30, VAL="new30"), - ] - finally: Utils.drop_table(session, table_name) @@ -4701,7 +4683,9 @@ def test_write_table_with_overwrite_condition(session): reason="overwrite_condition is a SQL feature", run=False, ) -@pytest.mark.parametrize("invalid_mode", ["truncate", "errorifexists", "ignore"]) +@pytest.mark.parametrize( + "invalid_mode", ["append", "truncate", "errorifexists", "ignore"] +) def test_write_table_with_overwrite_condition_edge_cases(session, invalid_mode): """Test overwrite_condition edge cases: table not exists, and invalid modes.""" table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) @@ -4713,7 +4697,7 @@ def test_write_table_with_overwrite_condition_edge_cases(session, invalid_mode): [StructField("id", IntegerType()), StructField("val", StringType())] ), ) - df.write.mode("append").save_as_table( + df.write.mode("overwrite").save_as_table( table_name, overwrite_condition="id = 999" ) result = session.table(table_name).order_by("id").collect() @@ -4722,7 +4706,7 @@ def test_write_table_with_overwrite_condition_edge_cases(session, invalid_mode): # Edge case 2: Invalid mode raises ValueError with pytest.raises( ValueError, - match="'overwrite_condition' is only supported with mode='append' or mode='overwrite'", + match="'overwrite_condition' is only supported with mode='overwrite'", ): df.write.mode(invalid_mode).save_as_table( table_name, overwrite_condition="id = 1" From afe2dbe672e1089ee8e9620373a6f217a56cdf80 Mon Sep 17 00:00:00 2001 From: May Liu Date: Thu, 11 Dec 2025 12:06:13 -0800 Subject: [PATCH 5/5] elaborate selective overwrite behavior in docstring --- src/snowflake/snowpark/dataframe_writer.py | 10 +++++++--- tests/integ/test_dataframe.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/snowflake/snowpark/dataframe_writer.py b/src/snowflake/snowpark/dataframe_writer.py index 9c297a65f4..c1d2c4da41 100644 --- a/src/snowflake/snowpark/dataframe_writer.py +++ b/src/snowflake/snowpark/dataframe_writer.py @@ -271,7 +271,9 @@ def save_as_table( "append": Append data of this DataFrame to the existing table. Creates a table if it does not exist. - "overwrite": Overwrite the existing table by dropping old table. + "overwrite": Overwrite the existing table. By default, drops and recreates the table. + When ``overwrite_condition`` is specified, performs selective overwrite: deletes only + rows matching the condition, then inserts new data. "truncate": Overwrite the existing table by truncating old table. @@ -333,8 +335,10 @@ def save_as_table( Set to ``True`` if table exists, ``False`` if it doesn't, or ``None`` (default) for automatic detection. Primarily useful for "append", "truncate", and "overwrite" with overwrite_condition modes to avoid running query for automatic detection. overwrite_condition: Specifies the overwrite condition to perform atomic targeted delete-insert. - Can only be used when ``mode`` is "overwrite" and the table exists. Rows matching the - condition are deleted from the target table, then all rows from the DataFrame are inserted. + Can only be used when ``mode`` is "overwrite". When provided and the table exists, rows matching + the condition are atomically deleted and all rows from the DataFrame are inserted, preserving + non-matching rows. When not provided, the default "overwrite" behavior applies (drop and recreate table). + If the table does not exist, ``overwrite_condition`` is ignored and the table is created normally. Example 1:: diff --git a/tests/integ/test_dataframe.py b/tests/integ/test_dataframe.py index c2ec824c27..af24ea98e8 100644 --- a/tests/integ/test_dataframe.py +++ b/tests/integ/test_dataframe.py @@ -4568,7 +4568,7 @@ def test_write_table_with_clustering_keys_and_comment( @pytest.mark.xfail( "config.getoption('local_testing_mode', default=False)", - reason="overwrite_condition is a SQL feature", + reason="overwrite_condition is not supported in local testing mode", run=False, ) def test_write_table_with_overwrite_condition(session): @@ -4680,7 +4680,7 @@ def test_write_table_with_overwrite_condition(session): @pytest.mark.xfail( "config.getoption('local_testing_mode', default=False)", - reason="overwrite_condition is a SQL feature", + reason="overwrite_condition is a not supported in local testing mode", run=False, ) @pytest.mark.parametrize(