diff --git a/CHANGELOG.md b/CHANGELOG.md index dea46e404b..15ebd5461d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Release History +## 1.29.1 (2025-03-12) + +### Snowpark Python API Updates + +#### Bug Fixes + +- Fixed a bug in `DataFrameReader.dbapi` (PrPr) that prevents usage in stored procedure and snowbooks. + ## 1.29.0 (2025-03-05) ### Snowpark Python API Updates diff --git a/docs/source/snowpark/io.rst b/docs/source/snowpark/io.rst index 2b67bac047..e181af35d5 100644 --- a/docs/source/snowpark/io.rst +++ b/docs/source/snowpark/io.rst @@ -22,6 +22,7 @@ Input/Output DataFrameReader.avro DataFrameReader.csv + DataFrameReader.dbapi DataFrameReader.format DataFrameReader.json DataFrameReader.load diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 1733d5038b..3b190fb9c1 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "snowflake-snowpark-python" %} -{% set version = "1.29.0" %} +{% set version = "1.29.1" %} {% set noarch_build = (os.environ.get('SNOWFLAKE_SNOWPARK_PYTHON_NOARCH_BUILD', 'false')) == 'true' %} package: diff --git a/src/snowflake/snowpark/_internal/data_source_utils.py b/src/snowflake/snowpark/_internal/data_source_utils.py index b9c78f34d1..82fcd632f2 100644 --- a/src/snowflake/snowpark/_internal/data_source_utils.py +++ b/src/snowflake/snowpark/_internal/data_source_utils.py @@ -5,8 +5,10 @@ import datetime import decimal import logging +import os +import queue from enum import Enum -from typing import List, Any, Tuple, Protocol, Optional +from typing import List, Any, Tuple, Protocol, Optional, Set from snowflake.connector.options import pandas as pd from snowflake.snowpark._internal.utils import get_sorted_key_for_version @@ -294,7 +296,7 @@ def infer_data_source_schema( except Exception as exc: cursor.close() raise SnowparkDataframeReaderException( - f"Failed to infer Snowpark DataFrame schema from '{table_or_query}'." + f"Failed to infer Snowpark DataFrame schema from '{table_or_query}' due to {exc!r}." f" To avoid auto inference, you can manually specify the Snowpark DataFrame schema using 'custom_schema' in DataFrameReader.dbapi." f" Please check the stack trace for more details." ) from exc @@ -368,3 +370,15 @@ def output_type_handler(cursor, metadata): return cursor.var(oracledb.DB_TYPE_LONG, arraysize=cursor.arraysize) elif metadata.type_code == oracledb.DB_TYPE_BLOB: return cursor.var(oracledb.DB_TYPE_RAW, arraysize=cursor.arraysize) + + +def add_unseen_files_to_process_queue( + work_dir: str, set_of_files_already_added_in_queue: Set[str], queue: queue.Queue +): + """Add unseen files in the work_dir to the queue for processing.""" + # all files in the work_dir are parquet files, no subdirectory + all_files = set(os.listdir(work_dir)) + unseen = all_files - set_of_files_already_added_in_queue + for file in unseen: + queue.put(os.path.join(work_dir, file)) + set_of_files_already_added_in_queue.add(file) diff --git a/src/snowflake/snowpark/_internal/proto/ast.proto b/src/snowflake/snowpark/_internal/proto/ast.proto index 9ac89ca356..b2e71d4bbe 100644 --- a/src/snowflake/snowpark/_internal/proto/ast.proto +++ b/src/snowflake/snowpark/_internal/proto/ast.proto @@ -306,7 +306,7 @@ message PivotValue_Dataframe { DataframeRef v = 1; } -// const.ir:81 +// const.ir:84 message PythonTimeZone { google.protobuf.StringValue name = 1; int64 offset_seconds = 2; @@ -440,7 +440,7 @@ message Assign { VarId var_id = 4; } -// const.ir:28 +// const.ir:31 message BigDecimalVal { int64 scale = 1; google.protobuf.StringValue special = 2; @@ -450,8 +450,9 @@ message BigDecimalVal { // const.ir:24 message BigIntVal { - SrcPosition src = 1; - bytes v = 2; + bool is_negative = 1; + SrcPosition src = 2; + bytes v = 3; } message BinOp { @@ -476,7 +477,7 @@ message BinOp { } } -// const.ir:42 +// const.ir:45 message BinaryVal { SrcPosition src = 1; bytes v = 2; @@ -1292,7 +1293,7 @@ message DataframeWrite { SrcPosition src = 5; } -// const.ir:46 +// const.ir:49 message DatatypeVal { DataType datatype = 1; SrcPosition src = 2; @@ -1582,7 +1583,7 @@ message Flatten { SrcPosition src = 6; } -// const.ir:34 +// const.ir:37 message Float64Val { SrcPosition src = 1; double v = 2; @@ -1960,7 +1961,7 @@ message Pow { SrcPosition src = 3; } -// const.ir:65 +// const.ir:68 message PythonDateVal { int64 day = 1; int64 month = 2; @@ -1968,7 +1969,7 @@ message PythonDateVal { int64 year = 4; } -// const.ir:71 +// const.ir:74 message PythonTimeVal { int64 hour = 1; int64 microsecond = 2; @@ -1978,7 +1979,7 @@ message PythonTimeVal { PythonTimeZone tz = 6; } -// const.ir:54 +// const.ir:57 message PythonTimestampVal { int64 day = 1; int64 hour = 2; @@ -2198,7 +2199,7 @@ message StoredProcedure { bool strict = 21; } -// const.ir:38 +// const.ir:41 message StringVal { SrcPosition src = 1; string v = 2; diff --git a/src/snowflake/snowpark/dataframe_reader.py b/src/snowflake/snowpark/dataframe_reader.py index f05c70b495..4800b567b3 100644 --- a/src/snowflake/snowpark/dataframe_reader.py +++ b/src/snowflake/snowpark/dataframe_reader.py @@ -4,9 +4,8 @@ import datetime import decimal import functools -import multiprocessing as mp +import queue import os -import shutil import tempfile import time import traceback @@ -22,7 +21,6 @@ from dateutil import parser import sys from logging import getLogger -import queue from typing import Any, Dict, List, Literal, Optional, Tuple, Union, Callable import snowflake.snowpark @@ -62,6 +60,7 @@ DATA_SOURCE_SQL_COMMENT, generate_sql_with_predicates, output_type_handler, + add_unseen_files_to_process_queue, ) from snowflake.snowpark._internal.utils import ( INFER_SCHEMA_FORMAT_TYPES, @@ -1104,31 +1103,66 @@ def dbapi( predicates: Optional[List[str]] = None, session_init_statement: Optional[str] = None, ) -> DataFrame: - """Reads data from a database table using a DBAPI connection with optional partitioning, parallel processing, and query customization. - By default, the function reads the entire table at a time without a query timeout. - There are several ways to break data into small pieces and speed up ingestion, you can also combine them to acquire optimal performance: - 1.Use column, lower_bound, upper_bound and num_partitions at the same time when you need to split large tables into smaller partitions for parallel processing. These must all be specified together, otherwise error will be raised. - 2.Set max_workers to a proper positive integer. This defines the maximum number of processes and threads used for parallel execution. - 3.Adjusting fetch_size can optimize performance by reducing the number of round trips to the database. - 4.Use predicates to defining WHERE conditions for partitions, predicates will be ignored if column is specified to generate partition. - 5.Set custom_schema to avoid snowpark infer schema, custom_schema must have a matched column name with table in external data source. - You can also use session_init_statement to perform any SQL that you want to execute on external data source before fetching data. + """ + Reads data from a database table or query into a DataFrame using a DBAPI connection, + with support for optional partitioning, parallel processing, and query customization. + + There are multiple methods to partition data and accelerate ingestion. + These methods can be combined to achieve optimal performance: + + 1.Use column, lower_bound, upper_bound and num_partitions at the same time when you need to split large tables into smaller partitions for parallel processing. + These must all be specified together, otherwise error will be raised. + 2.Set max_workers to a proper positive integer. + This defines the maximum number of processes and threads used for parallel execution. + 3.Adjusting fetch_size can optimize performance by reducing the number of round trips to the database. + 4.Use predicates to defining WHERE conditions for partitions, + predicates will be ignored if column is specified to generate partition. + 5.Set custom_schema to avoid snowpark infer schema, custom_schema must have a matched + column name with table in external data source. + Args: - create_connection: a function that return a dbapi connection - table: Specifies the name of the table in the external data source. This parameter cannot be set simultaneously with the query parameter. - query: A valid SQL query to be used in the FROM clause. This parameter cannot be set simultaneously with the table parameter. - column: column name used to create partition, the column type must be numeric like int type or float type, or Date type. - lower_bound: lower bound of partition, decide the stride of partition along with upper_bound, this parameter does not filter out data. - upper_bound: upper bound of partition, decide the stride of partition along with lower_bound, this parameter does not filter out data. + create_connection: A callable that takes no arguments and returns a DB-API compatible database connection. + The callable must be picklable, as it will be passed to and executed in child processes. + table: The name of the table in the external data source. + This parameter cannot be used together with the `query` parameter. + query: A valid SQL query to be used as the data source in the FROM clause. + This parameter cannot be used together with the `table` parameter. + column: The column name used for partitioning the table. Partitions will be retrieved in parallel. + The column must be of a numeric type (e.g., int or float) or a date type. + When specifying `column`, `lower_bound`, `upper_bound`, and `num_partitions` must also be provided. + lower_bound: lower bound of partition, decide the stride of partition along with `upper_bound`. + This parameter does not filter out data. It must be provided when `column` is specified. + upper_bound: upper bound of partition, decide the stride of partition along with `lower_bound`. + This parameter does not filter out data. It must be provided when `column` is specified. num_partitions: number of partitions to create when reading in parallel from multiple processes and threads. + It must be provided when `column` is specified. max_workers: number of processes and threads used for parallelism. - query_timeout: timeout(seconds) for each query, default value is 0, which means never timeout. - fetch_size: batch size when fetching from external data source, which determine how many rows fetched per round trip. This improve performace for drivers that have a low default fetch size. + query_timeout: The timeout (in seconds) for each query execution. A default value of `0` means + the query will never time out. The timeout behavior can also be configured within + the `create_connection` method when establishing the database connection, depending on the capabilities + of the DBMS and its driver. + fetch_size: The number of rows to fetch per batch from the external data source. + This determines how many rows are retrieved in each round trip, + which can improve performance for drivers with a low default fetch size. custom_schema: a custom snowflake table schema to read data from external data source, the column names should be identical to corresponded column names external data source. This can be a schema string, for example: "id INTEGER, int_col INTEGER, text_col STRING", or StructType, for example: StructType([StructField("ID", IntegerType(), False)]) - predicates: a list of expressions suitable for inclusion in WHERE clauses, each defines a partition - session_init_statement: session initiation statements for external data source, this statement will be executed before fetch data from external data source, for example: "insert into test_table values (1, 'sample_data')" will insert data into test_table before fetch data from it. - Note: - column, lower_bound, upper_bound and num_partitions must be specified if any one of them is specified. + predicates: A list of expressions suitable for inclusion in WHERE clauses, where each expression defines a partition. + Partitions will be retrieved in parallel. + If both `column` and `predicates` are specified, `column` takes precedence. + session_init_statement: A SQL statement executed before fetching data from the external data source. + This can be used for session initialization tasks such as setting configurations. + For example, `"SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED"` can be used in SQL Server + to avoid row locks and improve read performance. + The `session_init_statement` is executed only once at the beginning of each partition read. + + Example:: + .. code-block:: python + + import oracledb + def create_oracledb_connection(): + connection = oracledb.connect(...) + return connection + + df = session.read.dbapi(create_oracledb_connection, table=...) """ if (not table and not query) or (table and query): raise SnowparkDataframeReaderException( @@ -1236,23 +1270,21 @@ def dbapi( ) try: - with mp.Manager() as process_manager, ProcessPoolExecutor( + with ProcessPoolExecutor( max_workers=max_workers ) as process_executor, ThreadPoolExecutor( max_workers=max_workers ) as thread_executor: thread_pool_futures, process_pool_futures = [], [] - parquet_file_queue = process_manager.Queue() def ingestion_thread_cleanup_callback(parquet_file_path, _): # clean the local temp file after ingestion to avoid consuming too much temp disk space - shutil.rmtree(parquet_file_path, ignore_errors=True) + os.remove(parquet_file_path) logger.debug("Starting to fetch data from the data source.") for partition_idx, query in enumerate(partitioned_queries): process_future = process_executor.submit( DataFrameReader._task_fetch_from_data_source_with_retry, - parquet_file_queue, create_connection, query, struct_schema, @@ -1265,8 +1297,22 @@ def ingestion_thread_cleanup_callback(parquet_file_path, _): ) process_pool_futures.append(process_future) # Monitor queue while tasks are running + + parquet_file_queue = ( + queue.Queue() + ) # maintain the queue of parquet files to process + set_of_files_already_added_in_queue = ( + set() + ) # maintain file names we have already put into queue while True: try: + # each process and per fetch will create a parquet with a unique file name + # we add unseen files to process queue + add_unseen_files_to_process_queue( + tmp_dir, + set_of_files_already_added_in_queue, + parquet_file_queue, + ) file = parquet_file_queue.get_nowait() logger.debug(f"Retrieved file from parquet queue: {file}") thread_future = thread_executor.submit( @@ -1301,8 +1347,15 @@ def ingestion_thread_cleanup_callback(parquet_file_path, _): else: unfinished_process_pool_futures.append(future) all_job_done = False - if all_job_done and parquet_file_queue.empty(): - # all jod is done and parquet file queue is empty, we finished all the fetch work + if ( + all_job_done + and parquet_file_queue.empty() + and len(os.listdir(tmp_dir)) == 0 + ): + # we finished all the fetch work based on the following 3 conditions: + # 1. all jod is done + # 2. parquet file queue is empty + # 3. no files in the temp work dir as they are all removed in thread future callback # now we just need to wait for all ingestion threads to complete logger.debug( "All jobs are done, and the parquet file queue is empty. Fetching work is complete." @@ -1464,9 +1517,12 @@ def _upload_and_copy_into_table( statements_params: Optional[Dict[str, str]] = None, ): file_name = os.path.basename(local_file) - put_query = ( - f"PUT {normalize_local_file(local_file)} " - f"@{snowflake_stage_name} OVERWRITE=TRUE {DATA_SOURCE_SQL_COMMENT}" + # SNOW-1975354: session.sql("PUT ...").collect() is not supported in stored procedure + self._session.file.put( + normalize_local_file(local_file), + f"{snowflake_stage_name}", + overwrite=True, + statement_params=statements_params, ) copy_into_table_query = f""" COPY INTO {snowflake_table_name} FROM @{snowflake_stage_name}/{file_name} @@ -1476,7 +1532,6 @@ def _upload_and_copy_into_table( ON_ERROR={on_error} {DATA_SOURCE_SQL_COMMENT} """ - self._session.sql(put_query).collect(statement_params=statements_params) self._session.sql(copy_into_table_query).collect( statement_params=statements_params ) @@ -1500,7 +1555,6 @@ def _upload_and_copy_into_table_with_retry( @staticmethod def _task_fetch_from_data_source( - parquet_file_queue: queue.Queue, create_connection: Callable[[], "Connection"], query: str, schema: StructType, @@ -1517,12 +1571,11 @@ def convert_to_parquet(fetched_data, fetch_idx): logger.debug( f"The DataFrame is empty, no parquet file is generated for partition {partition_idx} fetch {fetch_idx}." ) - return None + return path = os.path.join( tmp_dir, f"data_partition{partition_idx}_fetch{fetch_idx}.parquet" ) df.to_parquet(path) - return path conn = create_connection() # this is specified to pyodbc, need other way to manage timeout on other drivers @@ -1536,9 +1589,7 @@ def convert_to_parquet(fetched_data, fetch_idx): if fetch_size == 0: cursor.execute(query) result = cursor.fetchall() - parquet_file_path = convert_to_parquet(result, 0) - if parquet_file_path: - parquet_file_queue.put(parquet_file_path) + convert_to_parquet(result, 0) elif fetch_size > 0: cursor = cursor.execute(query) fetch_idx = 0 @@ -1546,16 +1597,13 @@ def convert_to_parquet(fetched_data, fetch_idx): rows = cursor.fetchmany(fetch_size) if not rows: break - parquet_file_path = convert_to_parquet(rows, fetch_idx) - if parquet_file_path: - parquet_file_queue.put(parquet_file_path) + convert_to_parquet(rows, fetch_idx) fetch_idx += 1 else: raise ValueError("fetch size cannot be smaller than 0") @staticmethod def _task_fetch_from_data_source_with_retry( - parquet_file_queue: queue.Queue, create_connection: Callable[[], "Connection"], query: str, schema: StructType, @@ -1568,7 +1616,6 @@ def _task_fetch_from_data_source_with_retry( ): DataFrameReader._retry_run( DataFrameReader._task_fetch_from_data_source, - parquet_file_queue, create_connection, query, schema, diff --git a/src/snowflake/snowpark/version.py b/src/snowflake/snowpark/version.py index 9f8f164efb..a51d6077e1 100644 --- a/src/snowflake/snowpark/version.py +++ b/src/snowflake/snowpark/version.py @@ -2,4 +2,4 @@ # Update this for the versions -VERSION = (1, 29, 0) +VERSION = (1, 29, 1) diff --git a/tests/ast/data/DataFrame.agg.test b/tests/ast/data/DataFrame.agg.test index 5433740466..886fb55f9e 100644 --- a/tests/ast/data/DataFrame.agg.test +++ b/tests/ast/data/DataFrame.agg.test @@ -523,4 +523,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.collect.test b/tests/ast/data/DataFrame.collect.test index 1fddf52a84..9fd96a517c 100644 --- a/tests/ast/data/DataFrame.collect.test +++ b/tests/ast/data/DataFrame.collect.test @@ -343,4 +343,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.count.test b/tests/ast/data/DataFrame.count.test index 1a64405564..904177ac6b 100644 --- a/tests/ast/data/DataFrame.count.test +++ b/tests/ast/data/DataFrame.count.test @@ -176,4 +176,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.count2.test b/tests/ast/data/DataFrame.count2.test index cb296d4c5a..31219acf7c 100644 --- a/tests/ast/data/DataFrame.count2.test +++ b/tests/ast/data/DataFrame.count2.test @@ -207,4 +207,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.create_or_replace.test b/tests/ast/data/DataFrame.create_or_replace.test index ce5f0691b8..94aaf6a505 100644 --- a/tests/ast/data/DataFrame.create_or_replace.test +++ b/tests/ast/data/DataFrame.create_or_replace.test @@ -631,4 +631,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.cross_join.lsuffix.test b/tests/ast/data/DataFrame.cross_join.lsuffix.test index 89837ff3ff..e4e86233a6 100644 --- a/tests/ast/data/DataFrame.cross_join.lsuffix.test +++ b/tests/ast/data/DataFrame.cross_join.lsuffix.test @@ -184,4 +184,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.cross_join.rsuffix.test b/tests/ast/data/DataFrame.cross_join.rsuffix.test index 6b1cc4df53..0de6f41e8c 100644 --- a/tests/ast/data/DataFrame.cross_join.rsuffix.test +++ b/tests/ast/data/DataFrame.cross_join.rsuffix.test @@ -184,4 +184,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.cross_join.suffix.test b/tests/ast/data/DataFrame.cross_join.suffix.test index a49c7488be..f18c4da93d 100644 --- a/tests/ast/data/DataFrame.cross_join.suffix.test +++ b/tests/ast/data/DataFrame.cross_join.suffix.test @@ -187,4 +187,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.describe.test b/tests/ast/data/DataFrame.describe.test index eae70a4862..530c5f920f 100644 --- a/tests/ast/data/DataFrame.describe.test +++ b/tests/ast/data/DataFrame.describe.test @@ -207,4 +207,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.flatten.test b/tests/ast/data/DataFrame.flatten.test index 8aba5dda91..06ce3db0f0 100644 --- a/tests/ast/data/DataFrame.flatten.test +++ b/tests/ast/data/DataFrame.flatten.test @@ -186,4 +186,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.indexers.test b/tests/ast/data/DataFrame.indexers.test index 3f3901ad33..ae4f56f08a 100644 --- a/tests/ast/data/DataFrame.indexers.test +++ b/tests/ast/data/DataFrame.indexers.test @@ -228,4 +228,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.inner.column.test b/tests/ast/data/DataFrame.join.inner.column.test index d47ed192ec..3afcbec698 100644 --- a/tests/ast/data/DataFrame.join.inner.column.test +++ b/tests/ast/data/DataFrame.join.inner.column.test @@ -260,4 +260,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.inner.column_list.test b/tests/ast/data/DataFrame.join.inner.column_list.test index 47c4c4fcb6..73cde850ee 100644 --- a/tests/ast/data/DataFrame.join.inner.column_list.test +++ b/tests/ast/data/DataFrame.join.inner.column_list.test @@ -216,4 +216,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.inner.column_list_predicate.test b/tests/ast/data/DataFrame.join.inner.column_list_predicate.test index e22c391c87..0de38bbc5b 100644 --- a/tests/ast/data/DataFrame.join.inner.column_list_predicate.test +++ b/tests/ast/data/DataFrame.join.inner.column_list_predicate.test @@ -353,4 +353,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.inner.predicate.test b/tests/ast/data/DataFrame.join.inner.predicate.test index 4701d106e6..98f5dc3c7c 100644 --- a/tests/ast/data/DataFrame.join.inner.predicate.test +++ b/tests/ast/data/DataFrame.join.inner.predicate.test @@ -312,4 +312,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.inner.predicate_rsuffix.test b/tests/ast/data/DataFrame.join.inner.predicate_rsuffix.test index ed0803bd50..96466cffb9 100644 --- a/tests/ast/data/DataFrame.join.inner.predicate_rsuffix.test +++ b/tests/ast/data/DataFrame.join.inner.predicate_rsuffix.test @@ -300,4 +300,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.left_outer.column.test b/tests/ast/data/DataFrame.join.left_outer.column.test index a5065acc21..cf3851952e 100644 --- a/tests/ast/data/DataFrame.join.left_outer.column.test +++ b/tests/ast/data/DataFrame.join.left_outer.column.test @@ -216,4 +216,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join.right_outer.predicate.test b/tests/ast/data/DataFrame.join.right_outer.predicate.test index b08db0ebcb..711527e5ef 100644 --- a/tests/ast/data/DataFrame.join.right_outer.predicate.test +++ b/tests/ast/data/DataFrame.join.right_outer.predicate.test @@ -237,4 +237,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.join_table_function.test b/tests/ast/data/DataFrame.join_table_function.test index b480f22512..bbd08d0f38 100644 --- a/tests/ast/data/DataFrame.join_table_function.test +++ b/tests/ast/data/DataFrame.join_table_function.test @@ -272,4 +272,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.natural_join.test b/tests/ast/data/DataFrame.natural_join.test index 857da61cad..20a31d476b 100644 --- a/tests/ast/data/DataFrame.natural_join.test +++ b/tests/ast/data/DataFrame.natural_join.test @@ -184,4 +184,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.pivot.test b/tests/ast/data/DataFrame.pivot.test index 1e53807fa2..092faa470e 100644 --- a/tests/ast/data/DataFrame.pivot.test +++ b/tests/ast/data/DataFrame.pivot.test @@ -781,4 +781,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.select_expr.test b/tests/ast/data/DataFrame.select_expr.test index 67f46b4d92..79c63c2779 100644 --- a/tests/ast/data/DataFrame.select_expr.test +++ b/tests/ast/data/DataFrame.select_expr.test @@ -378,4 +378,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.stat.test b/tests/ast/data/DataFrame.stat.test index 12e9943c2a..35a1692db7 100644 --- a/tests/ast/data/DataFrame.stat.test +++ b/tests/ast/data/DataFrame.stat.test @@ -1230,4 +1230,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.to_df.test b/tests/ast/data/DataFrame.to_df.test index f2b8e908b0..a148a82f95 100644 --- a/tests/ast/data/DataFrame.to_df.test +++ b/tests/ast/data/DataFrame.to_df.test @@ -183,4 +183,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.to_local_iterator.test b/tests/ast/data/DataFrame.to_local_iterator.test index 1334995cf5..b208ed6b6f 100644 --- a/tests/ast/data/DataFrame.to_local_iterator.test +++ b/tests/ast/data/DataFrame.to_local_iterator.test @@ -416,4 +416,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.to_pandas.test b/tests/ast/data/DataFrame.to_pandas.test index bc09b5c43a..92bdddc855 100644 --- a/tests/ast/data/DataFrame.to_pandas.test +++ b/tests/ast/data/DataFrame.to_pandas.test @@ -217,4 +217,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.to_pandas_batch.test b/tests/ast/data/DataFrame.to_pandas_batch.test index bddc9cd8c2..84ce7eece5 100644 --- a/tests/ast/data/DataFrame.to_pandas_batch.test +++ b/tests/ast/data/DataFrame.to_pandas_batch.test @@ -217,4 +217,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.unpivot.test b/tests/ast/data/DataFrame.unpivot.test index ad3e73cc7f..792f769fc9 100644 --- a/tests/ast/data/DataFrame.unpivot.test +++ b/tests/ast/data/DataFrame.unpivot.test @@ -244,4 +244,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataFrame.write.test b/tests/ast/data/DataFrame.write.test index d0c3caeb4b..de70b598e2 100644 --- a/tests/ast/data/DataFrame.write.test +++ b/tests/ast/data/DataFrame.write.test @@ -1666,4 +1666,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.cube.test b/tests/ast/data/Dataframe.cube.test index 8b9f854e91..e1cb194475 100644 --- a/tests/ast/data/Dataframe.cube.test +++ b/tests/ast/data/Dataframe.cube.test @@ -555,4 +555,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.distinct.test b/tests/ast/data/Dataframe.distinct.test index 8268ae6b85..5fd7bc7136 100644 --- a/tests/ast/data/Dataframe.distinct.test +++ b/tests/ast/data/Dataframe.distinct.test @@ -96,4 +96,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.drop_duplicates.test b/tests/ast/data/Dataframe.drop_duplicates.test index c23222feb9..58be2b96c3 100644 --- a/tests/ast/data/Dataframe.drop_duplicates.test +++ b/tests/ast/data/Dataframe.drop_duplicates.test @@ -301,4 +301,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.drop_duplicates_snow1874622.test b/tests/ast/data/Dataframe.drop_duplicates_snow1874622.test index 9685d9b5a5..da1f0d55d8 100644 --- a/tests/ast/data/Dataframe.drop_duplicates_snow1874622.test +++ b/tests/ast/data/Dataframe.drop_duplicates_snow1874622.test @@ -532,4 +532,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.filter.test b/tests/ast/data/Dataframe.filter.test index 4820ffcc89..14c4f5d50e 100644 --- a/tests/ast/data/Dataframe.filter.test +++ b/tests/ast/data/Dataframe.filter.test @@ -401,4 +401,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.getitem.test b/tests/ast/data/Dataframe.getitem.test index eea6e98486..e0ca107ed0 100644 --- a/tests/ast/data/Dataframe.getitem.test +++ b/tests/ast/data/Dataframe.getitem.test @@ -262,4 +262,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.group_by.test b/tests/ast/data/Dataframe.group_by.test index 23d09ba60f..4d607be6b0 100644 --- a/tests/ast/data/Dataframe.group_by.test +++ b/tests/ast/data/Dataframe.group_by.test @@ -555,4 +555,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.group_by_grouping_sets.test b/tests/ast/data/Dataframe.group_by_grouping_sets.test index b4326f367c..642890429c 100644 --- a/tests/ast/data/Dataframe.group_by_grouping_sets.test +++ b/tests/ast/data/Dataframe.group_by_grouping_sets.test @@ -1137,4 +1137,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.join.asof.test b/tests/ast/data/Dataframe.join.asof.test index d34eecabba..229f2a0036 100644 --- a/tests/ast/data/Dataframe.join.asof.test +++ b/tests/ast/data/Dataframe.join.asof.test @@ -759,4 +759,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.join.prefix.test b/tests/ast/data/Dataframe.join.prefix.test index 7ab7a67579..287e1d596c 100644 --- a/tests/ast/data/Dataframe.join.prefix.test +++ b/tests/ast/data/Dataframe.join.prefix.test @@ -838,4 +838,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.rollup.test b/tests/ast/data/Dataframe.rollup.test index 6172ee2f38..cdfdab48de 100644 --- a/tests/ast/data/Dataframe.rollup.test +++ b/tests/ast/data/Dataframe.rollup.test @@ -555,4 +555,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.show.test b/tests/ast/data/Dataframe.show.test index 240a9bf1e5..36448cbe16 100644 --- a/tests/ast/data/Dataframe.show.test +++ b/tests/ast/data/Dataframe.show.test @@ -393,4 +393,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.to_snowpark_pandas.test b/tests/ast/data/Dataframe.to_snowpark_pandas.test index fa65e95f73..34c199b57b 100644 --- a/tests/ast/data/Dataframe.to_snowpark_pandas.test +++ b/tests/ast/data/Dataframe.to_snowpark_pandas.test @@ -202,4 +202,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Dataframe.with_col_fns.test b/tests/ast/data/Dataframe.with_col_fns.test index 0c051262ba..0f2a0fe292 100644 --- a/tests/ast/data/Dataframe.with_col_fns.test +++ b/tests/ast/data/Dataframe.with_col_fns.test @@ -967,4 +967,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/DataframeNaFunctions.test b/tests/ast/data/DataframeNaFunctions.test index 90221f64a6..324eeda83b 100644 --- a/tests/ast/data/DataframeNaFunctions.test +++ b/tests/ast/data/DataframeNaFunctions.test @@ -866,4 +866,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/RelationalGroupedDataFrame.agg.test b/tests/ast/data/RelationalGroupedDataFrame.agg.test index b25f258e97..1c04df4dca 100644 --- a/tests/ast/data/RelationalGroupedDataFrame.agg.test +++ b/tests/ast/data/RelationalGroupedDataFrame.agg.test @@ -1199,4 +1199,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/RelationalGroupedDataFrame.test b/tests/ast/data/RelationalGroupedDataFrame.test index 6a831c03a5..a98debfb4e 100644 --- a/tests/ast/data/RelationalGroupedDataFrame.test +++ b/tests/ast/data/RelationalGroupedDataFrame.test @@ -1816,4 +1816,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Session.call.test b/tests/ast/data/Session.call.test index 6ab0076da2..9521510cb2 100644 --- a/tests/ast/data/Session.call.test +++ b/tests/ast/data/Session.call.test @@ -359,4 +359,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Session.create_dataframe.test b/tests/ast/data/Session.create_dataframe.test index d682e0a12b..5873ebb922 100644 --- a/tests/ast/data/Session.create_dataframe.test +++ b/tests/ast/data/Session.create_dataframe.test @@ -1041,4 +1041,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Session.create_dataframe_from_pandas.test b/tests/ast/data/Session.create_dataframe_from_pandas.test index 9ee75506c9..36fc9d1676 100644 --- a/tests/ast/data/Session.create_dataframe_from_pandas.test +++ b/tests/ast/data/Session.create_dataframe_from_pandas.test @@ -176,4 +176,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Session.flatten.test b/tests/ast/data/Session.flatten.test index 7a63a9a0e2..d25a3e6d7f 100644 --- a/tests/ast/data/Session.flatten.test +++ b/tests/ast/data/Session.flatten.test @@ -179,4 +179,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Session.table_function.test b/tests/ast/data/Session.table_function.test index b56efefbb1..3c76eca6f2 100644 --- a/tests/ast/data/Session.table_function.test +++ b/tests/ast/data/Session.table_function.test @@ -997,4 +997,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Table.delete.test b/tests/ast/data/Table.delete.test index 807c55ac57..f3b2091e6b 100644 --- a/tests/ast/data/Table.delete.test +++ b/tests/ast/data/Table.delete.test @@ -579,4 +579,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Table.drop_table.test b/tests/ast/data/Table.drop_table.test index 4c795040df..af89dce21d 100644 --- a/tests/ast/data/Table.drop_table.test +++ b/tests/ast/data/Table.drop_table.test @@ -99,4 +99,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Table.init.test b/tests/ast/data/Table.init.test index e02d1f0614..dd31bc5276 100644 --- a/tests/ast/data/Table.init.test +++ b/tests/ast/data/Table.init.test @@ -145,4 +145,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Table.merge.test b/tests/ast/data/Table.merge.test index 06fd8bde9f..12f2891e48 100644 --- a/tests/ast/data/Table.merge.test +++ b/tests/ast/data/Table.merge.test @@ -1564,4 +1564,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Table.sample.test b/tests/ast/data/Table.sample.test index 2bab87565a..aeb67c2887 100644 --- a/tests/ast/data/Table.sample.test +++ b/tests/ast/data/Table.sample.test @@ -183,4 +183,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/Table.update.test b/tests/ast/data/Table.update.test index f8e0b5c3f6..76bc8c9632 100644 --- a/tests/ast/data/Table.update.test +++ b/tests/ast/data/Table.update.test @@ -728,4 +728,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/case_when.test b/tests/ast/data/case_when.test index 47a209025f..3e00079f68 100644 --- a/tests/ast/data/case_when.test +++ b/tests/ast/data/case_when.test @@ -1948,4 +1948,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_alias.test b/tests/ast/data/col_alias.test index 0243c0f1a8..aa806a99f9 100644 --- a/tests/ast/data/col_alias.test +++ b/tests/ast/data/col_alias.test @@ -428,4 +428,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_asc.test b/tests/ast/data/col_asc.test index f5274b4dca..8603df2501 100644 --- a/tests/ast/data/col_asc.test +++ b/tests/ast/data/col_asc.test @@ -317,4 +317,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_between.test b/tests/ast/data/col_between.test index 4b5703c70a..136425f8cb 100644 --- a/tests/ast/data/col_between.test +++ b/tests/ast/data/col_between.test @@ -191,4 +191,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_binops.test b/tests/ast/data/col_binops.test index b2a5260a6c..1e47c53899 100644 --- a/tests/ast/data/col_binops.test +++ b/tests/ast/data/col_binops.test @@ -1674,4 +1674,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_bitops.test b/tests/ast/data/col_bitops.test index 3a7f6c7b48..3efe6d9de0 100644 --- a/tests/ast/data/col_bitops.test +++ b/tests/ast/data/col_bitops.test @@ -409,4 +409,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_cast.test b/tests/ast/data/col_cast.test index d41d59bef7..001ffba3b6 100644 --- a/tests/ast/data/col_cast.test +++ b/tests/ast/data/col_cast.test @@ -2737,4 +2737,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_cast_coll.test b/tests/ast/data/col_cast_coll.test index be8bdd9e7c..443a95cc5d 100644 --- a/tests/ast/data/col_cast_coll.test +++ b/tests/ast/data/col_cast_coll.test @@ -841,4 +841,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_desc.test b/tests/ast/data/col_desc.test index 32a519c13c..8773d5f388 100644 --- a/tests/ast/data/col_desc.test +++ b/tests/ast/data/col_desc.test @@ -315,4 +315,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_getitem.test b/tests/ast/data/col_getitem.test index 06938a767c..27c8f318ba 100644 --- a/tests/ast/data/col_getitem.test +++ b/tests/ast/data/col_getitem.test @@ -192,4 +192,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_in_.test b/tests/ast/data/col_in_.test index f5cb5551c1..0294e10107 100644 --- a/tests/ast/data/col_in_.test +++ b/tests/ast/data/col_in_.test @@ -375,4 +375,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_lit.test b/tests/ast/data/col_lit.test index 8886f6c1bd..b2ae0c9ad8 100644 --- a/tests/ast/data/col_lit.test +++ b/tests/ast/data/col_lit.test @@ -535,4 +535,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_literal.test b/tests/ast/data/col_literal.test index a85538e32a..a764097269 100644 --- a/tests/ast/data/col_literal.test +++ b/tests/ast/data/col_literal.test @@ -3852,4 +3852,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_null_nan.test b/tests/ast/data/col_null_nan.test index e1d2286051..e3b45c9ee5 100644 --- a/tests/ast/data/col_null_nan.test +++ b/tests/ast/data/col_null_nan.test @@ -422,4 +422,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_rbinops.test b/tests/ast/data/col_rbinops.test index e8d9a3bd38..99a7b2c864 100644 --- a/tests/ast/data/col_rbinops.test +++ b/tests/ast/data/col_rbinops.test @@ -808,4 +808,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_star.test b/tests/ast/data/col_star.test index 3ae97fe9bc..9bb2eca80a 100644 --- a/tests/ast/data/col_star.test +++ b/tests/ast/data/col_star.test @@ -134,4 +134,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_string.test b/tests/ast/data/col_string.test index 37cee15118..cc98342532 100644 --- a/tests/ast/data/col_string.test +++ b/tests/ast/data/col_string.test @@ -771,4 +771,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_try_cast.test b/tests/ast/data/col_try_cast.test index 3596680bbd..0bb5dc237a 100644 --- a/tests/ast/data/col_try_cast.test +++ b/tests/ast/data/col_try_cast.test @@ -2465,4 +2465,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_udf.test b/tests/ast/data/col_udf.test index 25f5f46789..e1d344544f 100644 --- a/tests/ast/data/col_udf.test +++ b/tests/ast/data/col_udf.test @@ -815,4 +815,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_unary_ops.test b/tests/ast/data/col_unary_ops.test index 42288dd692..64b882d632 100644 --- a/tests/ast/data/col_unary_ops.test +++ b/tests/ast/data/col_unary_ops.test @@ -226,4 +226,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/col_within_group.test b/tests/ast/data/col_within_group.test index faaf81e3cc..719d67c98d 100644 --- a/tests/ast/data/col_within_group.test +++ b/tests/ast/data/col_within_group.test @@ -424,4 +424,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_alias.test b/tests/ast/data/df_alias.test index 6c0618ff6e..c0e0213ad2 100644 --- a/tests/ast/data/df_alias.test +++ b/tests/ast/data/df_alias.test @@ -97,4 +97,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_analytics_functions.test b/tests/ast/data/df_analytics_functions.test index d8fadc01f3..6d744c4bc5 100644 --- a/tests/ast/data/df_analytics_functions.test +++ b/tests/ast/data/df_analytics_functions.test @@ -1009,4 +1009,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_col.test b/tests/ast/data/df_col.test index 4c09946097..5cf28db990 100644 --- a/tests/ast/data/df_col.test +++ b/tests/ast/data/df_col.test @@ -157,4 +157,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_copy.test b/tests/ast/data/df_copy.test index 150bd3a0fc..cd364ff1a1 100644 --- a/tests/ast/data/df_copy.test +++ b/tests/ast/data/df_copy.test @@ -883,4 +883,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_drop.test b/tests/ast/data/df_drop.test index 633e13b90b..6810587971 100644 --- a/tests/ast/data/df_drop.test +++ b/tests/ast/data/df_drop.test @@ -134,4 +134,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_except.test b/tests/ast/data/df_except.test index 9f10e13355..5280c7737b 100644 --- a/tests/ast/data/df_except.test +++ b/tests/ast/data/df_except.test @@ -139,4 +139,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_first.test b/tests/ast/data/df_first.test index e06df1030b..85c497173f 100644 --- a/tests/ast/data/df_first.test +++ b/tests/ast/data/df_first.test @@ -293,4 +293,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_intersect.test b/tests/ast/data/df_intersect.test index 2390b743b8..1f00a8916e 100644 --- a/tests/ast/data/df_intersect.test +++ b/tests/ast/data/df_intersect.test @@ -140,4 +140,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_limit.test b/tests/ast/data/df_limit.test index f5642c66a1..2aa620f984 100644 --- a/tests/ast/data/df_limit.test +++ b/tests/ast/data/df_limit.test @@ -268,4 +268,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_random_split.test b/tests/ast/data/df_random_split.test index 922adbcf22..0a5cd0e481 100644 --- a/tests/ast/data/df_random_split.test +++ b/tests/ast/data/df_random_split.test @@ -712,4 +712,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_sample.test b/tests/ast/data/df_sample.test index 16be32998c..72248bb29a 100644 --- a/tests/ast/data/df_sample.test +++ b/tests/ast/data/df_sample.test @@ -135,4 +135,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_sort.test b/tests/ast/data/df_sort.test index 84d5509824..c70a9c8366 100644 --- a/tests/ast/data/df_sort.test +++ b/tests/ast/data/df_sort.test @@ -672,4 +672,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/df_union.test b/tests/ast/data/df_union.test index 26e4a9f0be..f4f05200db 100644 --- a/tests/ast/data/df_union.test +++ b/tests/ast/data/df_union.test @@ -420,4 +420,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/functions.table_functions.test b/tests/ast/data/functions.table_functions.test index dd98a482fe..86d4f35610 100644 --- a/tests/ast/data/functions.table_functions.test +++ b/tests/ast/data/functions.table_functions.test @@ -1984,4 +1984,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/functions1.test b/tests/ast/data/functions1.test index b606a31b4b..f8e0bce3c5 100644 --- a/tests/ast/data/functions1.test +++ b/tests/ast/data/functions1.test @@ -19157,4 +19157,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/functions2.test b/tests/ast/data/functions2.test index d652e20652..aedb2c4777 100644 --- a/tests/ast/data/functions2.test +++ b/tests/ast/data/functions2.test @@ -25435,4 +25435,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/interval.test b/tests/ast/data/interval.test index 4bf2f37bbf..00f7c11217 100644 --- a/tests/ast/data/interval.test +++ b/tests/ast/data/interval.test @@ -1102,4 +1102,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/select.test b/tests/ast/data/select.test index e04dd878a8..2278232796 100644 --- a/tests/ast/data/select.test +++ b/tests/ast/data/select.test @@ -146,4 +146,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session.read.test b/tests/ast/data/session.read.test index 7b9d500ea5..23715f6a24 100644 --- a/tests/ast/data/session.read.test +++ b/tests/ast/data/session.read.test @@ -569,4 +569,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session.sql.test b/tests/ast/data/session.sql.test index d2c14c87fb..860ff05e98 100644 --- a/tests/ast/data/session.sql.test +++ b/tests/ast/data/session.sql.test @@ -156,4 +156,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_generator.test b/tests/ast/data/session_generator.test index 7e776a26be..1c4486626a 100644 --- a/tests/ast/data/session_generator.test +++ b/tests/ast/data/session_generator.test @@ -501,4 +501,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_range.test b/tests/ast/data/session_range.test index c836a8f950..1348c52093 100644 --- a/tests/ast/data/session_range.test +++ b/tests/ast/data/session_range.test @@ -183,4 +183,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_dq_abs_l.test b/tests/ast/data/session_table_dq_abs_l.test index 33eadff459..7eeacf0eaf 100644 --- a/tests/ast/data/session_table_dq_abs_l.test +++ b/tests/ast/data/session_table_dq_abs_l.test @@ -112,4 +112,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_dq_abs_s.test b/tests/ast/data/session_table_dq_abs_s.test index 8488d5afd0..d00a8bc2a6 100644 --- a/tests/ast/data/session_table_dq_abs_s.test +++ b/tests/ast/data/session_table_dq_abs_s.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_dq_rs_l.test b/tests/ast/data/session_table_dq_rs_l.test index 2d922e2dce..9cd312eeea 100644 --- a/tests/ast/data/session_table_dq_rs_l.test +++ b/tests/ast/data/session_table_dq_rs_l.test @@ -111,4 +111,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_dq_rs_s.test b/tests/ast/data/session_table_dq_rs_s.test index 46c89a3951..42f2ef2389 100644 --- a/tests/ast/data/session_table_dq_rs_s.test +++ b/tests/ast/data/session_table_dq_rs_s.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_dq_rt_l.test b/tests/ast/data/session_table_dq_rt_l.test index 02f4d837ab..ab96614e90 100644 --- a/tests/ast/data/session_table_dq_rt_l.test +++ b/tests/ast/data/session_table_dq_rt_l.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_dq_rt_s.test b/tests/ast/data/session_table_dq_rt_s.test index 8ffb7802af..d529b4bed7 100644 --- a/tests/ast/data/session_table_dq_rt_s.test +++ b/tests/ast/data/session_table_dq_rt_s.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_temp_table_cleanup.test b/tests/ast/data/session_table_temp_table_cleanup.test index c465ecdeca..2b754098f6 100644 --- a/tests/ast/data/session_table_temp_table_cleanup.test +++ b/tests/ast/data/session_table_temp_table_cleanup.test @@ -185,4 +185,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_uq_abs_l.test b/tests/ast/data/session_table_uq_abs_l.test index c9d13838cb..013d0d2b72 100644 --- a/tests/ast/data/session_table_uq_abs_l.test +++ b/tests/ast/data/session_table_uq_abs_l.test @@ -112,4 +112,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_uq_abs_s.test b/tests/ast/data/session_table_uq_abs_s.test index 293ec3369d..5757643369 100644 --- a/tests/ast/data/session_table_uq_abs_s.test +++ b/tests/ast/data/session_table_uq_abs_s.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_uq_rs_l.test b/tests/ast/data/session_table_uq_rs_l.test index 50b4da799a..d0672dcc1b 100644 --- a/tests/ast/data/session_table_uq_rs_l.test +++ b/tests/ast/data/session_table_uq_rs_l.test @@ -111,4 +111,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_uq_rs_s.test b/tests/ast/data/session_table_uq_rs_s.test index 9225cf6206..8849662810 100644 --- a/tests/ast/data/session_table_uq_rs_s.test +++ b/tests/ast/data/session_table_uq_rs_s.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_uq_rt_l.test b/tests/ast/data/session_table_uq_rt_l.test index 2fff6f4fb7..fec32cd5f1 100644 --- a/tests/ast/data/session_table_uq_rt_l.test +++ b/tests/ast/data/session_table_uq_rt_l.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_table_uq_rt_s.test b/tests/ast/data/session_table_uq_rt_s.test index aee5cd1aa5..beb016111b 100644 --- a/tests/ast/data/session_table_uq_rt_s.test +++ b/tests/ast/data/session_table_uq_rt_s.test @@ -110,4 +110,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/session_write_pandas.test b/tests/ast/data/session_write_pandas.test index 959075aa5b..b2baadb7c3 100644 --- a/tests/ast/data/session_write_pandas.test +++ b/tests/ast/data/session_write_pandas.test @@ -154,4 +154,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/shadowed_local_name.test b/tests/ast/data/shadowed_local_name.test index e0c23f067a..9b425a3ecc 100644 --- a/tests/ast/data/shadowed_local_name.test +++ b/tests/ast/data/shadowed_local_name.test @@ -220,4 +220,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/smoke1.test b/tests/ast/data/smoke1.test index 9b387b183b..cdadd14b70 100644 --- a/tests/ast/data/smoke1.test +++ b/tests/ast/data/smoke1.test @@ -1136,4 +1136,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/smoke2.test b/tests/ast/data/smoke2.test index 6eb5b4a375..78d415b73a 100644 --- a/tests/ast/data/smoke2.test +++ b/tests/ast/data/smoke2.test @@ -1219,4 +1219,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/sproc.test b/tests/ast/data/sproc.test index fdcf4d03eb..144b9624cb 100644 --- a/tests/ast/data/sproc.test +++ b/tests/ast/data/sproc.test @@ -2203,4 +2203,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/udaf.test b/tests/ast/data/udaf.test index 4b1b09fb26..ffaa33e10e 100644 --- a/tests/ast/data/udaf.test +++ b/tests/ast/data/udaf.test @@ -675,4 +675,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/udtf.test b/tests/ast/data/udtf.test index 2e9216fa74..decba45ec7 100644 --- a/tests/ast/data/udtf.test +++ b/tests/ast/data/udtf.test @@ -1880,4 +1880,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/ast/data/windows.test b/tests/ast/data/windows.test index 70a86a9658..e196dc5650 100644 --- a/tests/ast/data/windows.test +++ b/tests/ast/data/windows.test @@ -2158,4 +2158,5 @@ client_language { client_version { major: 1 minor: 29 + patch: 1 } diff --git a/tests/integ/test_data_source_api.py b/tests/integ/test_data_source_api.py index d4d765e56b..dc9919ae08 100644 --- a/tests/integ/test_data_source_api.py +++ b/tests/integ/test_data_source_api.py @@ -4,7 +4,6 @@ import functools import math import os -import queue import tempfile import time import datetime @@ -13,9 +12,7 @@ import pytest -from snowflake.snowpark._internal.utils import ( - TempObjectType, -) +from snowflake.snowpark._internal.utils import TempObjectType from snowflake.snowpark.dataframe_reader import _MAX_RETRY_TIME, DataFrameReader from snowflake.snowpark._internal.data_source_utils import ( DATA_SOURCE_DBAPI_SIGNATURE, @@ -66,10 +63,24 @@ ) from tests.utils import Utils, IS_WINDOWS -pytestmark = pytest.mark.skipif( - "config.getoption('local_testing_mode', default=False)", - reason="feature not available in local testing", -) +try: + import pandas # noqa: F401 + + is_pandas_available = True +except ImportError: + is_pandas_available = False + + +pytestmark = [ + pytest.mark.skipif( + "config.getoption('local_testing_mode', default=False)", + reason="feature not available in local testing", + ), + pytest.mark.skipif( + not is_pandas_available, + reason="pandas is not available", + ), +] SQL_SERVER_TABLE_NAME = "AllDataTypesTable" ORACLEDB_TABLE_NAME = "ALL_TYPES_TABLE" @@ -140,7 +151,6 @@ def test_dbapi_retry(session): SnowparkDataframeReaderException, match="\\[RuntimeError\\] Test error" ): DataFrameReader._task_fetch_from_data_source_with_retry( - parquet_file_queue=queue.Queue(), create_connection=sql_server_create_connection, query="SELECT * FROM test_table", schema=StructType([StructField("col1", IntegerType(), False)]), @@ -350,7 +360,7 @@ def assert_datasource_statement_params_run_query(*args, **kwargs): statement_parameters = kwargs.get("_statement_params") query = args[0] assert statement_parameters[STATEMENT_PARAMS_DATA_SOURCE] == "1" - if "select" not in query.lower(): + if "select" not in query.lower() and "put" not in query.lower(): assert DATA_SOURCE_SQL_COMMENT in query comment_showed += 1 nonlocal called @@ -368,8 +378,8 @@ def assert_datasource_statement_params_run_query(*args, **kwargs): ) assert df._plan.api_calls == [{"name": DATA_SOURCE_DBAPI_SIGNATURE}] assert ( - called == 4 and comment_showed == 4 - ) # 4 queries: create table, create stage, put file, copy into + called == 4 and comment_showed == 3 + ) # 4 queries: create table, create stage, put file, copy into, but we use session.read.put not supporting comment assert mock_telemetry.called assert df.collect() == sql_server_all_type_data @@ -516,7 +526,6 @@ def test_negative_case(session): def test_task_fetch_from_data_source_with_fetch_size( fetch_size, partition_idx, expected_error ): - parquet_file_queue = queue.Queue() schema = infer_data_source_schema( sql_server_create_connection_small_data(), SQL_SERVER_TABLE_NAME, @@ -532,7 +541,6 @@ def test_task_fetch_from_data_source_with_fetch_size( with tempfile.TemporaryDirectory() as tmp_dir: params = { - "parquet_file_queue": parquet_file_queue, "create_connection": sql_server_create_connection_small_data, "query": "SELECT * FROM test_table", "schema": schema, @@ -550,16 +558,12 @@ def test_task_fetch_from_data_source_with_fetch_size( DataFrameReader._task_fetch_from_data_source(**params) else: DataFrameReader._task_fetch_from_data_source(**params) - - file_idx = 0 - while not parquet_file_queue.empty(): - file_path = parquet_file_queue.get() + files = sorted(os.listdir(tmp_dir)) + for idx, file in enumerate(files): assert ( - f"data_partition{partition_idx}_fetch{file_idx}.parquet" - in file_path - ) - file_idx += 1 - assert file_idx == file_count + f"data_partition{partition_idx}_fetch{idx}.parquet" in file + ), f"file: {file} does not match" + assert len(files) == file_count def test_database_detector(): diff --git a/tests/integ/test_stored_procedure.py b/tests/integ/test_stored_procedure.py index 94a00f8c07..af09e6d4de 100644 --- a/tests/integ/test_stored_procedure.py +++ b/tests/integ/test_stored_procedure.py @@ -2016,3 +2016,44 @@ def plus1(session_, x): caplog.clear() run_test_case(caplog, version_override, expect_warning) + + +@pytest.mark.skipif( + "config.getoption('local_testing_mode', default=False)", + reason="data source is not supported in local testing", + run=False, +) +def test_datasource_put_file_and_copy_into_in_sproc(session): + # The tests session.file.put API as well as the COPY INTO sql executed inside stored proc + def upload_and_copy_into(session_): + from snowflake.snowpark._internal.utils import ( + random_name_for_temp_object, + TempObjectType, + ) + + table_name = random_name_for_temp_object(TempObjectType.TABLE) + stage_name = random_name_for_temp_object(TempObjectType.STAGE) + session_.sql(f"CREATE TEMPORARY TABLE {table_name} (col INT)").collect() + session_.sql(f"CREATE TEMPORARY STAGE {stage_name}").collect() + + file_name = "data.csv" + csv_filename = f"/tmp/{file_name}" + with open(csv_filename, "w") as file: + file.write("42\n") # Sample data + + session_.file.put( + csv_filename, + f"@{stage_name}", + overwrite=True, + ) + session_.sql( + f"COPY INTO {table_name} FROM @{stage_name}/{file_name} FILE_FORMAT = (TYPE = CSV)" + ).collect() + if session_.table(table_name).collect() == [(42,)]: + return "success" + else: + return "failure" + + # sproc execution + ingestion = sproc(upload_and_copy_into, return_type=StringType()) + assert ingestion() == "success"