Merge pull request #535 from duckdb/v1.4-andium

V1.4 andium

Author: Tishj

.github/workflows/LocalTesting.yml

@@ -55,10 +55,6 @@ jobs:
           sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
 
-      - name: Install required python packages
-        run: |
-          python3 -m pip install pyiceberg pytest pyarrow
-
       - name: Install CMake 3.x
         run: |
           sudo apt-get remove -y cmake cmake-data
@@ -69,6 +65,14 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.13' 
+
+      - name: Install required python packages
+        run: |
+          python3 -m pip install -r scripts/requirements.txt 
+
       - name: Check installed versions
         run: |
           ninja --version
@@ -105,4 +109,4 @@ jobs:
         env:
           ICEBERG_SERVER_AVAILABLE: 1
         run: |
-          python3 -m pytest test/python
+          python3 -m pytest test/python

.github/workflows/MainDistributionPipeline.yml

@@ -17,8 +17,8 @@ jobs:
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: iceberg
-      duckdb_version: v1.4.0
-      ci_tools_version: main
+      duckdb_version: v1.4.1
+      ci_tools_version: v1.4.1
       exclude_archs: 'windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads'
       extra_toolchains: 'python3'
 
@@ -29,8 +29,8 @@ jobs:
     secrets: inherit
     with:
       extension_name: iceberg
-      duckdb_version: v1.4.0
-      ci_tools_version: main
+      duckdb_version: v1.4.1
+      ci_tools_version: v1.4.1
       exclude_archs: 'windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads'
       deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}
       deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}

duckdb

@@ -2,7 +2,7 @@
 duckdb_extension_load(avro
 		LOAD_TESTS
 		GIT_URL https://github.com/duckdb/duckdb-avro
-		GIT_TAG 0c97a61781f63f8c5444cf3e0c6881ecbaa9fe13
+		GIT_TAG 7b75062f6345d11c5342c09216a75c57342c2e82
 )
 
 # Extension from this repo
@@ -17,7 +17,7 @@ duckdb_extension_load(icu)
 duckdb_extension_load(ducklake
         LOAD_TESTS
         GIT_URL https://github.com/duckdb/ducklake
-        GIT_TAG dbb022506e21c27fc4d4cd3d14995af89955401a
+        GIT_TAG f134ad86f2f6e7cdf4133086c38ecd9c48f1a772
 )
 
 
@@ -27,7 +27,7 @@ if (NOT MINGW)
     duckdb_extension_load(aws
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb-aws
-            GIT_TAG f855eb3dce37700bfd36fe906a683e4be17dcaf6
+            GIT_TAG 18803d5e55b9f9f6dda5047d0fdb4f4238b6801d
     )
 endif ()
 endif()

extension_config.cmake

@@ -1,4 +1,6 @@
 pyspark==3.5.0
 duckdb
-pyiceberg
-pyarrow
+pydantic==2.9.0
+pyiceberg==0.9.1
+pyarrow
+pytest

scripts/data_generators/iceberg-spark-runtime-3.5_2.12-1.4.1.jar

@@ -429,6 +429,10 @@ rest_api_objects::LoadTableResult IRCAPI::CommitNewTable(ClientContext &context,
 	try {
 		HTTPHeaders headers(*context.db);
 		headers.Insert("Content-Type", "application/json");
+		// if you are creating a table with stage create, you need vended credentials
+		if (catalog.attach_options.access_mode == IRCAccessDelegationMode::VENDED_CREDENTIALS) {
+			headers.Insert("X-Iceberg-Access-Delegation", "vended-credentials");
+		}
 		auto response =
 		    catalog.auth_handler->Request(RequestType::POST_REQUEST, context, url_builder, headers, create_table_json);
 		if (response->status != HTTPStatusCode::OK_200) {

scripts/requirements.txt

@@ -2,6 +2,7 @@
 
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/http_util.hpp"
+#include "duckdb/main/extension_helper.hpp"
 #include "duckdb/common/exception/http_exception.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/main/client_data.hpp"
@@ -30,6 +31,15 @@ const string &APIUtils::GetCURLCertPath() {
 unique_ptr<HTTPResponse> APIUtils::Request(RequestType request_type, ClientContext &context,
                                            const IRCEndpointBuilder &endpoint_builder, HTTPHeaders &headers,
                                            const string &data) {
+	// load httpfs since iceberg requests do not go through the file system api
+	if (!context.db.get()) {
+		throw InvalidConfigurationException("Context does not have database instance when loading Httpfs in Iceberg");
+	}
+	ExtensionHelper::AutoLoadExtension(*context.db, "httpfs");
+	if (!context.db->ExtensionIsLoaded("httpfs")) {
+		throw MissingExtensionException("The iceberg extension requires the httpfs extension to be loaded!");
+	}
+
 	auto &db = DatabaseInstance::GetDatabase(context);
 	string request_url = AddHttpHostIfMissing(endpoint_builder.GetURL());
 

src/catalog_api.cpp

@@ -319,7 +319,7 @@ bool IcebergMultiFileList::FileMatchesFilter(const IcebergManifestEntry &file) {
 		}
 
 		auto &filter = *it->second;
-		if (!IcebergPredicate::MatchBounds(filter, stats, IcebergTransform::Identity())) {
+		if (!IcebergPredicate::MatchBounds(context, filter, stats, IcebergTransform::Identity())) {
 			//! If any predicate fails, exclude the file
 			return false;
 		}
@@ -521,7 +521,7 @@ bool IcebergMultiFileList::ManifestMatchesFilter(const IcebergManifest &manifest
 		stats.has_null = field_summary.contains_null;
 		stats.has_not_null = true; // Not enough information in field_summary to determine if this should be false
 
-		if (!IcebergPredicate::MatchBounds(*table_filter, stats, field.transform)) {
+		if (!IcebergPredicate::MatchBounds(context, *table_filter, stats, field.transform)) {
 			return false;
 		}
 	}

src/common/api_utils.cpp

@@ -78,17 +78,21 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 	vector<LogicalType> types;
 
 	auto &current_partition_spec = table_info.table_metadata.GetLatestPartitionSpec();
+
 	{
+		child_list_t<Value> status_field;
 		// status: int - 0
 		names.push_back("status");
 		types.push_back(LogicalType::INTEGER);
-		field_ids.emplace_back("status", Value::INTEGER(STATUS));
+		status_field.emplace_back("__duckdb_field_id", Value::INTEGER(STATUS));
+		status_field.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", STATUS);
 		yyjson_mut_obj_add_strcpy(doc, field_obj, "name", "status");
 		yyjson_mut_obj_add_bool(doc, field_obj, "required", true);
 		yyjson_mut_obj_add_strcpy(doc, field_obj, "type", "int");
+		field_ids.emplace_back("status", Value::STRUCT(status_field));
 	}
 
 	{
@@ -137,9 +141,12 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 
 	auto child_fields_arr = yyjson_mut_arr(doc);
 	{
+		child_list_t<Value> content_field;
 		// content: int - 134
 		children.emplace_back("content", LogicalType::INTEGER);
-		data_file_field_ids.emplace_back("content", Value::INTEGER(CONTENT));
+		content_field.emplace_back("__duckdb_field_id", Value::INTEGER(CONTENT));
+		content_field.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
+		data_file_field_ids.emplace_back("content", Value::STRUCT(content_field));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, child_fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", CONTENT);
@@ -149,9 +156,12 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 	}
 
 	{
+		child_list_t<Value> file_path;
 		// file_path: string - 100
 		children.emplace_back("file_path", LogicalType::VARCHAR);
-		data_file_field_ids.emplace_back("file_path", Value::INTEGER(FILE_PATH));
+		file_path.emplace_back("__duckdb_field_id", Value::INTEGER(FILE_PATH));
+		file_path.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
+		data_file_field_ids.emplace_back("file_path", Value::STRUCT(file_path));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, child_fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", FILE_PATH);
@@ -161,9 +171,12 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 	}
 
 	{
+		child_list_t<Value> file_format;
 		// file_format: string - 101
 		children.emplace_back("file_format", LogicalType::VARCHAR);
-		data_file_field_ids.emplace_back("file_format", Value::INTEGER(FILE_FORMAT));
+		file_format.emplace_back("__duckdb_field_id", Value::INTEGER(FILE_FORMAT));
+		file_format.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
+		data_file_field_ids.emplace_back("file_format", Value::STRUCT(file_format));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, child_fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", FILE_FORMAT);
@@ -173,9 +186,12 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 	}
 
 	{
+		child_list_t<Value> partition;
 		// partition: struct(...) - 102
 		children.emplace_back("partition", PartitionStructType(table_info, manifest_file));
-		data_file_field_ids.emplace_back("partition", Value::INTEGER(PARTITION));
+		partition.emplace_back("__duckdb_field_id", Value::INTEGER(PARTITION));
+		partition.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
+		data_file_field_ids.emplace_back("partition", Value::STRUCT(partition));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, child_fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", PARTITION);
@@ -190,9 +206,12 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 	}
 
 	{
+		child_list_t<Value> record_count;
 		// record_count: long - 103
 		children.emplace_back("record_count", LogicalType::BIGINT);
-		data_file_field_ids.emplace_back("record_count", Value::INTEGER(RECORD_COUNT));
+		record_count.emplace_back("__duckdb_field_id", Value::INTEGER(RECORD_COUNT));
+		record_count.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
+		data_file_field_ids.emplace_back("record_count", Value::STRUCT(record_count));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, child_fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", RECORD_COUNT);
@@ -202,9 +221,12 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 	}
 
 	{
+		child_list_t<Value> file_size_in_bytes;
 		// file_size_in_bytes: long - 104
 		children.emplace_back("file_size_in_bytes", LogicalType::BIGINT);
-		data_file_field_ids.emplace_back("file_size_in_bytes", Value::INTEGER(FILE_SIZE_IN_BYTES));
+		file_size_in_bytes.emplace_back("__duckdb_field_id", Value::INTEGER(FILE_SIZE_IN_BYTES));
+		file_size_in_bytes.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
+		data_file_field_ids.emplace_back("file_size_in_bytes", Value::STRUCT(file_size_in_bytes));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, child_fields_arr);
 		yyjson_mut_obj_add_uint(doc, field_obj, "id", FILE_SIZE_IN_BYTES);
@@ -226,6 +248,7 @@ idx_t WriteToFile(IcebergTableInformation &table_info, const IcebergManifestFile
 		names.push_back("data_file");
 		types.push_back(LogicalType::STRUCT(std::move(children)));
 		data_file_field_ids.emplace_back("__duckdb_field_id", Value::INTEGER(DATA_FILE));
+		data_file_field_ids.emplace_back("__duckdb_nullable", Value::BOOLEAN(false));
 		field_ids.emplace_back("data_file", Value::STRUCT(data_file_field_ids));
 
 		auto field_obj = yyjson_mut_arr_add_obj(doc, fields_arr);