Backport ClickHouse#86832 to 25.8: Return fallback for schema resolution for a manifest file entry

Author: robot-clickhouse

src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp

@@ -22,6 +22,9 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 
+#include <Common/logger_useful.h>
+
+
 namespace DB::ErrorCodes
 {
     extern const int ICEBERG_SPECIFICATION_VIOLATION;
@@ -143,7 +146,7 @@ ManifestFileContent::ManifestFileContent(
     const String & manifest_file_name,
     Int32 format_version_,
     const String & common_path,
-    const IcebergSchemaProcessor & schema_processor,
+    IcebergSchemaProcessor & schema_processor,
     Int64 inherited_sequence_number,
     Int64 inherited_snapshot_id,
     const String & table_location,
@@ -196,6 +199,8 @@ ManifestFileContent::ManifestFileContent(
     const Poco::JSON::Object::Ptr & schema_object = json.extract<Poco::JSON::Object::Ptr>();
     Int32 manifest_schema_id = schema_object->getValue<int>(f_schema_id);
 
+    schema_processor.addIcebergTableSchema(schema_object);
+
     for (size_t i = 0; i != partition_specification->size(); ++i)
     {
         auto partition_specification_field = partition_specification->getObject(static_cast<UInt32>(i));
@@ -262,13 +267,22 @@ ManifestFileContent::ManifestFileContent(
         const auto schema_id_opt = schema_processor.tryGetSchemaIdForSnapshot(snapshot_id);
         if (!schema_id_opt.has_value())
         {
-            throw Exception(
-                ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION,
-                "Cannot read Iceberg table: manifest file '{}' has entry with snapshot_id '{}' for which write file schema is unknown",
-                manifest_file_name,
-                snapshot_id);
+            /// Error logged but not thrown to avoid breaking whole query because of backward compatibility reasons.
+            /// That's actually an error because it can lead to incorrect query results, so we are creating an exception to put it to system.error_log.
+            try
+            {
+                throw Exception(
+                    ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION,
+                    "Cannot read Iceberg table: manifest file '{}' has entry with snapshot_id '{}' for which write file schema is unknown",
+                    manifest_file_name,
+                    snapshot_id);
+            }
+            catch (const Exception &)
+            {
+                tryLogCurrentException("ICEBERG_SPECIFICATION_VIOLATION", "", LogsLevel::error);
+            }
         }
-        const auto schema_id = schema_id_opt.value();
+        const auto schema_id = schema_id_opt.has_value() ? schema_id_opt.value() : manifest_schema_id;
 
         const auto file_path_key
             = manifest_file_deserializer.getValueFromRowByName(i, c_data_file_file_path, TypeIndex::String).safeGet<String>();

src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h

@@ -111,7 +111,7 @@ class ManifestFileContent : public boost::noncopyable
         const String & manifest_file_name,
         Int32 format_version_,
         const String & common_path,
-        const IcebergSchemaProcessor & schema_processor,
+        IcebergSchemaProcessor & schema_processor,
         Int64 inherited_sequence_number,
         Int64 inherited_snapshot_id,
         const std::string & table_location,