desperate step: add debug output to see what happens in CI

Author: zvonand

src/Interpreters/ClusterFunctionReadTask.cpp

@@ -8,6 +8,7 @@
 #include <Interpreters/ActionsDAG.h>
 #include <Storages/ObjectStorage/StorageObjectStorageSource.h>
 #include <Common/logger_useful.h>
+#include <iostream>
 
 namespace DB
 {
@@ -36,6 +37,13 @@ ClusterFunctionReadTaskResponse::ClusterFunctionReadTaskResponse(ObjectInfoPtr o
     const bool send_over_whole_archive = !context->getSettingsRef()[Setting::cluster_function_process_archive_on_multiple_nodes];
     path = send_over_whole_archive ? object->getPathOrPathToArchiveIfArchive() : object->getPath();
     absolute_path = object->getAbsolutePath();
+    
+    std::cerr << "[MASTER ClusterFunctionReadTaskResponse] object->getPath()=" << object->getPath()
+              << " object->getAbsolutePath()=" << (object->getAbsolutePath().has_value() ? object->getAbsolutePath().value() : "none")
+              << " object->getObjectStorage()=" << (object->getObjectStorage().has_value() ? "set" : "not_set")
+              << " extracted path=" << path
+              << " extracted absolute_path=" << (absolute_path.has_value() ? absolute_path.value() : "none")
+              << std::endl;
 }
 
 ClusterFunctionReadTaskResponse::ClusterFunctionReadTaskResponse(const std::string & path_)
@@ -52,6 +60,13 @@ ObjectInfoPtr ClusterFunctionReadTaskResponse::getObjectInfo() const
     object->data_lake_metadata = data_lake_metadata;
     object->file_meta_info = file_meta_info;
     object->absolute_path = absolute_path;
+    
+    std::cerr << "[WORKER ClusterFunctionReadTaskResponse::getObjectInfo] path=" << path
+              << " absolute_path=" << (absolute_path.has_value() ? absolute_path.value() : "none")
+              << " created object->getPath()=" << object->getPath()
+              << " object->getAbsolutePath()=" << (object->getAbsolutePath().has_value() ? object->getAbsolutePath().value() : "none")
+              << std::endl;
+    
     return object;
 }
 

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp

@@ -24,6 +24,8 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <Interpreters/Context.h>
+#include <iostream>
+#include <Disks/DiskType.h>
 
 #include <IO/CompressedReadBufferWrapper.h>
 #include <Interpreters/ExpressionActions.h>
@@ -339,8 +341,21 @@ ObjectInfoPtr IcebergIterator::next(size_t)
         auto [storage_to_use, resolved_key] = resolveObjectStorageForPath(
             persistent_components.table_location, manifest_file_entry.file_path, object_storage, secondary_storages, local_context);
 
+        std::cerr << "[MASTER IcebergIterator::next] table_location=" << persistent_components.table_location
+                  << " file_path=" << manifest_file_entry.file_path
+                  << " resolved_key=" << resolved_key 
+                  << " storage_type=" << (storage_to_use ? toString(storage_to_use->getType()) : "null")
+                  << " storage_desc=" << (storage_to_use ? storage_to_use->getDescription() : "null")
+                  << " storage_same_as_base=" << (storage_to_use == object_storage ? "true" : "false")
+                  << std::endl;
+        
         IcebergDataObjectInfoPtr object_info = std::make_shared<IcebergDataObjectInfo>(manifest_file_entry, storage_to_use, resolved_key);
 
+        std::cerr << "[MASTER IcebergIterator::next] created object_info: getPath()=" << object_info->getPath()
+                  << " getAbsolutePath()=" << (object_info->getAbsolutePath().has_value() ? object_info->getAbsolutePath().value() : "none")
+                  << " getObjectStorage()=" << (object_info->getObjectStorage().has_value() ? "set" : "not_set")
+                  << std::endl;
+        
         for (const auto & position_delete : defineDeletesSpan(manifest_file_entry, position_deletes_files, false))
             object_info->addPositionDeleteObject(position_delete);
 

src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp

@@ -1,4 +1,5 @@
 #include <optional>
+#include <iostream>
 #include <Storages/ObjectStorage/StorageObjectStorageCluster.h>
 
 #include <Common/Exception.h>
@@ -500,7 +501,14 @@ class TaskDistributor : public TaskIterator
     {
         auto task = task_distributor.getNextTask(number_of_current_replica);
         if (task)
+        {
+            std::cerr << "[MASTER StorageObjectStorageCluster] sending task to replica " << number_of_current_replica
+                      << " task->getPath()=" << task->getPath()
+                      << " task->getAbsolutePath()=" << (task->getAbsolutePath().has_value() ? task->getAbsolutePath().value() : "none")
+                      << " task->getObjectStorage()=" << (task->getObjectStorage().has_value() ? "set" : "not_set")
+                      << std::endl;
             return std::make_shared<ClusterFunctionReadTaskResponse>(std::move(task), context);
+        }
         return std::make_shared<ClusterFunctionReadTaskResponse>();
     }
 

src/Storages/ObjectStorage/StorageObjectStorageSource.cpp

@@ -39,7 +39,8 @@
 #endif
 
 #include <fmt/ranges.h>
-
+#include <iostream>
+#include <Disks/DiskType.h>
 
 namespace fs = std::filesystem;
 namespace ProfileEvents
@@ -520,6 +521,13 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade
     while (not_a_path || (query_settings.skip_empty_files && object_info->metadata->size_bytes == 0));
 
     ObjectStoragePtr storage_to_use = object_info->getObjectStorage().value_or(object_storage);
+    
+    std::cerr << "[WORKER createReader] object_info->getPath()=" << object_info->getPath()
+              << " object_info->relative_path=" << object_info->relative_path
+              << " object_info->getAbsolutePath()=" << (object_info->getAbsolutePath().has_value() ? object_info->getAbsolutePath().value() : "none")
+              << " storage_to_use_type=" << toString(storage_to_use->getType())
+              << " storage_to_use_desc=" << storage_to_use->getDescription()
+              << std::endl;
 
     QueryPipelineBuilder builder;
     std::shared_ptr<ISource> source;
@@ -1317,23 +1325,60 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::ReadTaskIterator
         if (!raw || raw->isEmpty())
             return nullptr;
 
-        object_info = raw->getObjectInfo();
+        std::cerr << "[WORKER ReadTaskIterator::next] received raw->path=" << raw->path
+                  << " raw->absolute_path=" << (raw->absolute_path.has_value() ? raw->absolute_path.value() : "none")
+                  << std::endl;
 
+        object_info = raw->getObjectInfo();
+        
+        std::cerr << "[WORKER ReadTaskIterator::next] after getObjectInfo: object_info->getPath()=" << object_info->getPath()
+                  << " object_info->getAbsolutePath()=" << (object_info->getAbsolutePath().has_value() ? object_info->getAbsolutePath().value() : "none")
+                  << " object_info->getObjectStorage()=" << (object_info->getObjectStorage().has_value() ? "set" : "not_set")
+                  << std::endl;
+
+        // The 'path' field from master is already the correctly resolved relative path.
+        // We should use it directly and NOT overwrite relative_path.
+        // Only resolve absolute_path if we need to determine which storage to use (for secondary storages).
+        object_info->object_storage_to_use = object_storage;
+        
         if (raw->absolute_path.has_value())
         {
             auto [storage_to_use, key]
                 = resolveObjectStorageForPath("", raw->absolute_path.value(), object_storage, secondary_storages, getContext());
 
-            if (!key.empty()) /// Otherwise not a valid key/path, maybe it is "retry_after_us". Store as is.
+            std::cerr << "[WORKER ReadTaskIterator::next] resolved absolute_path: key=" << key
+                      << " storage_type=" << (storage_to_use ? toString(storage_to_use->getType()) : "null")
+                      << " storage_desc=" << (storage_to_use ? storage_to_use->getDescription() : "null")
+                      << " storage_same_as_base=" << (storage_to_use == object_storage ? "true" : "false")
+                      << " original_relative_path=" << object_info->relative_path
+                      << std::endl;
+
+            if (!key.empty() && storage_to_use != object_storage)
             {
+                // File is in a different storage (secondary storage), use that storage
+                // BUT preserve the original relative_path from master - don't overwrite it!
                 object_info->object_storage_to_use = storage_to_use;
-                object_info->relative_path = key;
+                
+                std::cerr << "[WORKER ReadTaskIterator::next] SET: object_storage_to_use=secondary_storage"
+                          << " relative_path=" << object_info->relative_path << " (preserved from master)"
+                          << std::endl;
+            }
+            else
+            {
+                std::cerr << "[WORKER ReadTaskIterator::next] using default storage, relative_path=" << object_info->relative_path
+                          << " (preserved from master)" << std::endl;
             }
         }
         else
         {
-            object_info->object_storage_to_use = object_storage;
+            std::cerr << "[WORKER ReadTaskIterator::next] NO absolute_path: using default storage, relative_path=" << object_info->relative_path
+                      << " (preserved from master)" << std::endl;
         }
+        
+        std::cerr << "[WORKER ReadTaskIterator::next] FINAL: object_info->getPath()=" << object_info->getPath()
+                  << " object_info->relative_path=" << object_info->relative_path
+                  << " object_info->getObjectStorage()=" << (object_info->getObjectStorage().has_value() ? "set" : "not_set")
+                  << std::endl;
     }
     else
     {