Skip to content

Commit d350e7d

Browse files
committed
follow-up 1
1 parent 3841418 commit d350e7d

File tree

3 files changed

+14
-8
lines changed

3 files changed

+14
-8
lines changed

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ std::optional<ManifestFileEntry> SingleThreadIcebergKeysIterator::next()
141141
auto files = files_generator(current_manifest_file_content);
142142
while (internal_data_index < files.size())
143143
{
144-
const auto & manifest_file_entry = files[internal_data_index++];
144+
auto & manifest_file_entry = files[internal_data_index++];
145145
if ((manifest_file_entry.schema_id != previous_entry_schema) && (use_partition_pruning))
146146
{
147147
previous_entry_schema = manifest_file_entry.schema_id;
@@ -164,7 +164,13 @@ std::optional<ManifestFileEntry> SingleThreadIcebergKeysIterator::next()
164164
switch (pruning_status)
165165
{
166166
case PruningReturnStatus::NOT_PRUNED:
167+
{
168+
auto [storage_to_use, resolved_key] = resolveObjectStorageForPath(
169+
persistent_components.table_location, manifest_file_entry.file_path, object_storage, *secondary_storages, local_context);
170+
manifest_file_entry.storage_to_use = storage_to_use;
171+
manifest_file_entry.resolved_key = resolved_key;
167172
return manifest_file_entry;
173+
}
168174
case PruningReturnStatus::MIN_MAX_INDEX_PRUNED: {
169175
++min_max_index_pruned_files;
170176
break;
@@ -242,7 +248,6 @@ IcebergIterator::IcebergIterator(
242248
std::shared_ptr<SecondaryStorages> secondary_storages_)
243249
: filter_dag(filter_dag_ ? std::make_unique<ActionsDAG>(filter_dag_->clone()) : nullptr)
244250
, object_storage(std::move(object_storage_))
245-
, local_context(local_context_)
246251
, data_files_iterator(
247252
object_storage,
248253
local_context_,
@@ -336,11 +341,8 @@ ObjectInfoPtr IcebergIterator::next(size_t)
336341
Iceberg::ManifestFileEntry manifest_file_entry;
337342
if (blocking_queue.pop(manifest_file_entry))
338343
{
339-
// Resolve the data file path to get the correct storage and key
340-
auto [storage_to_use, resolved_key] = resolveObjectStorageForPath(
341-
persistent_components.table_location, manifest_file_entry.file_path, object_storage, *secondary_storages, local_context);
342-
343-
IcebergDataObjectInfoPtr object_info = std::make_shared<IcebergDataObjectInfo>(manifest_file_entry, storage_to_use, resolved_key);
344+
IcebergDataObjectInfoPtr object_info = std::make_shared<IcebergDataObjectInfo>(
345+
manifest_file_entry, manifest_file_entry.storage_to_use, manifest_file_entry.resolved_key);
344346

345347
for (const auto & position_delete : defineDeletesSpan(manifest_file_entry, position_deletes_files, false))
346348
object_info->addPositionDeleteObject(position_delete);

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,6 @@ class IcebergIterator : public IObjectIterator
103103
private:
104104
std::unique_ptr<ActionsDAG> filter_dag;
105105
ObjectStoragePtr object_storage;
106-
ContextPtr local_context;
107106
Iceberg::SingleThreadIcebergKeysIterator data_files_iterator;
108107
Iceberg::SingleThreadIcebergKeysIterator deletes_iterator;
109108
ConcurrentBoundedQueue<Iceberg::ManifestFileEntry> blocking_queue;

src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ struct ColumnInfo
2626
#include <Storages/KeyDescription.h>
2727
#include <Storages/MergeTree/KeyCondition.h>
2828
#include <Core/Field.h>
29+
#include <Disks/ObjectStorages/IObjectStorage_fwd.h>
2930

3031
#include <cstdint>
3132

@@ -84,6 +85,10 @@ struct ManifestFileEntry
8485
String file_format;
8586
std::optional<String> reference_data_file_path; // For position delete files only.
8687
std::optional<std::vector<Int32>> equality_ids;
88+
89+
// Resolved storage and key (set by SingleThreadIcebergKeysIterator)
90+
ObjectStoragePtr storage_to_use;
91+
String resolved_key;
8792
};
8893

8994
/**

0 commit comments

Comments
 (0)