Skip to content

Commit 07f9c57

Browse files
committed
try to fix
try to fix(2)
1 parent 8d5ee53 commit 07f9c57

File tree

4 files changed

+19
-15
lines changed

4 files changed

+19
-15
lines changed

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ namespace DB
3131
/// And we can get `ManifestFileContent` from cache by ManifestFileEntry.
3232
struct ManifestFileCacheKey
3333
{
34-
String manifest_file_path;
3534
String manifest_file_absolute_path;
3635
Int64 added_sequence_number;
3736
Int64 added_snapshot_id;
@@ -74,7 +73,7 @@ struct IcebergMetadataFilesCacheCell : private boost::noncopyable
7473
size_t total_size = 0;
7574
for (const auto & entry: manifest_file_cache_keys)
7675
{
77-
total_size += sizeof(ManifestFileCacheKey) + entry.manifest_file_path.capacity() + entry.manifest_file_absolute_path.capacity();
76+
total_size += sizeof(ManifestFileCacheKey) + entry.manifest_file_absolute_path.capacity();
7877
}
7978
return total_size;
8079
}

src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -285,11 +285,9 @@ ManifestFileContent::ManifestFileContent(
285285
}
286286
const auto schema_id = schema_id_opt.has_value() ? schema_id_opt.value() : manifest_schema_id;
287287

288-
const auto file_path_key
289-
= manifest_file_deserializer.getValueFromRowByName(i, c_data_file_file_path, TypeIndex::String).safeGet<String>();
290-
const auto file_path = makeAbsolutePath(
291-
table_location,
292-
manifest_file_deserializer.getValueFromRowByName(i, c_data_file_file_path, TypeIndex::String).safeGet<String>());
288+
const auto file_path_key_field = manifest_file_deserializer.getValueFromRowByName(i, c_data_file_file_path, TypeIndex::String);
289+
const auto file_path_key = file_path_key_field.safeGet<String>();
290+
const auto file_path = makeAbsolutePath(table_location, file_path_key);
293291

294292
/// NOTE: This is weird, because in manifest file partition looks like this:
295293
/// {

src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ ManifestFileCacheKeys getManifestList(
165165
{
166166
const std::string file_path
167167
= manifest_list_deserializer.getValueFromRowByName(i, f_manifest_path, TypeIndex::String).safeGet<std::string>();
168-
const auto manifest_file_name = makeAbsolutePath(persistent_table_components.table_location, file_path);
168+
const auto manifest_absolute_path = makeAbsolutePath(persistent_table_components.table_location, file_path);
169169
Int64 added_sequence_number = 0;
170170
auto added_snapshot_id = manifest_list_deserializer.getValueFromRowByName(i, f_added_snapshot_id);
171171
if (added_snapshot_id.isNull())
@@ -184,7 +184,7 @@ ManifestFileCacheKeys getManifestList(
184184
manifest_list_deserializer.getValueFromRowByName(i, f_content, TypeIndex::Int32).safeGet<Int32>());
185185
}
186186
manifest_file_cache_keys.emplace_back(
187-
file_path, manifest_file_name, added_sequence_number, added_snapshot_id.safeGet<Int64>(), content_type);
187+
manifest_absolute_path, added_sequence_number, added_snapshot_id.safeGet<Int64>(), content_type);
188188

189189
auto dump_row_metadata = [&]()->String { return manifest_list_deserializer.getContent(i); };
190190
insertRowToLogTable(

src/Storages/ObjectStorage/StorageObjectStorageSource.cpp

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1291,6 +1291,10 @@ StorageObjectStorageSource::ReadTaskIterator::ReadTaskIterator(
12911291
object->relative_path = key;
12921292
}
12931293
}
1294+
else
1295+
{
1296+
object->object_storage_to_use = object_storage;
1297+
}
12941298
buffer.push_back(object);
12951299
}
12961300
}
@@ -1315,17 +1319,20 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::ReadTaskIterator
13151319

13161320
object_info = raw->getObjectInfo();
13171321

1318-
if (object_info->getObjectStorage().has_value())
1319-
{
1320-
1321-
}
1322-
else if (raw->absolute_path.has_value())
1322+
if (raw->absolute_path.has_value())
13231323
{
13241324
auto [storage_to_use, key]
13251325
= resolveObjectStorageForPath("", raw->absolute_path.value(), object_storage, secondary_storages, getContext());
13261326

1327-
if (!key.empty()) /// Not a valid key/path, maybe it is "retry_after_us". Store as is.
1327+
if (!key.empty()) /// Otherwise not a valid key/path, maybe it is "retry_after_us". Store as is.
1328+
{
13281329
object_info->object_storage_to_use = storage_to_use;
1330+
object_info->relative_path = key;
1331+
}
1332+
}
1333+
else
1334+
{
1335+
object_info->object_storage_to_use = object_storage;
13291336
}
13301337
}
13311338
else

0 commit comments

Comments
 (0)