Skip to content

Commit 963e7fe

Browse files
committed
all integration tests work
1 parent 7f1cc56 commit 963e7fe

File tree

4 files changed

+11
-9
lines changed

4 files changed

+11
-9
lines changed

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ std::optional<ManifestFileEntry> SingleThreadIcebergKeysIterator::next()
137137
secondary_storages);
138138
internal_data_index = 0;
139139
}
140-
auto files = files_generator(current_manifest_file_content);
140+
const auto & files = files_generator(current_manifest_file_content);
141141
while (internal_data_index < files.size())
142142
{
143143
const auto & manifest_file_entry = files[internal_data_index++];

src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1093,7 +1093,7 @@ void IcebergMetadata::addDeleteTransformers(
10931093
}
10941094
/// Then we read the content of the delete file.
10951095
auto mutable_columns_for_set = block_for_set.cloneEmptyColumns();
1096-
std::unique_ptr<ReadBuffer> data_read_buffer = createReadBuffer(delete_file_object, object_storage, local_context, log);
1096+
std::unique_ptr<ReadBuffer> data_read_buffer = createReadBuffer(delete_file_object, delete_storage_to_use, local_context, log);
10971097
CompressionMethod compression_method = chooseCompressionMethod(delete_file.file_path, "auto");
10981098
auto delete_format = FormatFactory::instance().getInput(
10991099
delete_file.file_format,

src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,19 @@ std::optional<DeleteFileWriteResultWithStats> writeDataFiles(
168168
Field cur_value;
169169
col_data_filename.column->get(i, cur_value);
170170

171+
String original_path = cur_value.safeGet<String>();
171172
String path_without_namespace;
172-
if (cur_value.safeGet<String>().starts_with(configuration->getNamespace()))
173-
path_without_namespace = cur_value.safeGet<String>().substr(configuration->getNamespace().size());
174173

175-
if (!path_without_namespace.starts_with(configuration->getPathForRead().path))
174+
if (original_path.starts_with(configuration->getNamespace()))
175+
path_without_namespace = original_path.substr(configuration->getNamespace().size());
176+
else
177+
path_without_namespace = original_path;
178+
179+
if (!path_without_namespace.empty() && !path_without_namespace.starts_with(configuration->getPathForRead().path))
176180
{
177181
if (path_without_namespace.starts_with('/'))
178182
path_without_namespace = path_without_namespace.substr(1);
179-
else
183+
else if (!path_without_namespace.empty())
180184
path_without_namespace = "/" + path_without_namespace;
181185
}
182186
col_data_filename_without_namespaces->insert(path_without_namespace);

src/Storages/ObjectStorage/DataLakes/Iceberg/PositionDeleteTransform.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ void IcebergPositionDeleteTransform::initializeDeleteSources()
6767
{
6868
/// Skip position deletes that do not match the data file path.
6969
if (position_deletes_object.reference_data_file_path.has_value()
70-
&& position_deletes_object.reference_data_file_path != iceberg_data_path)
70+
&& position_deletes_object.reference_data_file_path.value() != iceberg_data_path)
7171
continue;
7272

7373
/// Resolve the position delete file path to get the correct storage and key
@@ -192,10 +192,8 @@ void IcebergBitmapPositionDeleteTransform::initialize()
192192
while (auto delete_chunk = delete_source->read())
193193
{
194194
int position_index = getColumnIndex(delete_source, IcebergPositionDeleteTransform::positions_column_name);
195-
int filename_index = getColumnIndex(delete_source, IcebergPositionDeleteTransform::data_file_path_column_name);
196195

197196
auto position_column = delete_chunk.getColumns()[position_index];
198-
auto filename_column = delete_chunk.getColumns()[filename_index];
199197

200198
for (size_t i = 0; i < delete_chunk.getNumRows(); ++i)
201199
{

0 commit comments

Comments
 (0)