Skip to content

Commit 28d17c8

Browse files
committed
fix all-local table path treatment. DO not require credentials for local DataLakeCatalog
1 parent b69dde6 commit 28d17c8

File tree

2 files changed

+20
-18
lines changed

2 files changed

+20
-18
lines changed

src/Databases/DataLake/DatabaseDataLake.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,8 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con
375375
LOG_DEBUG(log, "Has no credentials");
376376
}
377377
}
378-
else if (!lightweight && table_metadata.requiresCredentials())
378+
else if (!lightweight && table_metadata.requiresCredentials()
379+
&& table_metadata.getStorageType() != DatabaseDataLakeStorageType::Local)
379380
{
380381
throw Exception(
381382
ErrorCodes::BAD_ARGUMENTS,

src/Storages/ObjectStorage/Utils.cpp

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,12 @@ SchemeAuthorityKey::SchemeAuthorityKey(const std::string & uri)
182182
return;
183183
}
184184
authority = std::string(rest.substr(0, slash));
185-
key = std::string(rest.substr(++slash)); // do not keep leading '/'
185+
/// For file:// URIs, the path is absolute, so we need to keep the leading '/'
186+
/// e.g. file:///home/user/data -> scheme="file", authority="", key="/home/user/data"
187+
if (scheme == "file")
188+
key = std::string(rest.substr(slash));
189+
else
190+
key = std::string(rest.substr(++slash));
186191
return;
187192
}
188193

@@ -352,7 +357,7 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
352357
use_base_storage = true;
353358
}
354359
}
355-
360+
356361
if (!use_base_storage && (base_scheme_normalized == "s3" || base_scheme_normalized == "https" || base_scheme_normalized == "http"))
357362
{
358363
std::string normalized_table_location = table_location;
@@ -361,11 +366,11 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
361366
normalized_table_location = "s3://" + table_location_decomposed.authority + "/" + table_location_decomposed.key;
362367
}
363368
S3::URI base_s3_uri(normalized_table_location);
364-
369+
365370
if (s3URIMatches(s3_uri, base_s3_uri.bucket, base_s3_uri.endpoint, target_scheme_normalized))
366371
use_base_storage = true;
367372
}
368-
373+
369374
if (use_base_storage)
370375
return {base_storage, key_to_use};
371376

@@ -417,7 +422,7 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
417422
if (target_scheme_normalized == "hdfs")
418423
{
419424
bool use_base_storage = false;
420-
425+
421426
// Check if base_storage matches (only if it's HDFS)
422427
if (base_storage->getType() == ObjectStorageType::HDFS)
423428
{
@@ -430,13 +435,13 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
430435
base_endpoint = base_url.substr(0, pos);
431436
else
432437
base_endpoint = base_url;
433-
438+
434439
// For HDFS, compare endpoints (namenode addresses)
435440
std::string target_endpoint = target_scheme_normalized + "://" + target_decomposed.authority;
436-
441+
437442
if (base_endpoint == target_endpoint)
438443
use_base_storage = true;
439-
444+
440445
// Also check if table_location matches
441446
if (!use_base_storage && base_scheme_normalized == "hdfs")
442447
{
@@ -445,7 +450,7 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
445450
}
446451
}
447452
}
448-
453+
449454
if (use_base_storage)
450455
return {base_storage, target_decomposed.key};
451456
}
@@ -461,30 +466,26 @@ std::pair<DB::ObjectStoragePtr, std::string> resolveObjectStorageForPath(
461466
if (type_for_factory.empty())
462467
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unsupported storage scheme '{}' in path '{}'", target_scheme_normalized, path);
463468

464-
std::string key_to_use = target_decomposed.key;
465-
if (target_scheme_normalized == "file")
466-
key_to_use = "/" + target_decomposed.key; // file:///absolute/path/to/file -> key = /absolute/path/to/file (full POSIX path)
467-
468469
/// Handle storage types that need new storage creation
469470
return getOrCreateStorageAndKey(
470471
cache_key,
471-
key_to_use,
472+
target_decomposed.key,
472473
type_for_factory,
473474
secondary_storages,
474475
context,
475476
[&](Poco::Util::MapConfiguration & cfg, const std::string & config_prefix)
476477
{
477478
if (target_scheme_normalized == "file")
478479
{
479-
std::filesystem::path fs_path(key_to_use);
480+
std::filesystem::path fs_path(target_decomposed.key);
480481
std::filesystem::path parent = fs_path.parent_path();
481482
std::string dir_path = parent.string();
482-
483+
483484
if (dir_path.empty() || dir_path == "/")
484485
dir_path = "/";
485486
else if (dir_path.back() != '/')
486487
dir_path += '/';
487-
488+
488489
cfg.setString(config_prefix + ".path", dir_path);
489490
}
490491
else if (target_scheme_normalized == "abfs")

0 commit comments

Comments
 (0)