Skip to content

Commit 5ff409a

Browse files
ianton-ruianton-ru
committed
Fix for table schema, more tests
1 parent 45c0ce3 commit 5ff409a

File tree

11 files changed

+62
-15
lines changed

11 files changed

+62
-15
lines changed

‎src/Databases/DataLake/DatabaseDataLake.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con
326326

327327
auto [namespace_name, table_name] = DataLake::parseTableName(name);
328328

329-
if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata))
329+
if (!catalog->tryGetTableMetadata(namespace_name, table_name, context_, table_metadata))
330330
return nullptr;
331331

332332
if (ignore_if_not_iceberg && !table_metadata.isDefaultReadableTable())
@@ -632,15 +632,15 @@ ASTPtr DatabaseDataLake::getCreateDatabaseQuery() const
632632

633633
ASTPtr DatabaseDataLake::getCreateTableQueryImpl(
634634
const String & name,
635-
ContextPtr /* context_ */,
635+
ContextPtr context_,
636636
bool /* throw_on_error */) const
637637
{
638638
auto catalog = getCatalog();
639639
auto table_metadata = DataLake::TableMetadata().withLocation().withSchema();
640640

641641
const auto [namespace_name, table_name] = DataLake::parseTableName(name);
642642

643-
if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata))
643+
if (!catalog->tryGetTableMetadata(namespace_name, table_name, context_, table_metadata))
644644
{
645645
throw Exception(
646646
ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY, "Table `{}` doesn't exist", name);

‎src/Databases/DataLake/GlueCatalog.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@ bool GlueCatalog::existsTable(const std::string & database_name, const std::stri
276276
bool GlueCatalog::tryGetTableMetadata(
277277
const std::string & database_name,
278278
const std::string & table_name,
279+
DB::ContextPtr /* context_ */,
279280
TableMetadata & result) const
280281
{
281282
Aws::Glue::Model::GetTableRequest request;
@@ -394,9 +395,10 @@ bool GlueCatalog::tryGetTableMetadata(
394395
void GlueCatalog::getTableMetadata(
395396
const std::string & database_name,
396397
const std::string & table_name,
398+
DB::ContextPtr context_,
397399
TableMetadata & result) const
398400
{
399-
if (!tryGetTableMetadata(database_name, table_name, result))
401+
if (!tryGetTableMetadata(database_name, table_name, context_, result))
400402
{
401403
throw DB::Exception(
402404
DB::ErrorCodes::DATALAKE_DATABASE_ERROR,

‎src/Databases/DataLake/GlueCatalog.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,13 @@ class GlueCatalog final : public ICatalog, private DB::WithContext
4040
void getTableMetadata(
4141
const std::string & database_name,
4242
const std::string & table_name,
43+
DB::ContextPtr context_,
4344
TableMetadata & result) const override;
4445

4546
bool tryGetTableMetadata(
4647
const std::string & database_name,
4748
const std::string & table_name,
49+
DB::ContextPtr context_,
4850
TableMetadata & result) const override;
4951

5052
std::optional<StorageType> getStorageType() const override

‎src/Databases/DataLake/HiveCatalog.cpp‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,13 +96,21 @@ bool HiveCatalog::existsTable(const std::string & namespace_name, const std::str
9696
return true;
9797
}
9898

99-
void HiveCatalog::getTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const
99+
void HiveCatalog::getTableMetadata(
100+
const std::string & namespace_name,
101+
const std::string & table_name,
102+
DB::ContextPtr context_,
103+
TableMetadata & result) const
100104
{
101-
if (!tryGetTableMetadata(namespace_name, table_name, result))
105+
if (!tryGetTableMetadata(namespace_name, table_name, context_, result))
102106
throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog");
103107
}
104108

105-
bool HiveCatalog::tryGetTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const
109+
bool HiveCatalog::tryGetTableMetadata(
110+
const std::string & namespace_name,
111+
const std::string & table_name,
112+
DB::ContextPtr /* context_ */,
113+
TableMetadata & result) const
106114
{
107115
Apache::Hadoop::Hive::Table table;
108116

‎src/Databases/DataLake/HiveCatalog.h‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,17 @@ class HiveCatalog final : public ICatalog, private DB::WithContext
3838

3939
bool existsTable(const std::string & namespace_name, const std::string & table_name) const override;
4040

41-
void getTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const override;
42-
43-
bool tryGetTableMetadata(const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const override;
41+
void getTableMetadata(
42+
const std::string & namespace_name,
43+
const std::string & table_name,
44+
DB::ContextPtr context_,
45+
TableMetadata & result) const override;
46+
47+
bool tryGetTableMetadata(
48+
const std::string & namespace_name,
49+
const std::string & table_name,
50+
DB::ContextPtr context_,
51+
TableMetadata & result) const override;
4452

4553
std::optional<StorageType> getStorageType() const override;
4654

‎src/Databases/DataLake/ICatalog.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,14 @@
88
#include <Databases/DataLake/DatabaseDataLakeStorageType.h>
99
#include <Poco/JSON/Object.h>
1010

11+
namespace DB
12+
{
13+
14+
class Context;
15+
using ContextPtr = std::shared_ptr<const Context>;
16+
17+
}
18+
1119
namespace DataLake
1220
{
1321

@@ -150,13 +158,15 @@ class ICatalog
150158
virtual void getTableMetadata(
151159
const std::string & namespace_name,
152160
const std::string & table_name,
161+
DB::ContextPtr context,
153162
TableMetadata & result) const = 0;
154163

155164
/// Get table metadata in the given namespace.
156165
/// Return `false` if table does not exist, `true` otherwise.
157166
virtual bool tryGetTableMetadata(
158167
const std::string & namespace_name,
159168
const std::string & table_name,
169+
DB::ContextPtr context,
160170
TableMetadata & result) const = 0;
161171

162172
/// Get storage type, where Iceberg tables' data is stored.

‎src/Databases/DataLake/RestCatalog.cpp‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -532,17 +532,18 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas
532532
bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const
533533
{
534534
TableMetadata table_metadata;
535-
return tryGetTableMetadata(namespace_name, table_name, table_metadata);
535+
return tryGetTableMetadata(namespace_name, table_name, getContext(), table_metadata);
536536
}
537537

538538
bool RestCatalog::tryGetTableMetadata(
539539
const std::string & namespace_name,
540540
const std::string & table_name,
541+
DB::ContextPtr context_,
541542
TableMetadata & result) const
542543
{
543544
try
544545
{
545-
return getTableMetadataImpl(namespace_name, table_name, result);
546+
return getTableMetadataImpl(namespace_name, table_name, context_, result);
546547
}
547548
catch (...)
548549
{
@@ -554,15 +555,17 @@ bool RestCatalog::tryGetTableMetadata(
554555
void RestCatalog::getTableMetadata(
555556
const std::string & namespace_name,
556557
const std::string & table_name,
558+
DB::ContextPtr context_,
557559
TableMetadata & result) const
558560
{
559-
if (!getTableMetadataImpl(namespace_name, table_name, result))
561+
if (!getTableMetadataImpl(namespace_name, table_name, context_, result))
560562
throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog");
561563
}
562564

563565
bool RestCatalog::getTableMetadataImpl(
564566
const std::string & namespace_name,
565567
const std::string & table_name,
568+
DB::ContextPtr context_,
566569
TableMetadata & result) const
567570
{
568571
LOG_TEST(log, "Checking table {} in namespace {}", table_name, namespace_name);
@@ -623,7 +626,7 @@ bool RestCatalog::getTableMetadataImpl(
623626
if (result.requiresSchema())
624627
{
625628
// int format_version = metadata_object->getValue<int>("format-version");
626-
auto schema_processor = DB::Iceberg::IcebergSchemaProcessor(getContext());
629+
auto schema_processor = DB::Iceberg::IcebergSchemaProcessor(context_);
627630
auto id = DB::IcebergMetadata::parseTableSchema(metadata_object, schema_processor, log);
628631
auto schema = schema_processor.getClickhouseTableSchemaById(id);
629632
result.setSchema(*schema);

‎src/Databases/DataLake/RestCatalog.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,13 @@ class RestCatalog final : public ICatalog, private DB::WithContext
4242
void getTableMetadata(
4343
const std::string & namespace_name,
4444
const std::string & table_name,
45+
DB::ContextPtr context_,
4546
TableMetadata & result) const override;
4647

4748
bool tryGetTableMetadata(
4849
const std::string & namespace_name,
4950
const std::string & table_name,
51+
DB::ContextPtr context_,
5052
TableMetadata & result) const override;
5153

5254
std::optional<StorageType> getStorageType() const override;
@@ -126,6 +128,7 @@ class RestCatalog final : public ICatalog, private DB::WithContext
126128
bool getTableMetadataImpl(
127129
const std::string & namespace_name,
128130
const std::string & table_name,
131+
DB::ContextPtr context_,
129132
TableMetadata & result) const;
130133

131134
Config loadConfig();

‎src/Databases/DataLake/UnityCatalog.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,10 @@ DB::Names UnityCatalog::getTables() const
9191
void UnityCatalog::getTableMetadata(
9292
const std::string & namespace_name,
9393
const std::string & table_name,
94+
DB::ContextPtr context_,
9495
TableMetadata & result) const
9596
{
96-
if (!tryGetTableMetadata(namespace_name, table_name, result))
97+
if (!tryGetTableMetadata(namespace_name, table_name, context_, result))
9798
throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from unity catalog");
9899
}
99100

@@ -136,6 +137,7 @@ void UnityCatalog::getCredentials(const std::string & table_id, TableMetadata &
136137
bool UnityCatalog::tryGetTableMetadata(
137138
const std::string & schema_name,
138139
const std::string & table_name,
140+
DB::ContextPtr /* context_ */,
139141
TableMetadata & result) const
140142
{
141143
auto full_table_name = warehouse + "." + schema_name + "." + table_name;

‎src/Databases/DataLake/UnityCatalog.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,13 @@ class UnityCatalog final : public ICatalog, private DB::WithContext
3434
void getTableMetadata(
3535
const std::string & namespace_name,
3636
const std::string & table_name,
37+
DB::ContextPtr context_,
3738
TableMetadata & result) const override;
3839

3940
bool tryGetTableMetadata(
4041
const std::string & schema_name,
4142
const std::string & table_name,
43+
DB::ContextPtr context_,
4244
TableMetadata & result) const override;
4345

4446
std::optional<StorageType> getStorageType() const override { return std::nullopt; }

0 commit comments

Comments
 (0)