Skip to content
This repository was archived by the owner on Sep 27, 2019. It is now read-only.

Commit 5686479

Browse files
poojanilangekarpervazea
authored andcommitted
Schema - Layout decoupling (#1327)
* Added layout.h. Modified Runtime functions to handle row stores. * Modify TileGroup api * Changed Updater & Inserter to use Layout object * Remove calls from optimizer and executor * Removed all calls to schemas vector from TileGroup * Removed Calls to LocateTileAndColumn for TileGroup * Moved definition of column_map_type. Removed column_map from tile_group * Added GetInfo function * make check pases. Need to change a few more function calls * Convert PL_ASSERT -> PELOTON_ASSERT * Modified DataTable and LayoutTunerTest * Added Layout Test for codegen * Get the tests to actually use different layouts * Modified storage layer, builds successfully. Need to fix tests * Moved GetColumnMapStats -> GetColumnLayoutStats * Minor change in TableScanTranslator * Modify TransformTileGroup * Change layout.cpp to better handle HYBRID layouts * use std::make_shared * Modified tests to change calls to GetTileGroup() * make check passes. Need to add test with hybrid layout. * Modify print functions * Add TODOs for catalog * Added LayoutCatalog. Yet to modify TableCatalog * Added catalog functions * Added DeleteLayouts to delete all layouts of a table_oid * LayoutTunerTest fixed * Fix build failures * Access LayoutCatalog via the global Catalog object * Added Multi layout scan test * Fixed tests after catalog refactor * Failing LayoutCatalog test because of deserialization * Fix all tests * Added documentation * Style Fix * Address review comments + minor clean-up in the layout.h API * Change valid_layout_objects_ * Revert unused changes * Addressed Prashanth's initial comments * Modify CreateTable for tests + modify LayoutTuner * Address Pervaze's review comments. * Changed after Prashanth's review
1 parent d052644 commit 5686479

File tree

76 files changed

+2104
-597
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+2104
-597
lines changed

src/catalog/catalog.cpp

Lines changed: 76 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "catalog/index_catalog.h"
1919
#include "catalog/index_metrics_catalog.h"
2020
#include "catalog/language_catalog.h"
21+
#include "catalog/layout_catalog.h"
2122
#include "catalog/proc_catalog.h"
2223
#include "catalog/query_history_catalog.h"
2324
#include "catalog/query_metrics_catalog.h"
@@ -161,7 +162,7 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database,
161162
system_catalogs->GetSchemaCatalog()->InsertSchema(
162163
CATALOG_SCHEMA_OID, CATALOG_SCHEMA_NAME, pool_.get(), txn);
163164
system_catalogs->GetSchemaCatalog()->InsertSchema(
164-
DEFUALT_SCHEMA_OID, DEFUALT_SCHEMA_NAME, pool_.get(), txn);
165+
DEFAULT_SCHEMA_OID, DEFAULT_SCHEMA_NAME, pool_.get(), txn);
165166

166167
// Insert catalog tables into pg_table
167168
// pg_database record is shared across different databases
@@ -180,6 +181,9 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database,
180181
system_catalogs->GetTableCatalog()->InsertTable(
181182
COLUMN_CATALOG_OID, COLUMN_CATALOG_NAME, CATALOG_SCHEMA_NAME,
182183
database_oid, pool_.get(), txn);
184+
system_catalogs->GetTableCatalog()->InsertTable(
185+
LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, CATALOG_SCHEMA_NAME,
186+
database_oid, pool_.get(), txn);
183187
}
184188

185189
void Catalog::Bootstrap() {
@@ -295,7 +299,8 @@ ResultType Catalog::CreateTable(const std::string &database_name,
295299
const std::string &table_name,
296300
std::unique_ptr<catalog::Schema> schema,
297301
concurrency::TransactionContext *txn,
298-
bool is_catalog, oid_t tuples_per_tilegroup) {
302+
bool is_catalog, uint32_t tuples_per_tilegroup,
303+
peloton::LayoutType layout_type) {
299304
if (txn == nullptr)
300305
throw CatalogException("Do not have transaction to create table " +
301306
table_name);
@@ -348,7 +353,8 @@ ResultType Catalog::CreateTable(const std::string &database_name,
348353
bool adapt_table = false;
349354
auto table = storage::TableFactory::GetDataTable(
350355
database_object->GetDatabaseOid(), table_oid, schema.release(),
351-
table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog);
356+
table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog,
357+
layout_type);
352358
database->AddTable(table, is_catalog);
353359
// put data table object into rw_object_set
354360
txn->RecordCreate(database_object->GetDatabaseOid(), table_oid, INVALID_OID);
@@ -555,6 +561,44 @@ ResultType Catalog::CreateIndex(
555561
return ResultType::SUCCESS;
556562
}
557563

564+
std::shared_ptr<const storage::Layout> Catalog::CreateLayout(
565+
oid_t database_oid, oid_t table_oid, const column_map_type &column_map,
566+
concurrency::TransactionContext *txn) {
567+
auto storage_manager = storage::StorageManager::GetInstance();
568+
auto database = storage_manager->GetDatabaseWithOid(database_oid);
569+
auto table = database->GetTableWithOid(table_oid);
570+
571+
oid_t layout_oid = table->GetNextLayoutOid();
572+
// Ensure that the new layout
573+
PELOTON_ASSERT(layout_oid < INVALID_OID);
574+
auto new_layout = std::shared_ptr<const storage::Layout>(
575+
new const storage::Layout(column_map, layout_oid));
576+
577+
// Add the layout the pg_layout table
578+
auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog();
579+
bool result =
580+
pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn);
581+
if (!result) {
582+
LOG_ERROR("Failed to create a new layout for table %u", table_oid);
583+
return nullptr;
584+
}
585+
return new_layout;
586+
}
587+
588+
std::shared_ptr<const storage::Layout> Catalog::CreateDefaultLayout(
589+
oid_t database_oid, oid_t table_oid, const column_map_type &column_map,
590+
concurrency::TransactionContext *txn) {
591+
auto new_layout = CreateLayout(database_oid, table_oid, column_map, txn);
592+
// If the layout creation was successful, set it as the default
593+
if (new_layout != nullptr) {
594+
auto storage_manager = storage::StorageManager::GetInstance();
595+
auto database = storage_manager->GetDatabaseWithOid(database_oid);
596+
auto table = database->GetTableWithOid(table_oid);
597+
table->SetDefaultLayout(new_layout);
598+
}
599+
return new_layout;
600+
}
601+
558602
//===----------------------------------------------------------------------===//
559603
// DROP FUNCTIONS
560604
//===----------------------------------------------------------------------===//
@@ -700,7 +744,6 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid,
700744
auto table_object = database_object->GetTableObject(table_oid);
701745
auto index_objects = table_object->GetIndexObjects();
702746
LOG_TRACE("dropping #%d indexes", (int)index_objects.size());
703-
704747
// delete trigger and records in pg_trigger
705748
auto pg_trigger =
706749
catalog_map_[database_object->GetDatabaseOid()]->GetTriggerCatalog();
@@ -719,11 +762,15 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid,
719762
catalog_map_[database_object->GetDatabaseOid()]->GetColumnCatalog();
720763
pg_attribute->DeleteColumns(table_oid, txn);
721764

765+
// delete record in pg_layout
766+
auto pg_layout =
767+
catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog();
768+
pg_layout->DeleteLayouts(table_oid, txn);
769+
722770
// delete record in pg_table
723771
auto pg_table =
724772
catalog_map_[database_object->GetDatabaseOid()]->GetTableCatalog();
725773
pg_table->DeleteTable(table_oid, txn);
726-
727774
database->GetTableWithOid(table_oid);
728775
txn->RecordDrop(database_oid, table_oid, INVALID_OID);
729776

@@ -764,6 +811,30 @@ ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid,
764811
return ResultType::SUCCESS;
765812
}
766813

814+
ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid,
815+
oid_t layout_oid,
816+
concurrency::TransactionContext *txn) {
817+
// Check if the default_layout of the table is the same.
818+
// If true reset it to a row store.
819+
auto storage_manager = storage::StorageManager::GetInstance();
820+
auto database = storage_manager->GetDatabaseWithOid(database_oid);
821+
auto table = database->GetTableWithOid(table_oid);
822+
auto default_layout = table->GetDefaultLayout();
823+
824+
if (default_layout.GetOid() == layout_oid) {
825+
table->ResetDefaultLayout();
826+
}
827+
828+
auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog();
829+
if (!pg_layout->DeleteLayout(table_oid, layout_oid, txn)) {
830+
auto layout = table->GetDefaultLayout();
831+
LOG_DEBUG("Layout delete failed. Default layout id: %u", layout.GetOid());
832+
return ResultType::FAILURE;
833+
}
834+
835+
return ResultType::SUCCESS;
836+
}
837+
767838
//===--------------------------------------------------------------------===//
768839
// GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION
769840
//===--------------------------------------------------------------------===//

src/catalog/layout_catalog.cpp

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Peloton
4+
//
5+
// layout_catalog.cpp
6+
//
7+
// Identification: src/catalog/layout_catalog.cpp
8+
//
9+
// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#include "catalog/layout_catalog.h"
14+
15+
#include "catalog/catalog.h"
16+
#include "catalog/system_catalogs.h"
17+
#include "concurrency/transaction_context.h"
18+
#include "executor/logical_tile.h"
19+
#include "storage/data_table.h"
20+
#include "storage/layout.h"
21+
22+
namespace peloton {
23+
namespace catalog {
24+
25+
/** @brief Constructor invoked by the SystemsCatalog constructor.
26+
* @param pg_catalog The database to which this pg_layout belongs.
27+
*/
28+
LayoutCatalog::LayoutCatalog(
29+
storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool,
30+
UNUSED_ATTRIBUTE concurrency::TransactionContext *txn)
31+
: AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME,
32+
InitializeSchema().release(), pg_catalog) {
33+
// Add indexes for pg_attribute
34+
AddIndex({ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, LAYOUT_CATALOG_PKEY_OID,
35+
LAYOUT_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY);
36+
AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID,
37+
LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT);
38+
}
39+
/** @brief Destructor. Do nothing. Layouts will be dropped by DropTable. */
40+
LayoutCatalog::~LayoutCatalog() {}
41+
42+
/** @brief Initilailizes the schema for the pg_layout table.
43+
* @return unique_ptr of the schema for pg_layout.
44+
*/
45+
std::unique_ptr<catalog::Schema> LayoutCatalog::InitializeSchema() {
46+
const std::string primary_key_constraint_name = "primary_key";
47+
const std::string not_null_constraint_name = "not_null";
48+
49+
auto table_id_column = catalog::Column(
50+
type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER),
51+
"table_oid", true);
52+
table_id_column.AddConstraint(catalog::Constraint(
53+
ConstraintType::PRIMARY, primary_key_constraint_name));
54+
table_id_column.AddConstraint(
55+
catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name));
56+
57+
auto layout_oid_column = catalog::Column(
58+
type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER),
59+
"layout_oid", true);
60+
layout_oid_column.AddConstraint(catalog::Constraint(
61+
ConstraintType::PRIMARY, primary_key_constraint_name));
62+
layout_oid_column.AddConstraint(
63+
catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name));
64+
65+
auto num_columns_column = catalog::Column(
66+
type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER),
67+
"num_columns", true);
68+
num_columns_column.AddConstraint(
69+
catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name));
70+
71+
auto column_map_column = catalog::Column(
72+
type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR),
73+
"column_map", false);
74+
column_map_column.AddConstraint(
75+
catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name));
76+
77+
std::unique_ptr<catalog::Schema> column_catalog_schema(
78+
new catalog::Schema({table_id_column, layout_oid_column,
79+
num_columns_column, column_map_column}));
80+
81+
return column_catalog_schema;
82+
}
83+
84+
/** @brief Insert a layout into the pg_layout table.
85+
* @param table_oid oid of the table to which the new layout belongs.
86+
* @param layout layout to be added to the pg_layout table.
87+
* @param pool to allocate memory for the column_map column.
88+
* @param txn TransactionContext for adding the layout.
89+
* @return true on success.
90+
*/
91+
bool LayoutCatalog::InsertLayout(oid_t table_oid,
92+
std::shared_ptr<const storage::Layout> layout,
93+
type::AbstractPool *pool,
94+
concurrency::TransactionContext *txn) {
95+
// Create the tuple first
96+
std::unique_ptr<storage::Tuple> tuple(
97+
new storage::Tuple(catalog_table_->GetSchema(), true));
98+
99+
auto val0 = type::ValueFactory::GetIntegerValue(table_oid);
100+
auto val1 = type::ValueFactory::GetIntegerValue(layout->GetOid());
101+
auto val2 = type::ValueFactory::GetIntegerValue(layout->GetColumnCount());
102+
auto val3 = type::ValueFactory::GetVarcharValue(layout->SerializeColumnMap(),
103+
nullptr);
104+
105+
tuple->SetValue(LayoutCatalog::ColumnId::TABLE_OID, val0, pool);
106+
tuple->SetValue(LayoutCatalog::ColumnId::LAYOUT_OID, val1, pool);
107+
tuple->SetValue(LayoutCatalog::ColumnId::NUM_COLUMNS, val2, pool);
108+
tuple->SetValue(LayoutCatalog::ColumnId::COLUMN_MAP, val3, pool);
109+
110+
// Insert the tuple
111+
return InsertTuple(std::move(tuple), txn);
112+
}
113+
114+
/** @brief Delete a layout from the pg_layout table.
115+
* @param table_oid oid of the table to which the old layout belongs.
116+
* @param layout_oid oid of the layout to be deleted.
117+
* @param txn TransactionContext for deleting the layout.
118+
* @return true on success.
119+
*/
120+
bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid,
121+
concurrency::TransactionContext *txn) {
122+
oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid
123+
124+
std::vector<type::Value> values;
125+
values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy());
126+
values.push_back(type::ValueFactory::GetIntegerValue(layout_oid).Copy());
127+
128+
auto pg_table = Catalog::GetInstance()
129+
->GetSystemCatalogs(database_oid)
130+
->GetTableCatalog();
131+
132+
// delete column from cache
133+
auto table_object = pg_table->GetTableObject(table_oid, txn);
134+
table_object->EvictLayout(layout_oid);
135+
136+
return DeleteWithIndexScan(index_offset, values, txn);
137+
}
138+
139+
/** @brief Delete all layouts correponding to a table from the pg_layout.
140+
* @param table_oid oid of the table to delete all layouts.
141+
* @param txn TransactionContext for deleting the layouts.
142+
* @return true on success.
143+
*/
144+
bool LayoutCatalog::DeleteLayouts(oid_t table_oid,
145+
concurrency::TransactionContext *txn) {
146+
oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid
147+
std::vector<type::Value> values;
148+
values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy());
149+
150+
// delete layouts from cache
151+
auto pg_table = Catalog::GetInstance()
152+
->GetSystemCatalogs(database_oid)
153+
->GetTableCatalog();
154+
auto table_object = pg_table->GetTableObject(table_oid, txn);
155+
table_object->EvictAllLayouts();
156+
157+
return DeleteWithIndexScan(index_offset, values, txn);
158+
}
159+
160+
/** @brief Get all layouts correponding to a table from the pg_layout.
161+
* @param table_oid oid of the table to fetch all layouts.
162+
* @param txn TransactionContext for getting the layouts.
163+
* @return unordered_map containing a layout_oid -> layout mapping.
164+
*/
165+
const std::unordered_map<oid_t, std::shared_ptr<const storage::Layout>>
166+
LayoutCatalog::GetLayouts(oid_t table_oid,
167+
concurrency::TransactionContext *txn) {
168+
// Try to find the layouts in the cache
169+
auto pg_table = Catalog::GetInstance()
170+
->GetSystemCatalogs(database_oid)
171+
->GetTableCatalog();
172+
auto table_object = pg_table->GetTableObject(table_oid, txn);
173+
PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid);
174+
auto layout_objects = table_object->GetLayouts(true);
175+
if (layout_objects.size() != 0) {
176+
return layout_objects;
177+
}
178+
179+
// Cache miss, get from pg_catalog
180+
std::vector<oid_t> column_ids(all_column_ids);
181+
oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid
182+
std::vector<type::Value> values;
183+
values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy());
184+
185+
auto result_tiles =
186+
GetResultWithIndexScan(column_ids, index_offset, values, txn);
187+
188+
for (auto &tile : (*result_tiles)) { // Iterate through the result_tiles
189+
for (auto tuple_id : *tile) {
190+
oid_t layout_oid =
191+
tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID)
192+
.GetAs<oid_t>();
193+
oid_t num_columns =
194+
tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS)
195+
.GetAs<oid_t>();
196+
std::string column_map_str =
197+
tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP)
198+
.ToString();
199+
auto column_map =
200+
storage::Layout::DeserializeColumnMap(num_columns, column_map_str);
201+
auto layout_object =
202+
std::make_shared<const storage::Layout>(column_map, layout_oid);
203+
table_object->InsertLayout(layout_object);
204+
}
205+
}
206+
207+
return table_object->GetLayouts();
208+
}
209+
210+
/** @brief Get the layout by layout_oid from the pg_layout.
211+
* @param table_oid oid of the table to fetch the layout.
212+
* @param layout_oid oid of the layout being queried.
213+
* @param txn TransactionContext for getting the layout.
214+
* @return shared_ptr corresponding to the layout_oid if found.
215+
* nullptr otherwise.
216+
*/
217+
std::shared_ptr<const storage::Layout> LayoutCatalog::GetLayoutWithOid(
218+
oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) {
219+
auto table_layouts = GetLayouts(table_oid, txn);
220+
for (const auto &layout_entry : table_layouts) {
221+
if (layout_entry.second->GetOid() == layout_oid) {
222+
return layout_entry.second;
223+
}
224+
}
225+
return nullptr;
226+
}
227+
228+
} // namespace catalog
229+
} // namespace peloton

0 commit comments

Comments
 (0)