Skip to content
This repository was archived by the owner on Sep 27, 2019. It is now read-only.

Commit f710495

Browse files
linmagitlinmagit
authored
Merge branch 'master' into clusterer
2 parents 3682c75 + 8b8f68d commit f710495

File tree

13 files changed

+270
-59
lines changed

13 files changed

+270
-59
lines changed

‎script/installation/packages.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ fi
3838
unset UNAME
3939
DISTRO=$(echo $DISTRO | tr "[:lower:]" "[:upper:]")
4040
TMPDIR=/tmp
41-
TF_VERSION="1.4.0"
4241
TF_TYPE="cpu"
4342

4443

@@ -115,20 +114,24 @@ if [ "$DISTRO" = "UBUNTU" ]; then
115114
PKG_CMAKE="cmake"
116115
PKG_LLVM="llvm-3.7"
117116
PKG_CLANG="clang-3.7"
117+
TF_VERSION="1.4.0"
118118

119119
# Fix for cmake name change on Ubuntu 14.x and 16.x plus --force-yes deprecation
120120
if [ "$MAJOR_VER" == "14" ]; then
121121
PKG_CMAKE="cmake3"
122122
FORCE_Y="--force-yes"
123+
TF_VERSION="1.4.0"
123124
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp34-cp34m-linux_x86_64.whl"
124125
fi
125126
if [ "$MAJOR_VER" == "16" ]; then
127+
TF_VERSION="1.5.0"
126128
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
127129
fi
128130
# Fix for llvm on Ubuntu 17.x
129131
if [ "$MAJOR_VER" == "17" ]; then
130132
PKG_LLVM="llvm-3.9"
131133
PKG_CLANG="clang-3.8"
134+
TF_VERSION="1.5.0"
132135
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
133136
fi
134137
TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
@@ -201,6 +204,7 @@ elif [[ "$DISTRO" == *"FEDORA"* ]]; then
201204
26) LLVM="llvm";;
202205
*) LLVM="llvm4.0";;
203206
esac
207+
TF_VERSION="1.5.0"
204208
TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
205209
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp36-cp36m-linux_x86_64.whl"
206210
LinkerConfigCmd="sudo ldconfig"
@@ -316,6 +320,7 @@ elif [ "$DISTRO" = "DARWIN" ]; then
316320
echo "Installing homebrew..."
317321
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
318322
fi
323+
TF_VERSION="1.4.0"
319324
TFBinaryURL="https://storage.googleapis.com/tensorflow/mac/${TF_TYPE}/tensorflow-${TF_VERSION}-py3-none-any.whl"
320325
TFCApiFile="libtensorflow-${TF_TYPE}-darwin-x86_64-${TF_VERSION}.tar.gz"
321326
LinkerConfigCmd="sudo update_dyld_shared_cache"

‎src/include/common/exception.h

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ enum class ExceptionType {
5757
CONNECTION = 21, // connection related
5858
SYNTAX = 22, // syntax related
5959
SETTINGS = 23, // settings related
60-
BINDER = 24, // settings related
61-
NETWORK = 25
60+
BINDER = 24, // binder related
61+
NETWORK = 25, // network related
62+
OPTIMIZER = 26 // optimizer related
6263
};
6364

6465
class Exception : public std::runtime_error {
@@ -125,6 +126,8 @@ class Exception : public std::runtime_error {
125126
return "Syntax";
126127
case ExceptionType::SETTINGS:
127128
return "Settings";
129+
case ExceptionType::OPTIMIZER:
130+
return "Optimizer";
128131
default:
129132
return "Unknown";
130133
}
@@ -452,4 +455,12 @@ class BinderException : public Exception {
452455
BinderException(std::string msg) : Exception(ExceptionType::BINDER, msg) {}
453456
};
454457

458+
class OptimizerException : public Exception {
459+
OptimizerException() = delete;
460+
461+
public:
462+
OptimizerException(std::string msg)
463+
: Exception(ExceptionType::OPTIMIZER, msg) {}
464+
};
465+
455466
} // namespace peloton

‎src/include/optimizer/group.h

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,18 @@ class Group {
7373

7474
std::shared_ptr<ColumnStats> GetStats(std::string column_name);
7575

76-
void AddStats(std::string column_name, std::shared_ptr<ColumnStats> stats);
77-
78-
bool HasColumnStats(std::string column_name);
76+
void AddStats(std::string column_name, std::shared_ptr<ColumnStats> stats);
77+
78+
bool HasColumnStats(std::string column_name);
79+
80+
/*
81+
* HasExpressions - Determines whether or not a lowest cost expression exists
82+
* for this group
83+
*
84+
* properties: the property set used to search the lowest cost expressions
85+
* return: true if an expression is found, false otherwise
86+
*/
87+
bool HasExpressions(const std::shared_ptr<PropertySet> &properties) const;
7988

8089
void SetNumRows(size_t num_rows) { num_rows_ = num_rows; }
8190

‎src/include/optimizer/optimize_context.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,12 @@
1212

1313
#pragma once
1414

15+
#include "common/timer.h"
16+
1517
#include "optimizer/property_set.h"
1618
#include "optimizer/optimizer_task.h"
1719
#include "optimizer/optimizer_task_pool.h"
20+
#include "settings/settings_manager.h"
1821

1922
namespace peloton {
2023
namespace optimizer {

‎src/include/optimizer/optimizer.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,19 @@ class Optimizer : public AbstractOptimizer {
124124
GroupID id, std::shared_ptr<PropertySet> required_props,
125125
std::vector<expression::AbstractExpression *> required_cols);
126126

127+
/* ExecuteTaskStack - Execute elements of given optimization task stack
128+
* and ensure that we do not go beyond the time limit (unless if one plan has
129+
* not been generated yet)
130+
*
131+
* task_stack: the optimizer's task stack to iterate through
132+
* root_group_id: the root group id to check if we have generated a plan or
133+
*not
134+
* root_context: the OptimizerContext to use that maintains required
135+
*properties
136+
*/
137+
void ExecuteTaskStack(OptimizerTaskStack &task_stack, int root_group_id,
138+
std::shared_ptr<OptimizeContext> root_context);
139+
127140
//////////////////////////////////////////////////////////////////////////////
128141
/// Metadata
129142
OptimizerMetadata metadata_;

‎src/include/optimizer/optimizer_metadata.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@
1111
//===----------------------------------------------------------------------===//
1212

1313
#pragma once
14+
15+
#include "common/timer.h"
1416
#include "optimizer/memo.h"
1517
#include "optimizer/group_expression.h"
1618
#include "optimizer/rule.h"
19+
#include "settings/settings_manager.h"
1720

1821
namespace peloton {
1922
namespace catalog {
@@ -26,9 +29,17 @@ class RuleSet;
2629

2730
class OptimizerMetadata {
2831
public:
32+
OptimizerMetadata()
33+
: timeout_limit(settings::SettingsManager::GetInt(
34+
settings::SettingId::task_execution_timeout)),
35+
timer(Timer<std::milli>()) {}
36+
2937
Memo memo;
3038
RuleSet rule_set;
3139
OptimizerTaskPool *task_pool;
40+
catalog::CatalogCache *catalog_cache;
41+
unsigned int timeout_limit;
42+
Timer<std::milli> timer;
3243
concurrency::TransactionContext* txn;
3344

3445
void SetTaskPool(OptimizerTaskPool *task_pool) {

‎src/include/settings/settings.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
//
1111
//===----------------------------------------------------------------------===//
1212

13-
1413
//===----------------------------------------------------------------------===//
1514
// FILE LOCATIONS
1615
//===----------------------------------------------------------------------===//
@@ -196,9 +195,12 @@ SETTING_bool(hash_join_bloom_filter,
196195
true,
197196
true, true)
198197

198+
SETTING_int(task_execution_timeout,
199+
"Maximum allowed length of time (in ms) for task "
200+
"execution step of optimizer, "
201+
"assuming one plan has been found (default 5000)",
202+
5000, true, true)
203+
199204
//===----------------------------------------------------------------------===//
200205
// GENERAL
201206
//===----------------------------------------------------------------------===//
202-
203-
204-

‎src/optimizer/group.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ GroupExpression *Group::GetBestExpression(
6262
return nullptr;
6363
}
6464

65+
bool Group::HasExpressions(
66+
const std::shared_ptr<PropertySet> &properties) const {
67+
const auto &it = lowest_cost_expressions_.find(properties);
68+
return (it != lowest_cost_expressions_.end());
69+
}
70+
6571
std::shared_ptr<ColumnStats> Group::GetStats(std::string column_name) {
6672
if (!stats_.count(column_name)) {
6773
return nullptr;

‎src/optimizer/optimizer.cpp

Lines changed: 66 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
#include "catalog/manager.h"
1919
#include "catalog/table_catalog.h"
2020

21+
#include "common/exception.h"
22+
2123
#include "optimizer/binding.h"
2224
#include "optimizer/operator_visitor.h"
2325
#include "optimizer/properties.h"
@@ -74,24 +76,19 @@ void Optimizer::OptimizeLoop(int root_group_id,
7476

7577
task_stack->Push(new BottomUpRewrite(
7678
root_group_id, root_context, RewriteRuleSetName::UNNEST_SUBQUERY, false));
77-
while (!task_stack->Empty()) {
78-
auto task = task_stack->Pop();
79-
task->execute();
80-
}
79+
80+
ExecuteTaskStack(*task_stack, root_group_id, root_context);
8181

8282
// Perform optimization after the rewrite
8383
task_stack->Push(new OptimizeGroup(metadata_.memo.GetGroupByID(root_group_id),
8484
root_context));
85+
8586
// Derive stats for the only one logical expression before optimizing
8687
task_stack->Push(new DeriveStats(
8788
metadata_.memo.GetGroupByID(root_group_id)->GetLogicalExpression(),
8889
ExprSet{}, root_context));
8990

90-
// TODO: Add timer for early stop
91-
while (!task_stack->Empty()) {
92-
auto task = task_stack->Pop();
93-
task->execute();
94-
}
91+
ExecuteTaskStack(*task_stack, root_group_id, root_context);
9592
}
9693

9794
shared_ptr<planner::AbstractPlan> Optimizer::BuildPelotonPlanTree(
@@ -124,7 +121,11 @@ shared_ptr<planner::AbstractPlan> Optimizer::BuildPelotonPlanTree(
124121
// Get the physical properties the final plan must output
125122
auto query_info = GetQueryInfo(parse_tree);
126123

127-
OptimizeLoop(root_id, query_info.physical_props);
124+
try {
125+
OptimizeLoop(root_id, query_info.physical_props);
126+
} catch (OptimizerException &e) {
127+
LOG_WARN("Optimize Loop ended prematurely: %s", e.what());
128+
}
128129

129130
try {
130131
auto best_plan = ChooseBestPlan(root_id, query_info.physical_props,
@@ -240,29 +241,29 @@ shared_ptr<GroupExpression> Optimizer::InsertQueryTree(
240241
}
241242

242243
QueryInfo Optimizer::GetQueryInfo(parser::SQLStatement *tree) {
243-
auto GetQueryInfoHelper = [](
244-
std::vector<unique_ptr<expression::AbstractExpression>> &select_list,
245-
std::unique_ptr<parser::OrderDescription> &order_info,
246-
std::vector<expression::AbstractExpression *> &output_exprs,
247-
std::shared_ptr<PropertySet> &physical_props) {
248-
// Extract output column
249-
for (auto &expr : select_list) output_exprs.push_back(expr.get());
250-
251-
// Extract sort property
252-
if (order_info != nullptr) {
253-
std::vector<expression::AbstractExpression *> sort_exprs;
254-
std::vector<bool> sort_ascending;
255-
for (auto &expr : order_info->exprs) {
256-
sort_exprs.push_back(expr.get());
257-
}
258-
for (auto &type : order_info->types) {
259-
sort_ascending.push_back(type == parser::kOrderAsc);
260-
}
261-
if (!sort_exprs.empty())
262-
physical_props->AddProperty(
263-
std::make_shared<PropertySort>(sort_exprs, sort_ascending));
264-
}
265-
};
244+
auto GetQueryInfoHelper =
245+
[](std::vector<unique_ptr<expression::AbstractExpression>> &select_list,
246+
std::unique_ptr<parser::OrderDescription> &order_info,
247+
std::vector<expression::AbstractExpression *> &output_exprs,
248+
std::shared_ptr<PropertySet> &physical_props) {
249+
// Extract output column
250+
for (auto &expr : select_list) output_exprs.push_back(expr.get());
251+
252+
// Extract sort property
253+
if (order_info != nullptr) {
254+
std::vector<expression::AbstractExpression *> sort_exprs;
255+
std::vector<bool> sort_ascending;
256+
for (auto &expr : order_info->exprs) {
257+
sort_exprs.push_back(expr.get());
258+
}
259+
for (auto &type : order_info->types) {
260+
sort_ascending.push_back(type == parser::kOrderAsc);
261+
}
262+
if (!sort_exprs.empty())
263+
physical_props->AddProperty(
264+
std::make_shared<PropertySort>(sort_exprs, sort_ascending));
265+
}
266+
};
266267

267268
std::vector<expression::AbstractExpression *> output_exprs;
268269
std::shared_ptr<PropertySet> physical_props = std::make_shared<PropertySet>();
@@ -280,7 +281,8 @@ QueryInfo Optimizer::GetQueryInfo(parser::SQLStatement *tree) {
280281
output_exprs, physical_props);
281282
break;
282283
}
283-
default:;
284+
default:
285+
;
284286
}
285287

286288
return QueryInfo(output_exprs, physical_props);
@@ -336,5 +338,35 @@ unique_ptr<planner::AbstractPlan> Optimizer::ChooseBestPlan(
336338
LOG_TRACE("Finish Choosing best plan for group %d", id);
337339
return plan;
338340
}
341+
342+
void Optimizer::ExecuteTaskStack(
343+
OptimizerTaskStack &task_stack, int root_group_id,
344+
std::shared_ptr<OptimizeContext> root_context) {
345+
auto root_group = metadata_.memo.GetGroupByID(root_group_id);
346+
auto &timer = metadata_.timer;
347+
const auto timeout_limit = metadata_.timeout_limit;
348+
const auto &required_props = root_context->required_prop;
349+
350+
if (timer.GetInvocations() == 0) {
351+
timer.Start();
352+
}
353+
// Iterate through the task stack
354+
while (!task_stack.Empty()) {
355+
// Check to see if we have at least one plan, and if we have exceeded our
356+
// timeout limit
357+
if (timer.GetDuration() >= timeout_limit &&
358+
root_group->HasExpressions(required_props)) {
359+
throw OptimizerException("Optimizer task execution duration " +
360+
std::to_string(timer.GetDuration()) +
361+
" exceeds timeout limit " +
362+
std::to_string(timeout_limit));
363+
}
364+
timer.Reset();
365+
auto task = task_stack.Pop();
366+
task->execute();
367+
timer.Stop();
368+
}
369+
}
370+
339371
} // namespace optimizer
340372
} // namespace peloton

0 commit comments

Comments
 (0)