Skip to content

Add tests for loss-guided expand #62

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 9, 2024
Merged

Add tests for loss-guided expand #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3a9d2df
minor refactoring; adding tests
Aug 6, 2024
a046944
remove dead comments
Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 8 additions & 14 deletions plugin/sycl/tree/hist_updater.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,12 @@ void HistUpdater<GradientSumT>::BuildLocalHistograms(
template<typename GradientSumT>
void HistUpdater<GradientSumT>::BuildNodeStats(
const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
builder_monitor_.Start("BuildNodeStats");
for (auto const& entry : qexpand_depth_wise_) {
int nid = entry.nid;
this->InitNewNode(nid, gmat, gpair, *p_fmat, *p_tree);
this->InitNewNode(nid, gmat, gpair, *p_tree);
// add constraints
if (!(*p_tree)[nid].IsLeftChild() && !(*p_tree)[nid].IsRoot()) {
// it's a right child
Expand Down Expand Up @@ -232,7 +231,6 @@ void HistUpdater<GradientSumT>::SplitSiblings(
template<typename GradientSumT>
void HistUpdater<GradientSumT>::ExpandWithDepthWise(
const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
int num_leaves = 0;
Expand All @@ -249,7 +247,7 @@ void HistUpdater<GradientSumT>::ExpandWithDepthWise(
hist_rows_adder_->AddHistRows(this, &sync_ids, p_tree);
BuildLocalHistograms(gmat, p_tree, gpair);
hist_synchronizer_->SyncHistograms(this, sync_ids, p_tree);
BuildNodeStats(gmat, p_fmat, p_tree, gpair);
BuildNodeStats(gmat, p_tree, gpair);

EvaluateAndApplySplits(gmat, p_tree, &num_leaves, depth,
&temp_qexpand_depth);
Expand All @@ -270,7 +268,6 @@ void HistUpdater<GradientSumT>::ExpandWithDepthWise(
template<typename GradientSumT>
void HistUpdater<GradientSumT>::ExpandWithLossGuide(
const common::GHistIndexMatrix& gmat,
DMatrix* p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
builder_monitor_.Start("ExpandWithLossGuide");
Expand All @@ -280,10 +277,10 @@ void HistUpdater<GradientSumT>::ExpandWithLossGuide(
ExpandEntry node(ExpandEntry::kRootNid, p_tree->GetDepth(ExpandEntry::kRootNid));
BuildHistogramsLossGuide(node, gmat, p_tree, gpair);

this->InitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_fmat, *p_tree);

this->InitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_tree);
this->EvaluateSplits({node}, gmat, *p_tree);
node.split.loss_chg = snode_host_[ExpandEntry::kRootNid].best.loss_chg;
// LOG(FATAL) << node.split.loss_chg;

qexpand_loss_guided_->push(node);
++num_leaves;
Expand All @@ -305,9 +302,7 @@ void HistUpdater<GradientSumT>::ExpandWithLossGuide(
e.best.DefaultLeft(), e.weight, left_leaf_weight,
right_leaf_weight, e.best.loss_chg, e.stats.GetHess(),
e.best.left_sum.GetHess(), e.best.right_sum.GetHess());

this->ApplySplit({candidate}, gmat, p_tree);

const int cleft = (*p_tree)[nid].LeftChild();
const int cright = (*p_tree)[nid].RightChild();

Expand All @@ -320,8 +315,8 @@ void HistUpdater<GradientSumT>::ExpandWithLossGuide(
BuildHistogramsLossGuide(right_node, gmat, p_tree, gpair);
}

this->InitNewNode(cleft, gmat, gpair, *p_fmat, *p_tree);
this->InitNewNode(cright, gmat, gpair, *p_fmat, *p_tree);
this->InitNewNode(cleft, gmat, gpair, *p_tree);
this->InitNewNode(cright, gmat, gpair, *p_tree);
bst_uint featureid = snode_host_[nid].best.SplitIndex();
tree_evaluator_.AddSplit(nid, cleft, cright, featureid,
snode_host_[cleft].weight, snode_host_[cright].weight);
Expand Down Expand Up @@ -356,9 +351,9 @@ void HistUpdater<GradientSumT>::Update(

this->InitData(gmat, gpair_device, *p_fmat, *p_tree);
if (param_.grow_policy == xgboost::tree::TrainParam::kLossGuide) {
ExpandWithLossGuide(gmat, p_fmat, p_tree, gpair_device);
ExpandWithLossGuide(gmat, p_tree, gpair_device);
} else {
ExpandWithDepthWise(gmat, p_fmat, p_tree, gpair_device);
ExpandWithDepthWise(gmat, p_tree, gpair_device);
}

for (int nid = 0; nid < p_tree->NumNodes(); ++nid) {
Expand Down Expand Up @@ -842,7 +837,6 @@ void HistUpdater<GradientSumT>::InitNewNode(int nid,
const common::GHistIndexMatrix& gmat,
const USMVector<GradientPair,
MemoryType::on_device> &gpair,
const DMatrix& fmat,
const RegTree& tree) {
builder_monitor_.Start("InitNewNode");

Expand Down
4 changes: 0 additions & 4 deletions plugin/sycl/tree/hist_updater.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,9 @@ class HistUpdater {
void InitNewNode(int nid,
const common::GHistIndexMatrix& gmat,
const USMVector<GradientPair, MemoryType::on_device> &gpair,
const DMatrix& fmat,
const RegTree& tree);

void ExpandWithDepthWise(const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);

Expand All @@ -169,7 +167,6 @@ class HistUpdater {
RegTree *p_tree);

void BuildNodeStats(const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);

Expand All @@ -188,7 +185,6 @@ class HistUpdater {
std::vector<ExpandEntry>* temp_qexpand_depth);

void ExpandWithLossGuide(const common::GHistIndexMatrix& gmat,
DMatrix* p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device>& gpair);

Expand Down
70 changes: 66 additions & 4 deletions tests/cpp/plugin/test_sycl_hist_updater.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,8 @@ class TestHistUpdater : public HistUpdater<GradientSumT> {
auto TestInitNewNode(int nid,
const common::GHistIndexMatrix& gmat,
const USMVector<GradientPair, MemoryType::on_device> &gpair,
const DMatrix& fmat,
const RegTree& tree) {
HistUpdater<GradientSumT>::InitNewNode(nid, gmat, gpair, fmat, tree);
HistUpdater<GradientSumT>::InitNewNode(nid, gmat, gpair, tree);
return HistUpdater<GradientSumT>::snode_host_[nid];
}

Expand All @@ -67,6 +66,13 @@ class TestHistUpdater : public HistUpdater<GradientSumT> {
RegTree* p_tree) {
HistUpdater<GradientSumT>::ApplySplit(nodes, gmat, p_tree);
}

auto TestExpandWithLossGuide(const common::GHistIndexMatrix& gmat,
DMatrix *p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
HistUpdater<GradientSumT>::ExpandWithLossGuide(gmat, p_tree, gpair);
}
};

void GenerateRandomGPairs(::sycl::queue* qu, GradientPair* gpair_ptr, size_t num_rows, bool has_neg_hess) {
Expand Down Expand Up @@ -295,7 +301,7 @@ void TestHistUpdaterInitNewNode(const xgboost::tree::TrainParam& param, float sp
auto& row_idxs = row_set_collection->Data();
const size_t* row_idxs_ptr = row_idxs.DataConst();
updater.TestBuildHistogramsLossGuide(node, gmat, &tree, gpair);
const auto snode = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_fmat, tree);
const auto snode = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, tree);

GradStats<GradientSumT> grad_stat;
{
Expand Down Expand Up @@ -354,7 +360,7 @@ void TestHistUpdaterEvaluateSplits(const xgboost::tree::TrainParam& param) {
auto& row_idxs = row_set_collection->Data();
const size_t* row_idxs_ptr = row_idxs.DataConst();
const auto* hist = updater.TestBuildHistogramsLossGuide(node, gmat, &tree, gpair);
const auto snode_init = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_fmat, tree);
const auto snode_init = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, tree);

const auto snode_updated = updater.TestEvaluateSplits({node}, gmat, tree);
auto best_loss_chg = snode_updated[0].best.loss_chg;
Expand Down Expand Up @@ -479,6 +485,53 @@ void TestHistUpdaterApplySplit(const xgboost::tree::TrainParam& param, float spa

}

template <typename GradientSumT>
void TestHistUpdaterExpandWithLossGuide(const xgboost::tree::TrainParam& param) {
const size_t num_rows = 3;
const size_t num_columns = 1;
const size_t n_bins = 16;

Context ctx;
ctx.UpdateAllowUnknown(Args{{"device", "sycl"}});

DeviceManager device_manager;
auto qu = device_manager.GetQueue(ctx.Device());

std::vector<float> data = {7, 3, 15};
auto p_fmat = GetDMatrixFromData(data, num_rows, num_columns);

DeviceMatrix dmat;
dmat.Init(qu, p_fmat.get());
common::GHistIndexMatrix gmat;
gmat.Init(qu, &ctx, dmat, n_bins);

std::vector<GradientPair> gpair_host = {{1, 2}, {3, 1}, {1, 1}};
USMVector<GradientPair, MemoryType::on_device> gpair(&qu, gpair_host);

RegTree tree;
FeatureInteractionConstraintHost int_constraints;
TestHistUpdater<GradientSumT> updater(&ctx, qu, param, int_constraints, p_fmat.get());
updater.SetHistSynchronizer(new BatchHistSynchronizer<GradientSumT>());
updater.SetHistRowsAdder(new BatchHistRowsAdder<GradientSumT>());
auto* row_set_collection = updater.TestInitData(gmat, gpair, *p_fmat, tree);

updater.TestExpandWithLossGuide(gmat, p_fmat.get(), &tree, gpair);

const auto& nodes = tree.GetNodes();
std::vector<float> ans(data.size());
for (size_t data_idx = 0; data_idx < data.size(); ++data_idx) {
size_t node_idx = 0;
while (!nodes[node_idx].IsLeaf()) {
node_idx = data[data_idx] < nodes[node_idx].SplitCond() ? nodes[node_idx].LeftChild() : nodes[node_idx].RightChild();
}
ans[data_idx] = nodes[node_idx].LeafValue();
}

ASSERT_NEAR(ans[0], -0.15, 1e-6);
ASSERT_NEAR(ans[1], -0.45, 1e-6);
ASSERT_NEAR(ans[2], -0.15, 1e-6);
}

TEST(SyclHistUpdater, Sampling) {
xgboost::tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"subsample", "0.7"}});
Expand Down Expand Up @@ -546,4 +599,13 @@ TEST(SyclHistUpdater, ApplySplitDence) {
TestHistUpdaterApplySplit<double>(param, 0.0, (1u << 16) + 1);
}

TEST(SyclHistUpdater, ExpandWithLossGuide) {
xgboost::tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"max_depth", "2"},
{"grow_policy", "lossguide"}});

TestHistUpdaterExpandWithLossGuide<float>(param);
TestHistUpdaterExpandWithLossGuide<double>(param);
}

} // namespace xgboost::sycl::tree
Loading