Skip to content

Commit bbeecc0

Browse files
neNasko1Atanas Dimitrovjameslamb
authored
[c++] Fix dump_model() information for root node (#6569)
Co-authored-by: Atanas Dimitrov <nasko119@abv.bg> Co-authored-by: James Lamb <jaylamb20@gmail.com>
1 parent 562157e commit bbeecc0

16 files changed

+228
-51
lines changed

‎include/LightGBM/cuda/cuda_tree.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ class CUDATree : public Tree {
7777
const data_size_t* used_data_indices,
7878
data_size_t num_data, double* score) const override;
7979

80-
inline void AsConstantTree(double val) override;
80+
inline void AsConstantTree(double val, int count) override;
8181

8282
const int* cuda_leaf_parent() const { return cuda_leaf_parent_; }
8383

‎include/LightGBM/tree.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -228,13 +228,14 @@ class Tree {
228228
shrinkage_ = 1.0f;
229229
}
230230

231-
virtual inline void AsConstantTree(double val) {
231+
virtual inline void AsConstantTree(double val, int count = 0) {
232232
num_leaves_ = 1;
233233
shrinkage_ = 1.0f;
234234
leaf_value_[0] = val;
235235
if (is_linear_) {
236236
leaf_const_[0] = val;
237237
}
238+
leaf_count_[0] = count;
238239
}
239240

240241
/*! \brief Serialize this object to string*/
@@ -563,7 +564,7 @@ inline void Tree::Split(int leaf, int feature, int real_feature,
563564
leaf_parent_[leaf] = new_node_idx;
564565
leaf_parent_[num_leaves_] = new_node_idx;
565566
// save current leaf value to internal node before change
566-
internal_weight_[new_node_idx] = leaf_weight_[leaf];
567+
internal_weight_[new_node_idx] = left_weight + right_weight;
567568
internal_value_[new_node_idx] = leaf_value_[leaf];
568569
internal_count_[new_node_idx] = left_cnt + right_cnt;
569570
leaf_value_[leaf] = std::isnan(left_value) ? 0.0f : left_value;

‎python-package/lightgbm/basic.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3906,7 +3906,7 @@ def _get_split_feature(
39063906
return feature_name
39073907

39083908
def _is_single_node_tree(tree: Dict[str, Any]) -> bool:
3909-
return set(tree.keys()) == {"leaf_value"}
3909+
return set(tree.keys()) == {"leaf_value", "leaf_count"}
39103910

39113911
# Create the node record, and populate universal data members
39123912
node: Dict[str, Union[int, str, None]] = OrderedDict()

‎src/boosting/gbdt.cpp

+4-1
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,10 @@ bool GBDT::TrainOneIter(const score_t* gradients, const score_t* hessians) {
427427
score_updater->AddScore(init_scores[cur_tree_id], cur_tree_id);
428428
}
429429
}
430-
new_tree->AsConstantTree(init_scores[cur_tree_id]);
430+
new_tree->AsConstantTree(init_scores[cur_tree_id], num_data_);
431+
} else {
432+
// extend init_scores with zeros
433+
new_tree->AsConstantTree(0, num_data_);
431434
}
432435
}
433436
// add model

‎src/boosting/rf.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ class RF : public GBDT {
168168
output = init_scores_[cur_tree_id];
169169
}
170170
}
171-
new_tree->AsConstantTree(output);
171+
new_tree->AsConstantTree(output, num_data_);
172172
MultiplyScore(cur_tree_id, (iter_ + num_init_iteration_));
173173
UpdateScore(new_tree.get(), cur_tree_id);
174174
MultiplyScore(cur_tree_id, 1.0 / (iter_ + num_init_iteration_ + 1));

‎src/io/cuda/cuda_tree.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -330,9 +330,10 @@ void CUDATree::SyncLeafOutputFromCUDAToHost() {
330330
CopyFromCUDADeviceToHost<double>(leaf_value_.data(), cuda_leaf_value_, leaf_value_.size(), __FILE__, __LINE__);
331331
}
332332

333-
void CUDATree::AsConstantTree(double val) {
334-
Tree::AsConstantTree(val);
333+
void CUDATree::AsConstantTree(double val, int count) {
334+
Tree::AsConstantTree(val, count);
335335
CopyFromHostToCUDADevice<double>(cuda_leaf_value_, &val, 1, __FILE__, __LINE__);
336+
CopyFromHostToCUDADevice<int>(cuda_leaf_count_, &count, 1, __FILE__, __LINE__);
336337
}
337338

338339
} // namespace LightGBM

‎src/io/cuda/cuda_tree.cu

+2-2
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ __global__ void SplitKernel( // split information
9494
split_gain[new_node_index] = static_cast<float>(cuda_split_info->gain);
9595
} else if (thread_index == 4) {
9696
// save current leaf value to internal node before change
97-
internal_weight[new_node_index] = leaf_weight[leaf_index];
97+
internal_weight[new_node_index] = cuda_split_info->left_sum_hessians + cuda_split_info->right_sum_hessians;
9898
leaf_weight[leaf_index] = cuda_split_info->left_sum_hessians;
9999
} else if (thread_index == 5) {
100100
internal_value[new_node_index] = leaf_value[leaf_index];
@@ -210,7 +210,7 @@ __global__ void SplitCategoricalKernel( // split information
210210
split_gain[new_node_index] = static_cast<float>(cuda_split_info->gain);
211211
} else if (thread_index == 4) {
212212
// save current leaf value to internal node before change
213-
internal_weight[new_node_index] = leaf_weight[leaf_index];
213+
internal_weight[new_node_index] = cuda_split_info->left_sum_hessians + cuda_split_info->right_sum_hessians;
214214
leaf_weight[leaf_index] = cuda_split_info->left_sum_hessians;
215215
} else if (thread_index == 5) {
216216
internal_value[new_node_index] = leaf_value[leaf_index];

‎src/io/tree.cpp

+12-9
Original file line numberDiff line numberDiff line change
@@ -416,12 +416,15 @@ std::string Tree::ToJSON() const {
416416
str_buf << "\"num_cat\":" << num_cat_ << "," << '\n';
417417
str_buf << "\"shrinkage\":" << shrinkage_ << "," << '\n';
418418
if (num_leaves_ == 1) {
419+
str_buf << "\"tree_structure\":{";
420+
str_buf << "\"leaf_value\":" << leaf_value_[0] << ", " << '\n';
419421
if (is_linear_) {
420-
str_buf << "\"tree_structure\":{" << "\"leaf_value\":" << leaf_value_[0] << ", " << "\n";
421-
str_buf << LinearModelToJSON(0) << "}" << "\n";
422+
str_buf << "\"leaf_count\":" << leaf_count_[0] << ", " << '\n';
423+
str_buf << LinearModelToJSON(0);
422424
} else {
423-
str_buf << "\"tree_structure\":{" << "\"leaf_value\":" << leaf_value_[0] << "}" << '\n';
425+
str_buf << "\"leaf_count\":" << leaf_count_[0];
424426
}
427+
str_buf << "}" << '\n';
425428
} else {
426429
str_buf << "\"tree_structure\":" << NodeToJSON(0) << '\n';
427430
}
@@ -731,6 +734,12 @@ Tree::Tree(const char* str, size_t* used_len) {
731734
is_linear_ = false;
732735
}
733736

737+
if (key_vals.count("leaf_count")) {
738+
leaf_count_ = CommonC::StringToArrayFast<int>(key_vals["leaf_count"], num_leaves_);
739+
} else {
740+
leaf_count_.resize(num_leaves_);
741+
}
742+
734743
#ifdef USE_CUDA
735744
is_cuda_tree_ = false;
736745
#endif // USE_CUDA
@@ -793,12 +802,6 @@ Tree::Tree(const char* str, size_t* used_len) {
793802
leaf_weight_.resize(num_leaves_);
794803
}
795804

796-
if (key_vals.count("leaf_count")) {
797-
leaf_count_ = CommonC::StringToArrayFast<int>(key_vals["leaf_count"], num_leaves_);
798-
} else {
799-
leaf_count_.resize(num_leaves_);
800-
}
801-
802805
if (key_vals.count("decision_type")) {
803806
decision_type_ = CommonC::StringToArrayFast<int8_t>(key_vals["decision_type"], num_leaves_ - 1);
804807
} else {

‎src/treelearner/cuda/cuda_leaf_splits.cpp

+5-2
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,14 @@ void CUDALeafSplits::InitValues(
3838
const double lambda_l1, const double lambda_l2,
3939
const score_t* cuda_gradients, const score_t* cuda_hessians,
4040
const data_size_t* cuda_bagging_data_indices, const data_size_t* cuda_data_indices_in_leaf,
41-
const data_size_t num_used_indices, hist_t* cuda_hist_in_leaf, double* root_sum_hessians) {
41+
const data_size_t num_used_indices, hist_t* cuda_hist_in_leaf,
42+
double* root_sum_gradients, double* root_sum_hessians) {
4243
cuda_gradients_ = cuda_gradients;
4344
cuda_hessians_ = cuda_hessians;
4445
cuda_sum_of_gradients_buffer_.SetValue(0);
4546
cuda_sum_of_hessians_buffer_.SetValue(0);
4647
LaunchInitValuesKernal(lambda_l1, lambda_l2, cuda_bagging_data_indices, cuda_data_indices_in_leaf, num_used_indices, cuda_hist_in_leaf);
48+
CopyFromCUDADeviceToHost<double>(root_sum_gradients, cuda_sum_of_gradients_buffer_.RawData(), 1, __FILE__, __LINE__);
4749
CopyFromCUDADeviceToHost<double>(root_sum_hessians, cuda_sum_of_hessians_buffer_.RawData(), 1, __FILE__, __LINE__);
4850
SynchronizeCUDADevice(__FILE__, __LINE__);
4951
}
@@ -53,11 +55,12 @@ void CUDALeafSplits::InitValues(
5355
const int16_t* cuda_gradients_and_hessians,
5456
const data_size_t* cuda_bagging_data_indices,
5557
const data_size_t* cuda_data_indices_in_leaf, const data_size_t num_used_indices,
56-
hist_t* cuda_hist_in_leaf, double* root_sum_hessians,
58+
hist_t* cuda_hist_in_leaf, double* root_sum_gradients, double* root_sum_hessians,
5759
const score_t* grad_scale, const score_t* hess_scale) {
5860
cuda_gradients_ = reinterpret_cast<const score_t*>(cuda_gradients_and_hessians);
5961
cuda_hessians_ = nullptr;
6062
LaunchInitValuesKernal(lambda_l1, lambda_l2, cuda_bagging_data_indices, cuda_data_indices_in_leaf, num_used_indices, cuda_hist_in_leaf, grad_scale, hess_scale);
63+
CopyFromCUDADeviceToHost<double>(root_sum_gradients, cuda_sum_of_gradients_buffer_.RawData(), 1, __FILE__, __LINE__);
6164
CopyFromCUDADeviceToHost<double>(root_sum_hessians, cuda_sum_of_hessians_buffer_.RawData(), 1, __FILE__, __LINE__);
6265
SynchronizeCUDADevice(__FILE__, __LINE__);
6366
}

‎src/treelearner/cuda/cuda_leaf_splits.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,14 @@ class CUDALeafSplits {
4444
const score_t* cuda_gradients, const score_t* cuda_hessians,
4545
const data_size_t* cuda_bagging_data_indices,
4646
const data_size_t* cuda_data_indices_in_leaf, const data_size_t num_used_indices,
47-
hist_t* cuda_hist_in_leaf, double* root_sum_hessians);
47+
hist_t* cuda_hist_in_leaf, double* root_sum_gradients, double* root_sum_hessians);
4848

4949
void InitValues(
5050
const double lambda_l1, const double lambda_l2,
5151
const int16_t* cuda_gradients_and_hessians,
5252
const data_size_t* cuda_bagging_data_indices,
5353
const data_size_t* cuda_data_indices_in_leaf, const data_size_t num_used_indices,
54-
hist_t* cuda_hist_in_leaf, double* root_sum_hessians,
54+
hist_t* cuda_hist_in_leaf, double* root_sum_gradients, double* root_sum_hessians,
5555
const score_t* grad_scale, const score_t* hess_scale);
5656

5757
void InitValues();

‎src/treelearner/cuda/cuda_single_gpu_tree_learner.cpp

+16-10
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ void CUDASingleGPUTreeLearner::Init(const Dataset* train_data, bool is_constant_
6666
leaf_best_split_default_left_.resize(config_->num_leaves, 0);
6767
leaf_num_data_.resize(config_->num_leaves, 0);
6868
leaf_data_start_.resize(config_->num_leaves, 0);
69+
leaf_sum_gradients_.resize(config_->num_leaves, 0.0f);
6970
leaf_sum_hessians_.resize(config_->num_leaves, 0.0f);
7071

7172
if (!boosting_on_cuda_) {
@@ -122,6 +123,7 @@ void CUDASingleGPUTreeLearner::BeforeTrain() {
122123
cuda_data_partition_->cuda_data_indices(),
123124
root_num_data,
124125
cuda_histogram_constructor_->cuda_hist_pointer(),
126+
&leaf_sum_gradients_[0],
125127
&leaf_sum_hessians_[0],
126128
cuda_gradient_discretizer_->grad_scale_ptr(),
127129
cuda_gradient_discretizer_->hess_scale_ptr());
@@ -137,6 +139,7 @@ void CUDASingleGPUTreeLearner::BeforeTrain() {
137139
cuda_data_partition_->cuda_data_indices(),
138140
root_num_data,
139141
cuda_histogram_constructor_->cuda_hist_pointer(),
142+
&leaf_sum_gradients_[0],
140143
&leaf_sum_hessians_[0]);
141144
}
142145
leaf_num_data_[0] = root_num_data;
@@ -162,6 +165,12 @@ Tree* CUDASingleGPUTreeLearner::Train(const score_t* gradients,
162165
const bool track_branch_features = !(config_->interaction_constraints_vector.empty());
163166
std::unique_ptr<CUDATree> tree(new CUDATree(config_->num_leaves, track_branch_features,
164167
config_->linear_tree, config_->gpu_device_id, has_categorical_feature_));
168+
// set the root value by hand, as it is not handled by splits
169+
tree->SetLeafOutput(0, CUDALeafSplits::CalculateSplittedLeafOutput<true, false>(
170+
leaf_sum_gradients_[smaller_leaf_index_], leaf_sum_hessians_[smaller_leaf_index_],
171+
config_->lambda_l1, config_->lambda_l2, config_->path_smooth,
172+
static_cast<data_size_t>(num_data_), 0));
173+
tree->SyncLeafOutputFromHostToCUDA();
165174
for (int i = 0; i < config_->num_leaves - 1; ++i) {
166175
global_timer.Start("CUDASingleGPUTreeLearner::ConstructHistogramForLeaf");
167176
const data_size_t num_data_in_smaller_leaf = leaf_num_data_[smaller_leaf_index_];
@@ -293,8 +302,6 @@ Tree* CUDASingleGPUTreeLearner::Train(const score_t* gradients,
293302
best_split_info);
294303
}
295304

296-
double sum_left_gradients = 0.0f;
297-
double sum_right_gradients = 0.0f;
298305
cuda_data_partition_->Split(best_split_info,
299306
best_leaf_index_,
300307
right_leaf_index,
@@ -313,10 +320,10 @@ Tree* CUDASingleGPUTreeLearner::Train(const score_t* gradients,
313320
&leaf_data_start_[right_leaf_index],
314321
&leaf_sum_hessians_[best_leaf_index_],
315322
&leaf_sum_hessians_[right_leaf_index],
316-
&sum_left_gradients,
317-
&sum_right_gradients);
323+
&leaf_sum_gradients_[best_leaf_index_],
324+
&leaf_sum_gradients_[right_leaf_index]);
318325
#ifdef DEBUG
319-
CheckSplitValid(best_leaf_index_, right_leaf_index, sum_left_gradients, sum_right_gradients);
326+
CheckSplitValid(best_leaf_index_, right_leaf_index);
320327
#endif // DEBUG
321328
smaller_leaf_index_ = (leaf_num_data_[best_leaf_index_] < leaf_num_data_[right_leaf_index] ? best_leaf_index_ : right_leaf_index);
322329
larger_leaf_index_ = (smaller_leaf_index_ == best_leaf_index_ ? right_leaf_index : best_leaf_index_);
@@ -374,6 +381,7 @@ void CUDASingleGPUTreeLearner::ResetConfig(const Config* config) {
374381
leaf_best_split_default_left_.resize(config_->num_leaves, 0);
375382
leaf_num_data_.resize(config_->num_leaves, 0);
376383
leaf_data_start_.resize(config_->num_leaves, 0);
384+
leaf_sum_gradients_.resize(config_->num_leaves, 0.0f);
377385
leaf_sum_hessians_.resize(config_->num_leaves, 0.0f);
378386
}
379387
cuda_histogram_constructor_->ResetConfig(config);
@@ -562,9 +570,7 @@ void CUDASingleGPUTreeLearner::SelectFeatureByNode(const Tree* tree) {
562570
#ifdef DEBUG
563571
void CUDASingleGPUTreeLearner::CheckSplitValid(
564572
const int left_leaf,
565-
const int right_leaf,
566-
const double split_sum_left_gradients,
567-
const double split_sum_right_gradients) {
573+
const int right_leaf) {
568574
std::vector<data_size_t> left_data_indices(leaf_num_data_[left_leaf]);
569575
std::vector<data_size_t> right_data_indices(leaf_num_data_[right_leaf]);
570576
CopyFromCUDADeviceToHost<data_size_t>(left_data_indices.data(),
@@ -585,9 +591,9 @@ void CUDASingleGPUTreeLearner::CheckSplitValid(
585591
sum_right_gradients += host_gradients_[index];
586592
sum_right_hessians += host_hessians_[index];
587593
}
588-
CHECK_LE(std::fabs(sum_left_gradients - split_sum_left_gradients), 1e-6f);
594+
CHECK_LE(std::fabs(sum_left_gradients - leaf_sum_gradients_[left_leaf]), 1e-6f);
589595
CHECK_LE(std::fabs(sum_left_hessians - leaf_sum_hessians_[left_leaf]), 1e-6f);
590-
CHECK_LE(std::fabs(sum_right_gradients - split_sum_right_gradients), 1e-6f);
596+
CHECK_LE(std::fabs(sum_right_gradients - leaf_sum_gradients_[right_leaf]), 1e-6f);
591597
CHECK_LE(std::fabs(sum_right_hessians - leaf_sum_hessians_[right_leaf]), 1e-6f);
592598
}
593599
#endif // DEBUG

‎src/treelearner/cuda/cuda_single_gpu_tree_learner.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,7 @@ class CUDASingleGPUTreeLearner: public SerialTreeLearner {
7171

7272
#ifdef DEBUG
7373
void CheckSplitValid(
74-
const int left_leaf, const int right_leaf,
75-
const double sum_left_gradients, const double sum_right_gradients);
74+
const int left_leaf, const int right_leaf);
7675
#endif // DEBUG
7776

7877
void RenewDiscretizedTreeLeaves(CUDATree* cuda_tree);
@@ -103,6 +102,7 @@ class CUDASingleGPUTreeLearner: public SerialTreeLearner {
103102
std::vector<uint8_t> leaf_best_split_default_left_;
104103
std::vector<data_size_t> leaf_num_data_;
105104
std::vector<data_size_t> leaf_data_start_;
105+
std::vector<double> leaf_sum_gradients_;
106106
std::vector<double> leaf_sum_hessians_;
107107
int smaller_leaf_index_;
108108
int larger_leaf_index_;

‎src/treelearner/serial_tree_learner.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,12 @@ Tree* SerialTreeLearner::Train(const score_t* gradients, const score_t *hessians
201201
auto tree_ptr = tree.get();
202202
constraints_->ShareTreePointer(tree_ptr);
203203

204+
// set the root value by hand, as it is not handled by splits
205+
tree->SetLeafOutput(0, FeatureHistogram::CalculateSplittedLeafOutput<true, true, true, false>(
206+
smaller_leaf_splits_->sum_gradients(), smaller_leaf_splits_->sum_hessians(),
207+
config_->lambda_l1, config_->lambda_l2, config_->max_delta_step,
208+
BasicConstraint(), config_->path_smooth, static_cast<data_size_t>(num_data_), 0));
209+
204210
// root leaf
205211
int left_leaf = 0;
206212
int cur_depth = 1;

0 commit comments

Comments
 (0)