Skip to content

Commit 180e733

Browse files
xaduprexadupre
authored
Fixes early stopping with XGBoost 2.0 (#597)
* add one unit test to investigate a bug Signed-off-by: xadupre <[email protected]> * add one more test Signed-off-by: xadupre <[email protected]> * remove unnecessary print Signed-off-by: xadupre <[email protected]> * update CI Signed-off-by: Xavier Dupre <[email protected]> * ci Signed-off-by: Xavier Dupre <[email protected]> * remove removed files Signed-off-by: Xavier Dupre <[email protected]> * update test Signed-off-by: Xavier Dupre <[email protected]> * fix early stopping Signed-off-by: Xavier Dupre <[email protected]> * fix rf models Signed-off-by: Xavier Dupre <[email protected]> * remaining merge issue Signed-off-by: Xavier Dupre <[email protected]> --------- Signed-off-by: xadupre <[email protected]> Signed-off-by: Xavier Dupre <[email protected]>
1 parent 7858f9f commit 180e733

File tree

8 files changed

+517
-17
lines changed

8 files changed

+517
-17
lines changed

‎.azure-pipelines/linux-conda-CI.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@ jobs:
2525
numpy.version: ''
2626
scipy.version: ''
2727

28-
Python311-1150-RT1160-xgb175-lgbm40:
28+
Python311-1150-RT1163-xgb175-lgbm40:
2929
python.version: '3.11'
3030
ONNX_PATH: 'onnx==1.15.0'
31-
ONNXRT_PATH: 'onnxruntime==1.16.2'
31+
ONNXRT_PATH: 'onnxruntime==1.16.3'
3232
COREML_PATH: NONE
3333
lightgbm.version: '>=4.0'
34-
xgboost.version: '==1.7.5'
34+
xgboost.version: '>=1.7.5,<2'
3535
numpy.version: ''
3636
scipy.version: ''
3737

@@ -41,7 +41,7 @@ jobs:
4141
ONNXRT_PATH: 'onnxruntime==1.16.2'
4242
COREML_PATH: NONE
4343
lightgbm.version: '>=4.0'
44-
xgboost.version: '==1.7.5'
44+
xgboost.version: '>=1.7.5,<2'
4545
numpy.version: ''
4646
scipy.version: ''
4747

@@ -51,7 +51,7 @@ jobs:
5151
ONNXRT_PATH: 'onnxruntime==1.15.1'
5252
COREML_PATH: NONE
5353
lightgbm.version: '<4.0'
54-
xgboost.version: '==1.7.5'
54+
xgboost.version: '>=1.7.5,<2'
5555
numpy.version: ''
5656
scipy.version: ''
5757

@@ -61,7 +61,7 @@ jobs:
6161
ONNXRT_PATH: 'onnxruntime==1.14.0'
6262
COREML_PATH: NONE
6363
lightgbm.version: '<4.0'
64-
xgboost.version: '==1.7.5'
64+
xgboost.version: '>=1.7.5,<2'
6565
numpy.version: ''
6666
scipy.version: ''
6767

@@ -71,7 +71,7 @@ jobs:
7171
ONNXRT_PATH: 'onnxruntime==1.15.1'
7272
COREML_PATH: NONE
7373
lightgbm.version: '>=4.0'
74-
xgboost.version: '==1.7.5'
74+
xgboost.version: '>=1.7.5,<2'
7575
numpy.version: ''
7676
scipy.version: '==1.8.0'
7777

‎.azure-pipelines/win32-conda-CI.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ jobs:
1515
strategy:
1616
matrix:
1717

18+
Python311-1150-RT1163:
19+
python.version: '3.11'
20+
ONNX_PATH: 'onnx==1.15.0'
21+
ONNXRT_PATH: 'onnxruntime==1.16.3'
22+
COREML_PATH: NONE
23+
numpy.version: ''
24+
xgboost.version: '2.0.2'
25+
1826
Python311-1150-RT1162:
1927
python.version: '3.11'
2028
ONNX_PATH: 'onnx==1.15.0'

‎CHANGELOGS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## 1.12.0
44

5+
* Fix early stopping for XGBClassifier and xgboost > 2
6+
[#597](https://github.com/onnx/onnxmltools/pull/597)
57
* Fix discrepancies with XGBRegressor and xgboost > 2
68
[#670](https://github.com/onnx/onnxmltools/pull/670)
79
* Support count:poisson for XGBRegressor

‎onnxmltools/convert/xgboost/common.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ def get_xgb_params(xgb_node):
3232
bs = float(config["learner"]["learner_model_param"]["base_score"])
3333
# xgboost >= 2.0
3434
params["base_score"] = bs
35+
36+
bst = xgb_node.get_booster()
37+
if hasattr(bst, "best_ntree_limit"):
38+
params["best_ntree_limit"] = bst.best_ntree_limit
39+
if "gradient_booster" in config["learner"]:
40+
gbp = config["learner"]["gradient_booster"]["gbtree_model_param"]
41+
if "num_trees" in gbp:
42+
params["best_ntree_limit"] = int(gbp["num_trees"])
3543
return params
3644

3745

‎onnxmltools/convert/xgboost/operator_converters/XGBoost.py‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,20 @@ def common_members(xgb_node, inputs):
4040
params = XGBConverter.get_xgb_params(xgb_node)
4141
objective = params["objective"]
4242
base_score = params["base_score"]
43+
if hasattr(xgb_node, "best_ntree_limit"):
44+
best_ntree_limit = xgb_node.best_ntree_limit
45+
elif hasattr(xgb_node, "best_iteration"):
46+
best_ntree_limit = xgb_node.best_iteration + 1
47+
else:
48+
best_ntree_limit = params.get("best_ntree_limit", None)
4349
if base_score is None:
4450
base_score = 0.5
4551
booster = xgb_node.get_booster()
4652
# The json format was available in October 2017.
4753
# XGBoost 0.7 was the first version released with it.
4854
js_tree_list = booster.get_dump(with_stats=True, dump_format="json")
4955
js_trees = [json.loads(s) for s in js_tree_list]
50-
return objective, base_score, js_trees
56+
return objective, base_score, js_trees, best_ntree_limit
5157

5258
@staticmethod
5359
def _get_default_tree_attribute_pairs(is_classifier):
@@ -231,17 +237,17 @@ def _get_default_tree_attribute_pairs():
231237
def convert(scope, operator, container):
232238
xgb_node = operator.raw_operator
233239
inputs = operator.inputs
234-
objective, base_score, js_trees = XGBConverter.common_members(xgb_node, inputs)
240+
objective, base_score, js_trees, best_ntree_limit = XGBConverter.common_members(
241+
xgb_node, inputs
242+
)
235243

236244
if objective in ["reg:gamma", "reg:tweedie"]:
237245
raise RuntimeError("Objective '{}' not supported.".format(objective))
238246

239247
attr_pairs = XGBRegressorConverter._get_default_tree_attribute_pairs()
240248
attr_pairs["base_values"] = [base_score]
241249

242-
bst = xgb_node.get_booster()
243-
best_ntree_limit = getattr(bst, "best_ntree_limit", len(js_trees))
244-
if best_ntree_limit < len(js_trees):
250+
if best_ntree_limit and best_ntree_limit < len(js_trees):
245251
js_trees = js_trees[:best_ntree_limit]
246252

247253
XGBConverter.fill_tree_attributes(
@@ -289,7 +295,9 @@ def convert(scope, operator, container):
289295
xgb_node = operator.raw_operator
290296
inputs = operator.inputs
291297

292-
objective, base_score, js_trees = XGBConverter.common_members(xgb_node, inputs)
298+
objective, base_score, js_trees, best_ntree_limit = XGBConverter.common_members(
299+
xgb_node, inputs
300+
)
293301

294302
params = XGBConverter.get_xgb_params(xgb_node)
295303
n_estimators = get_n_estimators_classifier(xgb_node, params, js_trees)
@@ -305,8 +313,9 @@ def convert(scope, operator, container):
305313
else:
306314
ncl = (max(attr_pairs["class_treeids"]) + 1) // n_estimators
307315

308-
bst = xgb_node.get_booster()
309-
best_ntree_limit = getattr(bst, "best_ntree_limit", len(js_trees)) * ncl
316+
best_ntree_limit = best_ntree_limit or len(js_trees)
317+
if ncl > 0:
318+
best_ntree_limit *= ncl
310319
if 0 < best_ntree_limit < len(js_trees):
311320
js_trees = js_trees[:best_ntree_limit]
312321
attr_pairs = XGBClassifierConverter._get_default_tree_attribute_pairs()

0 commit comments

Comments
 (0)