Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[python-package] scikit-learn fit() methods: add eval_X, eval_y, deprecate eval_set #6857

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

[python-package] scikit-learn fit() methods: add eval_X, eval_y, deprecate eval_set #6857

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b29d5c1
ENH add eval_X, eval_y, deprecate eval_set
lorentzenchr Mar 9, 2025
5a95c9a
CLN eval_X, eval_y in all fit signatures
lorentzenchr Mar 9, 2025
379cfdf
ENH eval_X and eval_y in _DaskLGBMModel
lorentzenchr Mar 9, 2025
5dd3171
CLN appease ruff/linting
lorentzenchr Mar 9, 2025
06d87a8
FIX _validate_eval_set_Xy
lorentzenchr Mar 9, 2025
05f20b2
FIX dask
lorentzenchr Mar 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions python-package/lightgbm/dask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
_lgbmmodel_doc_custom_eval_note,
_lgbmmodel_doc_fit,
_lgbmmodel_doc_predict,
_validate_eval_set_Xy,
)

__all__ = [
Expand Down Expand Up @@ -318,6 +319,13 @@ def _train_part(
if eval_class_weight:
kwargs["eval_class_weight"] = [eval_class_weight[i] for i in eval_component_idx]

if local_eval_set is None:
local_eval_X = None
local_eval_y = None
else:
local_eval_X = tuple(X for X, y in local_eval_set)
local_eval_y = tuple(y for X, y in local_eval_set)

model = model_factory(**params)
if remote_socket is not None:
remote_socket.release()
Expand All @@ -329,7 +337,8 @@ def _train_part(
sample_weight=weight,
init_score=init_score,
group=group,
eval_set=local_eval_set,
eval_X=local_eval_X,
eval_y=local_eval_y,
eval_sample_weight=local_eval_sample_weight,
eval_init_score=local_eval_init_score,
eval_group=local_eval_group,
Expand All @@ -342,7 +351,8 @@ def _train_part(
label,
sample_weight=weight,
init_score=init_score,
eval_set=local_eval_set,
eval_X=local_eval_X,
eval_y=local_eval_y,
eval_sample_weight=local_eval_sample_weight,
eval_init_score=local_eval_init_score,
eval_names=local_eval_names,
Expand Down Expand Up @@ -422,6 +432,8 @@ def _train(
group: Optional[_DaskVectorLike] = None,
eval_set: Optional[List[Tuple[_DaskMatrixLike, _DaskCollection]]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_DaskMatrixLike, Tuple[_DaskMatrixLike]]] = None,
eval_y: Optional[Union[_DaskCollection, Tuple[_DaskCollection]]] = None,
eval_sample_weight: Optional[List[_DaskVectorLike]] = None,
eval_class_weight: Optional[List[Union[dict, str]]] = None,
eval_init_score: Optional[List[_DaskCollection]] = None,
Expand Down Expand Up @@ -461,6 +473,10 @@ def _train(
of ``evals_result_`` and ``best_score_`` will be empty dictionaries.
eval_names : list of str, or None, optional (default=None)
Names of eval_set.
eval_X : Dask Array or Dask DataFrame, tuple thereof or None, optional (default=None)
Feature matrix or tuple thereof, e.g. `(X_val0, X_val1)`, to use as validation sets.
eval_y : Dask Array or Dask DataFrame, tuple thereof or None, optional (default=None)
Target values or tuple thereof, i.g. `(y_val0, y_val1)`, to use as validation sets.
eval_sample_weight : list of Dask Array or Dask Series, or None, optional (default=None)
Weights for each validation set in eval_set. Weights should be non-negative.
eval_class_weight : list of dict or str, or None, optional (default=None)
Expand Down Expand Up @@ -570,6 +586,7 @@ def _train(
for i in range(n_parts):
parts[i]["init_score"] = init_score_parts[i]

eval_set = _validate_eval_set_Xy(eval_set=eval_set, eval_X=eval_X, eval_y=eval_y)
# evals_set will to be re-constructed into smaller lists of (X, y) tuples, where
# X and y are each delayed sub-lists of original eval dask Collections.
if eval_set:
Expand Down Expand Up @@ -1049,6 +1066,8 @@ def _lgb_dask_fit(
group: Optional[_DaskVectorLike] = None,
eval_set: Optional[List[Tuple[_DaskMatrixLike, _DaskCollection]]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_DaskMatrixLike, Tuple[_DaskMatrixLike]]] = None,
eval_y: Optional[Union[_DaskCollection, Tuple[_DaskCollection]]] = None,
eval_sample_weight: Optional[List[_DaskVectorLike]] = None,
eval_class_weight: Optional[List[Union[dict, str]]] = None,
eval_init_score: Optional[List[_DaskCollection]] = None,
Expand Down Expand Up @@ -1076,6 +1095,8 @@ def _lgb_dask_fit(
group=group,
eval_set=eval_set,
eval_names=eval_names,
eval_X=eval_X,
eval_y=eval_y,
eval_sample_weight=eval_sample_weight,
eval_class_weight=eval_class_weight,
eval_init_score=eval_init_score,
Expand Down Expand Up @@ -1182,6 +1203,8 @@ def fit( # type: ignore[override]
init_score: Optional[_DaskCollection] = None,
eval_set: Optional[List[Tuple[_DaskMatrixLike, _DaskCollection]]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_DaskMatrixLike, Tuple[_DaskMatrixLike]]] = None,
eval_y: Optional[Union[_DaskCollection, Tuple[_DaskCollection]]] = None,
eval_sample_weight: Optional[List[_DaskVectorLike]] = None,
eval_class_weight: Optional[List[Union[dict, str]]] = None,
eval_init_score: Optional[List[_DaskCollection]] = None,
Expand All @@ -1197,6 +1220,8 @@ def fit( # type: ignore[override]
init_score=init_score,
eval_set=eval_set,
eval_names=eval_names,
eval_X=eval_X,
eval_y=eval_y,
eval_sample_weight=eval_sample_weight,
eval_class_weight=eval_class_weight,
eval_init_score=eval_init_score,
Expand Down Expand Up @@ -1386,6 +1411,8 @@ def fit( # type: ignore[override]
init_score: Optional[_DaskVectorLike] = None,
eval_set: Optional[List[Tuple[_DaskMatrixLike, _DaskCollection]]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_DaskMatrixLike, Tuple[_DaskMatrixLike]]] = None,
eval_y: Optional[Union[_DaskCollection, Tuple[_DaskCollection]]] = None,
eval_sample_weight: Optional[List[_DaskVectorLike]] = None,
eval_init_score: Optional[List[_DaskVectorLike]] = None,
eval_metric: Optional[_LGBM_ScikitEvalMetricType] = None,
Expand All @@ -1400,6 +1427,8 @@ def fit( # type: ignore[override]
init_score=init_score,
eval_set=eval_set,
eval_names=eval_names,
eval_X=eval_X,
eval_y=eval_y,
eval_sample_weight=eval_sample_weight,
eval_init_score=eval_init_score,
eval_metric=eval_metric,
Expand Down Expand Up @@ -1555,6 +1584,8 @@ def fit( # type: ignore[override]
group: Optional[_DaskVectorLike] = None,
eval_set: Optional[List[Tuple[_DaskMatrixLike, _DaskCollection]]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_DaskMatrixLike, Tuple[_DaskMatrixLike]]] = None,
eval_y: Optional[Union[_DaskCollection, Tuple[_DaskCollection]]] = None,
eval_sample_weight: Optional[List[_DaskVectorLike]] = None,
eval_init_score: Optional[List[_DaskVectorLike]] = None,
eval_group: Optional[List[_DaskVectorLike]] = None,
Expand All @@ -1572,6 +1603,8 @@ def fit( # type: ignore[override]
group=group,
eval_set=eval_set,
eval_names=eval_names,
eval_X=eval_X,
eval_y=eval_y,
eval_sample_weight=eval_sample_weight,
eval_init_score=eval_init_score,
eval_group=eval_group,
Expand Down
53 changes: 51 additions & 2 deletions python-package/lightgbm/sklearn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""Scikit-learn wrapper interface for LightGBM."""

import copy
import warnings
from inspect import signature
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
Expand All @@ -13,6 +14,7 @@
_MULTICLASS_OBJECTIVES,
Booster,
Dataset,
LGBMDeprecationWarning,
LightGBMError,
_choose_param_value,
_ConfigAliases,
Expand Down Expand Up @@ -338,11 +340,16 @@ def __call__(
sum(group) = n_samples.
For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
eval_set : list or None, optional (default=None)
eval_set : list or None, optional (default=None) (deprecated)
A list of (X, y) tuple pairs to use as validation sets.
This is deprecated, use `eval_X` and `eval_y` instead.
eval_names : list of str, or None, optional (default=None)
Names of eval_set.
eval_sample_weight : {eval_sample_weight_shape}
eval_X : {X_shape} or tuple or None, optional (default=None)
Feature matrix or tuple thereof, e.g. `(X_val0, X_val1)`, to use as validation sets.
eval_y : {y_shape} or tuple or None, optional (default=None)
Target values or tuple thereof, i.g. `(y_val0, y_val1)`, to use as validation sets.
eval_sample_weight : {eval_sample_weight_shape} or tuple or None (default=None)
Weights of eval data. Weights should be non-negative.
eval_class_weight : list or None, optional (default=None)
Class weights of eval data.
Expand Down Expand Up @@ -483,6 +490,33 @@ def _extract_evaluation_meta_data(
raise TypeError(f"{name} should be dict or list")


def _validate_eval_set_Xy(eval_set, eval_X, eval_y):
"""Validate eval args.

Returns
-------
eval_set
"""
if eval_set is not None:
msg = "The argument 'eval_set' is deprecated, use 'eval_X' and 'eval_y' instead."
warnings.warn(msg, category=LGBMDeprecationWarning, stacklevel=2)
if eval_X is not None or eval_y is not None:
raise ValueError("Specify either 'eval_set' or 'eval_X' and 'eval_y', but not both.")
return eval_set
if (eval_X is None) != (eval_y is None):
raise ValueError("You must specify eval_X and eval_y, not just one of them.")
if eval_set is None and eval_X is not None:
if isinstance(eval_X, tuple) != isinstance(eval_y, tuple):
raise ValueError("If eval_X is a tuple, y_val must be a tuple of same length, and vice versa.")
if isinstance(eval_X, tuple) and len(eval_X) != len(eval_y):
raise ValueError("If eval_X is a tuple, y_val must be a tuple of same length, and vice versa.")
if isinstance(eval_X, tuple):
eval_set = list(zip(eval_X, eval_y))
else:
eval_set = [(eval_X, eval_y)]
return eval_set


class LGBMModel(_LGBMModelBase):
"""Implementation of the scikit-learn API for LightGBM."""

Expand Down Expand Up @@ -913,6 +947,8 @@ def fit(
group: Optional[_LGBM_GroupType] = None,
eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_LGBM_ScikitMatrixLike, Tuple[_LGBM_ScikitMatrixLike]]] = None,
eval_y: Optional[Union[_LGBM_LabelType, Tuple[_LGBM_LabelType]]] = None,
eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
eval_class_weight: Optional[List[float]] = None,
eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
Expand Down Expand Up @@ -987,6 +1023,7 @@ def fit(
)

valid_sets: List[Dataset] = []
eval_set = _validate_eval_set_Xy(eval_set=eval_set, eval_X=eval_X, eval_y=eval_y)
if eval_set is not None:
if isinstance(eval_set, tuple):
eval_set = [eval_set]
Expand Down Expand Up @@ -1386,6 +1423,8 @@ def fit( # type: ignore[override]
init_score: Optional[_LGBM_InitScoreType] = None,
eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_LGBM_ScikitMatrixLike, Tuple[_LGBM_ScikitMatrixLike]]] = None,
eval_y: Optional[Union[_LGBM_LabelType, Tuple[_LGBM_LabelType]]] = None,
eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
eval_metric: Optional[_LGBM_ScikitEvalMetricType] = None,
Expand All @@ -1401,6 +1440,8 @@ def fit( # type: ignore[override]
sample_weight=sample_weight,
init_score=init_score,
eval_set=eval_set,
eval_X=eval_X,
eval_y=eval_y,
eval_names=eval_names,
eval_sample_weight=eval_sample_weight,
eval_init_score=eval_init_score,
Expand Down Expand Up @@ -1499,6 +1540,8 @@ def fit( # type: ignore[override]
init_score: Optional[_LGBM_InitScoreType] = None,
eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_LGBM_ScikitMatrixLike, Tuple[_LGBM_ScikitMatrixLike]]] = None,
eval_y: Optional[Union[_LGBM_LabelType, Tuple[_LGBM_LabelType]]] = None,
eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
eval_class_weight: Optional[List[float]] = None,
eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
Expand Down Expand Up @@ -1564,6 +1607,8 @@ def fit( # type: ignore[override]
init_score=init_score,
eval_set=valid_sets,
eval_names=eval_names,
eval_X=eval_X,
eval_y=eval_y,
eval_sample_weight=eval_sample_weight,
eval_class_weight=eval_class_weight,
eval_init_score=eval_init_score,
Expand Down Expand Up @@ -1745,6 +1790,8 @@ def fit( # type: ignore[override]
group: Optional[_LGBM_GroupType] = None,
eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
eval_names: Optional[List[str]] = None,
eval_X: Optional[Union[_LGBM_ScikitMatrixLike, Tuple[_LGBM_ScikitMatrixLike]]] = None,
eval_y: Optional[Union[_LGBM_LabelType, Tuple[_LGBM_LabelType]]] = None,
eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
eval_group: Optional[List[_LGBM_GroupType]] = None,
Expand Down Expand Up @@ -1785,6 +1832,8 @@ def fit( # type: ignore[override]
group=group,
eval_set=eval_set,
eval_names=eval_names,
eval_X=eval_X,
eval_y=eval_y,
eval_sample_weight=eval_sample_weight,
eval_init_score=eval_init_score,
eval_group=eval_group,
Expand Down
34 changes: 34 additions & 0 deletions tests/python_package_test/test_sklearn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from sklearn.utils.validation import check_is_fitted

import lightgbm as lgb
from lightgbm.basic import LGBMDeprecationWarning
from lightgbm.compat import (
DASK_INSTALLED,
DATATABLE_INSTALLED,
Expand Down Expand Up @@ -2043,3 +2044,36 @@ def test_classifier_fit_detects_classes_every_time():
assert model.objective_ == "multiclass"
model.fit(X, y_bin)
assert model.objective_ == "binary"


def test_eval_set_deprecation():
"""Test use of eval_set raises deprecation warning."""
X, y = make_synthetic_regression(n_samples=10)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)
gbm = lgb.LGBMRegressor()
msg = "The argument 'eval_set' is deprecated.*"
with pytest.warns(LGBMDeprecationWarning, match=msg):
gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)])


def test_eval_X_eval_y_eval_set_equivalence():
"""Test that eval_X and eval_y are equivalent to eval_set."""
X, y = make_synthetic_regression()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
cbs = [lgb.early_stopping(2)]
gbm1 = lgb.LGBMRegressor()
gbm1.fit(X_train, y_train, eval_set=[(X_test, y_test)], callbacks=cbs)
gbm2 = lgb.LGBMRegressor()
gbm2.fit(X_train, y_train, eval_X=X_test, eval_y=y_test, callbacks=cbs)
np.testing.assert_allclose(gbm1.predict(X), gbm2.predict(X))

# 2 evaluation sets
n = X_test.shape[0]
X_test1, X_test2 = X_test[: n // 2], X_test[n // 2 :]
y_test1, y_test2 = y_test[: n // 2], y_test[n // 2 :]
gbm1 = lgb.LGBMRegressor()
gbm1.fit(X_train, y_train, eval_set=[(X_test1, y_test1), (X_test2, y_test2)], callbacks=cbs)
gbm2 = lgb.LGBMRegressor()
gbm2.fit(X_train, y_train, eval_X=(X_test1, X_test2), eval_y=(y_test1, y_test2), callbacks=cbs)
np.testing.assert_allclose(gbm1.predict(X), gbm2.predict(X))
assert gbm1.evals_result_["valid_0"]["l2"][0] == pytest.approx(gbm2.evals_result_["valid_0"]["l2"][0])
Loading