Skip to content

Commit 7349a33

Browse files
ClaudioSalvatoreArcidiaconoClaudioSalvatoreArcidiacono
committed
enable auto early stopping
1 parent 5f261c4 commit 7349a33

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+4957
-735
lines changed

‎.appveyor.yml

-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ clone_depth: 5
2323

2424
install:
2525
- git submodule update --init --recursive # get `external_libs` folder
26-
- set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix)
2726
- set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
2827
- set PYTHON_VERSION=%CONFIGURATION%
2928
- set CONDA_ENV="test-env"

‎.ci/check_python_dists.sh

+31
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,35 @@ if { test "${TASK}" = "bdist" || test "${METHOD}" = "wheel"; }; then
1717
check-wheel-contents ${DIST_DIR}/*.whl || exit -1
1818
fi
1919

20+
PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)")
21+
if [ $PY_MINOR_VER -gt 7 ]; then
22+
echo "pydistcheck..."
23+
pip install pydistcheck
24+
if { test "${TASK}" = "cuda" || test "${METHOD}" = "wheel"; }; then
25+
pydistcheck \
26+
--inspect \
27+
--ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
28+
--max-allowed-size-uncompressed '60M' \
29+
--max-allowed-files 800 \
30+
${DIST_DIR}/* || exit -1
31+
elif { test $(uname -m) = "aarch64"; }; then
32+
pydistcheck \
33+
--inspect \
34+
--ignore 'compiled-objects-have-debug-symbols' \
35+
--max-allowed-size-compressed '5M' \
36+
--max-allowed-size-uncompressed '15M' \
37+
--max-allowed-files 800 \
38+
${DIST_DIR}/* || exit -1
39+
else
40+
pydistcheck \
41+
--inspect \
42+
--max-allowed-size-compressed '5M' \
43+
--max-allowed-size-uncompressed '15M' \
44+
--max-allowed-files 800 \
45+
${DIST_DIR}/* || exit -1
46+
fi
47+
else
48+
echo "skipping pydistcheck (does not support Python 3.${PY_MINOR_VER})"
49+
fi
50+
2051
echo "done checking Python package distributions"

‎.ci/lint-cpp.sh

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/sh
2+
3+
echo "running cpplint"
4+
cpplint \
5+
--filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length \
6+
--recursive ./src ./include ./R-package ./swig ./tests \
7+
|| exit -1
8+
echo "done running cpplint"
9+
10+
echo "running cmakelint"
11+
cmake_files=$(
12+
find . -name CMakeLists.txt -o -path "./cmake/*.cmake" \
13+
| grep -v external_libs
14+
)
15+
cmakelint \
16+
--linelength=120 \
17+
--filter=-convention/filename,-package/stdargs,-readability/wonkycase \
18+
${cmake_files} \
19+
|| exit -1
20+
echo "done running cmakelint"

‎.ci/lint-python.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/sh
2+
3+
echo "running ruff"
4+
ruff check \
5+
--config=./python-package/pyproject.toml \
6+
. \
7+
|| exit -1
8+
echo "done running ruff"
9+
10+
echo "running isort"
11+
isort \
12+
--check-only \
13+
--settings-path=./python-package/pyproject.toml \
14+
. \
15+
|| exit -1
16+
echo "done running isort"
17+
18+
echo "running mypy"
19+
mypy \
20+
--config-file=./python-package/pyproject.toml \
21+
./python-package \
22+
|| true
23+
echo "done running mypy"

‎.ci/test.sh

+69-45
Original file line numberDiff line numberDiff line change
@@ -66,30 +66,22 @@ if [[ $TASK == "swig" ]]; then
6666
fi
6767

6868
if [[ $TASK == "lint" ]]; then
69+
cd ${BUILD_DIRECTORY}
6970
conda create -q -y -n $CONDA_ENV \
7071
${CONDA_PYTHON_REQUIREMENT} \
7172
cmakelint \
7273
cpplint \
73-
flake8 \
7474
isort \
7575
mypy \
76-
pydocstyle \
77-
"r-lintr>=3.0"
76+
'r-lintr>=3.0' \
77+
ruff
7878
source activate $CONDA_ENV
7979
echo "Linting Python code"
80-
flake8 \
81-
--ignore=E501,W503 \
82-
--exclude=./.nuget,./external_libs,./python-package/build \
83-
. || exit -1
84-
pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
85-
isort . --check-only || exit -1
86-
mypy --ignore-missing-imports python-package/ || true
80+
sh ${BUILD_DIRECTORY}/.ci/lint-python.sh || exit -1
8781
echo "Linting R code"
8882
Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1
8983
echo "Linting C++ code"
90-
cpplint --filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length --recursive ./src ./include ./R-package ./swig ./tests || exit -1
91-
cmake_files=$(find . -name CMakeLists.txt -o -path "*/cmake/*.cmake")
92-
cmakelint --linelength=120 --filter=-convention/filename,-package/stdargs,-readability/wonkycase ${cmake_files} || exit -1
84+
sh ${BUILD_DIRECTORY}/.ci/lint-cpp.sh || exit -1
9385
exit 0
9486
fi
9587

@@ -153,21 +145,23 @@ if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
153145
fi
154146

155147
if [[ $TASK == "sdist" ]]; then
156-
cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
157-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
158-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
148+
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
149+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
150+
pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
159151
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
160-
cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
152+
cp $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1
161153
fi
162154
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
163155
exit 0
164156
elif [[ $TASK == "bdist" ]]; then
165157
if [[ $OS_NAME == "macos" ]]; then
166-
cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --python-tag py3 || exit -1
167-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
168-
mv dist/lightgbm-$LGB_VER-py3-none-macosx.whl dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl
158+
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit -1
159+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
160+
mv \
161+
./dist/*.whl \
162+
dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl || exit -1
169163
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
170-
cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
164+
cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1
171165
fi
172166
else
173167
ARCH=$(uname -m)
@@ -176,37 +170,51 @@ elif [[ $TASK == "bdist" ]]; then
176170
else
177171
PLATFORM="manylinux2014_$ARCH"
178172
fi
179-
cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --integrated-opencl --plat-name=$PLATFORM --python-tag py3 || exit -1
180-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
173+
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1
174+
mv \
175+
./dist/*.whl \
176+
./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit -1
177+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
181178
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
182-
cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
179+
cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1
183180
fi
184181
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
185182
export LIGHTGBM_TEST_DUAL_CPU_GPU=1
186183
fi
187-
pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
184+
pip install --user $BUILD_DIRECTORY/dist/*.whl || exit -1
188185
pytest $BUILD_DIRECTORY/tests || exit -1
189186
exit 0
190187
fi
191188

192-
mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
189+
# temporarily pin pip to versions that support 'pip install --install-option'
190+
# ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287
191+
if [[ $METHOD == "pip" ]]; then
192+
pip install 'pip<23.1'
193+
fi
193194

194195
if [[ $TASK == "gpu" ]]; then
195196
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
196197
grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done
197198
if [[ $METHOD == "pip" ]]; then
198-
cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
199-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
200-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1
199+
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
200+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
201+
pip install \
202+
--user \
203+
-v \
204+
--install-option=--gpu \
205+
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
206+
|| exit -1
201207
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
202208
exit 0
203209
elif [[ $METHOD == "wheel" ]]; then
204-
cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --gpu || exit -1
205-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
206-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
210+
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --gpu || exit -1
211+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
212+
pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
207213
pytest $BUILD_DIRECTORY/tests || exit -1
208214
exit 0
209215
elif [[ $METHOD == "source" ]]; then
216+
mkdir $BUILD_DIRECTORY/build
217+
cd $BUILD_DIRECTORY/build
210218
cmake -DUSE_GPU=ON ..
211219
fi
212220
elif [[ $TASK == "cuda" ]]; then
@@ -216,43 +224,59 @@ elif [[ $TASK == "cuda" ]]; then
216224
sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' $BUILD_DIRECTORY/include/LightGBM/config.h
217225
grep -q 'gpu_use_dp = true' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done
218226
if [[ $METHOD == "pip" ]]; then
219-
cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
220-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
221-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1
227+
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
228+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
229+
pip install \
230+
--user \
231+
-v \
232+
--install-option=--cuda \
233+
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
234+
|| exit -1
222235
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
223236
exit 0
224237
elif [[ $METHOD == "wheel" ]]; then
225-
cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --cuda || exit -1
226-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
227-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
238+
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit -1
239+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
240+
pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
228241
pytest $BUILD_DIRECTORY/tests || exit -1
229242
exit 0
230243
elif [[ $METHOD == "source" ]]; then
244+
mkdir $BUILD_DIRECTORY/build
245+
cd $BUILD_DIRECTORY/build
231246
cmake -DUSE_CUDA=ON ..
232247
fi
233248
elif [[ $TASK == "mpi" ]]; then
234249
if [[ $METHOD == "pip" ]]; then
235-
cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
236-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
237-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
250+
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
251+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
252+
pip install \
253+
--user \
254+
-v \
255+
--install-option=--mpi \
256+
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
257+
|| exit -1
238258
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
239259
exit 0
240260
elif [[ $METHOD == "wheel" ]]; then
241-
cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --mpi || exit -1
242-
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
243-
pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
261+
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --mpi || exit -1
262+
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
263+
pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
244264
pytest $BUILD_DIRECTORY/tests || exit -1
245265
exit 0
246266
elif [[ $METHOD == "source" ]]; then
267+
mkdir $BUILD_DIRECTORY/build
268+
cd $BUILD_DIRECTORY/build
247269
cmake -DUSE_MPI=ON -DUSE_DEBUG=ON ..
248270
fi
249271
else
272+
mkdir $BUILD_DIRECTORY/build
273+
cd $BUILD_DIRECTORY/build
250274
cmake ..
251275
fi
252276

253277
make _lightgbm -j4 || exit -1
254278

255-
cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1
279+
cd $BUILD_DIRECTORY && sh ./build-python.sh install --precompile --user || exit -1
256280
pytest $BUILD_DIRECTORY/tests || exit -1
257281

258282
if [[ $TASK == "regular" ]]; then

‎.ci/test_r_package.sh

+13-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ fi
1717
R_MAJOR_VERSION=( ${R_VERSION//./ } )
1818
if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
1919
export R_MAC_VERSION=3.6.3
20-
export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.pkg
20+
export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg
2121
export R_LINUX_VERSION="3.6.3-1bionic"
2222
export R_APT_REPO="bionic-cran35/"
2323
elif [[ "${R_MAJOR_VERSION}" == "4" ]]; then
@@ -77,13 +77,14 @@ fi
7777

7878
# Installing R precompiled for Mac OS 10.11 or higher
7979
if [[ $OS_NAME == "macos" ]]; then
80+
brew update-reset && brew update
8081
if [[ $R_BUILD_TYPE == "cran" ]]; then
8182
brew install automake || exit -1
8283
fi
8384
brew install \
8485
checkbashisms \
8586
qpdf || exit -1
86-
brew install --cask basictex || exit -1
87+
brew install basictex || exit -1
8788
export PATH="/Library/TeX/texbin:$PATH"
8889
sudo tlmgr --verify-repo=none update --self || exit -1
8990
sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit -1
@@ -117,6 +118,16 @@ if [[ $OS_NAME == "macos" ]]; then
117118
fi
118119
fi
119120

121+
# fix for issue where CRAN was not returning {lattice} when using R 3.6
122+
# "Warning: dependency ‘lattice’ is not available"
123+
#
124+
# refs for that MRAN snapshot:
125+
# * https://cran.r-project.org/web/packages/checkpoint/readme/README.html
126+
# * https://help.codeocean.com/en/articles/3087704-using-mran-snapshots-to-install-archived-r-packages
127+
if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
128+
Rscript --vanilla -e "install.packages('lattice', repos = 'https://cran.microsoft.com/snapshot/2020-04-23/', lib = '${R_LIB_PATH}')"
129+
fi
130+
120131
# Manually install Depends and Imports libraries + 'knitr', 'RhpcBLASctl', 'rmarkdown', 'testthat'
121132
# to avoid a CI-time dependency on devtools (for devtools::install_deps())
122133
# NOTE: testthat is not required when running rchk

0 commit comments

Comments
 (0)