Skip to content

Commit a01916a

Browse files
Ten0Ten0
committed
Merge remote-tracking branch 'upstream/master' into 6021-fix_single_row_contention
2 parents 063008a + 252828f commit a01916a

File tree

129 files changed

+2978
-983
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

129 files changed

+2978
-983
lines changed

‎.appveyor.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 4.1.0.99.{build}
1+
version: 4.3.0.{build}
22

33
image: Visual Studio 2015
44
platform: x64

‎.ci/install-clang-devel.sh

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
3+
# [description]
4+
#
5+
# Installs a development version of clang and the other LLVM tools.
6+
#
7+
8+
set -e -E -u -o pipefail
9+
10+
CLANG_VERSION=${1}
11+
12+
apt-get autoremove -y --purge \
13+
clang-* \
14+
libclang-* \
15+
libunwind-* \
16+
llvm-*
17+
18+
apt-get update -y
19+
apt-get install --no-install-recommends -y \
20+
gnupg \
21+
lsb-release \
22+
software-properties-common \
23+
wget
24+
25+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
26+
27+
# ref: https://apt.llvm.org/
28+
add-apt-repository -y "deb http://apt.llvm.org/unstable/ llvm-toolchain main"
29+
add-apt-repository -y "deb-src http://apt.llvm.org/unstable/ llvm-toolchain main"
30+
31+
apt-get install -y --no-install-recommends \
32+
clang-${CLANG_VERSION} \
33+
clangd-${CLANG_VERSION} \
34+
clang-format-${CLANG_VERSION} \
35+
clang-tidy-${CLANG_VERSION} \
36+
clang-tools-${CLANG_VERSION} \
37+
lldb-${CLANG_VERSION} \
38+
lld-${CLANG_VERSION} \
39+
llvm-${CLANG_VERSION}-dev \
40+
llvm-${CLANG_VERSION}-tools \
41+
libomp-${CLANG_VERSION}-dev \
42+
libc++-${CLANG_VERSION}-dev \
43+
libc++abi-${CLANG_VERSION}-dev \
44+
libclang-common-${CLANG_VERSION}-dev \
45+
libclang-${CLANG_VERSION}-dev \
46+
libclang-cpp${CLANG_VERSION}-dev \
47+
libunwind-${CLANG_VERSION}-dev
48+
49+
# overwriting the stuff in /usr/bin is simpler and more reliable than
50+
# updating PATH, LD_LIBRARY_PATH, etc.
51+
cp --remove-destination /usr/lib/llvm-${CLANG_VERSION}/bin/* /usr/bin/
52+
53+
# per https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang
54+
#
55+
# clang was built to use libc++: for a version built to default to libstdc++
56+
# (as shipped by Fedora/Debian/Ubuntu), add -stdlib=libc++ to CXX
57+
# and install the libcxx-devel/libc++-dev package.
58+
mkdir -p "${HOME}/.R"
59+
60+
cat << EOF > "${HOME}/.R/Makevars"
61+
CXX += -stdlib=libc++
62+
CXX11 += -stdlib=libc++
63+
CXX14 += -stdlib=libc++
64+
CXX17 += -stdlib=libc++
65+
CXX20 += -stdlib=libc++
66+
EOF
67+
68+
echo ""
69+
echo "done installing clang"
70+
clang --version
71+
echo ""

‎.ci/lint-cpp.sh

+1-2
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,7 @@ get_omp_pragmas_without_num_threads() {
3030
--include='*.h' \
3131
--include='*.hpp' \
3232
'pragma omp parallel' \
33-
| grep -v ' num_threads' \
34-
| grep -v 'openmp_wrapper.h'
33+
| grep -v ' num_threads'
3534
}
3635
PROBLEMATIC_LINES=$(
3736
get_omp_pragmas_without_num_threads

‎.ci/lint_r_code.R

+1-2
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,7 @@ LINTERS_TO_USE <- list(
7878
, "true_false" = lintr::T_and_F_symbol_linter()
7979
, "undesirable_function" = lintr::undesirable_function_linter(
8080
fun = c(
81-
"cat" = "CRAN forbids the use of cat() in packages except in special cases. Use message() or warning()."
82-
, "cbind" = paste0(
81+
"cbind" = paste0(
8382
"cbind is an unsafe way to build up a data frame. merge() or direct "
8483
, "column assignment is preferred."
8584
)

‎.ci/setup.sh

+8
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,14 @@ else # Linux
5454
sudo apt-get install --no-install-recommends -y \
5555
clang \
5656
libomp-dev
57+
elif [[ $COMPILER == "clang-17" ]]; then
58+
sudo apt-get install wget
59+
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
60+
sudo apt-add-repository deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
61+
sudo apt-add-repository deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
62+
sudo apt-get update
63+
sudo apt-get install -y clang-17
64+
sudo apt-get install --no-install-recommends -y libomp-17-dev
5765
fi
5866

5967
export LANG="en_US.UTF-8"

‎.ci/test-python-oldest.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
#
88
echo "installing lightgbm's dependencies"
99
pip install \
10+
'cffi==1.15.1' \
1011
'dataclasses' \
11-
'numpy==1.12.0' \
12+
'numpy==1.16.6' \
1213
'pandas==0.24.0' \
14+
'pyarrow==6.0.1' \
1315
'scikit-learn==0.18.2' \
1416
'scipy==0.19.0' \
1517
|| exit -1

‎.ci/test.sh

+9-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
66
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
77
export CXX=clang++
88
export CC=clang
9+
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang-17" ]]; then
10+
export CXX=clang++-17
11+
export CC=clang-17
912
fi
1013

1114
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
@@ -130,11 +133,13 @@ fi
130133
# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
131134
mamba create -q -y -n $CONDA_ENV \
132135
${CONSTRAINED_DEPENDENCIES} \
136+
cffi \
133137
cloudpickle \
134138
joblib \
135139
matplotlib \
136140
numpy \
137141
psutil \
142+
pyarrow \
138143
pytest \
139144
${CONDA_PYTHON_REQUIREMENT} \
140145
python-graphviz \
@@ -314,12 +319,14 @@ matplotlib.use\(\"Agg\"\)\
314319
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit -1 # run all notebooks
315320

316321
# importing the library should succeed even if all optional dependencies are not present
317-
conda uninstall --force --yes \
318-
dask \
322+
conda uninstall -n $CONDA_ENV --force --yes \
323+
cffi \
324+
dask-core \
319325
distributed \
320326
joblib \
321327
matplotlib \
322328
psutil \
329+
pyarrow \
323330
python-graphviz \
324331
scikit-learn || exit -1
325332
python -c "import lightgbm" || exit -1

‎.ci/test_r_package.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -127,13 +127,13 @@ if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
127127
Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')"
128128
fi
129129

130-
# Manually install Depends and Imports libraries + 'knitr', 'RhpcBLASctl', 'rmarkdown', 'testthat'
130+
# Manually install Depends and Imports libraries + 'knitr', 'markdown', 'RhpcBLASctl', 'testthat'
131131
# to avoid a CI-time dependency on devtools (for devtools::install_deps())
132132
# NOTE: testthat is not required when running rchk
133133
if [[ "${TASK}" == "r-rchk" ]]; then
134-
packages="c('data.table', 'jsonlite', 'knitr', 'Matrix', 'R6', 'RhpcBLASctl', 'rmarkdown')"
134+
packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl')"
135135
else
136-
packages="c('data.table', 'jsonlite', 'knitr', 'Matrix', 'R6', 'RhpcBLASctl', 'rmarkdown', 'testthat')"
136+
packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl', 'testthat')"
137137
fi
138138
compile_from_source="both"
139139
if [[ $OS_NAME == "macos" ]]; then

‎.ci/test_r_package_valgrind.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#!/bin/bash
22

3-
RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit -1
3+
RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com')" || exit -1
44
sh build-cran-package.sh \
55
--r-executable=RDvalgrind \
6+
--no-build-vignettes \
67
|| exit -1
8+
79
RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1
810

911
cd R-package/tests
@@ -68,7 +70,7 @@ bytes_possibly_lost=$(
6870
| tr -d ","
6971
)
7072
echo "valgrind found ${bytes_possibly_lost} bytes possibly lost"
71-
if [[ ${bytes_possibly_lost} -gt 352 ]]; then
73+
if [[ ${bytes_possibly_lost} -gt 1056 ]]; then
7274
exit -1
7375
fi
7476

‎.ci/test_r_package_windows.ps1

+1-1
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT
124124
Write-Output "Done installing Rtools"
125125

126126
Write-Output "Installing dependencies"
127-
$packages = "c('data.table', 'jsonlite', 'knitr', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'rmarkdown', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
127+
$packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
128128
Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $?
129129

130130
Write-Output "Building R package"

‎.ci/test_windows.ps1

+2
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,14 @@ conda install brotlipy
5252

5353
conda update -q -y conda
5454
conda create -q -y -n $env:CONDA_ENV `
55+
cffi `
5556
cloudpickle `
5657
joblib `
5758
matplotlib `
5859
numpy `
5960
pandas `
6061
psutil `
62+
pyarrow `
6163
pytest `
6264
"python=$env:PYTHON_VERSION[build=*cpython]" `
6365
python-graphviz `

‎.github/CODEOWNERS

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@
77
# offer a reasonable automatic best-guess
88

99
# catch-all rule (this only gets matched if no rules below match)
10-
* @guolinke @jameslamb @shiyu1994 @jmoralez
10+
* @guolinke @jameslamb @shiyu1994 @jmoralez @borchero

‎.github/workflows/lock.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@ jobs:
1818
action:
1919
runs-on: ubuntu-latest
2020
steps:
21-
- uses: dessant/lock-threads@v4
21+
- uses: dessant/lock-threads@v5
2222
with:
2323
github-token: ${{ github.token }}
2424
# after how many days of inactivity should a closed issue/PR be locked?
25-
issue-inactive-days: '90'
26-
pr-inactive-days: '90'
25+
issue-inactive-days: '365'
26+
pr-inactive-days: '365'
2727
# do not close feature request issues...
2828
# we close those but track them in https://github.com/microsoft/LightGBM/issues/2302
29-
exclude-any-issue-labels: '"feature request"'
29+
exclude-any-issue-labels: 'feature request'
3030
# what labels should be removed prior to locking?
3131
remove-issue-labels: 'awaiting response,awaiting review,blocking,in progress'
3232
remove-pr-labels: 'awaiting response,awaiting review,blocking,in progress'
@@ -42,3 +42,4 @@ jobs:
4242
# what shoulld the locking status be?
4343
issue-lock-reason: 'resolved'
4444
pr-lock-reason: 'resolved'
45+
process-only: 'issues, prs'

‎.github/workflows/r_package.yml

+21-42
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ jobs:
246246
- name: Install packages
247247
shell: bash
248248
run: |
249-
RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
249+
RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
250250
sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }}
251251
RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit -1
252252
- name: Run tests with sanitizers
@@ -258,10 +258,21 @@ jobs:
258258
cat ./tests.log
259259
exit ${exit_code}
260260
test-r-debian-clang:
261-
name: r-package (debian, R-devel, clang)
261+
name: r-package (debian, R-devel, clang-${{ matrix.clang-version }})
262262
timeout-minutes: 60
263+
strategy:
264+
fail-fast: false
265+
matrix:
266+
# list of versions tested in CRAN "Additional Checks":
267+
# https://cran.r-project.org/web/checks/check_issue_kinds.html
268+
clang-version:
269+
- 16
270+
- 17
271+
- 18
263272
runs-on: ubuntu-latest
264273
container: rhub/debian-clang-devel
274+
env:
275+
DEBIAN_FRONTEND: noninteractive
265276
steps:
266277
- name: Install Git before checkout
267278
shell: bash
@@ -276,53 +287,21 @@ jobs:
276287
with:
277288
fetch-depth: 5
278289
submodules: true
279-
- name: update to clang 15
280-
shell: bash
290+
- name: install clang
281291
run: |
282-
# remove clang stuff that comes installed in the image
283-
apt-get autoremove -y --purge \
284-
clang-* \
285-
libclang-* \
286-
libunwind-* \
287-
llvm-*
288-
#
289-
# replace it all with clang-15
290-
apt-get update -y
291-
apt-get install --no-install-recommends -y \
292-
gnupg \
293-
lsb-release \
294-
software-properties-common \
295-
wget
296-
#
297-
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
298-
#
299-
add-apt-repository "deb http://apt.llvm.org/unstable/ llvm-toolchain main"
300-
apt-get install -y --no-install-recommends \
301-
clang-15 \
302-
clangd-15 \
303-
clang-format-15 \
304-
clang-tidy-15 \
305-
clang-tools-15 \
306-
lldb-15 \
307-
lld-15 \
308-
llvm-15-dev \
309-
llvm-15-tools \
310-
libomp-15-dev \
311-
libc++-15-dev \
312-
libc++abi-15-dev \
313-
libclang-common-15-dev \
314-
libclang-15-dev \
315-
libclang-cpp15-dev \
316-
libunwind-15-dev
317-
# overwrite everything in /usr/bin with the new v15 versions
318-
cp --remove-destination /usr/lib/llvm-15/bin/* /usr/bin/
292+
./.ci/install-clang-devel.sh ${{ matrix.clang-version }}
319293
- name: Install packages and run tests
320294
shell: bash
321295
run: |
322296
export PATH=/opt/R-devel/bin/:${PATH}
323-
Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
297+
Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
324298
sh build-cran-package.sh
325299
R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1
300+
echo ""
301+
echo "install logs:"
302+
echo ""
303+
cat lightgbm.Rcheck/00install.out
304+
echo ""
326305
if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then
327306
echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
328307
exit -1

‎.github/workflows/static_analysis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ jobs:
6666
- name: Install packages
6767
shell: bash
6868
run: |
69-
Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'roxygen2', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
69+
Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'roxygen2', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
7070
sh build-cran-package.sh || exit -1
7171
R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit -1
7272
- name: Test documentation

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ _Pvt_Extensions
266266
*.out
267267
*.app
268268
/windows/LightGBM.VC.db
269-
lightgbm
269+
/lightgbm
270270
/testlightgbm
271271

272272
# Created by https://www.gitignore.io/api/python

0 commit comments

Comments
 (0)