Skip to content

Commit a22da27

Browse files
sh1ngsh1ng
authored
python 3.11 support (#879)
* use nvidia/cuda:11.6.1-cudnn8-devel-centos7 * use nvcr.io * remove RUN yum update -y * fix dependencies * python 3.11 * remove scikit-learn and another fixes * update dependencies * fix pip install * remove tests * remove tests * fix import, skip multi-gpu tests * add @pytest.mark.skip * remove import * fix import * skip tests, python 3.11 * sphinx upgrade * fix sphinx version, disable conda pkg * coment out pyarrow * pytest upgrade * fix failing tests * fix test * disable pylint * fix s3 upload * fix s3 upload * jenkins changes
1 parent aaf7795 commit a22da27

File tree

60 files changed

+229
-4852
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+229
-4852
lines changed

‎.gitmodules‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,6 @@
77
[submodule "py3nvml"]
88
path = py3nvml
99
url = https://github.com/h2oai/py3nvml
10-
[submodule "scikit-learn"]
11-
path = scikit-learn
12-
url = https://github.com/h2oai/scikit-learn.git
1310
[submodule "tests/googletest"]
1411
path = tests/googletest
1512
url = https://github.com/google/googletest.git

‎Dockerfile-build‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ ENV GIT_VERSION=2.17.0
2828
#
2929
RUN yum install -y epel-release
3030

31-
RUN yum update -y
31+
# RUN yum update -y
3232

3333
RUN yum install centos-release-scl-rh -y
3434

@@ -154,26 +154,26 @@ RUN \
154154
# Arrow
155155
# Need to enable c99 manually https://github.com/numpy/numpy/issues/14147
156156
RUN bash -c 'if [ `arch` = "ppc64le" ]; then \
157-
git clone https://github.com/apache/arrow.git && \
158-
cd $HOME/arrow/cpp && \
159-
git checkout tags/apache-arrow-0.17.1 && \
157+
git clone https://github.com/apache/arrow.git && \
158+
cd $HOME/arrow/cpp && \
159+
git checkout tags/apache-arrow-0.17.1 && \
160160
yum install -y boost-devel && \
161-
CFLAGS=-std=c99 pip install numpy==1.19.2 cython==0.29.14 scipy==1.5.2 && \
162-
cmake -DARROW_CXXFLAGS="-lutil" -DARROW_PYTHON=on && make -j && make install && \
163-
cd $HOME/arrow/python && \
164-
ARROW_HOME=/usr/local python setup.py install && \
165-
yum install -y libjpeg-devel;\
166-
fi'
161+
CFLAGS=-std=c99 pip install numpy==1.19.2 cython==0.29.14 scipy==1.5.2 && \
162+
cmake -DARROW_CXXFLAGS="-lutil" -DARROW_PYTHON=on && make -j && make install && \
163+
cd $HOME/arrow/python && \
164+
ARROW_HOME=/usr/local python setup.py install && \
165+
yum install -y libjpeg-devel;\
166+
fi'
167167

168168

169169
#
170170
# Install Python requirements
171171
#
172172
RUN echo ${python_version}
173173

174-
RUN pip install -U pip==20.1.1
174+
RUN pip install -U pip==23.3.2
175175

176-
RUN pip install numpy==1.19.2 scipy==1.5.2 setuptools==49.2.0
176+
RUN pip install numpy==1.23.5 scipy==1.10.1 setuptools==58.5.3
177177

178178
COPY src/interface_py/requirements_buildonly.txt requirements_buildonly.txt
179179

@@ -193,17 +193,17 @@ ENV LANGUAGE en_US:en
193193
# Could compile with these as well: -DBOOST_COMPUTE_USE_OFFLINE_CACHE=OFF -DBOOST_COMPUTE_THREAD_SAFE=ON
194194
RUN \
195195
export CUDA_HOME=/usr/local/cuda/ && \
196-
yum install -y opencl-headers icu libicu-devel bzip2 bzip2-devel zlib-devel python-devel && \
197-
wget https://s3.amazonaws.com/0xdata-public/boost/boost_1_72_0.tar.bz2 && \
198-
tar xjf boost_1_72_0.tar.bz2 && \
199-
cd boost_1_72_0 && \
196+
yum install -y opencl-headers icu libicu-devel bzip2 bzip2-devel zlib-devel python-devel && \
197+
wget https://github.com/boostorg/boost/releases/download/boost-1.82.0/boost-1.82.0.tar.gz && \
198+
tar -zxvf boost-1.82.0.tar.gz && \
199+
cd boost-1.82.0 && \
200200
export PYTHONPATH=/opt/h2oai/h2o4gpu/python/ && \
201201
./bootstrap.sh --prefix=/opt/boost/ --with-python=python3 && \
202-
export CPPFLAGS="-I/opt/h2oai/h2o4gpu/python/include/python${python_version}m/ -fPIC" && \
203-
export C_INCLUDE_PATH="/opt/h2oai/h2o4gpu/python/include/python${python_version}m/" ; export CPLUS_INCLUDE_PATH="/opt/h2oai/h2o4gpu/python/include/python${python_version}m/" && \
204-
./b2 link=static -a -d0 install --prefix=/opt/boost/ --with=all -j 20 cxxflags="-fPIC -I /opt/h2oai/h2o4gpu/python/include/python${python_version}m/" && \
205-
cd /usr/include ; rm -rf boost ; ln -s /opt/boost/include/boost . && \
206-
cd /usr/lib64/ ; rm -rf libboost* ; cp -a /opt/boost/lib/* . && \
202+
export CPPFLAGS="-I/opt/h2oai/h2o4gpu/python/include/python${python_version}/ -fPIC" && \
203+
export C_INCLUDE_PATH="/opt/h2oai/h2o4gpu/python/include/python${python_version}/" && export CPLUS_INCLUDE_PATH="/opt/h2oai/h2o4gpu/python/include/python${python_version}m/" && \
204+
./b2 link=static -a -d0 install --prefix=/opt/boost/ --with=all -j 20 cxxflags="-fPIC -I /opt/h2oai/h2o4gpu/python/include/python${python_version}/" && \
205+
cd /usr/include && rm -rf boost && ln -s /opt/boost/include/boost . && \
206+
cd /usr/lib64/ && rm -rf libboost* && cp -a /opt/boost/lib/* . && \
207207
cd /
208208

209209
# yum install -y boost boost-devel boost-system boost-filesystem boost-thread
@@ -214,7 +214,7 @@ RUN chmod -R o+rwx /opt/h2oai/h2o4gpu/python
214214
RUN chmod -R o+rwx /root
215215
RUN yum install -y hdf5-devel
216216
RUN bash -c 'if [ `arch` == "ppc64le" ]; then \
217-
yum install -y ocl-icd; \
217+
yum install -y ocl-icd; \
218218
fi'
219219

220220
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/nvidia

‎Dockerfile-runtime-multi-gpu‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ RUN if [ "$use_miniconda" = "1" ] ; then wget https://repo.anaconda.com/minicond
6262

6363
ENV PATH=/opt/h2oai/h2o4gpu/python/bin:$PATH
6464

65-
RUN pip install -U pip==20.1.1
65+
RUN pip install -U pip==23.3.2
6666

6767
#
6868
# PPC64 specific - certain libs/whl don't support PPC64LE

‎Dockerfile-runtime-single-gpu‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,15 @@ RUN yum install -y gcc gcc-c++ libgcc libstdc++ libgomp glibc
2222
RUN yum install -y \
2323
make \
2424
ncurses-devel \
25-
zlib-devel \
25+
zlib-devel \
2626
epel-release \
2727
zeromq-devel \
28-
wget \
29-
blas-devel \
28+
wget \
29+
blas-devel \
3030
openblas-devel \
3131
libpng-devel \
3232
freetype-devel \
33-
bzip2 \
33+
bzip2 \
3434
libffi-devel
3535

3636
# Arrow
@@ -51,7 +51,7 @@ RUN if [ "$use_miniconda" = "1" ] ; then wget https://repo.anaconda.com/minicond
5151

5252
ENV PATH=/opt/h2oai/h2o4gpu/python/bin:$PATH
5353

54-
RUN pip install -U pip==20.1.1
54+
RUN pip install -U pip==23.3.2
5555

5656
#
5757
# PPC64 specific - certain libs/whl don't support PPC64LE
@@ -62,27 +62,27 @@ WORKDIR $HOME
6262
# Arrow
6363
# Need to enable c99 manually https://github.com/numpy/numpy/issues/14147
6464
RUN bash -c 'if [ `arch` = "ppc64le" ]; then \
65-
ln -s /usr/bin/cmake3 /usr/bin/cmake && \
66-
git clone https://github.com/apache/arrow.git && \
67-
cd $HOME/arrow/cpp && \
65+
ln -s /usr/bin/cmake3 /usr/bin/cmake && \
66+
git clone https://github.com/apache/arrow.git && \
67+
cd $HOME/arrow/cpp && \
6868
git checkout tags/apache-arrow-0.17.1 && \
6969
CFLAGS=-std=c99 pip install numpy==1.19.2 cython==0.29.14 scipy==1.5.2 && \
7070
export ARROW_BUILD_TYPE=release && \
7171
export ARROW_HOME=/usr/local && \
72-
cmake -DARROW_CXXFLAGS="-lutil" -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DARROW_PYTHON=on -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE -DARROW_BUILD_TESTS=OFF && make -j && make install && \
73-
cd $HOME/arrow/python && \
74-
python setup.py install; \
75-
fi'
72+
cmake -DARROW_CXXFLAGS="-lutil" -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DARROW_PYTHON=on -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE -DARROW_BUILD_TESTS=OFF && make -j && make install && \
73+
cd $HOME/arrow/python && \
74+
python setup.py install; \
75+
fi'
7676

7777
WORKDIR /
7878

7979
# Add requirements
8080
COPY src/interface_py/requirements_runtime_demos_single_gpu.txt requirements_runtime_demos_single_gpu.txt
8181
COPY src/interface_py/requirements_test.txt requirements_test.txt
8282
RUN \
83-
chmod a+rwx requirements*.txt && \
84-
python -m pip install -r requirements_runtime_demos_single_gpu.txt && \
85-
python -m pip install -r requirements_test.txt
83+
chmod a+rwx requirements*.txt && \
84+
python -m pip install -r requirements_runtime_demos_single_gpu.txt && \
85+
python -m pip install -r requirements_test.txt
8686

8787
RUN mkdir -p /etc/OpenCL/vendors && \
8888
echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
@@ -96,7 +96,7 @@ ENV LD_LIBRARY_PATH_CUDA=$CUDA_HOME/lib64/:$CUDA_HOME/lib/:/usr/local/cuda/lib64
9696

9797
# Add a canned jupyter notebook demo to the container
9898
RUN \
99-
mkdir -p jupyter/demos
99+
mkdir -p jupyter/demos
100100
COPY examples/py/demos/H2O4GPU_Ridge.ipynb /jupyter/demos/H2O4GPU_Ridge.ipynb
101101
COPY examples/py/demos/H2O4GPU_LinearRegression.ipynb /jupyter/demos/H2O4GPU_LinearRegression.ipynb
102102
COPY examples/py/demos/H2O4GPU_GBM.ipynb /jupyter/demos/H2O4GPU_GBM.ipynb
@@ -110,29 +110,29 @@ COPY examples/py/demos/H2O4GPU_PCA.ipynb /jupyter/demos/H2O4GPU_PCA.ipynb
110110
COPY examples/py/demos/H2O4GPU_Daal_LinearRegression.ipynb /jupyter/demos/H2O4GPU_Daal_LinearRegression.ipynb
111111
COPY examples/py/demos/figures /jupyter/demos/figures
112112
RUN \
113-
cd /jupyter/demos && \
114-
chmod -R a+rwx /jupyter && \
115-
mkdir /scikit_learn_data && \
116-
chmod -R a+rwx /scikit_learn_data && \
117-
mkdir -p /jupyter/scikit_learn_data/covertype && \
118-
chmod -R a+rwx /jupyter/scikit_learn_data/covertype && \
119-
mkdir -p /jupyter/scikit_learn_data/lfw_home && \
120-
chmod -R a+rwx /jupyter/scikit_learn_data/lfw_home && \
121-
HOME=/jupyter jupyter notebook --generate-config && \
122-
sed -i "s/#c.NotebookApp.token = '<generated>'/c.NotebookApp.token = 'h2o'/" /jupyter/.jupyter/jupyter_notebook_config.py && \
123-
chmod -R a+rwx /jupyter/.jupyter
113+
cd /jupyter/demos && \
114+
chmod -R a+rwx /jupyter && \
115+
mkdir /scikit_learn_data && \
116+
chmod -R a+rwx /scikit_learn_data && \
117+
mkdir -p /jupyter/scikit_learn_data/covertype && \
118+
chmod -R a+rwx /jupyter/scikit_learn_data/covertype && \
119+
mkdir -p /jupyter/scikit_learn_data/lfw_home && \
120+
chmod -R a+rwx /jupyter/scikit_learn_data/lfw_home && \
121+
HOME=/jupyter jupyter notebook --generate-config && \
122+
sed -i "s/#c.NotebookApp.token = '<generated>'/c.NotebookApp.token = 'h2o'/" /jupyter/.jupyter/jupyter_notebook_config.py && \
123+
chmod -R a+rwx /jupyter/.jupyter
124124

125125
# Add shell wrapper
126126
COPY scripts/run.sh /run.sh
127127
RUN \
128-
chmod a+rwx run.sh
128+
chmod a+rwx run.sh
129129

130130
ARG h2o4gpu_VERSION
131131
ARG h2o4gpu_COMMIT
132132
ARG DOCKER_VERSION_TAG
133133
LABEL \
134-
h2o4gpu_commit="$h2o4gpu_COMMIT" \
135-
docker_version_tag="$DOCKER_VERSION_TAG"
134+
h2o4gpu_commit="$h2o4gpu_COMMIT" \
135+
docker_version_tag="$DOCKER_VERSION_TAG"
136136

137137
ENTRYPOINT ["./run.sh"]
138138
EXPOSE 8888

‎Makefile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ sync_open_data:
5454
# DEPENDENCY MANAGEMENT TARGETS
5555
#########################################
5656

57-
deps-install-with-all-pkgs: all_deps_install fullinstall-xgboost fullinstall-xgboost-prev fullinstall-lightgbm libsklearn
57+
deps-install-with-all-pkgs: all_deps_install fullinstall-xgboost fullinstall-xgboost-prev fullinstall-lightgbm
5858

59-
deps-install-with-build-pkgs: build_deps_install fullinstall-xgboost fullinstall-xgboost-prev fullinstall-lightgbm libsklearn
59+
deps-install-with-build-pkgs: build_deps_install fullinstall-xgboost fullinstall-xgboost-prev fullinstall-lightgbm
6060

6161
submodule_update:
6262
@echo "---- Fetch submudule dependencies ---- "
@@ -100,18 +100,18 @@ cpp:
100100
cp _ch2o4gpu_*pu.so ../src/interface_c/ && \
101101
cp ch2o4gpu_*pu.py ../src/interface_py/h2o4gpu/libs;
102102

103-
py: apply-sklearn_simple build/VERSION.txt
103+
py: build/VERSION.txt
104104
$(MAKE) -j all -C src/interface_py
105105

106106
.PHONY: xgboost_prev
107107
xgboost_prev:
108108
@echo "----- Building XGboost previous version target $(XGBOOST_TARGET) -----"
109-
cd xgboost_prev; sed -i -e 's/35;50;52;60;61;70/35;37;50;52;53;60;61;62;70/g' cmake/Utils.cmake; $(XGB_PROLOGUE) 'make -f Makefile2 PYTHON=$(PYTHON) CXX=$(XGB_CXX) CC=$(XGB_CC) $(XGBOOST_TARGET)'
109+
cd xgboost_prev; sed -i -e 's/35;50;52;60;61;70/35;37;50;52;53;60;61;62;70;75/g' cmake/Utils.cmake; $(XGB_PROLOGUE) 'make -f Makefile2 PYTHON=$(PYTHON) CXX=$(XGB_CXX) CC=$(XGB_CC) $(XGBOOST_TARGET)'
110110

111111
.PHONY: xgboost
112112
xgboost:
113113
@echo "----- Building XGboost target $(XGBOOST_TARGET) -----"
114-
cd xgboost; sed -i -e 's/35;50;52;60;61;70/35;37;50;52;53;60;61;62;70/g' cmake/Utils.cmake; $(XGB_PROLOGUE) 'make -f Makefile2 PYTHON=$(PYTHON) CXX=$(XGB_CXX) CC=$(XGB_CC) $(XGBOOST_TARGET)'
114+
cd xgboost; sed -i -e 's/35;50;52;60;61;70/35;37;50;52;53;60;61;62;70;75/g' cmake/Utils.cmake; $(XGB_PROLOGUE) 'make -f Makefile2 PYTHON=$(PYTHON) CXX=$(XGB_CXX) CC=$(XGB_CC) $(XGBOOST_TARGET)'
115115

116116
fullinstall-xgboost: nccl xgboost install_xgboost
117117

0 commit comments

Comments
 (0)