Skip to content

Commit 9faf703

Browse files
HartornHartorn
committed
feat: adding generation script + first images
1 parent 489f6cb commit 9faf703

File tree

10 files changed

+927
-0
lines changed

10 files changed

+927
-0
lines changed

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.history

‎add-python-debian.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
. "$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)/utils.sh"
3+
4+
# DEBIAN_VERSION="buster"
5+
# DEBIAN_VARIANT="slim"
6+
# PYTHON_VERSION="3.7"
7+
8+
function install_python() {
9+
output_file=$1
10+
python_version=$2
11+
# Creating temp folder and entering it
12+
temp_folder
13+
source_url="https://raw.githubusercontent.com/docker-library/python/master/${python_version}/Dockerfile"
14+
wget --quiet ${source_url}
15+
# Skip 6 first lines (comment)
16+
tail -n +6 Dockerfile >Dockerfile_trunc
17+
# Remove CMD
18+
sed -ie 's/^CMD .*$//g' Dockerfile_trunc
19+
# Remove FROM
20+
sed -ie 's/^FROM .*$//g' Dockerfile_trunc
21+
22+
# Comment under are to keep this version
23+
PYTHON_PRECISE_VERSION=$(cat Dockerfile_trunc | grep 'ENV PYTHON_VERSION' | sed -e 's/ENV PYTHON_VERSION \(.*\)$/\1/g')
24+
PIP_PRECISE_VERSION=$(cat Dockerfile_trunc | grep 'ENV PYTHON_PIP_VERSION' | sed -e 's/ENV PYTHON_PIP_VERSION \(.*\)$/\1/g')
25+
26+
echo '' >>"${output_file}"
27+
echo '# Dockerfile generated fragment to install Python and Pip' >>"${output_file}"
28+
echo "# Source: ${source_url}" >>"${output_file}"
29+
echo "# Python: ${PYTHON_PRECISE_VERSION}" >>"${output_file}"
30+
echo "# Pip: ${PIP_PRECISE_VERSION}" >>"${output_file}"
31+
echo "" >>"${output_file}"
32+
33+
cat Dockerfile_trunc >>"${output_file}"
34+
35+
# Now, to avoid GPG problems
36+
# https://github.com/f-secure-foundry/usbarmory-debian-base_image/issues/9
37+
sed -i 's/^\(.*&&.*export GNUPGHOME="$(mktemp -d)" \)/\1\\\n# Fix to avoid GPG server problem\n\t\&\& echo "disable-ipv6" >> ${GNUPGHOME}\/dirmngr.conf /' "${output_file}"
38+
39+
# Exiting temp folder and removing it
40+
cleanup_folder
41+
}

‎apt-utils.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
. "$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)/utils.sh"
3+
4+
function apt_install_packages() {
5+
output_file=$1
6+
packages=${@:2}
7+
echo 'RUN apt update \' >>${output_file}
8+
echo "&& apt install ${packages} -y --no-install-recommends \\" >>${output_file}
9+
echo "&& apt autoclean \\" >>${output_file}
10+
echo "&& apt clean \\" >>${output_file}
11+
echo "&& rm -rf /var/lib/apt/lists/*" >>${output_file}
12+
}
13+
14+
function apt_install_temp_packages() {
15+
output_file=$1
16+
packages=${@:2}
17+
echo 'RUN savedAptMark="$(apt-mark showmanual)" \' >>${output_file}
18+
echo '&& apt update \' >>${output_file}
19+
echo "&& apt install ${packages} -y --no-install-recommends \\" >>${output_file}
20+
}
21+
22+
function apt_clean_temp_packages() {
23+
output_file=$1
24+
25+
cat >>"${output_file}" <<-'EOF'
26+
&& apt-mark auto '.*' > /dev/null \
27+
&& apt-mark manual $savedAptMark \
28+
&& apt purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
29+
&& apt autoclean \
30+
&& apt clean \
31+
&& rm -rf /var/lib/apt/lists/*
32+
EOF
33+
}

‎build-dockerfiles.sh‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/bash
2+
set -e
3+
. "$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)/add-python-debian.sh"
4+
. "$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)/apt-utils.sh"
5+
6+
# We build associative array, with key name and value base image
7+
8+
# We want same base version with, or without cuda
9+
# For now, it will be ubuntu 18.04
10+
declare -A config_from_type
11+
config_from_type["cpu"]='ubuntu:18.04@sha256:3235326357dfb65f1781dbc4df3b834546d8bf914e82cce58e6e6b676e23ce8f'
12+
config_from_type["gpu"]='nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04@sha256:557de4ba2cb674029ffb602bed8f748d44d59bb7db9daa746ea72a102406d3ec'
13+
14+
TENSORT_RT_6_PACKAGES="libnvinfer6=6.0.1-1+cuda10.1 libnvinfer-dev=6.0.1-1+cuda10.1 libnvinfer-plugin6=6.0.1-1+cuda10.1 libnvinfer-plugin-dev=6.0.1-1+cuda10.1"
15+
16+
# Not sure on this : should gcc and g++ be included ?
17+
# Useful for lot's of python install packages, let's go for yes
18+
INSTALL_PACKAGES="gcc g++ libgomp1 libopenblas-dev libomp-dev"
19+
20+
# This are the temp package to install, when building packages or deps
21+
BUILD_PACKAGES="gcc g++ make cmake git gfortran"
22+
23+
# MKL-DNN (or OneDNN now) version to use
24+
ONE_DNN_VERSION="v0.21.5"
25+
26+
# From https://github.com/docker-library/python/
27+
# Here, we give link to raw content on github, on master
28+
29+
for python_version in "3.7/buster/slim" "3.8/buster/slim"; do
30+
for type in "cpu" "gpu"; do
31+
echo "Building Python ${python_version} for ${type}"
32+
folder="$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)/dockerfiles/${python_version}/${type}"
33+
mkdir -p ${folder}
34+
output_file="${folder}/Dockerfile"
35+
# Build FROM directive
36+
echo "# DO NOT MODIFY MANUALLY" >"${output_file}"
37+
echo "# GENERATED FROM SCRIPTS" >>"${output_file}"
38+
echo "FROM ${config_from_type[$type]}" >>"${output_file}"
39+
echo '' >>"${output_file}"
40+
# Add DEBIAN_FRONTEND noninteractive to avoid tzdata prompt
41+
echo '# Avoid tzdata interactive action' >>"${output_file}"
42+
echo 'ENV DEBIAN_FRONTEND noninteractive' >>"${output_file}"
43+
echo '' >>"${output_file}"
44+
# Adding python to the image, based on remote dockerfile from https://github.com/docker-library/python
45+
echo "Getting Python build from source from official dockerfile in Github"
46+
echo "# Adding Python to image" >>"${output_file}"
47+
install_python ${output_file} ${python_version}
48+
echo '' >>"${output_file}"
49+
# Adding TensorRT if it's cuda image
50+
if [ "$type" == "gpu" ]; then
51+
echo "Adding TensorRT support"
52+
# Looks like ML CUDA is already installed
53+
# Keeping link as reminder
54+
# http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
55+
echo "# Installing TensorRT for cuda 10.1" >>"${output_file}"
56+
# TODO try to install TensorRT 7 ?
57+
apt_install_packages ${output_file} "${TENSORT_RT_6_PACKAGES}"
58+
echo "" >>"${output_file}"
59+
fi
60+
61+
echo "Adding OneDNN to the dockerfile"
62+
echo "# Adding MKL-DNN (now OneDNN) to the image" >>"${output_file}"
63+
apt_install_temp_packages ${output_file} "${BUILD_PACKAGES}"
64+
echo "&& git clone https://github.com/01org/mkl-dnn.git -b ${ONE_DNN_VERSION} --depth 1 \\" >>"${output_file}"
65+
echo "&& cd mkl-dnn/scripts && ./prepare_mkl.sh && cd .. \\" >>"${output_file}"
66+
echo "&& mkdir -p build && cd build && cmake .. && make \\" >>"${output_file}"
67+
echo "&& make install \\" >>"${output_file}"
68+
apt_clean_temp_packages ${output_file}
69+
echo "" >>"${output_file}"
70+
echo 'ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/local/lib' >>"${output_file}"
71+
echo "" >>"${output_file}" >>"${output_file}"
72+
done
73+
done

‎dockerfiles/3.7/buster/slim/cpu/Dockerfile‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
# DO NOT MODIFY MANUALLY
2+
# GENERATED FROM SCRIPTS
3+
FROM ubuntu:18.04@sha256:3235326357dfb65f1781dbc4df3b834546d8bf914e82cce58e6e6b676e23ce8f
4+
5+
# Avoid tzdata interactive action
6+
ENV DEBIAN_FRONTEND noninteractive
7+
8+
# Adding Python to image
9+
10+
# Dockerfile generated fragment to install Python and Pip
11+
# Source: https://raw.githubusercontent.com/docker-library/python/master/3.7/buster/slim/Dockerfile
12+
# Python: 3.7.7
13+
# Pip: 20.1
14+
15+
16+
17+
18+
# ensure local python is preferred over distribution python
19+
ENV PATH /usr/local/bin:$PATH
20+
21+
# http://bugs.python.org/issue19846
22+
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
23+
ENV LANG C.UTF-8
24+
25+
# runtime dependencies
26+
RUN apt-get update && apt-get install -y --no-install-recommends \
27+
ca-certificates \
28+
netbase \
29+
&& rm -rf /var/lib/apt/lists/*
30+
31+
ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
32+
ENV PYTHON_VERSION 3.7.7
33+
34+
RUN set -ex \
35+
\
36+
&& savedAptMark="$(apt-mark showmanual)" \
37+
&& apt-get update && apt-get install -y --no-install-recommends \
38+
dpkg-dev \
39+
gcc \
40+
libbluetooth-dev \
41+
libbz2-dev \
42+
libc6-dev \
43+
libexpat1-dev \
44+
libffi-dev \
45+
libgdbm-dev \
46+
liblzma-dev \
47+
libncursesw5-dev \
48+
libreadline-dev \
49+
libsqlite3-dev \
50+
libssl-dev \
51+
make \
52+
tk-dev \
53+
uuid-dev \
54+
wget \
55+
xz-utils \
56+
zlib1g-dev \
57+
# as of Stretch, "gpg" is no longer included by default
58+
$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \
59+
\
60+
&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
61+
&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
62+
&& export GNUPGHOME="$(mktemp -d)" \
63+
# Fix to avoid GPG server problem
64+
&& echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \
65+
&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
66+
&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
67+
&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
68+
&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
69+
&& mkdir -p /usr/src/python \
70+
&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
71+
&& rm python.tar.xz \
72+
\
73+
&& cd /usr/src/python \
74+
&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
75+
&& ./configure \
76+
--build="$gnuArch" \
77+
--enable-loadable-sqlite-extensions \
78+
--enable-optimizations \
79+
--enable-option-checking=fatal \
80+
--enable-shared \
81+
--with-system-expat \
82+
--with-system-ffi \
83+
--without-ensurepip \
84+
&& make -j "$(nproc)" \
85+
# setting PROFILE_TASK makes "--enable-optimizations" reasonable: https://bugs.python.org/issue36044 / https://github.com/docker-library/python/issues/160#issuecomment-509426916
86+
PROFILE_TASK='-m test.regrtest --pgo \
87+
test_array \
88+
test_base64 \
89+
test_binascii \
90+
test_binhex \
91+
test_binop \
92+
test_bytes \
93+
test_c_locale_coercion \
94+
test_class \
95+
test_cmath \
96+
test_codecs \
97+
test_compile \
98+
test_complex \
99+
test_csv \
100+
test_decimal \
101+
test_dict \
102+
test_float \
103+
test_fstring \
104+
test_hashlib \
105+
test_io \
106+
test_iter \
107+
test_json \
108+
test_long \
109+
test_math \
110+
test_memoryview \
111+
test_pickle \
112+
test_re \
113+
test_set \
114+
test_slice \
115+
test_struct \
116+
test_threading \
117+
test_time \
118+
test_traceback \
119+
test_unicode \
120+
' \
121+
&& make install \
122+
&& ldconfig \
123+
\
124+
&& apt-mark auto '.*' > /dev/null \
125+
&& apt-mark manual $savedAptMark \
126+
&& find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \
127+
| awk '/=>/ { print $(NF-1) }' \
128+
| sort -u \
129+
| xargs -r dpkg-query --search \
130+
| cut -d: -f1 \
131+
| sort -u \
132+
| xargs -r apt-mark manual \
133+
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
134+
&& rm -rf /var/lib/apt/lists/* \
135+
\
136+
&& find /usr/local -depth \
137+
\( \
138+
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
139+
-o \
140+
\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
141+
\) -exec rm -rf '{}' + \
142+
&& rm -rf /usr/src/python \
143+
\
144+
&& python3 --version
145+
146+
# make some useful symlinks that are expected to exist
147+
RUN cd /usr/local/bin \
148+
&& ln -s idle3 idle \
149+
&& ln -s pydoc3 pydoc \
150+
&& ln -s python3 python \
151+
&& ln -s python3-config python-config
152+
153+
# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
154+
ENV PYTHON_PIP_VERSION 20.1
155+
# https://github.com/pypa/get-pip
156+
ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/1fe530e9e3d800be94e04f6428460fc4fb94f5a9/get-pip.py
157+
ENV PYTHON_GET_PIP_SHA256 ce486cddac44e99496a702aa5c06c5028414ef48fdfd5242cd2fe559b13d4348
158+
159+
RUN set -ex; \
160+
\
161+
savedAptMark="$(apt-mark showmanual)"; \
162+
apt-get update; \
163+
apt-get install -y --no-install-recommends wget; \
164+
\
165+
wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
166+
echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum --check --strict -; \
167+
\
168+
apt-mark auto '.*' > /dev/null; \
169+
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; \
170+
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
171+
rm -rf /var/lib/apt/lists/*; \
172+
\
173+
python get-pip.py \
174+
--disable-pip-version-check \
175+
--no-cache-dir \
176+
"pip==$PYTHON_PIP_VERSION" \
177+
; \
178+
pip --version; \
179+
\
180+
find /usr/local -depth \
181+
\( \
182+
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
183+
-o \
184+
\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
185+
\) -exec rm -rf '{}' +; \
186+
rm -f get-pip.py
187+
188+
189+
190+
# Adding MKL-DNN (now OneDNN) to the image
191+
RUN savedAptMark="$(apt-mark showmanual)" \
192+
&& apt update \
193+
&& apt install gcc g++ make cmake git gfortran -y --no-install-recommends \
194+
&& git clone https://github.com/01org/mkl-dnn.git -b v0.21.5 --depth 1 \
195+
&& cd mkl-dnn/scripts && ./prepare_mkl.sh && cd .. \
196+
&& mkdir -p build && cd build && cmake .. && make \
197+
&& make install \
198+
&& apt-mark auto '.*' > /dev/null \
199+
&& apt-mark manual $savedAptMark \
200+
&& apt purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
201+
&& apt autoclean \
202+
&& apt clean \
203+
&& rm -rf /var/lib/apt/lists/*
204+
205+
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/local/lib
206+

0 commit comments

Comments
 (0)