From e3134bf3d3e98a282dcbcce7d80b8dbe61d59a91 Mon Sep 17 00:00:00 2001 From: Stephen Spencer Date: Tue, 25 Oct 2022 13:25:07 -0500 Subject: [PATCH 1/3] ok. the source sig verification bits are not working.. Something for later. --- Dockerfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb53545..aeaaa10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,6 @@ LABEL stage RUNTIME ############################################################################### FROM runtime as build-setup -ADD gnupg/pubring.gpg gnupg/trustdb.gpg /root/.gnupg/ - RUN set -ex \ && mkdir -p /root/.gnupg \ && chmod 700 /root/.gnupg \ @@ -76,13 +74,8 @@ RUN set -ex \ ARG PYTHON_VERSION RUN \ - set -ex; \ - curl -sL -o /python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" \ - && curl -sL -o /python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ - && gpg --keyserver ha.pool.sks-keyservers.net --refresh-keys 2>&1 | egrep -v 'requesting key|not changed' || sleep 6000 \ - && gpg --batch --verify /python.tar.xz.asc /python.tar.xz \ - && mkdir -p /usr/src/python \ - && tar -xJC /usr/src/python --strip-components=1 -f /python.tar.xz + mkdir -p /usr/src/python \ + && tar -xJC /usr/src/python --strip-components=1 -f <( curl -sL "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" ) LABEL stage BUILD-SETUP From 34f93113d9b78b6c0f6ba566f362a60e4e3f8d19 Mon Sep 17 00:00:00 2001 From: Stephen Spencer Date: Thu, 27 Oct 2022 10:45:26 -0500 Subject: [PATCH 2/3] update build to use buildx cacheing directives and perform multi-arch builds --- Dockerfile | 68 +++++++++++++++++++++++++++++++++++++----------------- Makefile | 9 +++++--- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index aeaaa10..8ad77e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ ############################################################################### #FROM gcr.io/google-containers/debian-base-amd64:v2.0.0 as runtime -FROM debian:buster-slim as runtime +FROM debian:buster-slim as base ENV PATH /usr/local/bin:$PATH @@ -62,24 +62,42 @@ RUN set -ex \ LABEL stage RUNTIME +############################################################################### +FROM scratch as runtime + +COPY --from=base / / + +############################################################################### +FROM alpine as source-download + +ARG PYTHON_VERSION + +ENV SRCDIR /python +RUN apk add curl +RUN mkdir -p /python /build \ + && tar -xJC ${SRCDIR} --strip-components=1 -f <( curl -sL "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" ) + + ############################################################################### FROM runtime as build-setup -RUN set -ex \ - && mkdir -p /root/.gnupg \ - && chmod 700 /root/.gnupg \ - && buildDeps='libsqlite3-dev zlib1g-dev libexpat1-dev libssl-dev xz-utils dpkg-dev binutils libbz2-dev libreadline-dev libffi-dev libncurses5 libncurses5-dev libncursesw5 openssl curl gnupg' \ - && apt-get -qq update; apt-get -qq -y install ${buildDeps} +WORKDIR /python ARG PYTHON_VERSION -RUN \ - mkdir -p /usr/src/python \ - && tar -xJC /usr/src/python --strip-components=1 -f <( curl -sL "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" ) +RUN apt-get update +RUN apt-get -y install --no-install-recommends \ + libsqlite3-dev zlib1g-dev libexpat1-dev \ + libssl-dev xz-utils dpkg-dev binutils libbz2-dev \ + libreadline-dev libffi-dev libncurses5 \ + libncurses5-dev libncursesw5 openssl \ + gcc g++ make autoconf libtool \ + dpkg-dev + +# COPY --from=source-download /${PYTHON_VERSION} /python LABEL stage BUILD-SETUP -LABEL version ${PYTHON_VERSION} ############################################################################### FROM build-setup as builder @@ -90,21 +108,33 @@ ENV LANG C.UTF-8 ENV CFLAGS -I/usr/include/openssl -RUN set -ex \ - && cd /usr/src/python \ +WORKDIR /build + +RUN --mount=type=bind,from=source-download,target=/python,source=/python \ + --mount=type=cache,target=/tmp \ + --mount=type=cache,target=/var/tmp \ + --mount=type=cache,target=/var/log \ + --mount=type=cache,target=/root \ + set -ex \ && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ && [ $(( ` echo $PYTHON_VERSION | cut -d"." -f1 ` )) -lt 3 ] && BUILD_ARGS="" \ - ; ./configure \ + ; ../python/configure \ --build="$gnuArch" \ --enable-loadable-sqlite-extensions \ --enable-shared \ --with-system-expat \ --with-system-ffi \ - --without-ensurepip ${BUILD_ARGS} \ - && make -j $(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) \ + --without-ensurepip ${BUILD_ARGS} + +RUN --mount=type=bind,from=source-download,target=/python,source=/python \ + --mount=type=cache,target=/tmp \ + --mount=type=cache,target=/var/tmp \ + --mount=type=cache,target=/var/log \ + --mount=type=cache,target=/root \ + make -j $(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) \ && make install - RUN set -ex \ +RUN set -ex \ find /usr/local -type f -name "*.so" -exec strip --strip-unneeded {} + \ & ldconfig \ & find /usr/local -depth \ @@ -134,10 +164,9 @@ FROM builder as post-build ENV PYTHON_PIP_VERSION 19.1.1 -COPY ./ipython_config.py / +ADD https://bootstrap.pypa.io/get-pip.py . RUN set -ex; ldconfig -RUN set -ex; curl -sL -o get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; RUN set -ex; python get-pip.py \ --disable-pip-version-check \ --no-cache-dir; \ @@ -145,9 +174,6 @@ RUN set -ex; python get-pip.py \ # "pip==$PYTHON_PIP_VERSION"; -RUN mkdir -p $HOME/.ipython/profile_default ; -RUN mv ipython_config.py $HOME/.ipython/profile_default/. ; - RUN set -ex; \ find /usr/local -depth \ \( \ diff --git a/Makefile b/Makefile index 9b5e5f3..3e5ec7c 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,11 @@ PYTHON_VERSION ?= 3.6.2 TAG = ${PYTHON_VERSION}-wee IMAGE_TAG = ${IMAGE}:${TAG}${TAG_SUFFIX} +LATEST = ${IMAGE}:latest BUILD_ARGS = +PLATFORMS ?= linux/amd64 ifdef PKG_PROXY PROXY_ARGS := --build-arg=http_proxy=${PKG_PROXY} --build-arg=https_proxy=${PKG_PROXY} @@ -34,6 +36,7 @@ ifdef LTO TAG := ${TAG}-lto endif + ifndef DOCKERFILE DOCKERFILE := ./Dockerfile endif @@ -42,10 +45,10 @@ TAG_SUFFIX ?= build-image: @echo building ${IMAGE_TAG} - @docker build ${PROXY_ARGS} -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" -t ${IMAGE_TAG} . + docker buildx build ${PROXY_ARGS} --cache-from=type=registry,ref=${LATEST} --cache-to=type=registry,ref=${LATEST},mode=max -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --platform ${PLATFORMS} --load -t ${IMAGE_TAG} . push-image: @echo pushing ${IMAGE_TAG} - @docker push ${IMAGE_TAG} + docker buildx build ${PROXY_ARGS} --cache-from=type=registry,ref=${LATEST} --cache-to=type=registry,ref=${LATEST},mode=max -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --platform ${PLATFORMS} --push -t ${IMAGE_TAG} . -image: build-image push-image +image: push-image From 33c89260ad06e148594ec2e2f1716fa87fc72347 Mon Sep 17 00:00:00 2001 From: "Stephen D. Spencer" Date: Mon, 19 Dec 2022 12:20:27 -0600 Subject: [PATCH 3/3] ... --- Dockerfile | 39 ++++++++++++++++++--------------------- Makefile | 14 +++++++++----- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ad77e3..868a9a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# vim: ft=Dockerfile ts=4 sw=4 expandtab +# vim: ft=dockerfile ts=4 sw=4 expandtab ############################################################################### # # Multi-stage Python 3.x build @@ -47,7 +47,7 @@ RUN set -ex \ && apt update \ && apt -y upgrade \ && apt-mark unhold apt libcap2 libsemanage1 passwd \ - && apt-get install --no-install-recommends -qq -y libsqlite3-0 zlib1g libexpat1 bash procps less libbz2-1.0 netcat-openbsd git binutils \ + && apt-get install --no-install-recommends -qq -y ca-certificates libsqlite3-0 zlib1g libexpat1 bash procps less libbz2-1.0 netcat-openbsd git binutils \ && find /usr -type f -name "*.so" -exec strip --strip-unneeded {} + \ && apt-get remove -qq --allow-remove-essential --purge -y -qq \ binutils e2fsprogs e2fslibs libx11-6 libx11-data \ @@ -65,12 +65,21 @@ LABEL stage RUNTIME ############################################################################### FROM scratch as runtime +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +COPY ./init-functions /lib/lsb/ + COPY --from=base / / ############################################################################### FROM alpine as source-download ARG PYTHON_VERSION +ENV PYTHON_VERSION ${PYTHON_VERSION} ENV SRCDIR /python RUN apk add curl @@ -83,8 +92,6 @@ FROM runtime as build-setup WORKDIR /python -ARG PYTHON_VERSION - RUN apt-get update RUN apt-get -y install --no-install-recommends \ libsqlite3-dev zlib1g-dev libexpat1-dev \ @@ -94,9 +101,6 @@ RUN apt-get -y install --no-install-recommends \ gcc g++ make autoconf libtool \ dpkg-dev -# COPY --from=source-download /${PYTHON_VERSION} /python - - LABEL stage BUILD-SETUP ############################################################################### @@ -110,12 +114,9 @@ ENV CFLAGS -I/usr/include/openssl WORKDIR /build -RUN --mount=type=bind,from=source-download,target=/python,source=/python \ - --mount=type=cache,target=/tmp \ - --mount=type=cache,target=/var/tmp \ - --mount=type=cache,target=/var/log \ - --mount=type=cache,target=/root \ - set -ex \ +COPY --from=source-download /python /python + +RUN set -ex \ && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ && [ $(( ` echo $PYTHON_VERSION | cut -d"." -f1 ` )) -lt 3 ] && BUILD_ARGS="" \ ; ../python/configure \ @@ -126,12 +127,7 @@ RUN --mount=type=bind,from=source-download,target=/python,source=/python \ --with-system-ffi \ --without-ensurepip ${BUILD_ARGS} -RUN --mount=type=bind,from=source-download,target=/python,source=/python \ - --mount=type=cache,target=/tmp \ - --mount=type=cache,target=/var/tmp \ - --mount=type=cache,target=/var/log \ - --mount=type=cache,target=/root \ - make -j $(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) \ +RUN make -j $(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) \ && make install RUN set -ex \ @@ -161,7 +157,7 @@ LABEL version ${PYTHON_VERSION} FROM builder as post-build # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 19.1.1 +ENV PYTHON_PIP_VERSION 22.2.2 ADD https://bootstrap.pypa.io/get-pip.py . @@ -191,10 +187,11 @@ LABEL stage POST-BUILD LABEL version ${PYTHON_VERSION} ############################################################################### -FROM runtime as final +FROM runtime as release COPY --from=post-build /usr/local /usr/local COPY --from=post-build /root/* /root/ + RUN /sbin/ldconfig LABEL stage FINAL diff --git a/Makefile b/Makefile index 3e5ec7c..519d02a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: image push-image build-image optimal +.PHONY: image push-image build-image optimal buil-builder include .version @@ -9,10 +9,11 @@ TAG = ${PYTHON_VERSION}-wee IMAGE_TAG = ${IMAGE}:${TAG}${TAG_SUFFIX} LATEST = ${IMAGE}:latest -BUILD_ARGS = PLATFORMS ?= linux/amd64 +TARGETS := --platform ${PLATFORMS} + ifdef PKG_PROXY PROXY_ARGS := --build-arg=http_proxy=${PKG_PROXY} --build-arg=https_proxy=${PKG_PROXY} else @@ -43,12 +44,15 @@ endif TAG_SUFFIX ?= -build-image: +build-builder: + docker buildx build ${PROXY_ARGS} ${TARGETS} --cache-from=type=registry,ref=${LATEST} --cache-to=type=registry,ref=${LATEST},mode=max -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --target builder -t ${IMAGE_TAG} . + +build-image: build-builder @echo building ${IMAGE_TAG} - docker buildx build ${PROXY_ARGS} --cache-from=type=registry,ref=${LATEST} --cache-to=type=registry,ref=${LATEST},mode=max -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --platform ${PLATFORMS} --load -t ${IMAGE_TAG} . + docker buildx build ${PROXY_ARGS} ${TARGETS} --cache-from=type=registry,ref=${LATEST} -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --load -t ${IMAGE_TAG} . push-image: @echo pushing ${IMAGE_TAG} - docker buildx build ${PROXY_ARGS} --cache-from=type=registry,ref=${LATEST} --cache-to=type=registry,ref=${LATEST},mode=max -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --platform ${PLATFORMS} --push -t ${IMAGE_TAG} . + docker buildx build ${PROXY_ARGS} ${TARGETS} --cache-from=type=registry,ref=${LATEST} -f ${DOCKERFILE} --build-arg=PYTHON_VERSION=${PYTHON_VERSION} --build-arg=BUILD_ARGS="${BUILD_ARGS}" --push -t ${IMAGE_TAG} . image: push-image