From cd3f3b1c483e17a0c174d2bfa80d61a5b0f72d2e Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 30 Aug 2017 21:19:18 -0700 Subject: [PATCH] Use "apt-get source --compile" to build for Debian architectures upstream doesn't --- .travis.yml | 21 +++++++++- 10/Dockerfile | 77 +++++++++++++++++++++++++++++----- 9.2/Dockerfile | 79 +++++++++++++++++++++++++++++------ 9.3/Dockerfile | 79 +++++++++++++++++++++++++++++------ 9.4/Dockerfile | 79 +++++++++++++++++++++++++++++------ 9.5/Dockerfile | 79 +++++++++++++++++++++++++++++------ 9.6/Dockerfile | 79 +++++++++++++++++++++++++++++------ Dockerfile-debian.template | 79 +++++++++++++++++++++++++++++------ generate-stackbrew-library.sh | 10 +---- update.sh | 9 +++- 10 files changed, 496 insertions(+), 95 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0138f2269..b6527eb6c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,16 +3,22 @@ services: docker env: - VERSION=9.6 + - VERSION=9.6 FORCE_DEB_BUILD=1 - VERSION=9.6 VARIANT=alpine - VERSION=9.5 + - VERSION=9.5 FORCE_DEB_BUILD=1 - VERSION=9.5 VARIANT=alpine - VERSION=9.4 + - VERSION=9.4 FORCE_DEB_BUILD=1 - VERSION=9.4 VARIANT=alpine - VERSION=9.3 + - VERSION=9.3 FORCE_DEB_BUILD=1 - VERSION=9.3 VARIANT=alpine - VERSION=9.2 + - VERSION=9.2 FORCE_DEB_BUILD=1 - VERSION=9.2 VARIANT=alpine - VERSION=10 + - VERSION=10 FORCE_DEB_BUILD=1 - VERSION=10 VARIANT=alpine install: @@ -24,8 +30,19 @@ before_script: - image="postgres:${VERSION}${VARIANT:+-${VARIANT}}" script: - - travis_retry docker build -t "$image" . - - ~/official-images/test/run.sh "$image" + - | + ( + set -Eeuo pipefail + set -x + if [ -n "${FORCE_DEB_BUILD:+x}" ]; then + [ "$(dpkg --print-architecture)" = 'amd64' ] + grep -qE 'amd64[|]' Dockerfile + sed -ri -e 's/amd64[|]//g' Dockerfile + ! grep -qE 'amd64[|]' Dockerfile + fi + travis_retry docker build -t "$image" . + ~/official-images/test/run.sh "$image" + ) after_script: - docker images diff --git a/10/Dockerfile b/10/Dockerfile index c43731bcd4..600cacc225 100644 --- a/10/Dockerfile +++ b/10/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,18 +49,73 @@ RUN set -ex; \ ENV PG_MAJOR 10 ENV PG_VERSION 10~beta3-1.pgdg90+1 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64|i386|ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/9.2/Dockerfile b/9.2/Dockerfile index 6293703b57..1f8aceb157 100644 --- a/9.2/Dockerfile +++ b/9.2/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,19 +49,74 @@ RUN set -ex; \ ENV PG_MAJOR 9.2 ENV PG_VERSION 9.2.22-1.pgdg80+1 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - postgresql-contrib-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64|i386|ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/9.3/Dockerfile b/9.3/Dockerfile index 8a5fdc258e..1dc28b79b4 100644 --- a/9.3/Dockerfile +++ b/9.3/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,19 +49,74 @@ RUN set -ex; \ ENV PG_MAJOR 9.3 ENV PG_VERSION 9.3.18-1.pgdg80+1 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - postgresql-contrib-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64|i386|ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/9.4/Dockerfile b/9.4/Dockerfile index ee407bd7f7..487eecebe5 100644 --- a/9.4/Dockerfile +++ b/9.4/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,19 +49,74 @@ RUN set -ex; \ ENV PG_MAJOR 9.4 ENV PG_VERSION 9.4.13-1.pgdg80+1 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - postgresql-contrib-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64|i386|ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/9.5/Dockerfile b/9.5/Dockerfile index 6ee68e41c9..4c2f32e2c7 100644 --- a/9.5/Dockerfile +++ b/9.5/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,19 +49,74 @@ RUN set -ex; \ ENV PG_MAJOR 9.5 ENV PG_VERSION 9.5.8-1.pgdg80+1 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - postgresql-contrib-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64|i386|ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/9.6/Dockerfile b/9.6/Dockerfile index 60fa83cc74..cf2ddf725a 100644 --- a/9.6/Dockerfile +++ b/9.6/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,19 +49,74 @@ RUN set -ex; \ ENV PG_MAJOR 9.6 ENV PG_VERSION 9.6.4-1.pgdg80+1 -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - postgresql-contrib-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + amd64|i386|ppc64el) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template index 50f45ff119..1fcfe98cfc 100644 --- a/Dockerfile-debian.template +++ b/Dockerfile-debian.template @@ -15,7 +15,7 @@ RUN set -ex; \ RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres # grab gosu for easy step-down from root -ENV GOSU_VERSION 1.7 +ENV GOSU_VERSION 1.10 RUN set -x \ && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ @@ -49,19 +49,74 @@ RUN set -ex; \ ENV PG_MAJOR %%PG_MAJOR%% ENV PG_VERSION %%PG_VERSION%% -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ %%DEBIAN_SUITE%%-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list - -RUN apt-get update \ - && apt-get install -y postgresql-common \ - && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ - && apt-get install -y \ - postgresql-$PG_MAJOR=$PG_VERSION \ - postgresql-contrib-$PG_MAJOR=$PG_VERSION \ - && rm -rf /var/lib/apt/lists/* +RUN set -ex; \ + \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "$dpkgArch" in \ + %%ARCH_LIST%%) \ +# arches officialy built by upstream + echo "deb http://apt.postgresql.org/pub/repos/apt/ %%DEBIAN_SUITE%%-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + ;; \ + *) \ +# we're on an architecture upstream doesn't officially build for +# let's build binaries from their published source packages + echo "deb-src http://apt.postgresql.org/pub/repos/apt/ %%DEBIAN_SUITE%%-pgdg main $PG_MAJOR" > /etc/apt/sources.list.d/pgdg.list; \ + \ + tempDir="$(mktemp -d)"; \ + cd "$tempDir"; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + \ +# build .deb files from upstream's source packages (which are verified by apt-get) + apt-get update; \ + apt-get build-dep -y \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ + DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile \ + postgresql-common pgdg-keyring \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + ; \ +# we don't remove APT lists here because they get re-downloaded and removed later + \ +# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies +# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) + apt-mark showmanual | xargs apt-mark auto > /dev/null; \ + apt-mark manual $savedAptMark; \ + \ +# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) + ls -lAFh; \ + dpkg-scanpackages . > Packages; \ + grep '^Package: ' Packages; \ + echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list; \ +# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") +# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) +# ... +# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) + apt-get -o Acquire::GzipIndexes=false update; \ + ;; \ + esac; \ + \ + apt-get install -y postgresql-common; \ + sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ + apt-get install -y \ + "postgresql-$PG_MAJOR=$PG_VERSION" \ + "postgresql-contrib-$PG_MAJOR=$PG_VERSION" \ + ; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ + if [ -n "$tempDir" ]; then \ +# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) + apt-get purge -y --auto-remove; \ + rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi # make the sample config easier to munge (and "correct by default") -RUN mv -v /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample /usr/share/postgresql/ \ - && ln -sv ../postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ \ +RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \ + && ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \ && sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 28c4e4fd9b..1d1aec4e65 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -85,18 +85,12 @@ for version in "${versions[@]}"; do ) versionParent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/Dockerfile")" - versionArches=() - # http://apt.postgresql.org/pub/repos/apt/dists/jessie-pgdg/main/ - for arch in amd64 i386 ppc64le; do - if [[ " ${parentRepoToArches[$versionParent]} " =~ " $arch " ]]; then - versionArches+=( "$arch" ) - fi - done + versionArches="${parentRepoToArches[$versionParent]}" echo cat <<-EOE Tags: $(join ', ' "${versionAliases[@]}") - Architectures: $(join ', ' "${versionArches[@]}") + Architectures: $(join ', ' $versionArches) GitCommit: $commit Directory: $version EOE diff --git a/update.sh b/update.sh index 3e7c047443..e89ac58fcf 100755 --- a/update.sh +++ b/update.sh @@ -32,13 +32,16 @@ packagesBase='http://apt.postgresql.org/pub/repos/apt/dists/' osspUuidVersion='1.6.2' osspUuidHash='11a615225baa5f8bb686824423f50e4427acd3f70d394765bdff32801f0fd5b0' -declare -A suitePackageList +declare -A suitePackageList=() suiteArches=() travisEnv= for version in "${versions[@]}"; do suite="${debianSuite[$version]}" if [ -z "${suitePackageList["$suite"]:+isset}" ]; then suitePackageList["$suite"]="$(curl -fsSL "${packagesBase}/${suite}-pgdg/main/binary-amd64/Packages.bz2" | bunzip2)" fi + if [ -z "${suiteArches["$suite"]:+isset}" ]; then + suiteArches["$suite"]="$(curl -fsSL "${packagesBase}/${suite}-pgdg/Release" | gawk -F ':[[:space:]]+' '$1 == "Architectures" { gsub(/[[:space:]]+/, "|", $2); print $2 }')" + fi versionList="$(echo "${suitePackageList["$suite"]}"; curl -fsSL "${packagesBase}/${suite}-pgdg/${version}/binary-amd64/Packages.bz2" | bunzip2)" fullVersion="$(echo "$versionList" | awk -F ': ' '$1 == "Package" { pkg = $2 } $1 == "Version" && pkg == "postgresql-'"$version"'" { print $2; exit }' || true)" @@ -48,6 +51,7 @@ for version in "${versions[@]}"; do sed -e 's/%%PG_MAJOR%%/'"$version"'/g;' \ -e 's/%%PG_VERSION%%/'"$fullVersion"'/g' \ -e 's/%%DEBIAN_SUITE%%/'"$suite"'/g' \ + -e 's/%%ARCH_LIST%%/'"${suiteArches["$suite"]}"'/g' \ Dockerfile-debian.template > "$version/Dockerfile" if [ "$version" = '10' ]; then # postgresql-contrib-10 package does not exist, but is provided by postgresql-10 @@ -109,7 +113,8 @@ for version in "${versions[@]}"; do travisEnv="\n - VERSION=$version VARIANT=$variant$travisEnv" done - travisEnv='\n - VERSION='"$version$travisEnv" + travisEnv="\n - VERSION=$version FORCE_DEB_BUILD=1$travisEnv" + travisEnv="\n - VERSION=$version$travisEnv" done travis="$(awk -v 'RS=\n\n' '$1 == "env:" { $0 = "env:'"$travisEnv"'" } { printf "%s%s", $0, RS }' .travis.yml)"