diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env diff --git a/Dockerfile b/Dockerfile index 54859a4..f115693 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,8 @@ FROM alpine:latest -RUN apk update \ - && apk add coreutils \ - && apk add postgresql-client \ - && apk add python3 py3-pip && pip3 install --upgrade pip && pip3 install awscli \ - && apk add openssl \ - && rm -rf /var/cache/apk/* +RUN apk update +RUN apk add coreutils postgresql-client python3 py3-pip openssl --no-cache +RUN pip3 install awscli COPY entrypoint.sh /entrypoint.sh diff --git a/README.md b/README.md index 10b8cb4..483dfe1 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ metadata: name: postgresql-backup namespace: shared spec: + # at minute 0 past every 8th hour schedule: 0 */8 * * * jobTemplate: spec: @@ -77,40 +78,44 @@ spec: spec: containers: - name: postgresql-backup - image: nikitakoschelenko/postgres2s3:15.1 - env: - - name: POSTGRES_HOST - value: postgresql.shared - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgresql-backup-secret - key: POSTGRES_USER - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgresql-backup-secret - key: POSTGRES_PASSWORD - - name: S3_ENDPOINT - value: http://minio.shared:9000/ - - name: S3_ACCESS_KEY - valueFrom: - secretKeyRef: - name: postgresql-backup-secret - key: S3_ACCESS_KEY - - name: S3_SECRET_KEY - valueFrom: - secretKeyRef: - name: postgresql-backup-secret - key: S3_SECRET_KEY - - name: S3_BUCKET - value: backups - - name: S3_PREFIX - value: postresql/backup- - - name: ENCRYPTION_PASSWORD - valueFrom: - secretKeyRef: - name: postgresql-backup-secret - key: ENCRYPTION_PASSWORD + image: nikitakoschelenko/postgres2s3:15.1-rc.1 + # use envFrom to load Secrets and ConfigMaps into environment variables + envFrom: + - configMapRef: + name: postgresql-backup-configmap + - secretRef: + name: postgresql-backup-secret restartPolicy: OnFailure ``` + +Use config map for not-secret configuration data: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql-backup-configmap + namespace: shared +data: + POSTGRES_HOST: postgresql.shared + S3_ENDPOINT: http://minio.shared:9000/ + S3_BUCKET: backups + S3_FILE_PREFIX: postresql/backup- +``` + +Use secrets for things which are actually secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-backup-secret + namespace: shared +type: Opaque +data: + # base64 encode the values stored in a Kubernetes Secret: $ pbpaste | base64 | pbcopy + # the --decode flag is convenient: $ pbpaste | base64 --decode + POSTGRES_USER: cG9zdGdyZXM= + POSTGRES_PASSWORD: cG9zdGdyZXNwdw== + S3_ACCESS_KEY: YWNjZXNzS2V5 + S3_SECRET_KEY: c2VjcmV0S2V5 + ENCRYPTION_PASSWORD: c3VwZXJzZWNyZXRwYXNzd29yZA== +``` diff --git a/entrypoint.sh b/entrypoint.sh index 37dd2c3..43d7894 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,72 +1,63 @@ #! /bin/sh -if [[ -z "$POSTGRES_HOST" ]]; then - echo "POSTGRES_HOST environment variable is required" - exit 1 -fi - -POSTGRES_PORT="${POSTGRES_PORT:-5432}" - -if [[ -z "$POSTGRES_USER" ]]; then - echo "POSTGRES_USER environment variable is required" - exit 1 -fi - -if [[ -z "$POSTGRES_PASSWORD" ]]; then - echo "POSTGRES_PASSWORD environment variable is required" - exit 1 -fi - -if [[ -z "$S3_ENDPOINT" ]]; then - echo "S3_ENDPOINT environment variable is required" - exit 1 -fi - -if [[ -z "$S3_ACCESS_KEY" ]]; then - echo "S3_ACCESS_KEY environment variable is required" - exit 1 -fi - -if [[ -z "$S3_SECRET_KEY" ]]; then - echo "S3_SECRET_KEY environment variable is required" - exit 1 -fi - -if [[ -z "$S3_BUCKET" ]]; then - echo "S3_BUCKET environment variable is required" - exit 1 -fi - -S3_FILE_PREFIX="${S3_FILE_PREFIX:-backup-}" - +# with these settings, certain common errors will cause the script to immediately fail, explicitly and loudly +set -euo pipefail + +# required environment variables +: "${POSTGRES_HOST}" +: "${POSTGRES_USER}" +: "${POSTGRES_PASSWORD}" +: "${S3_ENDPOINT}" +: "${S3_ACCESS_KEY}" +: "${S3_SECRET_KEY}" +: "${S3_BUCKET}" + +# not required environment variables with default values +: "${POSTGRES_PORT:=5432}" +: "${S3_FILE_PREFIX:=backup-}" +: "${PG_DUMPALL_EXTRA_ARGS:=}" +: "${OPENSSL_ENC_EXTRA_ARGS:=}" +: "${AWS_S3_CP_EXTRA_ARGS:=}" + +# print pg_dumpall version pg_dumpall -V + echo "Creating a dump of all databases..." -SOURCE_FILE="output.bak.gz" -DESTINATION_FILE="${S3_FILE_PREFIX}$(date +"%Y-%m-%dT%H:%M:%SZ").bak.gz" +LOCAL_FILE="output.bak.gz" +REMOTE_FILE="${S3_FILE_PREFIX}$(date +"%Y-%m-%dT%H:%M:%SZ").bak.gz" +# PGPASSWORD is required for pg_dumpall export PGPASSWORD=$POSTGRES_PASSWORD -pg_dumpall -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $PG_DUMPALL_EXTRA_ARGS | gzip > $SOURCE_FILE + +# dump all databases and gzip to file +pg_dumpall -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $PG_DUMPALL_EXTRA_ARGS | gzip > $LOCAL_FILE echo "Dump created" +# if encryption password is set and not an empty string if [[ -z "$ENCRYPTION_PASSWORD" ]]; then echo "Encryption disabled" else echo "Encryption of the dump..." - openssl enc -aes-256-cbc -pbkdf2 -iter 20000 -in $SOURCE_FILE -out ${SOURCE_FILE}.enc -k $ENCRYPTION_PASSWORD $OPENSSL_ENC_EXTRA_ARGS + # encrypt local dump + openssl enc -aes-256-cbc -pbkdf2 -iter 20000 -in $LOCAL_FILE -out ${LOCAL_FILE}.enc -k $ENCRYPTION_PASSWORD $OPENSSL_ENC_EXTRA_ARGS - SOURCE_FILE="${SOURCE_FILE}.enc" - DESTINATION_FILE="${DESTINATION_FILE}.enc" + # update file extensions + LOCAL_FILE="${LOCAL_FILE}.enc" + REMOTE_FILE="${REMOTE_FILE}.enc" echo "Dump encrypted" fi echo "Uploading the dump to S3..." +# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY is required for aws export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY export AWS_SECRET_ACCESS_KEY=$S3_SECRET_KEY -aws s3 cp $SOURCE_FILE s3://$S3_BUCKET/$DESTINATION_FILE --endpoint-url $S3_ENDPOINT $AWS_S3_CP_EXTRA_ARGS + +# upload local dump to s3 +aws s3 cp $LOCAL_FILE s3://$S3_BUCKET/$REMOTE_FILE --endpoint-url $S3_ENDPOINT $AWS_S3_CP_EXTRA_ARGS echo "Dump uploaded" diff --git a/pg_extract.sh b/pg_extract.sh new file mode 100644 index 0000000..1197729 --- /dev/null +++ b/pg_extract.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# https://stackoverflow.com/a/48866503 + +[ $# -lt 2 ] && { echo "Usage: $0 "; exit 1; } + +sed "/connect.*$2/,\$!d" $1 | sed "/PostgreSQL database dump complete/,\$d"