Merge pull request #1 from silinternational/develop

Develop

.dockerignore

@@ -0,0 +1,4 @@
+*.aes
+codeship-services.yml
+codeship-steps.yml
+dockercfg

.gitignore

@@ -0,0 +1,7 @@
+# local config files
+aws.env
+local.env
+
+*.aes
+
+dockercfg

Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.15
+
+RUN apk update \
+ && apk add --no-cache \
+            bash \
+            postgresql12-client
+
+COPY application/ /data/
+WORKDIR /data
+
+CMD ["./entrypoint.sh"]

Makefile

@@ -0,0 +1,14 @@
+start: restore backup
+
+restore: db
+	docker-compose up -d restore
+
+backup: db
+	docker-compose up -d backup
+
+db:
+	docker-compose up -d db
+
+clean:
+	docker-compose kill
+	docker system prune -f

README.md

@@ -0,0 +1,53 @@
+# postgresql-backup-restore-fs
+Service to backup and/or restore a PostgreSQL database to/from a local filesystem directory
+
+## How to use it
+1. Create a directory on the filesystem to hold your backups
+2. Supply all appropriate environment variables
+3. Run a backup and check your directory for that backup
+
+### Environment variables
+`MODE` Valid values: `backup`, `restore`
+
+`DB_HOST` hostname of the database server
+
+`DB_NAME` name of the database
+
+`DB_OPTIONS` optional arguments to supply to the backup or restore commands
+
+`DB_ROOTPASSWORD` password for the `DB_ROOTUSER`
+
+`DB_ROOTUSER` database administrative user, typically "postgres" for PostgreSQL databases
+
+`DB_USERPASSWORD` password for the `DB_USER`
+
+`DB_USER` user that accesses the database (PostgreSQL "role")
+
+`BACKUP_DIR` e.g., _/path/to/database-backups_ **NOTE: no trailing slash**
+
+>**Versioning of the backup file is left as an exercise for the user.  This script will overwrite an existing backup file.**
+
+## Docker Hub
+This image is built automatically on Docker Hub as [silintl/postgresql-backup-restore-fs](https://hub.docker.com/r/silintl/postgresql-backup-restore-fs/)
+
+## Playing with it locally
+You'll need [Docker](https://www.docker.com/get-docker), [Docker Compose](https://docs.docker.com/compose/install/), and [Make](https://www.gnu.org/software/make/).
+
+1. cd .../postgres-backup-restore-fs
+2. mkdir ./mybackups
+3. cp -p test/world.sql.gz ./mybackups
+4. `make db`  # creates the Postgres DB server
+5. `make restore`  # restores the DB dump file
+6. `docker ps -a`  # get the Container ID of the exited restore container
+7. `docker logs <containerID>`  # review the restoration log messages
+8. `make backup`  # create a new DB dump file
+9. `docker ps -a`  # get the Container ID of the exited backup container
+10. `docker logs <containerID>`  # review the backup log messages
+11. `make restore`  # restore the DB dump file from the new backup
+12. `docker ps -a`  # get the Container ID of the exited restore container
+13. `docker logs <containerID>`  # review the restoration log messages
+14. `make clean`  # remove containers and network
+15. `docker volume ls`  # find the volume ID of the Postgres data container
+16. `docker volume rm <volumeID>`  # remove the data volume
+17. `docker images`  # list existing images
+18. `docker image rm <imageID ...>`  # remove images no longer needed

application/backup.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env sh
+
+STATUS=0
+
+echo "postgresql-backup-restore-fs: backup: Started"
+
+echo "postgresql-backup-restore-fs: Backing up ${DB_NAME}"
+
+start=$(date +%s)
+$(PGPASSWORD=${DB_USERPASSWORD} pg_dump --host=${DB_HOST} --username=${DB_USER} --create --clean ${DB_OPTIONS} --dbname=${DB_NAME} > /tmp/${DB_NAME}.sql) || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "postgresql-backup-restore-fs: FATAL: Backup of ${DB_NAME} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "postgresql-backup-restore-fs: Backup of ${DB_NAME} completed in $(expr ${end} - ${start}) seconds, ($(stat -c %s /tmp/${DB_NAME}.sql) bytes)."
+fi
+
+start=$(date +%s)
+gzip -f /tmp/${DB_NAME}.sql || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "postgresql-backup-restore-fs: FATAL: Compressing backup of ${DB_NAME} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "postgresql-backup-restore-fs: Compressing backup of ${DB_NAME} completed in $(expr ${end} - ${start}) seconds."
+fi
+
+start=$(date +%s)
+mv /tmp/${DB_NAME}.sql.gz ${BACKUP_DIR} || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "postgresql-backup-restore-fs: FATAL: Copy backup to ${BACKUP_DIR} of ${DB_NAME} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "postgresql-backup-restore-fs: Copy backup to ${BACKUP_DIR} of ${DB_NAME} completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "postgresql-backup-restore-fs: backup: Completed"

application/entrypoint.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env sh
+
+# hostname:port:database:username:password
+echo ${DB_HOST}:*:*:${DB_USER}:${DB_USERPASSWORD}      > /root/.pgpass
+echo ${DB_HOST}:*:*:${DB_ROOTUSER}:${DB_ROOTPASSWORD} >> /root/.pgpass
+chmod 600 /root/.pgpass
+
+STATUS=0
+
+case "${MODE}" in
+    backup|restore)
+        /data/${MODE}.sh || STATUS=$?
+        ;;
+    *)
+        echo postgresql-backup-restore-fs: FATAL: Unknown MODE: ${MODE}
+        exit 1
+esac
+
+if [ $STATUS -ne 0 ]; then
+    echo postgresql-backup-restore-fs: Non-zero exit: $STATUS
+fi
+
+exit $STATUS

application/restore.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env sh
+
+STATUS=0
+
+echo "postgresql-backup-restore-fs: restore: Started"
+
+# Ensure the database user exists.
+echo "postgresql-backup-restore-fs: checking for DB user ${DB_USER}"
+result=$(psql --host=${DB_HOST} --username=${DB_ROOTUSER} --command='\du' | grep ${DB_USER})
+if [ -z "${result}" ]; then
+    result=$(psql --host=${DB_HOST} --username=${DB_ROOTUSER} --command="create role ${DB_USER} with login password '${DB_USERPASSWORD}' inherit;")
+    if [ "${result}" != "CREATE ROLE" ]; then
+        message="Create role command failed: ${result}"
+        echo "postgresql-backup-restore-fs: FATAL: ${message}"
+        exit 1
+    fi
+fi
+
+# Delete database if it exists.
+echo "postgresql-backup-restore-fs: checking for DB ${DB_NAME}"
+result=$(psql --host=${DB_HOST} --username=${DB_ROOTUSER} --list | grep ${DB_NAME})
+if [ -z "${result}" ]; then
+    message="Database "${DB_NAME}" on host "${DB_HOST}" does not exist."
+    echo "postgresql-backup-restore-fs: INFO: ${message}"
+else
+    echo "postgresql-backup-restore-fs: deleting database ${DB_NAME}"
+    result=$(psql --host=${DB_HOST} --dbname=postgres --username=${DB_ROOTUSER} --command="DROP DATABASE ${DB_NAME};")
+    if [ "${result}" != "DROP DATABASE" ]; then
+        message="Create database command failed: ${result}"
+        echo "postgresql-backup-restore-fs: FATAL: ${message}"
+        exit 1
+    fi
+fi
+
+echo "postgresql-backup-restore-fs: copying database ${DB_NAME} backup from ${BACKUP_DIR}"
+start=$(date +%s)
+cp ${BACKUP_DIR}/${DB_NAME}.sql.gz /tmp/${DB_NAME}.sql.gz || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "postgresql-backup-restore-fs: FATAL: Copy backup of ${DB_NAME} from ${BACKUP_DIR} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "postgresql-backup-restore-fs: Copy backup of ${DB_NAME} from ${BACKUP_DIR} completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "postgresql-backup-restore-fs: decompressing backup of ${DB_NAME}"
+start=$(date +%s)
+gunzip -f /tmp/${DB_NAME}.sql.gz || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "postgresql-backup-restore-fs: FATAL: Decompressing backup of ${DB_NAME} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "postgresql-backup-restore-fs: Decompressing backup of ${DB_NAME} completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "postgresql-backup-restore-fs: restoring ${DB_NAME}"
+start=$(date +%s)
+psql --host=${DB_HOST} --username=${DB_ROOTUSER} --dbname=postgres ${DB_OPTIONS}  < /tmp/${DB_NAME}.sql || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "postgresql-backup-restore-fs: FATAL: Restore of ${DB_NAME} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "postgresql-backup-restore-fs: Restore of ${DB_NAME} completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "postgresql-backup-restore-fs: restore: Completed"
+exit $STATUS

codeship-services.yml

@@ -0,0 +1,5 @@
+app:
+  build:
+    image: silintl/postgresql-backup-restore-fs
+    dockerfile: ./Dockerfile
+  cached: true

codeship-steps.yml

@@ -0,0 +1,22 @@
+- name: push_branch
+  service: app
+  type: push
+  image_name: silintl/postgresql-backup-restore-fs
+  image_tag: "{{.Branch}}"
+  exclude: (master|main|snyk*)
+  registry: https://index.docker.io/v1/
+  encrypted_dockercfg_path: dockercfg.encrypted
+
+- name: push_latest
+  service: app
+  type: push
+  image_name: silintl/postgresql-backup-restore-fs
+  image_tag: "latest"
+  tag: main
+  registry: https://index.docker.io/v1/
+  encrypted_dockercfg_path: dockercfg.encrypted
+
+#- name: test
+#  service: app
+#  command: echo "Image was tested"
+

docker-compose.yml

@@ -0,0 +1,54 @@
+version: '2'
+services:
+    data:
+        image: silintl/data-volume:latest
+        volumes:
+          - ./application:/data
+          - ./mybackups:/backup
+
+    # See https://hub.docker.com/_/postgres/ for details of the postgres image.
+    # POSTGRES_PASSWORD - superuser password for PostgreSQL
+    # POSTGRES_USER - superuser (default is 'postgres')
+    # POSTGRES_DB - name of default database (default is value of POSTGRES_USER)
+    db:
+        image: postgres:11.15-alpine3.15
+        volumes_from:
+            - data
+        ports:
+            - "5432"
+        environment:
+            POSTGRES_PASSWORD: r00tp@ss!
+
+    # DB_HOST - hostname of the database server
+    # DB_ROOTUSER - administrative user for the database server
+    # DB_ROOTPASSWORD - password for the DB_ROOTUSER
+    # DB_USER - user that accesses the database
+    # DB_USERPASSWORD - password for the DB_USER
+    # DB_NAME - name of database to back up/restore
+    restore:
+        build: ./
+        volumes_from:
+            - data
+        environment:
+            DB_HOST: db
+            DB_ROOTUSER: postgres
+            DB_ROOTPASSWORD: r00tp@ss!
+            DB_USER: dbuser
+            DB_USERPASSWORD: dbuserpass
+            DB_NAME: world
+            BACKUP_DIR: /backup
+            MODE: restore
+
+    backup:
+        build: ./
+        volumes_from:
+            - data
+        environment:
+            DB_HOST: db
+            DB_ROOTUSER: postgres
+            DB_ROOTPASSWORD: r00tp@ss!
+            DB_USER: dbuser
+            DB_USERPASSWORD: dbuserpass
+            DB_NAME: world
+            BACKUP_DIR: /backup
+            MODE: backup

dockercfg.encrypted

@@ -0,0 +1,2 @@
+codeship:v2
+UpJlDtV3e4lG4+iN9VCPPW77GF8m/mtjHCMaQXfaKtl76F1N3f1Kb3+OuVVsOczWIH2pCvW5PivbcFVSaGD7t7qAuvrqJikuKKEzT0y0s6nNJe7lC5VWVaXadx7kFKJOXO4+1QXBakzf28Jwxf+okpPuJhygd5Vm9e0WpNwgAq3ggouUlg9qh6wUNlZvnbMLT58rICiu1D12wVkZdXaMLiKX9IDG4VER9MB9Sc7EjMEcigm1CFT9BwHUIvi2cyHr29Qua04qo+J1jwnkMHdJxprFZkOmKKXEiyMqgX7Om3P8m589uiHRr15SI0Iz

test/README

@@ -0,0 +1,18 @@
+World
+=====
+
+World is a port of the example database available for MySQL on the mysql.com
+website.
+
+It is a simple 1:1 port and no attempt has been made to redesign the schema to
+better suit PostgreSQL.
+
+It is useful for comparison purposes and for learning how to execute simple
+queries and create simple tables, but is not particularly useful for exercising
+the advanced features of PostgreSQL or benchmarking.
+
+---
+(Taken from http://pgfoundry.org/projects/dbsamples/ on July 10, 2018.)
+
+The example database was loaded by hand and then backed up with the backup.sh
+script.  The resulting backup file was copied to this directory.