diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b940f6f..79e17ada 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,9 @@ jobs: steps: - checkout + - run: sudo apt install shellcheck + - run: cat <(git grep -El '^#!.*sh\b') <(git ls-files | grep -E '.sh$') | sort -u | grep -v '/wait-for-it.sh$' | xargs shellcheck --exclude=SC2016 + - run: git submodule update -i - run: | @@ -14,7 +17,9 @@ jobs: DOMAIN=local SYSADMIN_EMAIL=no-reply@getodk.org' > .env - - run: docker-compose build + - run: touch ./files/allow-postgres14-upgrade + + - run: docker compose build - run: # we allow a long retry period for the first check because the first-run @@ -22,8 +27,8 @@ jobs: name: Verify frontend and backend load command: | set -x - docker-compose up -d - CONTAINER_NAME=$(docker inspect -f '{{.Name}}' $(docker-compose ps -q nginx) | cut -c2-) + docker compose up -d + CONTAINER_NAME=$(docker inspect -f '{{.Name}}' $(docker compose ps -q nginx) | cut -c2-) docker run --network container:$CONTAINER_NAME \ appropriate/curl -4 --insecure --retry 30 --retry-delay 10 --retry-connrefused https://localhost/ \ | tee /dev/tty \ @@ -32,3 +37,4 @@ jobs: appropriate/curl -4 --insecure --retry 20 --retry-delay 2 --retry-connrefused https://localhost/v1/projects \ | tee /dev/tty \ | grep -q '\[\]' + docker compose exec -T service pm2 list | grep -c "online" | grep -q 4 || exit 1 diff --git a/.env.template b/.env.template index 78ad3963..049afcd9 100644 --- a/.env.template +++ b/.env.template @@ -10,3 +10,26 @@ SSL_TYPE=letsencrypt # Do not change if using SSL_TYPE=letsencrypt HTTP_PORT=80 HTTPS_PORT=443 + +# Optional: configure Node +# SERVICE_NODE_OPTIONS= + +# Optional: connect to a custom database server +# DB_HOST= +# DB_USER= +# DB_PASSWORD= +# DB_NAME= + +# Optional: configure a custom mail server +# EMAIL_FROM= +# EMAIL_HOST= +# EMAIL_PORT= +# EMAIL_SECURE= +# EMAIL_IGNORE_TLS= +# EMAIL_USER= +# EMAIL_PASSWORD= + +# Optional: configure error reporting +# SENTRY_ORG_SUBDOMAIN= +# SENTRY_KEY= +# SENTRY_PROJECT= diff --git a/.gitignore b/.gitignore index 1dfbc7fa..899810a0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,9 @@ *.swo /.env /version.txt + +/files/allow-postgres14-upgrade +/files/postgres14/upgrade/* +!/files/postgres14/upgrade/check-available-space + +/files/local/customssl/*.pem \ No newline at end of file diff --git a/client b/client index 0b55d7d9..d4c96482 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 0b55d7d9f98f145c919c6e3476813275c4b69af7 +Subproject commit d4c964826031477962b05b6d08dfa8fecec98db2 diff --git a/docker-compose.yml b/docker-compose.yml index 233bfc56..13b13099 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,33 @@ version: "3" services: - postgres: - image: "postgres:9.6" + postgres14: + build: + context: . + dockerfile: postgres14.dockerfile volumes: - - /var/lib/postgresql/data + - postgres14:/var/lib/odk/postgresql/14 environment: POSTGRES_USER: odk POSTGRES_PASSWORD: odk POSTGRES_DATABASE: odk restart: always + postgres: + # This service upgrades from postgres 9.6 to 14. + # The legacy name must be maintained to allow access to the anonymous volume. + build: + context: . + dockerfile: postgres-upgrade.dockerfile + volumes: + - /var/lib/postgresql/data + - postgres14:/var/lib/postgresql/14 + - ./files/postgres14/upgrade:/postgres14-upgrade + environment: + PGUSER: odk + POSTGRES_INITDB_ARGS: -U odk + POSTGRES_PASSWORD: odk + POSTGRES_DATABASE: odk mail: - image: "itsissa/namshi-smtp:4.92-8.deb10u6" + image: "ixdotai/smtp:v0.2.0" volumes: - ./files/dkim/config:/etc/exim4/_docker_additional_macros:ro - ./files/dkim/rsa.private:/etc/exim4/domain.key:ro @@ -23,7 +40,7 @@ services: dockerfile: service.dockerfile depends_on: - secrets - - postgres + - postgres14 - mail - pyxform - enketo @@ -32,9 +49,25 @@ services: - /data/transfer:/data/transfer environment: - DOMAIN=${DOMAIN} - - HTTPS_PORT=${HTTPS_PORT:-443} - SYSADMIN_EMAIL=${SYSADMIN_EMAIL} - command: [ "./wait-for-it.sh", "postgres:5432", "--", "./start-odk.sh" ] + - HTTPS_PORT=${HTTPS_PORT:-443} + - NODE_OPTIONS=${SERVICE_NODE_OPTIONS:-''} + - DB_HOST=${DB_HOST:-postgres14} + - DB_USER=${DB_USER:-odk} + - DB_PASSWORD=${DB_PASSWORD:-odk} + - DB_NAME=${DB_NAME:-odk} + - DB_SSL=${DB_SSL:-null} + - EMAIL_FROM=${EMAIL_FROM:-no-reply@${DOMAIN}} + - EMAIL_HOST=${EMAIL_HOST:-mail} + - EMAIL_PORT=${EMAIL_PORT:-25} + - EMAIL_SECURE=${EMAIL_SECURE:-false} + - EMAIL_IGNORE_TLS=${EMAIL_IGNORE_TLS:-true} + - EMAIL_USER=${EMAIL_USER:-''} + - EMAIL_PASSWORD=${EMAIL_PASSWORD:-''} + - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} + - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} + - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} + command: [ "./wait-for-it.sh", "${DB_HOST:-postgres14}:5432", "--", "./start-odk.sh" ] restart: always logging: driver: local @@ -46,9 +79,12 @@ services: - service - enketo environment: - - SSL_TYPE=${SSL_TYPE:-letsencrypt} - DOMAIN=${DOMAIN} - CERTBOT_EMAIL=${SYSADMIN_EMAIL} + - SSL_TYPE=${SSL_TYPE:-letsencrypt} + - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} + - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} + - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} ports: - "${HTTP_PORT:-80}:80" - "${HTTPS_PORT:-443}:443" @@ -85,7 +121,7 @@ services: - SUPPORT_EMAIL=${SYSADMIN_EMAIL} - HTTPS_PORT=${HTTPS_PORT:-443} enketo_redis_main: - image: redis:5 + image: redis:7.0.8 volumes: - ./files/enketo/redis-enketo-main.conf:/usr/local/etc/redis/redis.conf:ro - enketo_redis_main:/data @@ -94,7 +130,7 @@ services: - /usr/local/etc/redis/redis.conf restart: always enketo_redis_cache: - image: redis:5 + image: redis:7.0.8 volumes: - ./files/enketo/redis-enketo-cache.conf:/usr/local/etc/redis/redis.conf:ro - enketo_redis_cache:/data @@ -103,7 +139,8 @@ services: - /usr/local/etc/redis/redis.conf restart: always volumes: + secrets: transfer: + postgres14: enketo_redis_main: enketo_redis_cache: - secrets: diff --git a/enketo.dockerfile b/enketo.dockerfile index 7ce3d1f6..95a4e4b8 100644 --- a/enketo.dockerfile +++ b/enketo.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/enketo/enketo-express:5.0.2 +FROM ghcr.io/enketo/enketo-express:6.0.0 ENV ENKETO_SRC_DIR=/srv/src/enketo_express WORKDIR ${ENKETO_SRC_DIR} @@ -14,7 +14,8 @@ COPY files/enketo/config.json.template ${ENKETO_SRC_DIR}/config/config.json.temp COPY files/enketo/config.json.template ${ENKETO_SRC_DIR}/config/config.json COPY files/enketo/start-enketo.sh ${ENKETO_SRC_DIR}/start-enketo.sh -RUN apt-get update; apt-get install gettext-base +RUN apt-get update && \ + apt-get install gettext-base EXPOSE 8005 diff --git a/files/docker-compose@.service b/files/docker-compose@.service deleted file mode 100644 index 3c187b17..00000000 --- a/files/docker-compose@.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=%i via docker-compose -Requires=docker.service -After=docker.service - -[Service] -Restart=always -WorkingDirectory=/root/%i - -ExecStart=/usr/local/bin/docker-compose up -ExecStop=/usr/local/bin/docker-compose stop -v - -[Install] -WantedBy=multi-user.target - diff --git a/files/enketo/config.json.template b/files/enketo/config.json.template index 9a27b78c..d0f87fcd 100644 --- a/files/enketo/config.json.template +++ b/files/enketo/config.json.template @@ -8,7 +8,7 @@ "api key": "${API_KEY}", "authentication": { "type": "cookie", - "url": "https://${DOMAIN}:${HTTPS_PORT}/#/login?next={RETURNURL}" + "url": "${BASE_URL}/#/login?next={RETURNURL}" }, "name": "ODK Central", "server url": "${DOMAIN}" diff --git a/files/enketo/start-enketo.sh b/files/enketo/start-enketo.sh index d939e79a..b097ce40 100755 --- a/files/enketo/start-enketo.sh +++ b/files/enketo/start-enketo.sh @@ -2,8 +2,14 @@ CONFIG_PATH=${ENKETO_SRC_DIR}/config/config.json echo "generating enketo configuration.." -/bin/bash -c "SECRET=$(cat /etc/secrets/enketo-secret) LESS_SECRET=$(cat /etc/secrets/enketo-less-secret) API_KEY=$(cat /etc/secrets/enketo-api-key) envsubst '\$DOMAIN\$HTTPS_PORT:\$SECRET:\$LESS_SECRET:\$API_KEY:\$SUPPORT_EMAIL' < ${CONFIG_PATH}.template > $CONFIG_PATH" + +BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \ +SECRET=$(cat /etc/secrets/enketo-secret) \ +LESS_SECRET=$(cat /etc/secrets/enketo-less-secret) \ +API_KEY=$(cat /etc/secrets/enketo-api-key) \ +envsubst '$DOMAIN $BASE_URL $SECRET $LESS_SECRET $API_KEY $SUPPORT_EMAIL' \ + < "$CONFIG_PATH.template" \ + > "$CONFIG_PATH" echo "starting pm2/enketo.." pm2 start --no-daemon app.js -n enketo - diff --git a/files/nginx/certbot.conf b/files/nginx/certbot.conf deleted file mode 100644 index 44f3a877..00000000 --- a/files/nginx/certbot.conf +++ /dev/null @@ -1,13 +0,0 @@ -server { - # Listen only on port 81 for localhost, and nothing else. - server_name 127.0.0.1; - listen 127.0.0.1:81 default_server; - - charset utf-8; - - # Certbot's folder used for the ACME challenge response. - location ^~ /.well-known/acme-challenge { - default_type text/plain; - root /var/www/letsencrypt; - } -} \ No newline at end of file diff --git a/files/nginx/common-headers.nginx.conf b/files/nginx/common-headers.conf similarity index 100% rename from files/nginx/common-headers.nginx.conf rename to files/nginx/common-headers.conf diff --git a/files/nginx/default b/files/nginx/default deleted file mode 100644 index 8e724b89..00000000 --- a/files/nginx/default +++ /dev/null @@ -1,2 +0,0 @@ -variables_hash_max_size 2048; - diff --git a/files/nginx/inflate_body.lua b/files/nginx/inflate_body.lua deleted file mode 100644 index db90ce45..00000000 --- a/files/nginx/inflate_body.lua +++ /dev/null @@ -1,69 +0,0 @@ --- from http://www.pataliebre.net/howto-make-nginx-decompress-a-gzipped-request.html - -ngx.ctx.max_chunk_size = tonumber(ngx.var.max_chunk_size) -ngx.ctx.max_body_size = tonumber(ngx.var.max_body_size) - -function create_error_response (code, description) - local message = string.format('{"status":400,"statusReason":"Bad Request","code":%d,"exception":"","description":"%s","message":"HTTP 400 Bad Request"}', code, description) - ngx.status = ngx.HTTP_BAD_REQUEST - ngx.header.content_type = "application/json" - ngx.say(message) - ngx.exit(ngx.HTTP_OK) -end - -function inflate_chunk (stream, chunk) - return stream(chunk) -end - -function inflate_body (data) - local stream = require("zlib").inflate() - local buffer = "" - local chunk = "" - - for index = 0, data:len(), ngx.ctx.max_chunk_size do - chunk = string.sub(data, index, index + ngx.ctx.max_chunk_size - 1) - local status, output, eof, bytes_in, bytes_out = pcall(stream, chunk) - - if not status then - -- corrupted chunk - ngx.log(ngx.ERR, output) - create_error_response(4001, "Corrupted GZIP body") - end - - if bytes_in == 0 and bytes_out == 0 then - -- body is not gzip compressed - create_error_response(4002, "Invalid GZIP body") - end - - buffer = buffer .. output - - if bytes_out > ngx.ctx.max_body_size then - -- uncompressed body too large - create_error_response(4003, "Uncompressed body too large") - end - end - - return buffer -end - - -local content_encoding = ngx.req.get_headers()["Content-Encoding"] -if content_encoding == "gzip" then - ngx.req.read_body() - local data = ngx.req.get_body_data() - - if data == nil or data == '' then - local file = io.open(ngx.req.get_body_file(), "r") - data = file:read("*a") - file:close() - end - - if data ~= nil and data ~= '' then - local new_data = inflate_body(data) - - ngx.req.clear_header("Content-Encoding") - ngx.req.clear_header("Content-Length") - ngx.req.set_body_data(new_data) - end -end - diff --git a/files/nginx/odk-setup.sh b/files/nginx/odk-setup.sh deleted file mode 100644 index b022bf32..00000000 --- a/files/nginx/odk-setup.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -DHPATH=/etc/dh/nginx.pem -if [ ! -s "$DHPATH" ] && [ "$SSL_TYPE" != "upstream" ] -then - echo "diffie hellman private key does not exist; creating.." - openssl dhparam -out "$DHPATH" 2048 -fi - -SELFSIGN_BASEPATH=/etc/selfsign/live/$DOMAIN -if [ "$SSL_TYPE" = "selfsign" ] && [ ! -s "$SELFSIGN_BASEPATH/privkey.pem" ] -then - echo "self-signed cert requested but does not exist; creating.. (this could take a while)" - mkdir -p "$SELFSIGN_BASEPATH" - openssl req -x509 -newkey rsa:4086 \ - -subj "/C=XX/ST=XXXX/L=XXXX/O=XXXX/CN=localhost" \ - -keyout "$SELFSIGN_BASEPATH/privkey.pem" \ - -out "$SELFSIGN_BASEPATH/fullchain.pem" \ - -days 3650 -nodes -sha256 -fi - -echo "writing a new nginx configuration file.." -CNAME=$([ "$SSL_TYPE" = "customssl" ] && echo "local" || echo "$DOMAIN") \ -/bin/bash -c "envsubst '\$SSL_TYPE \$CNAME' < /usr/share/nginx/odk.conf.template > /etc/nginx/conf.d/odk.conf" - -if [ "$SSL_TYPE" = "letsencrypt" ] -then - echo "starting nginx with certbot.." - cp /usr/share/nginx/certbot.conf /etc/nginx/conf.d/certbot.conf - cp /usr/share/nginx/redirector.conf /etc/nginx/conf.d/redirector.conf - /bin/bash /scripts/start_nginx_certbot.sh -elif [ "$SSL_TYPE" = "upstream" ] -then - echo "starting nginx without local SSL to allow for upstream SSL.." - perl -i -ne 's/listen 443.*/listen 80;/; print if ! /ssl_/' /etc/nginx/conf.d/odk.conf - perl -i -pe 's/X-Forwarded-Proto \$scheme/X-Forwarded-Proto https/;' /etc/nginx/conf.d/odk.conf - rm -f /etc/nginx/conf.d/certbot.conf - rm -f /etc/nginx/conf.d/redirector.conf - nginx -g "daemon off;" -else - echo "starting nginx without certbot.." - rm -f /etc/nginx/conf.d/certbot.conf - rm -f /etc/nginx/conf.d/redirector.conf - nginx -g "daemon off;" -fi diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index d7e1c941..b599c3f7 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -14,7 +14,7 @@ server { server_tokens off; - include /usr/share/nginx/common-headers.nginx.conf; + include /usr/share/odk/nginx/common-headers.conf; client_max_body_size 100m; @@ -36,7 +36,7 @@ server { # Rules set to 'none' here would fallback to default-src if excluded. # They are included here to ease interpretation of violation reports. # - # Other security headers are identical to those in common-headers.nginx.conf; + # Other security headers are identical to those in common-headers.conf; # We can't just include that file here though, as it will set two Content-Security-Policy* headers add_header Referrer-Policy same-origin; add_header Strict-Transport-Security "max-age=63072000" always; @@ -49,11 +49,6 @@ server { proxy_pass http://service:8383; proxy_redirect off; - # set up request-body gzip decompression: - set $max_chunk_size 16384; # ~16KB - set $max_body_size 134217728; # ~128MB - rewrite_by_lua_file inflate_body.lua; - # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; proxy_buffering off; @@ -64,17 +59,16 @@ server { root /usr/share/nginx/html; location /version.txt { - include /usr/share/nginx/common-headers.nginx.conf; + include /usr/share/odk/nginx/common-headers.conf; add_header Cache-Control no-cache; } location /index.html { - include /usr/share/nginx/common-headers.nginx.conf; + include /usr/share/odk/nginx/common-headers.conf; add_header Cache-Control no-cache; } } location /csp-report { - proxy_pass https://o130137.ingest.sentry.io/api/1298632/security/?sentry_key=3cf75f54983e473da6bd07daddf0d2ee; + proxy_pass https://${SENTRY_ORG_SUBDOMAIN}.ingest.sentry.io/api/${SENTRY_PROJECT}/security/?sentry_key=${SENTRY_KEY}; } } - diff --git a/files/nginx/redirector.conf b/files/nginx/redirector.conf index 05f75648..08e33bd5 100644 --- a/files/nginx/redirector.conf +++ b/files/nginx/redirector.conf @@ -4,11 +4,12 @@ server { listen 80 default_server reuseport; listen [::]:80 default_server reuseport; - # Pass this particular URL off to the certbot server so it can properly - # respond to the Let's Encrypt ACME challenges for the HTTPS certificates. + # Anything requesting this particular URL should be served content from + # Certbot's folder so the HTTP-01 ACME challenges can be completed for the + # HTTPS certificates. location '/.well-known/acme-challenge' { default_type "text/plain"; - proxy_pass http://localhost:81; + root /var/www/letsencrypt; } # Everything else gets shunted over to HTTPS for each user defined @@ -16,4 +17,4 @@ server { location / { return 301 https://$http_host$request_uri; } -} \ No newline at end of file +} diff --git a/files/nginx/setup-odk.sh b/files/nginx/setup-odk.sh new file mode 100644 index 00000000..7e3d3aef --- /dev/null +++ b/files/nginx/setup-odk.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +DH_PATH=/etc/dh/nginx.pem +if [ "$SSL_TYPE" != "upstream" ] && [ ! -s "$DH_PATH" ]; then + openssl dhparam -out "$DH_PATH" 2048 +fi + +SELFSIGN_PATH="/etc/selfsign/live/$DOMAIN" +if [ "$SSL_TYPE" = "selfsign" ] && [ ! -s "$SELFSIGN_PATH/privkey.pem" ]; then + mkdir -p "$SELFSIGN_PATH" + openssl req -x509 -newkey rsa:4086 \ + -subj "/C=XX/ST=XXXX/L=XXXX/O=XXXX/CN=localhost" \ + -keyout "$SELFSIGN_PATH/privkey.pem" \ + -out "$SELFSIGN_PATH/fullchain.pem" \ + -days 3650 -nodes -sha256 +fi + +# start from fresh templates in case ssl type has changed +echo "writing fresh nginx templates..." +cp /usr/share/odk/nginx/redirector.conf /etc/nginx/conf.d/redirector.conf +CNAME=$( [ "$SSL_TYPE" = "customssl" ] && echo "local" || echo "$DOMAIN") \ +envsubst '$SSL_TYPE $CNAME $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT' \ + < /usr/share/odk/nginx/odk.conf.template \ + > /etc/nginx/conf.d/odk.conf + +if [ "$SSL_TYPE" = "letsencrypt" ]; then + echo "starting nginx for letsencrypt..." + /bin/bash /scripts/start_nginx_certbot.sh +else + if [ "$SSL_TYPE" = "upstream" ]; then + # no need for letsencrypt challenge reply or 80 to 443 redirection + rm -f /etc/nginx/conf.d/redirector.conf + # strip out all ssl_* directives + perl -i -ne 's/listen 443.*/listen 80;/; print if ! /ssl_/' /etc/nginx/conf.d/odk.conf + # force https because we expect SSL upstream + perl -i -pe 's/X-Forwarded-Proto \$scheme/X-Forwarded-Proto https/;' /etc/nginx/conf.d/odk.conf + echo "starting nginx for upstream ssl..." + else + # remove letsencrypt challenge reply, but keep 80 to 443 redirection + perl -i -ne 'print if $. < 7 || $. > 14' /etc/nginx/conf.d/redirector.conf + echo "starting nginx for custom ssl and self-signed certs..." + fi + nginx -g "daemon off;" +fi diff --git a/files/postgres/upgrade-postgres.sh b/files/postgres/upgrade-postgres.sh new file mode 100755 index 00000000..a12bfe2c --- /dev/null +++ b/files/postgres/upgrade-postgres.sh @@ -0,0 +1,124 @@ +#!/bin/bash -eu + +set -o pipefail + +flag_upgradeCompletedOk="$PGDATANEW/../.postgres14-upgrade-successful" +flag_deleteOldData_name="delete-old-data" +flag_deleteOldData_internal="/postgres14-upgrade/$flag_deleteOldData_name" +flag_oldDataDeleted="/postgres14-upgrade/old-data-deleted" + +logPrefix="$(basename "$0")" +log() { + echo "$(TZ=GMT date) [$logPrefix] $*" +} + +log "Checking for existing upgrade marker file..." +if [[ -f "$flag_upgradeCompletedOk" ]]; then + log "Upgrade has been run previously." + + if [[ -f "$flag_deleteOldData_internal" ]]; then + log "Deleting old data..." + rm "$flag_deleteOldData_internal" + # We cannot run ./delete_old_cluster.sh here, as it will try to: + # + # rm -rf '/var/lib/postgresql/data' + # + # This will fail with: + # + # rm: cannot remove '/var/lib/postgresql/data': Device or resource busy + # + # This is because that is the root of a docker volume. Instead, + # we must do our own manual equivalent: + if ! [[ -f ./delete_old_cluster.sh ]]; then + log "!!!" + log "!!! ERROR: file missing: delete_old_cluster.sh" + log "!!!" + log "!!! Upgrade may not have completed successfully." + log "!!!" + log "!!! Old data will not be deleted." + log "!!!" + exit 1 + fi + rm -rf /var/lib/postgresql/data/* + touch "$flag_oldDataDeleted" + log "Old data deleted." + elif [[ -f "$PGDATAOLD/PG_VERSION" ]]; then + log "!!!" + log "!!! WARNING: you still have old data from PostgreSQL 9.6" + log "!!!" + log "!!! This is taking up disk space: $(du -hs "$PGDATAOLD" 2>/dev/null | cut -f1)B" + log "!!!" + log "!!! Continue with the instructions at https://docs.getodk.org/central-upgrade/" + log "!!!" + fi +else + if [[ -f "$flag_deleteOldData_internal" ]]; then + log "!!!" + log "!!! ERROR: Deletion request file created, but upgrade has not yet run!" + log "!!!" + log "!!! Please email support@getodk.org for assistance." + log "!!!" + exit 1 + fi + + if ! [[ -f "$PGDATAOLD/PG_VERSION" ]]; then + log "No old data found." + elif [[ -f "$PGDATANEW/PG_VERSION" ]]; then + log "!!!" + log "!!! ERROR: New data found, but upgrade not flagged as complete." + log "!!!" + log "!!! Please email support@getodk.org for assistance." + log "!!!" + exit 1 + else ( + log "Upgrade not run previously; upgrading now..." + + log "From: $PGDATAOLD" + log " To: $PGDATANEW" + + # standard ENTRYPOINT/CMD combo from parent Dockerfile + if ! docker-upgrade pg_upgrade; then + log "!!!" + log "!!! pg_upgrade FAILED; dumping log files..." + log "!!!" + tail -n+1 pg_upgrade_*.log || log "No pg_upgrade log files found ¯\_(ツ)_/¯" + log "!!!" + log "!!! pg_upgrade FAILED; check above for clues." + log "!!!" + exit 1 + fi + + # see https://github.com/tianon/docker-postgres-upgrade/issues/16, + # https://github.com/tianon/docker-postgres-upgrade/issues/1 + cp "$PGDATAOLD/pg_hba.conf" "$PGDATANEW/pg_hba.conf" + + log "Starting postgres server for maintenance..." + gosu postgres pg_ctl -D "$PGDATANEW" -l logfile start + + # As recommended by pg_upgrade: + log "Updating extensions..." + psql -f update_extensions.sql + + # As recommended by pg_upgrade: + log "Regenerating optimizer statistics..." + /usr/lib/postgresql/14/bin/vacuumdb --all --analyze-in-stages + + log "Stopping postgres server..." + gosu postgres pg_ctl -D "$PGDATANEW" -m smart stop + + # pg_upgrade recommends running ./delete_old_cluster.sh, which + # just runs `rm -rf '/var/lib/postgresql/data'`. Doing this here + # can fail with "Device or resource busy". In addition, deleting + # the old data may be risky if the upgrade didn't complete + # perfectly. We can hedge our bets and make life easier by + # skipping cleanup here, and providing a docker command to prune + # the old, unused volume in the next odk-central version. + + log "Upgrade complete." + ) > >(tee --append "/postgres14-upgrade/upgrade-postgres.log" >&2) 2>&1 + fi + touch "$flag_upgradeCompletedOk" + touch "/postgres14-upgrade/upgrade-successful" +fi + +log "Complete." diff --git a/files/postgres14/start-postgres.sh b/files/postgres14/start-postgres.sh new file mode 100755 index 00000000..9e341e52 --- /dev/null +++ b/files/postgres14/start-postgres.sh @@ -0,0 +1,19 @@ +#!/bin/bash -eu +set -o pipefail + +flag_upgradeCompletedOk="$PGDATA/../.postgres14-upgrade-successful" + +logPrefix="$(basename "$0")" +log() { + echo "$(TZ=GMT date) [$logPrefix] $*" +} + +if ! [[ -f "$flag_upgradeCompletedOk" ]]; then + log "Waiting for upgrade to complete..." + while ! [[ -f "$flag_upgradeCompletedOk" ]]; do sleep 1; done + log "Upgrade complete." +fi + +log "Starting postgres..." +# call ENTRYPOINT + CMD from parent Docker image +docker-entrypoint.sh postgres diff --git a/files/postgres14/upgrade/check-available-space b/files/postgres14/upgrade/check-available-space new file mode 100755 index 00000000..e4e666ed --- /dev/null +++ b/files/postgres14/upgrade/check-available-space @@ -0,0 +1,54 @@ +#!/bin/bash -eu +set -o pipefail + +logPrefix="$(basename "$0")" +log() { + echo "[$logPrefix] $*" +} + +if ! sudo -n true 2>/dev/null; then + log + log "WARNING: sudo may be required" + log +fi + +dockerRootDir="$(docker info --format '{{print .DockerRootDir}}')" +free="$(df --output=avail "$dockerRootDir/volumes" | tail -n+2)" + +containerName="central_postgres_1" +# The --format argument is a Go template. Adding spaces between the +# curlies seems to affect the output. For more info on this +# wonderful language, see: +# * https://pkg.go.dev/text/template +# * https://devops.stackexchange.com/a/6242 +volume="$(docker inspect --format='{{range $index, $element := .Mounts}}{{if eq $element.Destination "/var/lib/postgresql/data"}}{{println $element.Name}}{{end}}{{end}}' "$containerName")" +if [[ "$volume" = "" ]]; then + log "!!!" + log "!!! Volume not found for container: $containerName !!!" + log "!!!" + log "!!! Cannot continue !!!" + log "!!!" + exit 1 +fi +mountpoint="$(docker volume inspect --format='{{.Mountpoint}}' "$volume")" +used="$(du -s "$mountpoint" | cut -f1)" + +log +log " Free space: $(numfmt --to=si "$((free*1000))")B" +log "Required space: $(numfmt --to=si "$((used*1000))")B" +log + +if [[ "$used" -lt "$free" ]]; then + log "You have enough space to upgrade." + log + log "Continue with the instructions at https://docs.getodk.org/central-upgrade/" + log +else + log "!!!" + log "!!! ERROR: You do not have enough space available to upgrade." + log "!!!" + log "!!! Increase your disk by at least $(numfmt --to=si -- "$(((used-free)*1000))")B and try again." + log "!!!" + log + exit 1 +fi diff --git a/files/prebuild/build-frontend.sh b/files/prebuild/build-frontend.sh index e3aef102..6122c321 100755 --- a/files/prebuild/build-frontend.sh +++ b/files/prebuild/build-frontend.sh @@ -1,4 +1,4 @@ #!/bin/bash -eu cd client -npm install --no-audit --fund=false --update-notifier=false +npm clean-install --no-audit --fund=false --update-notifier=false npm run build diff --git a/files/service/config.json.template b/files/service/config.json.template index fc3814eb..1e1f9d53 100644 --- a/files/service/config.json.template +++ b/files/service/config.json.template @@ -1,17 +1,24 @@ { "default": { "database": { - "host": "postgres", - "user": "odk", - "password": "odk", - "database": "odk" + "host": "${DB_HOST}", + "user": "${DB_USER}", + "password": "${DB_PASSWORD}", + "database": "${DB_NAME}", + "ssl": ${DB_SSL} }, "email": { - "serviceAccount": "no-reply@${DOMAIN}", + "serviceAccount": "${EMAIL_FROM}", "transport": "smtp", "transportOpts": { - "host": "mail", - "port": 25 + "host": "${EMAIL_HOST}", + "port": ${EMAIL_PORT}, + "secure": ${EMAIL_SECURE}, + "ignoreTLS": ${EMAIL_IGNORE_TLS}, + "auth": { + "user": "${EMAIL_USER}", + "pass": "${EMAIL_PASSWORD}" + } } }, "xlsform": { @@ -23,14 +30,14 @@ "apiKey": "${ENKETO_API_KEY}" }, "env": { - "domain": "https://${DOMAIN}:${HTTPS_PORT}", + "domain": "${BASE_URL}", "sysadminAccount": "${SYSADMIN_EMAIL}" }, "external": { "sentry": { - "orgSubdomain": "o130137", - "key": "3cf75f54983e473da6bd07daddf0d2ee", - "project": "1298632" + "orgSubdomain": "${SENTRY_ORG_SUBDOMAIN}", + "key": "${SENTRY_KEY}", + "project": "${SENTRY_PROJECT}" } } } diff --git a/files/service/odk-cmd b/files/service/odk-cmd index 99b80ab5..8cc62be0 100755 --- a/files/service/odk-cmd +++ b/files/service/odk-cmd @@ -1,3 +1,3 @@ #!/bin/sh -exec $(which node) /usr/odk/lib/bin/cli.js $@ +exec "$(command -v node)" /usr/odk/lib/bin/cli.js "$@" diff --git a/files/service/scripts/start-odk.sh b/files/service/scripts/start-odk.sh index 89e3487a..39b44cd8 100755 --- a/files/service/scripts/start-odk.sh +++ b/files/service/scripts/start-odk.sh @@ -1,11 +1,19 @@ #!/usr/bin/env bash -CONFIG_PATH=/usr/odk/config/local.json echo "generating local service configuration.." -/bin/bash -c "ENKETO_API_KEY=$(cat /etc/secrets/enketo-api-key) envsubst '\$DOMAIN:\$HTTPS_PORT:\$SYSADMIN_EMAIL:\$ENKETO_API_KEY' < /usr/share/odk/config.json.template > $CONFIG_PATH" -export SENTRY_RELEASE="$(cat sentry-versions/server)" -export SENTRY_TAGS="{ \"version.central\": \"$(cat sentry-versions/central)\", \"version.client\": \"$(cat sentry-versions/client)\" }" +ENKETO_API_KEY=$(cat /etc/secrets/enketo-api-key) \ +BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \ +envsubst '$DOMAIN $BASE_URL $SYSADMIN_EMAIL $ENKETO_API_KEY $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME $DB_SSL $EMAIL_FROM $EMAIL_HOST $EMAIL_PORT $EMAIL_SECURE $EMAIL_IGNORE_TLS $EMAIL_USER $EMAIL_PASSWORD $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT' \ + < /usr/share/odk/config.json.template \ + > /usr/odk/config/local.json + +SENTRY_RELEASE="$(cat sentry-versions/server)" +export SENTRY_RELEASE +# shellcheck disable=SC2089 +SENTRY_TAGS="{ \"version.central\": \"$(cat sentry-versions/central)\", \"version.client\": \"$(cat sentry-versions/client)\" }" +# shellcheck disable=SC2090 +export SENTRY_TAGS echo "running migrations.." node ./lib/bin/run-migrations @@ -25,9 +33,9 @@ cron -f & MEMTOT=$(vmstat -s | grep 'total memory' | awk '{ print $1 }') if [ "$MEMTOT" -gt "1100000" ] then - WORKER_COUNT=4 + export WORKER_COUNT=4 else - WORKER_COUNT=1 + export WORKER_COUNT=1 fi echo "using $WORKER_COUNT worker(s) based on available memory ($MEMTOT).." diff --git a/instructions.md b/instructions.md new file mode 100644 index 00000000..37a1b46c --- /dev/null +++ b/instructions.md @@ -0,0 +1,3 @@ +1. back up your data! +2. `./upgrade.sh` ++ something about pruning diff --git a/nginx.dockerfile b/nginx.dockerfile index fbbf7f70..7f3d5a9b 100644 --- a/nginx.dockerfile +++ b/nginx.dockerfile @@ -1,30 +1,26 @@ -FROM node:16.17.0 as intermediate +FROM node:16.19.1 as intermediate COPY ./ ./ RUN files/prebuild/write-version.sh RUN files/prebuild/build-frontend.sh -# make sure you have updated *.conf files when upgrading this -FROM jonasal/nginx-certbot:2.4.1 +# when upgrading, look for upstream changes to redirector.conf +# also, confirm setup-odk.sh strips out HTTP-01 ACME challenge location +FROM jonasal/nginx-certbot:4.2.0 EXPOSE 80 EXPOSE 443 VOLUME [ "/etc/dh", "/etc/selfsign", "/etc/nginx/conf.d" ] -ENTRYPOINT [ "/bin/bash", "/scripts/odk-setup.sh" ] +ENTRYPOINT [ "/bin/bash", "/scripts/setup-odk.sh" ] -RUN apt-get update; apt-get install -y openssl netcat nginx-extras lua-zlib +RUN apt-get update && apt-get install -y netcat-openbsd -RUN mkdir -p /etc/selfsign/live/local/ -COPY files/nginx/odk-setup.sh /scripts/ +RUN mkdir -p /usr/share/odk/nginx/ +COPY files/nginx/setup-odk.sh /scripts/ COPY files/local/customssl/*.pem /etc/customssl/live/local/ +COPY files/nginx/*.conf* /usr/share/odk/nginx/ -COPY files/nginx/default /etc/nginx/sites-enabled/ -COPY files/nginx/inflate_body.lua /usr/share/nginx/ -COPY files/nginx/odk.conf.template /usr/share/nginx/ -COPY files/nginx/common-headers.nginx.conf /usr/share/nginx/ -COPY files/nginx/certbot.conf /usr/share/nginx/ -COPY files/nginx/redirector.conf /usr/share/nginx/ -COPY --from=intermediate client/dist/ /usr/share/nginx/html/ -COPY --from=intermediate /tmp/version.txt /usr/share/nginx/html/ +COPY --from=intermediate client/dist/ /usr/share/nginx/html +COPY --from=intermediate /tmp/version.txt /usr/share/nginx/html diff --git a/postgres-upgrade.dockerfile b/postgres-upgrade.dockerfile new file mode 100644 index 00000000..a42d3c97 --- /dev/null +++ b/postgres-upgrade.dockerfile @@ -0,0 +1,16 @@ +# see: https://github.com/tianon/docker-postgres-upgrade/blob/master/9.6-to-14/Dockerfile +FROM tianon/postgres-upgrade:9.6-to-14 + +# This file is required to encourage human validation of the process. +# It's expected it will be provided by the sysadmin performing the upgrade. +# Docker build will fail if this file is missing. +COPY ./files/allow-postgres14-upgrade . + +COPY files/postgres/upgrade-postgres.sh /usr/local/bin/ + +# we can't rename/remap this directory, as it's an anonymous volume +ENV PGDATAOLD=/var/lib/postgresql/data + +# N.B. postgres is not started automatically in this image as we are overriding CMD. +ENTRYPOINT [] +CMD upgrade-postgres.sh diff --git a/postgres14.dockerfile b/postgres14.dockerfile new file mode 100644 index 00000000..25ba616b --- /dev/null +++ b/postgres14.dockerfile @@ -0,0 +1,17 @@ +FROM postgres:14 + +COPY files/postgres14/start-postgres.sh /usr/local/bin/ + +# PGDATA is originally declared in the parent Dockerfile, but points +# to an anonymous VOLUME declaration in the same file: +# +# ENV PGDATA /var/lib/postgresql/data +# ... +# VOLUME /var/lib/postgresql/data +# +# To ensure future accessibility, PGDATA must be stored _outside_ the +# anonymous volume. +ENV PGDATA /var/lib/odk/postgresql/14/data + +ENTRYPOINT [] +CMD start-postgres.sh diff --git a/secrets.dockerfile b/secrets.dockerfile index 85155a9f..1b575e58 100644 --- a/secrets.dockerfile +++ b/secrets.dockerfile @@ -1,2 +1,2 @@ -FROM node:16.17.0 +FROM node:16.19.1 COPY files/enketo/generate-secrets.sh ./ \ No newline at end of file diff --git a/server b/server index ad061f60..05ebccde 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit ad061f6088cf81c95bddf6f53310923cce9f76be +Subproject commit 05ebccdea1736240ceafa929399d25920c7bdb2f diff --git a/service.dockerfile b/service.dockerfile index 0aab723f..7e2034ee 100644 --- a/service.dockerfile +++ b/service.dockerfile @@ -1,4 +1,4 @@ -FROM node:16.17.0 as intermediate +FROM node:16.19.1 as intermediate COPY . . RUN mkdir /tmp/sentry-versions @@ -12,17 +12,17 @@ FROM node:16.17.0 WORKDIR /usr/odk -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list; \ - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg; \ - apt-get update; \ - apt-get install -y cron gettext postgresql-client-9.6 +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg && \ + apt-get update && \ + apt-get install -y cron gettext postgresql-client-14 COPY files/service/crontab /etc/cron.d/odk COPY server/package*.json ./ RUN npm clean-install --omit=dev --legacy-peer-deps --no-audit --fund=false --update-notifier=false -RUN npm install pm2@5.2.0 -g +RUN npm install pm2@5.2.2 -g COPY server/ ./ COPY files/service/scripts/ ./