diff --git a/.gitignore b/.gitignore index 41b75f03..098f26fb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ passwords.json /mikeapp/ docker-compose.override*.yaml dump.* - +tasks.org diff --git a/Makefile b/Makefile index 9b758dba..87e41e2c 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ help: @grep -h '^.PHONY: .* #' Makefile ${ROOT_DIR}/_scripts/Makefile.globals | sed 's/\.PHONY: \(.*\) # \(.*\)/make \1 \t- \2/' | expand -t20 include _scripts/Makefile.globals +include _scripts/Makefile.cd .PHONY: check-deps check-deps: @@ -34,8 +35,7 @@ build: find ./ | grep docker-compose.yaml$ | xargs dirname | xargs -iXX docker-compose --env-file=XX/${ENV_FILE} -f XX/docker-compose.yaml build .PHONY: open # Open the repository website README -open: - xdg-open https://github.com/enigmacurry/d.rymcg.tech#readme +open: readme .PHONY: status # Check status of all sub-projects status: diff --git a/README.md b/README.md index 1d33b26e..bf75125c 100644 --- a/README.md +++ b/README.md @@ -493,12 +493,16 @@ As alluded to earlier, this project offers multiple ways to control Docker: 1. Editing `.env` files by hand, and running `docker compose` - commands yourself. + commands yourself (this is a usable, but lower level, base + abstraction). 2. Running `make` targets that edit the `.env` files automatically and runs `docker compose` for you (this is the author's preferred - method). + method, and the one that most of the documentation will actually + use). 3. Running the `d.rymcg.tech` CLI script, which runs the `make` - targets from any working directory. + targets from any working directory. (This method also includes + extra features such as creating your own new projects from + templates.) All of these methods are compatible, and they will all get you to the same place. The Makefiles offer a more streamlined approach with a @@ -516,12 +520,27 @@ For all of the containers that you wish to install, do the following: `docker-compose.yaml` * Copy the example `.env-dist` to `.env` * Edit all of the variables in `.env` according to the example and comments. - * Follow the README for instructons to start the containers. + * Create a + [`docker-compose.override.yaml`](https://docs.docker.com/compose/extends/#multiple-compose-files) + file by hand, copying from the template given in + `docker-compose.instance.yaml` (If the project does not have this + file, you can skip this step.) This [ytt](https://carvel.dev/ytt/) + template is mainly used for the service container labels, and has + logic for choosing which Traefik middlewares to apply. So you just + need to remove (comment out) the lines that don't apply in your + case. The override files are not committed into git, as they are + normally dynamically generated by the Makefiles and rendering from + the template on the fly. If you want to maintain these files by + hand, you can remove the exclusion of them from the + [.gitignore](.gitignore) and commit them with your own forked + repository. + * Follow the README for instructions to start the containers. Generally, all you need to do is run: `docker compose up --build -d` (This is the same thing that `make install` does) -When using `docker compose` by hand, it uses the `.env` file name by default. -You can change this behaviour by specifying the `--env-file` argument. +When using `docker compose` by hand, it uses the `.env` file name by +default. To use any other filename, specify the `--env-file` argument +(eg. when deploying multiple instances). ### Using the Makefiles @@ -547,8 +566,9 @@ directory you are in. * The suffix of the .env filename, `_default`, refers to the [instance](#creating-multiple-instances-of-a-service) of the service (each instance has a different name, with `_default` being - the default name, and this is typical when you are only deploying a - single instance.) + the default name. This default name is typical only when you are + deploying a single instance, otherwise you should use a unique name + for each instance.) * Verify the configuration by looking at the contents of `.env_${DOCKER_CONTEXT}_default`. * Run `make install` to start the services. (this is the same thing as @@ -564,11 +584,11 @@ directory you are in. `make config` *does not literally* create a file named `.env`, but rather one based upon the current docker context: -`.env_${DOCKER_CONTEXT}_default`. This allows for different configurations to -coexist in the same directory. All of the makefile commands operate -assuming this contextual environment file name, not `.env`. To switch -between configs, you switch your current docker context: `docker -context use {CONTEXT}`. +`.env_${DOCKER_CONTEXT}_default`. This allows for different +configurations to coexist in the same directory. All of the `make` +commands operate assuming this contextual environment file name, *not* +`.env`. To switch between configs, you switch your current docker +context: `docker context use {CONTEXT}`. During `make config`, you will sometimes be asked to create HTTP Basic Authentication passwords, and these passwords can be *optionally* @@ -666,8 +686,9 @@ d.rymcg.tech cd Press `Ctrl-D` to exit the sub-shell and jump back to wherever you came from. -From any working directory, you can create a new, barebones, [external -project](#integrating-external-projects): +From any working directory, you can create a new, [external +project](#integrating-external-projects), based upon one of the +[included templates](_templates): ``` # This creates a new project directory in your current working directory: @@ -676,7 +697,7 @@ project](#integrating-external-projects): d.rymcg.tech create ``` -Open the README for any project in your web browser: +Open any project's README file directly in your web browser: ``` ## Open the main README @@ -693,7 +714,7 @@ your `~/.bashrc` *after* the `eval` line that loads the main `d.rymcg.tech` script): ``` -## Create a short alias for the Traefik project: +## Example project alias: creates a shorter command used just for the Traefik project: __d.rymcg.tech_project_alias traefik ``` @@ -709,14 +730,13 @@ project](#integrating-external-projects) (eg. named `mikeapp`), you can create a command alias for it: ``` -## External project alias: +## Example external project alias: __d.rymcg.tech_project_alias mikeapp ~/git/mikeapp ``` With this alias installed, instead of running `make -C ~/git/mikeapp install` you can now simply run `mikeapp install`. - If you want a different alias for the main script, you can add that too: ``` @@ -756,13 +776,14 @@ By default, all of the `make` targets will use the default environment, but you can tell it use the instance environment instead, by setting the `instance` (or `INSTANCE`) variable: -``` make instance=foo config # Configure a new or existing instance -named foo make instance=bar config # (Re)configures bar instance make -instance=foo install # This (re)installs only the foo instance make -instance=bar install # (Re)installs only bar instance make -instance=foo ps # This shows the containers status of the foo instance -make instance=foo stop # This stops the foo instance make instance=bar -destroy # This destroys only the bar instance +``` +make instance=foo config # Configure a new or existing instance named foo +make instance=bar config # (Re)configures bar instance +make instance=foo install # This (re)installs only the foo instance +make instance=bar install # (Re)installs only bar instance +make instance=foo ps # This shows the containers status of the foo instance +make instance=foo stop # This stops the foo instance +make instance=bar destroy # This destroys only the bar instance # Show the status of all instances of the current project subdirectory: make status diff --git a/_scripts/Makefile.cd b/_scripts/Makefile.cd new file mode 100644 index 00000000..ccfb609f --- /dev/null +++ b/_scripts/Makefile.cd @@ -0,0 +1,5 @@ +.PHONY: cd # Enter a Bash sub-shell and change working directory to the project root +cd: + @echo "Changing directory to ${CURDIR}" + @echo "Entering sub-shell (press Ctrl-D to pop back out)" + @bash --rcfile <(echo "source ~/.bashrc; unset PROMPT_COMMAND; PS1=\"[@] \$${PS1}\"") -i diff --git a/_scripts/Makefile.project-subproject b/_scripts/Makefile.project-subproject index 4b290c9f..17e3cb56 100644 --- a/_scripts/Makefile.project-subproject +++ b/_scripts/Makefile.project-subproject @@ -4,6 +4,7 @@ include ${ROOT_DIR}/_scripts/Makefile.build include ${ROOT_DIR}/_scripts/Makefile.install include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.clean +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects b/_scripts/Makefile.projects index f28bb750..abbbbec8 100644 --- a/_scripts/Makefile.projects +++ b/_scripts/Makefile.projects @@ -6,6 +6,7 @@ include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.open +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-custom-build b/_scripts/Makefile.projects-custom-build index c58350a7..9c00a1df 100644 --- a/_scripts/Makefile.projects-custom-build +++ b/_scripts/Makefile.projects-custom-build @@ -6,6 +6,7 @@ include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.open +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-custom-build-custom-install b/_scripts/Makefile.projects-custom-build-custom-install index c287a372..3b282b08 100644 --- a/_scripts/Makefile.projects-custom-build-custom-install +++ b/_scripts/Makefile.projects-custom-build-custom-install @@ -5,6 +5,7 @@ include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.open +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-custom-install b/_scripts/Makefile.projects-custom-install index e7084a75..52479939 100644 --- a/_scripts/Makefile.projects-custom-install +++ b/_scripts/Makefile.projects-custom-install @@ -5,6 +5,7 @@ include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.open +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-external b/_scripts/Makefile.projects-external index f28bb750..abbbbec8 100644 --- a/_scripts/Makefile.projects-external +++ b/_scripts/Makefile.projects-external @@ -6,6 +6,7 @@ include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.open +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-no-open b/_scripts/Makefile.projects-no-open index 781dd06b..a96cb43c 100644 --- a/_scripts/Makefile.projects-no-open +++ b/_scripts/Makefile.projects-no-open @@ -5,6 +5,7 @@ include ${ROOT_DIR}/_scripts/Makefile.install include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.clean +include ${ROOT_DIR}/_scripts/Makefile.cd include ${ROOT_DIR}/_scripts/Makefile.reconfigure include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/d.rymcg.tech b/_scripts/d.rymcg.tech index e4733357..c7bf95da 100755 --- a/_scripts/d.rymcg.tech +++ b/_scripts/d.rymcg.tech @@ -93,11 +93,10 @@ __create() { } __change-directory() { - echo "Entering sub-shell. Press Ctrl-D to pop back to the parent shell." if [[ $# -gt 0 ]]; then - /bin/bash --rcfile <(echo "cd ${ROOT_DIR}/$1") + __make $1 cd else - /bin/bash --rcfile <(echo "cd ${ROOT_DIR}") + __make -- cd fi echo "Exited sub-shell." } @@ -112,7 +111,7 @@ __make() { PROJECT_DIR=${ROOT_DIR}/${PROJECT_NAME} fi test -d "${PROJECT_DIR}" || fault "Project directory does not exist: ${PROJECT_DIR}" - make -C "${PROJECT_DIR}" "$@" + make --no-print-directory -C "${PROJECT_DIR}" "$@" else __list_projects error "Missing project name argument. Choose one from the above." @@ -358,6 +357,8 @@ main() { __context "$@";; script) __run_script "$@";; + status) + __make -- status | sed "s|\./|${ROOT_DIR}/|g";; *) fault "Invalid command" esac diff --git a/_scripts/docker_compose_override b/_scripts/docker_compose_override index 3aa2db2e..96cbe5df 100755 --- a/_scripts/docker_compose_override +++ b/_scripts/docker_compose_override @@ -9,11 +9,11 @@ fi BIN=$(dirname ${BASH_SOURCE}) source ${BIN}/funcs.sh ENV_FILE=$1; shift -YTT_DATA_ARGS="" test -f docker-compose.instance.yaml || (echo "Missing docker-compose.instance.yaml. This project has not been setup with an override template yet." && exit 1) DOCKER_CONTEXT=$(${BIN}/docker_context) +check_var DOCKER_CONTEXT instance=${instance:-${INSTANCE}} if [[ -n "${instance}" ]]; then CONTEXT_INSTANCE="${DOCKER_CONTEXT}_${instance}" @@ -22,28 +22,41 @@ else fi OVERRIDE="docker-compose.override_${CONTEXT_INSTANCE}.yaml" +YTT_DATA_ARGS="--data-value context=${DOCKER_CONTEXT}" + for var in "$@"; do ytt_var="${var}" value="" + #### The variable arguments have three forms: + #### name=VARIABLE_NAME # sets the name field to the value of VARIABLE_NAME in the .env file + #### name=:VARIABLE_NAME # sets the name field the literal string "VARIABLE_NAME" + #### name=@VARIABLE_NAME # sets the name field the literal string '${VARIABLE_NAME}' if [[ "${var}" == *"=:"* ]]; then # Get literal value from command line parts=(${var//=:/ }); var=${parts[0]}; value=${parts[@]:1}; ytt_var="${var}" - else + elif [[ "${var}" == *"=@"* ]]; then + # Get literal value from command line and wrap it as a variable name ('VAR' becomes literal '${VAR}') + parts=(${var//=@/ }); var=${parts[0]}; value="\${${parts[1]}}"; + ytt_var="${var}" + elif [[ "${var}" == *"="* ]]; then # Get value from env file - if [[ "${var}" == *"="* ]]; then - # Make it an alias to a var in the env file: - parts=(${var//=/ }); var_alias=${parts[0]}; var=${parts[@]:1}; - ytt_var="${var_alias}" - fi + parts=(${var//=/ }); var_alias=${parts[0]}; var=${parts[@]:1}; + ytt_var="${var_alias}" value="$(${BIN}/dotenv -f ${ENV_FILE} get $var)" || true + else + fault "Invalid arg var: ${var}" fi - test -z "${value}" && echo "${var} is blank." && continue + test -z "${value}" && echo "${var} is blank." ytt_var=$(echo "$ytt_var" | tr '[:upper:]' '[:lower:]') - YTT_DATA_ARGS="${YTT_DATA_ARGS} --data-value ${ytt_var}=\"${value}\"" + YTT_DATA_ARGS="${YTT_DATA_ARGS} --data-value ${ytt_var}='${value}'" done echo '#' ytt ${YTT_DATA_ARGS} \< docker-compose.instance.yaml \> ${OVERRIDE} -ytt ${YTT_DATA_ARGS} < docker-compose.instance.yaml > ${OVERRIDE} +cat < ${OVERRIDE} +## DO NOT EDIT - This Docker Compose override file is generated from the docker-compose.instance.yaml template. +## This file is automatically recreated whenever you run \`make config\` or \`make install\`. +EOF +ytt ${YTT_DATA_ARGS} < docker-compose.instance.yaml >> ${OVERRIDE} echo "Created docker-compose override: ${OVERRIDE}" diff --git a/_scripts/open b/_scripts/open index 663e63cc..25c7846e 100755 --- a/_scripts/open +++ b/_scripts/open @@ -59,6 +59,9 @@ EOF } if [[ -f passwords.json ]]; then URL_PASSWORD=$(jq -r '(.["'${CONTEXT_INSTANCE}'"][0].username) + ":" + (.["'${CONTEXT_INSTANCE}'"][0].url_encoded)' /dev/null 2>&1; then - ${command} "${@}" - else - if [[ $IMAGE_BUILT != "true" ]]; then - cat </dev/null 2>&1 && IMAGE_BUILT="true" -FROM alpine -RUN apk add -U openssl apache2-utils python3 -EOF + PASSWORD_JSON="" + HASHED_PASSWORDS=() + while true; do + ask_no_blank "Enter the username for HTTP Basic Authentication" USERNAME + ask "Enter the passphrase for ${USERNAME} (leave blank to generate a random passphrase)" PLAIN_PASSWORD + if [[ -z ${PLAIN_PASSWORD} ]]; then + PLAIN_PASSWORD=$(openssl rand -base64 30 | head -c 20) + echo "Plain text password for ${USERNAME} (save this): ${PLAIN_PASSWORD}" fi - docker run --rm -i ${WRAPPER_IMAGE} ${command} "${@}" + HASHED_PASSWORD=$(htpasswd -nb "${USERNAME}" "${PLAIN_PASSWORD}") + HASHED_PASSWORDS+=(${HASHED_PASSWORD}) + URL_ENCODED_PASSWORD=$(python3 -c "from urllib.parse import quote; print(quote('''${PLAIN_PASSWORD=}''', safe=''))") + echo "Hashed password: ${HASHED_PASSWORD}" + echo "Url encoded: https://${USERNAME}:${URL_ENCODED_PASSWORD}@example.com/..." + PASSWORD_JSON="${PASSWORD_JSON}, {\"username\": \"${USERNAME}\", \"password\": \"${PLAIN_PASSWORD}\", \"hashed_password\": \"${HASHED_PASSWORD}\", \"url_encoded\": \"$URL_ENCODED_PASSWORD\"}" + ${BIN}/confirm no "Would you like to create additional usernames (for the same access privilege)" "?" || break + done + + COMBINED_PASSWORD=$(echo $(IFS=, ; echo "${HASHED_PASSWORDS[*]}") | sed 's/\$/\$\$/g') + ${BIN}/dotenv -f ${ENV_FILE} set ${VAR}="${COMBINED_PASSWORD}" + echo "Set ${VAR}=${COMBINED_PASSWORD}" + + echo "" + if ${BIN}/confirm $([[ "${DEFAULT_SAVE_CLEARTEXT_PASSWORDS_JSON}" == "true" ]] && echo yes || echo no) "Would you like to export the usernames and cleartext passwords to the file passwords.json" "?"; then + PASSWORD_JSON="[${PASSWORD_JSON:2}]" + if [[ ! -f passwords.json ]]; then + echo '{}' | jq > passwords.json + fi + TMP_PASSWORD=$(mktemp) + ( + cat passwords.json && echo ${PASSWORD_JSON} | jq --argjson "${CONTEXT_INSTANCE}" "${PASSWORD_JSON}" '$ARGS.named' + ) | jq -s add > ${TMP_PASSWORD} && mv ${TMP_PASSWORD} passwords.json fi } -if [[ -n $(${BIN}/dotenv -f ${ENV_FILE} get ${var}) ]]; then - ${BIN}/confirm no "There is already a user auth string configured. Do you want to generate new users and passwords" "?" || exit 0 -fi +disable_http_basic_authentication() { + ${BIN}/reconfigure ${ENV_FILE} ${VAR}= + TMP_PASSWORD=$(mktemp) + cat passwords.json | jq "del(.\"${CONTEXT_INSTANCE}\")" > ${TMP_PASSWORD} && mv ${TMP_PASSWORD} passwords.json + exit 0 +} -PASSWORD_JSON="" -HASHED_PASSWORDS=() -while true; do - ask_no_blank "Enter the username for HTTP Basic Authentication" USERNAME - ask "Enter the passphrase for ${USERNAME} (leave blank to generate a random passphrase)" PLAIN_PASSWORD - if [[ -z ${PLAIN_PASSWORD} ]]; then - PLAIN_PASSWORD=$(wrapper openssl rand -base64 30 | head -c 20) - echo "Plain text password for ${USERNAME} (save this): ${PLAIN_PASSWORD}" - fi - HASHED_PASSWORD=$(wrapper htpasswd -nb "${USERNAME}" "${PLAIN_PASSWORD}") - HASHED_PASSWORDS+=(${HASHED_PASSWORD}) - URL_ENCODED_PASSWORD=$(wrapper python3 -c "from urllib.parse import quote; print(quote('''${PLAIN_PASSWORD=}''', safe=''))") - echo "Hashed password: ${HASHED_PASSWORD}" - echo "Url encoded: https://${USERNAME}:${URL_ENCODED_PASSWORD}@example.com/..." - PASSWORD_JSON="${PASSWORD_JSON}, {\"username\": \"${USERNAME}\", \"password\": \"${PLAIN_PASSWORD}\", \"hashed_password\": \"${HASHED_PASSWORD}\", \"url_encoded\": \"$URL_ENCODED_PASSWORD\"}" - ${BIN}/confirm no "Would you like to create additional usernames (for the same access privilege)" "?" || break -done -COMBINED_PASSWORD=$(echo $(IFS=, ; echo "${HASHED_PASSWORDS[*]}") | sed 's/\$/\$\$/g') -${BIN}/dotenv -f ${ENV_FILE} set ${var}="${COMBINED_PASSWORD}" -echo "Set ${var}=${COMBINED_PASSWORD}" +## Make new .env if it doesn't exist: +test -f ${ENV_FILE} || cp .env-dist ${ENV_FILE} -echo "" -if ${BIN}/confirm $([[ "${DEFAULT_SAVE_CLEARTEXT_PASSWORDS_JSON}" == "true" ]] && echo yes || echo no) "Would you like to export the usernames and cleartext passwords to the file passwords.json" "?"; then - PASSWORD_JSON="[${PASSWORD_JSON:2}]" - if [[ ! -f passwords.json ]]; then - echo '{}' | jq > passwords.json +CONTEXT_INSTANCE=$(basename $ENV_FILE | sed 's/.env_//') +DEFAULT_SAVE_CLEARTEXT_PASSWORDS_JSON=${DEFAULT_SAVE_CLEARTEXT_PASSWORDS_JSON:-false} + +if [[ ${1} =~ ^default= ]]; then + DEFAULT_ENABLED_AUTH=no + if [[ ${1} =~ ^default=yes$ ]] || [[ -n "$(${BIN}/dotenv -f ${ENV_FILE} get ${VAR})" ]]; then + DEFAULT_ENABLED_AUTH=yes fi - TMP_PASSWORD=$(mktemp) - ( - cat passwords.json && echo ${PASSWORD_JSON} | jq --argjson "${CONTEXT_INSTANCE}" "${PASSWORD_JSON}" '$ARGS.named' - ) | jq -s add > ${TMP_PASSWORD} && mv ${TMP_PASSWORD} passwords.json + echo "" + echo "This configuration has optional HTTP Basic Authentication, provided by Traefik." + echo "This authentication is applied *before* any other authentication that the application itself may provide." + ${BIN}/confirm ${DEFAULT_ENABLED_AUTH} "Do you want to require Traefik HTTP Basic Authentication" "?" || disable_http_basic_authentication fi +enable_http_basic_authentication diff --git a/cryptpad/.env-dist b/cryptpad/.env-dist index 8ba4b08d..1dae609f 100644 --- a/cryptpad/.env-dist +++ b/cryptpad/.env-dist @@ -1,5 +1,9 @@ +## Check for newer releases (but untested): https://github.com/xwiki-labs/cryptpad/releases CRYPTPAD_VERSION=v5.1.0-nginx + +## Each instance of cryptpad gets its own name. If you only have one, use "default". CRYPTPAD_INSTANCE= + # CryptPad is designed to serve its content over two domains. Account # passwords and cryptographic content is handled on the 'main' domain, # while the user interface is loaded on a 'sandbox' domain which can only @@ -7,6 +11,22 @@ CRYPTPAD_INSTANCE= CRYPTPAD_TRAEFIK_HOST=pad.example.com CRYPTPAD_SANDBOX_DOMAIN=pad-box.example.com +# Filter access by IP address source range (CIDR): +##Disallow all access: 0.0.0.0/32 +##Allow all access: 0.0.0.0/0 +CRYPTPAD_IP_SOURCERANGE=0.0.0.0/0 + +# HTTP Basic Authentication: +# Use `make config` to fill this in properly, or set this to blank to disable. +CRYPTPAD_HTTP_AUTH= + +# By default CryptPad allows remote domains to embed CryptPad documents in iframes. +# This behaviour can be blocked by changing CRYPTPAD_ALLOWED_ORIGINS from "*" to the +# sandbox domain, which must be permitted to load content from the main domain +# in order for CryptPad to work as expected. +#CRYPTPAD_ALLOWED_ORIGINS=https://pad-box.example.com +CRYPTPAD_ALLOWED_ORIGINS=* + ## You need to start cryptpad one time with the default ADMIN_KEY: ## Go to the site, register a new user. ## Go to account settings, find Public Signing Key, and replace it here in ADMIN_KEY: @@ -21,4 +41,4 @@ CRYPTPAD_DEFAULT_STORAGE_LIMIT="1000 * 1024 * 1024" CRYPTPAD_MAX_UPLOAD_SIZE="100 * 1024 * 1024" CRYPTPAD_BLOCK_DAILY_CHECK=true -CRYPTPAD_LOG_LEVEL=warn \ No newline at end of file +CRYPTPAD_LOG_LEVEL=warn diff --git a/cryptpad/Makefile b/cryptpad/Makefile index ec942ed0..5e4999fe 100644 --- a/cryptpad/Makefile +++ b/cryptpad/Makefile @@ -7,9 +7,24 @@ config-hook: @${BIN}/reconfigure ${ENV_FILE} CRYPTPAD_INSTANCE=${instance} @${BIN}/reconfigure_ask ${ENV_FILE} CRYPTPAD_TRAEFIK_HOST "Enter the main cryptpad domain name" pad${INSTANCE_URL_SUFFIX}.${ROOT_DOMAIN} @${BIN}/reconfigure_ask ${ENV_FILE} CRYPTPAD_SANDBOX_DOMAIN "Enter the sandbox domain name" pad-sandbox${INSTANCE_URL_SUFFIX}.${ROOT_DOMAIN} + @${BIN}/reconfigure_htpasswd ${ENV_FILE} CRYPTPAD_HTTP_AUTH default=no @echo "" @docker volume inspect ${PROJECT_INSTANCE}_config >/dev/null && ${BIN}/reconfigure_ask ${ENV_FILE} CRYPTPAD_ADMIN_KEY "Enter your admin account public signing key" "-" && ${BIN}/reconfigure_ask ${ENV_FILE} CRYPTPAD_ADMIN_EMAIL "If you want to offer your support to your users, enter an email address" "-" || echo -e "That's expected, if this is the first time you're installing cryptpad.\nFinish installation and create the admin account:\n make install\n make open # Your browser should open to the application page.\nClick \`Sign up\` and register for a new account.\nOnce logged in, click on your username in the upper right corner, and select \`Settings\`.\nCopy your account's Public Signing Key.\nAfterwards, re-run:\n make config # Enter your Public Signing Key when asked.\n make install\nLogin again and find all the admin functions in the \`Administration\` menu." .PHONY: shell shell: make --no-print-directory docker-compose-lifecycle-cmd EXTRA_ARGS="exec -it cryptpad bash" + +.PHONY: override-hook +override-hook: +#### This sets the override template variables for docker-compose.instance.yaml: +#### The template dynamically renders to docker-compose.override_{DOCKER_CONTEXT}_{INSTANCE}.yaml +#### These settings are used to automatically generate the service container labels, and traefik config, inside the template. +#### The variable arguments have three forms: `=` `=:` `=@` +#### name=VARIABLE_NAME # sets the template 'name' field to the value of VARIABLE_NAME found in the .env file +#### # (this hardcodes the value into docker-compose.override.yaml) +#### name=:VARIABLE_NAME # sets the template 'name' field to the literal string 'VARIABLE_NAME' +#### # (this hardcodes the string into docker-compose.override.yaml) +#### name=@VARIABLE_NAME # sets the template 'name' field to the literal string '${VARIABLE_NAME}' +#### # (used for regular docker-compose expansion of env vars by name.) + @${BIN}/docker_compose_override ${ENV_FILE} project=:cryptpad instance=@CRYPTPAD_INSTANCE traefik_host=@CRYPTPAD_TRAEFIK_HOST cryptpad_sandbox_domain=@CRYPTPAD_SANDBOX_DOMAIN http_auth=CRYPTPAD_HTTP_AUTH http_auth_var=@CRYPTPAD_HTTP_AUTH ip_sourcerange=@CRYPTPAD_IP_SOURCERANGE diff --git a/cryptpad/config/Dockerfile b/cryptpad/config/Dockerfile index 2841421c..bdccc9a1 100644 --- a/cryptpad/config/Dockerfile +++ b/cryptpad/config/Dockerfile @@ -2,6 +2,6 @@ FROM debian:stable-slim RUN apt-get -y update && apt-get install -y openssl gettext WORKDIR /template VOLUME /cryptpad/config -COPY config.template.js setup.sh ./ +COPY config.template.js setup.sh nginx.conf ./ RUN chmod a+x setup.sh CMD ["./setup.sh"] diff --git a/cryptpad/config/nginx.conf b/cryptpad/config/nginx.conf new file mode 100644 index 00000000..e6d164c8 --- /dev/null +++ b/cryptpad/config/nginx.conf @@ -0,0 +1,229 @@ +# This file is included strictly as an example of how Nginx can be configured +# to work with CryptPad. This example WILL NOT WORK AS IS. For best results, +# compare the sections of this configuration file against a working CryptPad +# installation (http server by the Nodejs process). If you are using CryptPad +# in production and require professional support please contact sales@cryptpad.fr + +server { + listen 80 http2; + + # CryptPad serves static assets over these two domains. + # `main_domain` is what users will enter in their address bar. + # Privileged computation such as key management is handled in this scope + # UI content is loaded via the `sandbox_domain`. + # "Content Security Policy" headers prevent content loaded via the sandbox + # from accessing privileged information. + # These variables must be different to take advantage of CryptPad's sandboxing techniques. + # In the event of an XSS vulnerability in CryptPad's front-end code + # this will limit the amount of information accessible to attackers. + set $main_domain "${CRYPTPAD_TRAEFIK_HOST}"; + set $sandbox_domain "${CRYPTPAD_SANDBOX_DOMAIN}"; + + # By default CryptPad allows remote domains to embed CryptPad documents in iframes. + # This behaviour can be blocked by changing $allowed_origins from "*" to the + # sandbox domain, which must be permitted to load content from the main domain + # in order for CryptPad to work as expected. + # + # An example is given below which can be uncommented if you want to block + # remote sites from including content from your server + set $allowed_origins "${CRYPTPAD_ALLOWED_ORIGINS}"; + + # CryptPad's dynamic content (websocket traffic and encrypted blobs) + # can be served over separate domains. Using dedicated domains (or subdomains) + # for these purposes allows you to move them to a separate machine at a later date + # if you find that a single machine cannot handle all of your users. + # If you don't use dedicated domains, this can be the same as $main_domain + # If you do, they can be added as exceptions to any rules which block connections to remote domains. + # You can find these variables referenced below in the relevant places + set $api_domain "api.${CRYPTPAD_TRAEFIK_HOST}"; + set $files_domain "files.${CRYPTPAD_TRAEFIK_HOST}"; + + # nginx doesn't let you set server_name via variables, so you need to hardcode your domains here + server_name ${CRYPTPAD_TRAEFIK_HOST} ${CRYPTPAD_SANDBOX_DOMAIN}; + + # replace with the IP address of your resolver + resolver 8.8.8.8 8.8.4.4 1.1.1.1 1.0.0.1 9.9.9.9 149.112.112.112 208.67.222.222 208.67.220.220; + + add_header X-XSS-Protection "1; mode=block"; + add_header X-Content-Type-Options nosniff; + add_header Access-Control-Allow-Origin "${allowed_origins}"; + # add_header X-Frame-Options "SAMEORIGIN"; + + # Opt out of Google's FLoC Network + add_header Permissions-Policy interest-cohort=(); + + # Enable SharedArrayBuffer in Firefox (for .xlsx export) + add_header Cross-Origin-Resource-Policy cross-origin; + add_header Cross-Origin-Embedder-Policy require-corp; + + # Insert the path to your CryptPad repository root here + root /home/cryptpad/cryptpad; + index index.html; + error_page 404 /customize.dist/404.html; + + # any static assets loaded with "ver=" in their URL will be cached for a year + if ($args ~ ver=) { + set $cacheControl max-age=31536000; + } + # This rule overrides the above caching directive and makes things somewhat less efficient. + # We had inverted them as an optimization, but Safari 16 introduced a bug that interpreted + # some important headers incorrectly when loading these files from cache. + # This is why we can't have nice things :( + if ($uri ~ ^(\/|.*\/|.*\.html)$) { + set $cacheControl no-cache; + } + + # Will not set any header if it is emptystring + add_header Cache-Control $cacheControl; + + # CSS can be dynamically set inline, loaded from the same domain, or from $main_domain + set $styleSrc "'unsafe-inline' 'self' https://${main_domain}"; + + # connect-src restricts URLs which can be loaded using script interfaces + # if you have configured your instance to use a dedicated $files_domain or $api_domain + # you will need to add them below as: https://${files_domain} and https://${api_domain} + set $connectSrc "'self' https://${main_domain} blob: wss://${api_domain} https://${sandbox_domain}"; + + # fonts can be loaded from data-URLs or the main domain + set $fontSrc "'self' data: https://${main_domain}"; + + # images can be loaded from anywhere, though we'd like to deprecate this as it allows the use of images for tracking + set $imgSrc "'self' data: blob: https://${main_domain}"; + + # frame-src specifies valid sources for nested browsing contexts. + # this prevents loading any iframes from anywhere other than the sandbox domain + set $frameSrc "'self' https://${sandbox_domain} blob:"; + + # specifies valid sources for loading media using video or audio + set $mediaSrc "blob:"; + + # defines valid sources for webworkers and nested browser contexts + # deprecated in favour of worker-src and frame-src + set $childSrc "https://${main_domain}"; + + # specifies valid sources for Worker, SharedWorker, or ServiceWorker scripts. + # supercedes child-src but is unfortunately not yet universally supported. + set $workerSrc "'self'"; + + # script-src specifies valid sources for javascript, including inline handlers + set $scriptSrc "'self' resource: https://${main_domain}"; + + # frame-ancestors specifies which origins can embed your CryptPad instance + # this must include 'self' and your main domain (over HTTPS) in order for CryptPad to work + # if you have enabled remote embedding via the admin panel then this must be more permissive. + # note: cryptpad.fr permits web pages served via https: and vector: (element desktop app) + set $frameAncestors "'self' https://${main_domain}"; + # set $frameAncestors "'self' https: vector:"; + + set $unsafe 0; + # the following assets are loaded via the sandbox domain + # they unfortunately still require exceptions to the sandboxing to work correctly. + if ($uri ~ ^\/(sheet|doc|presentation)\/inner.html.*$) { set $unsafe 1; } + if ($uri ~ ^\/common\/onlyoffice\/.*\/.*\.html.*$) { set $unsafe 1; } + + # everything except the sandbox domain is a privileged scope, as they might be used to handle keys + if ($host != $sandbox_domain) { set $unsafe 0; } + # this iframe is an exception. Office file formats are converted outside of the sandboxed scope + # because of bugs in Chromium-based browsers that incorrectly ignore headers that are supposed to enable + # the use of some modern APIs that we require when javascript is run in a cross-origin context. + # We've applied other sandboxing techniques to mitigate the risk of running WebAssembly in this privileged scope + if ($uri ~ ^\/unsafeiframe\/inner\.html.*$) { set $unsafe 1; } + + # privileged contexts allow a few more rights than unprivileged contexts, though limits are still applied + if ($unsafe) { + set $scriptSrc "'self' 'unsafe-eval' 'unsafe-inline' resource: https://${main_domain}"; + } + + # Finally, set all the rules you composed above. + add_header Content-Security-Policy "default-src 'none'; child-src $childSrc; worker-src $workerSrc; media-src $mediaSrc; style-src $styleSrc; script-src $scriptSrc; connect-src $connectSrc; font-src $fontSrc; img-src $imgSrc; frame-src $frameSrc; frame-ancestors $frameAncestors"; + + # The nodejs process can handle all traffic whether accessed over websocket or as static assets + # We prefer to serve static content from nginx directly and to leave the API server to handle + # the dynamic content that only it can manage. This is primarily an optimization + location ^~ /cryptpad_websocket { + proxy_pass http://localhost:3000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # WebSocket support (nginx 1.4) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + } + + location ^~ /customize.dist/ { + # This is needed in order to prevent infinite recursion between /customize/ and the root + } + # try to load customizeable content via /customize/ and fall back to the default content + # located at /customize.dist/ + # This is what allows you to override behaviour. + location ^~ /customize/ { + rewrite ^/customize/(.*)$ $1 break; + try_files /customize/$uri /customize.dist/$uri; + } + + # /api/config is loaded once per page load and is used to retrieve + # the caching variable which is applied to every other resource + # which is loaded during that session. + location ~ ^/api/.*$ { + proxy_pass http://localhost:3000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # These settings prevent both NGINX and the API server + # from setting the same headers and creating duplicates + proxy_hide_header Cross-Origin-Resource-Policy; + add_header Cross-Origin-Resource-Policy cross-origin; + proxy_hide_header Cross-Origin-Embedder-Policy; + add_header Cross-Origin-Embedder-Policy require-corp; + } + + # encrypted blobs are immutable and are thus cached for a year + location ^~ /blob/ { + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' "${allowed_origins}"; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'application/octet-stream; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + add_header X-Content-Type-Options nosniff; + add_header Cache-Control max-age=31536000; + add_header 'Access-Control-Allow-Origin' "${allowed_origins}"; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,Content-Length'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,Content-Length'; + try_files $uri =404; + } + + # the "block-store" serves encrypted payloads containing users' drive keys + # these payloads are unlocked via login credentials. They are mutable + # and are thus never cached. They're small enough that it doesn't matter, in any case. + location ^~ /block/ { + add_header X-Content-Type-Options nosniff; + add_header Cache-Control max-age=0; + try_files $uri =404; + } + + # This block provides an alternative means of loading content + # otherwise only served via websocket. This is solely for debugging purposes, + # and is thus not allowed by default. + #location ^~ /datastore/ { + #add_header Cache-Control max-age=0; + #try_files $uri =404; + #} + + # The nodejs server has some built-in forwarding rules to prevent + # URLs like /pad from resulting in a 404. This simply adds a trailing slash + # to a variety of applications. + location ~ ^/(register|login|settings|user|pad|drive|poll|slide|code|whiteboard|file|media|profile|contacts|todo|filepicker|debug|kanban|sheet|support|admin|notifications|teams|calendar|presentation|doc|form|report|convert|checkup)$ { + rewrite ^(.*)$ $1/ redirect; + } + + # Finally, serve anything the above exceptions don't govern. + try_files /customize/www/$uri /customize/www/$uri/index.html /www/$uri /www/$uri/index.html /customize/$uri; +} diff --git a/cryptpad/config/setup.sh b/cryptpad/config/setup.sh index d3af57e9..9d20c4af 100644 --- a/cryptpad/config/setup.sh +++ b/cryptpad/config/setup.sh @@ -1,16 +1,24 @@ #!/bin/bash CONFIG_DIR=/cryptpad/config +set -e create_config() { TEMPLATE=/template/config.template.js CONFIG=${CONFIG_DIR}/config.js + NGINX_TEMPLATE=/template/nginx.conf + NGINX_CONFIG=${CONFIG_DIR}/nginx.conf mkdir -p ${CONFIG_DIR} cat ${TEMPLATE} | envsubst > ${CONFIG} echo "[ ! ] GENERATED NEW CONFIG FILE ::: ${CONFIG}" [[ $PRINT_CONFIG == true ]] && cat ${CONFIG} + + cat ${NGINX_TEMPLATE} | envsubst '${CRYPTPAD_TRAEFIK_HOST} ${CRYPTPAD_SANDBOX_DOMAIN} ${CRYPTPAD_ALLOWED_ORIGINS}' > ${NGINX_CONFIG} + echo "[ ! ] GENERATED NEW CONFIG FILE ::: ${NGINX_CONFIG}" + [[ $PRINT_CONFIG == true ]] && cat ${NGINX_CONFIG} } create_config +exit 0 diff --git a/cryptpad/cryptpad/Dockerfile b/cryptpad/cryptpad/Dockerfile index 9eb200af..647c2728 100644 --- a/cryptpad/cryptpad/Dockerfile +++ b/cryptpad/cryptpad/Dockerfile @@ -1,5 +1,4 @@ ARG CRYPTPAD_VERSION FROM promasu/cryptpad:${CRYPTPAD_VERSION} -ENTRYPOINT ["/bin/sh", "/docker-entrypoint.sh"] -ADD start_cryptpad.sh /bin/start_cryptpad.sh -RUN chmod a+x /bin/start_cryptpad.sh +RUN ln -s /cryptpad/config/nginx.conf /etc/nginx/conf.d/cryptpad.conf +EXPOSE 80 diff --git a/cryptpad/cryptpad/start_cryptpad.sh b/cryptpad/cryptpad/start_cryptpad.sh deleted file mode 100644 index 841ba543..00000000 --- a/cryptpad/cryptpad/start_cryptpad.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -wait_for_file() { - [ ! -f $1 ] && echo "Waiting for $1 to exist ..." - until [ -f $1 ]; do sleep 1; done -} - -wait_for_file /cryptpad/config/config.js -/usr/bin/supervisord -n -c /etc/supervisord.conf diff --git a/cryptpad/docker-compose.instance.yaml b/cryptpad/docker-compose.instance.yaml new file mode 100644 index 00000000..90ed2741 --- /dev/null +++ b/cryptpad/docker-compose.instance.yaml @@ -0,0 +1,47 @@ +#! This is a ytt template file for docker-compose.override.yaml +#! References: +#! https://carvel.dev/ytt +#! https://docs.docker.com/compose/extends/#adding-and-overriding-configuration +#! https://github.com/enigmacurry/d.rymcg.tech#overriding-docker-composeyaml-per-instance + +#! ### Standard project vars: +#@ load("@ytt:data", "data") +#@ project = data.values.project +#@ instance = data.values.instance +#@ context = data.values.context +#@ traefik_host = data.values.traefik_host +#@ ip_sourcerange = data.values.ip_sourcerange +#@ enable_http_auth = len(data.values.http_auth.strip()) > 0 +#@ http_auth = data.values.http_auth_var +#@ enabled_middlewares = [] + +#! ### Cryptpad specific vars: +#@ cryptpad_sandbox_host = data.values.traefik_host + +#@yaml/text-templated-strings +services: + cryptpad: + #@ service = "cryptpad" + labels: + #! Services must opt-in to be proxied by Traefik: + - "traefik.enable=true" + #! 'router' is the fully qualified key in traefik for this router/service: project + instance + service + #@ router = "{}-{}-{}".format(project,instance,service) + #! The host matching router rule: + - "traefik.http.routers.(@= router @).rule=Host(`(@= traefik_host @)`, `(@= cryptpad_sandbox_host @)`)" + - "traefik.http.routers.(@= router @).entrypoints=websecure" + #@ enabled_middlewares.append("{}-ipwhitelist".format(router)) + - "traefik.http.middlewares.(@= router @)-ipwhitelist.ipwhitelist.sourcerange=(@= ip_sourcerange @)" + #@ if enable_http_auth: + #@ enabled_middlewares.append("{}-basicauth".format(router)) + - "traefik.http.middlewares.(@= router @)-basicauth.basicauth.users=(@= http_auth @)" + #@ end + + #! Rewrite CORS headers - For some reason cryptpad is duplicating the CORS headers + #! This makes chromium browsers not load, so this rewrite rule de-duplicates the headers + #! (actually the value was cross-origin, which seems wrong to me, same-site seems safer) + - traefik.http.middlewares.(@= router @)-headers.headers.customresponseheaders.Cross-Origin-Resource-Policy=same-site + #@ enabled_middlewares.append("{}-headers".format(router)) + + #! Apply all middlewares (do this at the end!) + - "traefik.http.routers.(@= router @).middlewares=(@= ','.join(enabled_middlewares) @)" diff --git a/cryptpad/docker-compose.yaml b/cryptpad/docker-compose.yaml index 4d0f3269..0967995f 100644 --- a/cryptpad/docker-compose.yaml +++ b/cryptpad/docker-compose.yaml @@ -15,6 +15,9 @@ services: security_opt: - no-new-privileges:true environment: + - CRYPTPAD_TRAEFIK_HOST + - CRYPTPAD_SANDBOX_DOMAIN + - CRYPTPAD_ALLOWED_ORIGINS - CPAD_MAIN_DOMAIN=${CRYPTPAD_TRAEFIK_HOST} - CPAD_SANDBOX_DOMAIN=${CRYPTPAD_SANDBOX_DOMAIN} - ADMIN_KEY=${CRYPTPAD_ADMIN_KEY} @@ -33,12 +36,12 @@ services: context: cryptpad args: CRYPTPAD_VERSION: ${CRYPTPAD_VERSION} + #image: promasu/cryptpad:${CRYPTPAD_VERSION} restart: unless-stopped - command: ['start_cryptpad.sh'] environment: - CPAD_MAIN_DOMAIN=${CRYPTPAD_TRAEFIK_HOST} - CPAD_SANDBOX_DOMAIN=${CRYPTPAD_SANDBOX_DOMAIN} - # Traefik can't use HTTP2 to communicate with cryptpat_websocket + # Traefik can't use HTTP2 to communicate with cryptpad_websocket # A workaround is disabling HTTP2 in Nginx - CPAD_HTTP2_DISABLE=true - CPAD_REALIP_RECURSIVE=on @@ -53,68 +56,18 @@ services: - config:/cryptpad/config - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro - labels: - ## Enable traefik - - traefik.enable=true - ## Declare service - ## Change the port if you enabled TLS - - traefik.http.services.cryptpad-${CRYPTPAD_INSTANCE:-default}.loadbalancer.server.port=80 - # HTTPS router rules - - traefik.http.routers.cryptpad-${CRYPTPAD_INSTANCE:-default}-https.entrypoints=websecure - - traefik.http.routers.cryptpad-${CRYPTPAD_INSTANCE:-default}-https.rule=Host(`${CRYPTPAD_TRAEFIK_HOST}`, `${CRYPTPAD_SANDBOX_DOMAIN}`) - - traefik.http.routers.cryptpad-${CRYPTPAD_INSTANCE:-default}-https.service=cryptpad-${CRYPTPAD_INSTANCE:-default} - # Rewrite CORS headers - For some reason cryptpad is duplicating the CORS headers - # This makes chromium browsers not load, so this rewrite rule de-duplicates the headers - # (actually the value was cross-origin, which seems wrong to me, same-site seems safer) - - traefik.http.middlewares.cryptpad-${CRYPTPAD_INSTANCE:-default}-headers.headers.customresponseheaders.Cross-Origin-Resource-Policy=same-site - - traefik.http.routers.cryptpad-${CRYPTPAD_INSTANCE:-default}-https.middlewares=cryptpad-${CRYPTPAD_INSTANCE:-default}-headers@docker + ## All labels defined in docker-compose.instance.yaml + labels: [] ulimits: nofile: soft: 1000000 hard: 1000000 - security_opt: - - no-new-privileges:true - cap_drop: - - ALL - cap_add: - - SETGID - - SETUID - - CHOWN - ### DAC_OVERRIDE needed to open /var/log/nginx/error.log - ### However, nginx is set to log errors to stderr, so this is not needed? - # - DAC_OVERRIDE - ### Unused capabilities: - # - SYS_CHROOT - # - AUDIT_WRITE - # - FOWNER - # - AUDIT_CONTROL - # - AUDIT_READ - # - BLOCK_SUSPEND - # - DAC_READ_SEARCH - # - FSETID - # - IPC_LOCK - # - IPC_OWNER - # - KILL - # - LEASE - # - LINUX_IMMUTABLE - # - MAC_ADMIN - # - MAC_OVERRIDE - # - MKNOD - # - NET_ADMIN - # - NET_BIND_SERVICE - # - NET_BROADCAST - # - NET_RAW - # - SETFCAP - # - SETPCAP - # - SYS_ADMIN - # - SYS_BOOT - # - SYSLOG - # - SYS_MODULE - # - SYS_NICE - # - SYS_PACCT - # - SYS_PTRACE - # - SYS_RAWIO - # - SYS_RESOURCE - # - SYS_TIME - # - SYS_TTY_CONFIG - # - WAKE_ALARM + # security_opt: + # - no-new-privileges:true + # cap_drop: + # - ALL + # cap_add: + # - SETGID + # - SETUID + # - CHOWN + # - DAC_OVERRIDE diff --git a/gitea/.env-dist b/gitea/.env-dist index 69ab1dd4..a05c750c 100644 --- a/gitea/.env-dist +++ b/gitea/.env-dist @@ -1,6 +1,14 @@ GITEA_TRAEFIK_HOST=git.example.com GITEA_SSH_PORT=2222 +# The name of this instance. If there is only one instance, use 'default'. +GITEA_INSTANCE= + +# Filter access by IP address source range (CIDR): +##Disallow all access: 0.0.0.0/32 +##Allow all access: 0.0.0.0/0 +GITEA_IP_SOURCERANGE=0.0.0.0/0 + ## Gitea environment config overrides:: ## See https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini ## See https://docs.gitea.io/en-us/config-cheat-sheet/ @@ -9,6 +17,7 @@ GITEA_SSH_PORT=2222 ## Note: The *final* configuration for gitea is always the ini file: /etc/gitea/app.ini ## docker exec -it gitea cat /etc/gitea/app.ini APP_NAME="git thing" +GITEA__server__ROOT_URL= GITEA__server__DISABLE_SSH=false GITEA__service__DISABLE_REGISTRATION=true GITEA__service__REQUIRE_SIGNIN_VIEW=true diff --git a/gitea/Makefile b/gitea/Makefile index 34c95251..82bc58cc 100644 --- a/gitea/Makefile +++ b/gitea/Makefile @@ -5,4 +5,7 @@ include ${ROOT_DIR}/_scripts/Makefile.instance .PHONY: config-hook config-hook: @${BIN}/reconfigure_ask ${ENV_FILE} GITEA_TRAEFIK_HOST "Enter your gitea domain name" git${INSTANCE_URL_SUFFIX}.${ROOT_DOMAIN} - @${BIN}/reconfigure_ask ${ENV_FILE} APP_NAME "Enter the service description" "git ${instance}" + @${BIN}/reconfigure ${ENV_FILE} GITEA_INSTANCE=$${instance:-default} + @${BIN}/reconfigure_ask ${ENV_FILE} APP_NAME "Enter the service description" "$${instance:-default} - git hosting" + @PUBLIC_HTTPS_PORT="$$(${BIN}/dotenv -f ${ROOT_DIR}/${ROOT_ENV} get PUBLIC_HTTPS_PORT)"; test -n "$${PUBLIC_HTTPS_PORT}" && ${BIN}/reconfigure ${ENV_FILE} "GITEA__server__ROOT_URL=https://$$(${BIN}/dotenv -f ${ENV_FILE} get GITEA_TRAEFIK_HOST):$${PUBLIC_HTTPS_PORT}" || ${BIN}/reconfigure ${ENV_FILE} "GITEA__server__ROOT_URL=https://$$(${BIN}/dotenv -f ${ENV_FILE} get GITEA_TRAEFIK_HOST)" + diff --git a/gitea/docker-compose.yaml b/gitea/docker-compose.yaml index c79bccdd..b09ae681 100644 --- a/gitea/docker-compose.yaml +++ b/gitea/docker-compose.yaml @@ -6,7 +6,7 @@ services: image: gitea/gitea:1-rootless restart: always environment: - - GITEA__server__ROOT_URL=https://${GITEA_TRAEFIK_HOST} + - GITEA__server__ROOT_URL - GITEA__server__SSH_DOMAIN=${GITEA_TRAEFIK_HOST} - GITEA__server__SSH_PORT=${GITEA_SSH_PORT} - GITEA__server__SSH_LISTEN_PORT=22 @@ -40,6 +40,8 @@ services: - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-web.entrypoints=websecure" - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-web.service=gitea-${GITEA_INSTANCE:-default}-web" - "traefik.http.services.gitea-${GITEA_INSTANCE:-default}-web.loadbalancer.server.port=3000" + - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-ipwhitelist.ipwhitelist.sourcerange=${GITEA_IP_SOURCERANGE}" + - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-web.middlewares=gitea-${GITEA_INSTANCE:-default}-ipwhitelist" ## SSH - "traefik.tcp.routers.gitea-${GITEA_INSTANCE:-default}-ssh.rule=HostSNI(`*`)" - "traefik.tcp.routers.gitea-${GITEA_INSTANCE:-default}-ssh.entrypoints=ssh" @@ -51,14 +53,14 @@ services: - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-logout1.headers.customresponseheaders.Set-Cookie=gitea_incredible=deleted; Max-Age=0" - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-logout-redirect1.redirectregex.regex=.*" - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-logout-redirect1.redirectregex.replacement=https://${GITEA_TRAEFIK_HOST}/logout2" - - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-logout1.middlewares=gitea-${GITEA_INSTANCE:-default}-logout1,gitea-${GITEA_INSTANCE:-default}-logout-redirect1" + - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-logout1.middlewares=gitea-${GITEA_INSTANCE:-default}-ipwhitelist,gitea-${GITEA_INSTANCE:-default}-logout1,gitea-${GITEA_INSTANCE:-default}-logout-redirect1" ## Logout phase 2 (deletes 'i_like_gitea' cookie): - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-logout2.rule=Host(`${GITEA_TRAEFIK_HOST}`) && Path(`/logout2`)" - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-logout2.entrypoints=websecure" - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-logout2.headers.customresponseheaders.Set-Cookie=i_like_gitea=deleted; Max-Age=0" - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-logout-redirect2.redirectregex.regex=.*" - "traefik.http.middlewares.gitea-${GITEA_INSTANCE:-default}-logout-redirect2.redirectregex.replacement=https://${GITEA_TRAEFIK_HOST}/user/login" - - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-logout2.middlewares=gitea-${GITEA_INSTANCE:-default}-logout2,gitea-${GITEA_INSTANCE:-default}-logout-redirect2" + - "traefik.http.routers.gitea-${GITEA_INSTANCE:-default}-logout2.middlewares=gitea-${GITEA_INSTANCE:-default}-ipwhitelist,gitea-${GITEA_INSTANCE:-default}-logout2,gitea-${GITEA_INSTANCE:-default}-logout-redirect2" volumes: diff --git a/whoami/.env-dist b/whoami/.env-dist index c9378378..b6817b43 100644 --- a/whoami/.env-dist +++ b/whoami/.env-dist @@ -1,13 +1,19 @@ # The domain name for the whoami service: WHOAMI_TRAEFIK_HOST=whoami.example.com -# The whoami instance name (shown in all responses) -WHOAMI_NAME= + +# The name of this instance. If there is only one instance, use 'default'. +WHOAMI_INSTANCE= + # Filter access by IP address source range (CIDR): -##Disallow all access: -#WHOAMI_IP_SOURCERANGE="0.0.0.0/32" -##Allow all access: -WHOAMI_IP_SOURCERANGE="0.0.0.0/0" +##Disallow all access: 0.0.0.0/32 +##Allow all access: 0.0.0.0/0 +WHOAMI_IP_SOURCERANGE=0.0.0.0/0 +# HTTP Basic Authentication: +# Use `make config` to fill this in properly, or set this to blank to disable. +WHOAMI_HTTP_AUTH= + +## You can run the whoami service as any user/group: WHOAMI_UID=54321 WHOAMI_GID=54321 -WHOAMI_INSTANCE= \ No newline at end of file + diff --git a/whoami/Makefile b/whoami/Makefile index 05402d8b..1481b467 100644 --- a/whoami/Makefile +++ b/whoami/Makefile @@ -4,6 +4,24 @@ include ${ROOT_DIR}/_scripts/Makefile.instance .PHONY: config-hook config-hook: +#### This interactive configuration wizard creates the .env_{DOCKER_CONTEXT}_{INSTANCE} config file using .env-dist as the template: +#### reconfigure_ask asks the user a question to set the variable into the .env file, and with a provided default value. +#### reconfigure sets the value of a variable in the .env file without asking. +#### reconfigure_htpasswd will configure the HTTP Basic Authentication setting the var name and with a provided default value. @${BIN}/reconfigure_ask ${ENV_FILE} WHOAMI_TRAEFIK_HOST "Enter the whoami domain name" whoami${INSTANCE_URL_SUFFIX}.${ROOT_DOMAIN} - @${BIN}/reconfigure_ask ${ENV_FILE} WHOAMI_NAME "Enter a unique name to display in all responses" $${instance:-whoami} @${BIN}/reconfigure ${ENV_FILE} WHOAMI_INSTANCE=$${instance:-default} + @${BIN}/reconfigure_htpasswd ${ENV_FILE} WHOAMI_HTTP_AUTH default=no + +.PHONY: override-hook +override-hook: +#### This sets the override template variables for docker-compose.instance.yaml: +#### The template dynamically renders to docker-compose.override_{DOCKER_CONTEXT}_{INSTANCE}.yaml +#### These settings are used to automatically generate the service container labels, and traefik config, inside the template. +#### The variable arguments have three forms: `=` `=:` `=@` +#### name=VARIABLE_NAME # sets the template 'name' field to the value of VARIABLE_NAME found in the .env file +#### # (this hardcodes the value into docker-compose.override.yaml) +#### name=:VARIABLE_NAME # sets the template 'name' field to the literal string 'VARIABLE_NAME' +#### # (this hardcodes the string into docker-compose.override.yaml) +#### name=@VARIABLE_NAME # sets the template 'name' field to the literal string '${VARIABLE_NAME}' +#### # (used for regular docker-compose expansion of env vars by name.) + @${BIN}/docker_compose_override ${ENV_FILE} project=:whoami instance=@WHOAMI_INSTANCE traefik_host=@WHOAMI_TRAEFIK_HOST http_auth=WHOAMI_HTTP_AUTH http_auth_var=@WHOAMI_HTTP_AUTH ip_sourcerange=@WHOAMI_IP_SOURCERANGE diff --git a/whoami/README.md b/whoami/README.md index 0785a1f5..d5e3db2c 100644 --- a/whoami/README.md +++ b/whoami/README.md @@ -1,10 +1,8 @@ # whoami -[whoami](https://github.com/traefik/whoami) is a tiny Go webserver that prints -os information and HTTP request to output. - -The [docker-compose.yaml](docker-compose.yaml) contains several examples of -Traefik middleware, and can be used as a template for other services. +[whoami](https://github.com/traefik/whoami) is a tiny Go webserver +that prints os information and HTTP request to output. It is useful as +a basic deployment and connectivity test. ## Config @@ -12,24 +10,32 @@ Traefik middleware, and can be used as a template for other services. make config ``` -Or you can just edit `.env_${DOCKER_CONTEXT}_default` directly, set -`WHOAMI_TRAEFIK_HOST` to the domain you want to host the `whoami` -service on. - +This will ask you to enter the domain name to use, and whether or not +you want to configure a username/password via HTTP Basic +Authentication. It automatically saves your responses into the +configuration file `.env_{DOCKER_CONTEXT}`. -## Start +## Install ``` make install ``` -Or you can just run `docker-compose up -d` - -## Stop +## Open ``` -make stop +make open ``` -Or you can just run `docker-compose down` +This will automatically open the page in your web browser, and will +prefill the password if you enabled it (and chose to store it in +`passwords.json`). + +## Destroy + +``` +make destroy +``` +This completely removes the container (and would also delete all its +volumes; but `whoami` hasn't got any data to store.) diff --git a/whoami/docker-compose.instance.yaml b/whoami/docker-compose.instance.yaml new file mode 100644 index 00000000..386408c1 --- /dev/null +++ b/whoami/docker-compose.instance.yaml @@ -0,0 +1,46 @@ +#! This is a ytt template file for docker-compose.override.yaml +#! References: +#! https://carvel.dev/ytt +#! https://docs.docker.com/compose/extends/#adding-and-overriding-configuration +#! https://github.com/enigmacurry/d.rymcg.tech#overriding-docker-composeyaml-per-instance + +#! ### Standard project vars: +#@ load("@ytt:data", "data") +#@ project = data.values.project +#@ instance = data.values.instance +#@ context = data.values.context +#@ traefik_host = data.values.traefik_host +#@ ip_sourcerange = data.values.ip_sourcerange +#@ enable_http_auth = len(data.values.http_auth.strip()) > 0 +#@ http_auth = data.values.http_auth_var +#@ enabled_middlewares = [] + +#@yaml/text-templated-strings +services: + whoami: + #@ service = "whoami" + labels: + #! Services must opt-in to be proxied by Traefik: + - "traefik.enable=true" + + #! 'router' is the fully qualified key in traefik for this router/service: project + instance + service + #@ router = "{}-{}-{}".format(project,instance,service) + + #! The host matching router rule: + - "traefik.http.routers.(@= router @).rule=Host(`(@= traefik_host @)`)" + - "traefik.http.routers.(@= router @).entrypoints=websecure" + #@ enabled_middlewares.append("{}-ipwhitelist".format(router)) + - "traefik.http.middlewares.(@= router @)-ipwhitelist.ipwhitelist.sourcerange=(@= ip_sourcerange @)" + + #@ if enable_http_auth: + #@ enabled_middlewares.append("{}-basicauth".format(router)) + - "traefik.http.middlewares.(@= router @)-basicauth.basicauth.users=(@= http_auth @)" + #@ end + + #! Override the default port that whoami binds to, so that it lives in userspace >1024: + #! You don't normally need to do this, as long as your image has + #! an EXPOSE directive in it, Traefik will autodetect it, but this is how you can override it: + - "traefik.http.services.(@= router @).loadbalancer.server.port=8000" + + #! Apply all middlewares (do this at the end!) + - "traefik.http.routers.(@= router @).middlewares=(@= ','.join(enabled_middlewares) @)" diff --git a/whoami/docker-compose.yaml b/whoami/docker-compose.yaml index a3c6c715..ef75cebd 100644 --- a/whoami/docker-compose.yaml +++ b/whoami/docker-compose.yaml @@ -9,44 +9,10 @@ services: - ALL sysctls: - net.ipv4.ip_unprivileged_port_start=1024 - command: --port 8000 --name ${WHOAMI_NAME:-${WHOAMI_TRAEFIK_HOST}} + command: --port 8000 --name ${WHOAMI_INSTANCE:-default} user: ${WHOAMI_UID}:${WHOAMI_GID} restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.services.whoami-${WHOAMI_INSTANCE:-default}.loadbalancer.server.port=8000" - - ### Normally whoami responds on any path with any HTTP method, but if we - ### make all the routers path and/or method selective, then we can block - ### certain requests easily by default (404). - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}.rule=Host(`${WHOAMI_TRAEFIK_HOST}`) && (Path(`/`) || Path(`/health`))" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}.entrypoints=websecure" - - "traefik.http.middlewares.whoami-${WHOAMI_INSTANCE:-default}-whitelist.ipwhitelist.sourcerange=${WHOAMI_IP_SOURCERANGE}" - #- "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}.middlewares=whoami-whitelist,geoip@file" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}.middlewares=whoami-${WHOAMI_INSTANCE:-default}-whitelist" - - ## Lots of various rules for demo/documentation purposes.. - - # Block /forbidden (403 Forbidden; Using blockpath plugin middleware with catch all regex): - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-forbidden.rule=Host(`${WHOAMI_TRAEFIK_HOST}`) && PathPrefix(`/forbidden`)" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-forbidden.entrypoints=websecure" - - "traefik.http.middlewares.whoami-${WHOAMI_INSTANCE:-default}-forbidden.plugin.blockpath.regex=.*" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-forbidden.middlewares=whoami-${WHOAMI_INSTANCE:-default}-forbidden" - - # Allow /just-get (but only to GET requests): - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-just-get.rule=Host(`${WHOAMI_TRAEFIK_HOST}`) && Path(`/just-get`) && Method(`GET`)" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-just-get.entrypoints=websecure" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-just-get.middlewares=whoami-${WHOAMI_INSTANCE:-default}-whitelist" - - # Allow /test/ and add custom request and response headers: - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-test.rule=Host(`${WHOAMI_TRAEFIK_HOST}`) && PathPrefix(`/test/`)" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-test.entrypoints=websecure" - - "traefik.http.middlewares.testHeader.headers.customrequestheaders.X-Script-Name=testing-123" - - "traefik.http.middlewares.testHeader.headers.customresponseheaders.X-Custom-Response-Header=yeppp" - - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-test.middlewares=testHeader,whoami-${WHOAMI_INSTANCE:-default}-whitelist" - - # Test forward authentication on secondary domain: auth.whoami.example.com - # - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-auth.rule=Host(`auth.${WHOAMI_TRAEFIK_HOST}`)" - # - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-auth.entrypoints=websecure" - # - "traefik.http.routers.whoami-${WHOAMI_INSTANCE:-default}-auth.middlewares=traefik-forward-auth" - + # All labels are defined in the template: docker-compose.instance.yaml + # The labels will merge together here from the template output: + # docker-compose.override_{DOCKER_CONTEXT}_{INSTANCE}.yaml + labels: []