From e44c3789465b38ab9ce9e7352c666a03d5797ec2 Mon Sep 17 00:00:00 2001 From: iAlex97 Date: Thu, 24 Feb 2022 23:07:58 +0200 Subject: [PATCH 01/96] Fix/gb us postcode import (#1) * use gzipped csv instead of sql dumps for uk and us postcodes * add test case for GB postcode import --- .github/workflows/ci.yml | 13 +++++++++++++ 4.0/init.sh | 8 ++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e5227b..9c9aeb8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,6 +154,19 @@ jobs: sleep 35 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" + - name: Check GB postcode import + working-directory: .github/workflows + run: |- + docker run -i --rm \ + -e PBF_URL=https://download.geofabrik.de/europe/great-britain/england/rutland-latest.osm.pbf \ + -e IMPORT_GB_POSTCODES="true" \ + -p 8009:8080 \ + nominatim & + sleep 345 + ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=LE15+8TX" + ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+3SY" + ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+4ES" + - name: Login to DockerHub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} uses: docker/login-action@v1 diff --git a/4.0/init.sh b/4.0/init.sh index f4b73d19..2926e387 100755 --- a/4.0/init.sh +++ b/4.0/init.sh @@ -13,19 +13,19 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - curl https://nominatim.org/data/gb_postcode_data.sql.gz -L -o ${PROJECT_DIR}/gb_postcode_data.sql.gz + curl https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked - ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcode_data.sql.gz + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz else \ echo "Skipping optional GB postcode import" fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - curl https://nominatim.org/data/us_postcode_data.sql.gz -L -o ${PROJECT_DIR}/us_postcode_data.sql.gz + curl https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked - ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcode_data.sql.gz + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz else echo "Skipping optional US postcode import" fi; From 2e8428ac4bb1c21a3e01fe1cdf8a13e67c174a44 Mon Sep 17 00:00:00 2001 From: ialex Date: Fri, 25 Feb 2022 14:05:54 +0200 Subject: [PATCH 02/96] delete gzip after import --- 4.0/init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/4.0/init.sh b/4.0/init.sh index 2926e387..2ce17f29 100755 --- a/4.0/init.sh +++ b/4.0/init.sh @@ -101,6 +101,7 @@ rm /etc/postgresql/12/main/conf.d/postgres-import.conf echo "Deleting downloaded dumps in ${PROJECT_DIR}" rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz # nominatim needs the tokenizer configuration in the project directory to start up From 9fad7f2172ddb6f71d8f555abfaaddefa55ab974 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Fri, 25 Feb 2022 19:21:06 +0100 Subject: [PATCH 03/96] add iAlex97 as contributor [skip ci] https://github.com/mediagis/nominatim-docker/pull/301 --- .all-contributorsrc | 10 ++++++++++ README.md | 1 + 2 files changed, 11 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 2d145480..21b53fbd 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -380,6 +380,16 @@ "code", "doc" ] + }, + { + "login": "iAlex97", + "name": "iAlex97", + "avatar_url": "https://avatars.githubusercontent.com/u/12383594?v=4", + "profile": "https://github.com/iAlex97", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 6, diff --git a/README.md b/README.md index 0c5192b2..8c585f96 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
norcis
💻
SClo
💻 📖
Chris
💻 📖 +
iAlex97
💻 ⚠️ From 3dc4d6665012b1149310e4c386ad1955f1f9dad0 Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Thu, 17 Mar 2022 15:45:56 +0000 Subject: [PATCH 04/96] #245 Pass environment variables to all sudo commands Not just "nominatim import" This enables passing NOMINATIM_DATABASE_DSN --- 4.0/init.sh | 18 +++++++++--------- 4.0/start.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/4.0/init.sh b/4.0/init.sh index 2ce17f29..96183af2 100755 --- a/4.0/init.sh +++ b/4.0/init.sh @@ -57,13 +57,13 @@ if [ ! -f /var/lib/postgresql/12/main/PG_VERSION ]; then fi sudo service postgresql start && \ -sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -u postgres createuser -s nominatim && \ -sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -u postgres createuser -SDR www-data && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ -sudo -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ -sudo -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ -sudo -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" chown -R nominatim:nominatim ${PROJECT_DIR} @@ -72,10 +72,10 @@ sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then echo "Importing Tiger address data" - sudo -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz fi -sudo -u nominatim nominatim admin --check-database +sudo -E -u nominatim nominatim admin --check-database if [ "$REPLICATION_URL" != "" ]; then sudo -E -u nominatim nominatim replication --init @@ -85,14 +85,14 @@ if [ "$REPLICATION_URL" != "" ]; then else if [ "$FREEZE" = "true" ]; then echo "Freezing database" - sudo -u nominatim nominatim freeze + sudo -E -u nominatim nominatim freeze fi fi # gather statistics for query planner to potentially improve query performance # see, https://github.com/osm-search/Nominatim/issues/1023 # and https://github.com/osm-search/Nominatim/issues/1139 -sudo -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" sudo service postgresql stop diff --git a/4.0/start.sh b/4.0/start.sh index c3f2f8d5..263e35e5 100755 --- a/4.0/start.sh +++ b/4.0/start.sh @@ -38,7 +38,7 @@ fi service postgresql start -cd ${PROJECT_DIR} && sudo -u nominatim nominatim refresh --website --functions +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions service apache2 start From dfa9de5335ac30e2caf2a76f4cfb84d8b8a83115 Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Thu, 17 Mar 2022 15:49:53 +0000 Subject: [PATCH 05/96] #307 Add datrie to enable new ICU tokenizer --- 4.0/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/4.0/Dockerfile b/4.0/Dockerfile index 8c9a47b2..b6b5088d 100644 --- a/4.0/Dockerfile +++ b/4.0/Dockerfile @@ -51,6 +51,7 @@ RUN true \ python3-dotenv \ python3-psutil \ python3-jinja2 \ + python3-datrie \ python3-icu git \ python3-argparse-manpage \ # Misc. From 44297df91e3cac8aeecfad48b2fa27bb4e41f34e Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Fri, 18 Mar 2022 08:08:44 +0100 Subject: [PATCH 06/96] add mausch as contributor [skip ci] https://github.com/mediagis/nominatim-docker/pull/309 https://github.com/mediagis/nominatim-docker/pull/307 --- .all-contributorsrc | 9 +++++++++ README.md | 1 + 2 files changed, 10 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 21b53fbd..3dcea458 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -390,6 +390,15 @@ "code", "test" ] + }, + { + "login": "mausch", + "name": "Mauricio Scheffer", + "avatar_url": "https://avatars.githubusercontent.com/u/95194?v=4", + "profile": "http://bugsquash.blogspot.com/", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 6, diff --git a/README.md b/README.md index 8c585f96..15dcab15 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
SClo
💻 📖
Chris
💻 📖
iAlex97
💻 ⚠️ +
Mauricio Scheffer
💻 From fac1354f5c8edd67c61e9a3aa41e854c4e75076c Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 23 Mar 2022 14:47:26 +0100 Subject: [PATCH 07/96] re-enable the arm build --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77fd3240..1db4d7b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,11 +153,7 @@ jobs: - name: Build and push production docker image to Dockerhub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} run: |- - # we disable the arm build until these issues are resolved - # https://github.com/curl/curl/issues/7932 - # https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/1951279 - #docker buildx build --platform linux/amd64,linux/arm64 --push \ - docker buildx build --platform linux/amd64 --push \ + docker buildx build --platform linux/amd64,linux/arm64 --push \ -t mediagis/nominatim:${{ matrix.nominatim.version }} \ -t mediagis/nominatim:${{ matrix.nominatim.version }}-${{ github.sha }} . working-directory: ${{ matrix.nominatim.version }} From d055eef70264527e400b5974b4f59dbc5926847d Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 23 Mar 2022 21:38:02 +0100 Subject: [PATCH 08/96] add table of contents to README.md --- 4.0/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/4.0/README.md b/4.0/README.md index fd4843e9..d330eb7a 100644 --- a/4.0/README.md +++ b/4.0/README.md @@ -1,5 +1,23 @@ # Nominatim Docker (Nominatim version 4.0) +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + +*** + ## Automatic import Download the required data, initialize the database and start nominatim in one go From df31d779442dd0df8a33f223c780c1ef719863bc Mon Sep 17 00:00:00 2001 From: UntitleDude Date: Wed, 30 Mar 2022 09:46:12 +0200 Subject: [PATCH 09/96] ./config.sh: line 26: [{FILE}: No such file or directory Hi, I've found out that config.sh contains a bug while retrieving $REPLICATION_URL in order to update the database. Turns out, shell language was not respected on line 26 (spaces between [, ] and expressions). Starting the container with the pre-requisite parameters give you this warning / error message : `./config.sh: line 26: [https://download.geofabrik.de/europe/monaco-updates/: No such file or directory` Simple fix here. --- 4.0/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.0/config.sh b/4.0/config.sh index 46a5d8be..8ee322d2 100755 --- a/4.0/config.sh +++ b/4.0/config.sh @@ -16,7 +16,7 @@ fi reg_num='^[0-9]+$' if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then - if ["$REPLICATION_URL" = ""]; then + if [ "$REPLICATION_URL" = "" ]; then echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" exit 1 fi From 45d5f8c6e3210686ad9086e4078a7a1ba93b438c Mon Sep 17 00:00:00 2001 From: UntitleDude Date: Wed, 30 Mar 2022 10:28:25 +0200 Subject: [PATCH 10/96] (2) ./config.sh: line 26: [{FILE}: No such file or directory Second line that's erronous in bash. See #313 --- 4.0/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.0/config.sh b/4.0/config.sh index 8ee322d2..4bb9c6e0 100755 --- a/4.0/config.sh +++ b/4.0/config.sh @@ -23,7 +23,7 @@ if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} fi if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then - if ["$REPLICATION_URL" = ""]; then + if [ "$REPLICATION_URL" = "" ]; then echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" exit 1 fi From 2311e7b1def77b3b866c0789feade8937a13d49e Mon Sep 17 00:00:00 2001 From: Stefanic <4499284+Stefanic@users.noreply.github.com> Date: Mon, 18 Apr 2022 07:13:47 +0200 Subject: [PATCH 11/96] Country grid download fix (#320) --- 3.0/Dockerfile | 2 +- 3.1/Dockerfile | 2 +- 3.2/Dockerfile | 4 ++-- 3.3/Dockerfile | 2 +- 3.4/Dockerfile | 2 +- 3.5/Dockerfile | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/3.0/Dockerfile b/3.0/Dockerfile index b0547b5b..ccb22a4f 100644 --- a/3.0/Dockerfile +++ b/3.0/Dockerfile @@ -42,7 +42,7 @@ COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf # Load initial data ARG PBF_DATA=https://download.geofabrik.de/europe/monaco-latest.osm.pbf RUN curl -L -f $PBF_DATA --create-dirs -o /app/src/data.osm.pbf -RUN curl https://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz +RUN curl https://nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz RUN service postgresql start && \ sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -u postgres createuser -s nominatim && \ sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -u postgres createuser -SDR www-data && \ diff --git a/3.1/Dockerfile b/3.1/Dockerfile index 841de651..47bfa90d 100644 --- a/3.1/Dockerfile +++ b/3.1/Dockerfile @@ -41,7 +41,7 @@ COPY local.php /app/src/build/settings/local.php COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf # Load initial data -RUN curl https://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz +RUN curl https://nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz EXPOSE 5432 EXPOSE 8080 diff --git a/3.2/Dockerfile b/3.2/Dockerfile index b29a323a..d5a99084 100644 --- a/3.2/Dockerfile +++ b/3.2/Dockerfile @@ -41,7 +41,7 @@ COPY local.php /app/src/build/settings/local.php COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf # Load initial data -RUN curl https://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz +RUN curl https://nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz EXPOSE 5432 EXPOSE 8080 @@ -49,4 +49,4 @@ EXPOSE 8080 COPY start.sh /app/start.sh COPY startapache.sh /app/startapache.sh COPY startpostgres.sh /app/startpostgres.sh -COPY init.sh /app/init.sh \ No newline at end of file +COPY init.sh /app/init.sh diff --git a/3.3/Dockerfile b/3.3/Dockerfile index f5df1dfa..b4dfc7e5 100644 --- a/3.3/Dockerfile +++ b/3.3/Dockerfile @@ -39,7 +39,7 @@ COPY local.php /app/src/build/settings/local.php COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf # Load initial data -RUN curl https://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz +RUN curl https://nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz RUN sed -i -e 's|bin/python|bin/python3|' /app/src/utils/*.py RUN chmod o=rwx /app/src/build diff --git a/3.4/Dockerfile b/3.4/Dockerfile index 7e8523c0..f0474da5 100644 --- a/3.4/Dockerfile +++ b/3.4/Dockerfile @@ -39,7 +39,7 @@ COPY local.php /app/src/build/settings/local.php COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf # Load initial data -RUN curl https://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz +RUN curl https://nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz RUN chmod o=rwx /app/src/build EXPOSE 5432 diff --git a/3.5/Dockerfile b/3.5/Dockerfile index 5d3556bb..47c6a156 100644 --- a/3.5/Dockerfile +++ b/3.5/Dockerfile @@ -101,7 +101,7 @@ RUN if [ "$with_postcodes_us" = "" ]; then \ curl https://www.nominatim.org/data/us_postcode_data.sql.gz > /app/src/data/us_postcode_data.sql.gz; \ fi; -RUN curl https://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz +RUN curl https://nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz RUN true \ # Remove development and unused packages. From d9fced865b11b65096f478ad089d280ff9e2283a Mon Sep 17 00:00:00 2001 From: Stefanic <4499284+Stefanic@users.noreply.github.com> Date: Mon, 27 Jun 2022 19:00:09 +0200 Subject: [PATCH 12/96] PBF download supports resume --- 3.7/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3.7/init.sh b/3.7/init.sh index 0adb26dc..395abb9a 100755 --- a/3.7/init.sh +++ b/3.7/init.sh @@ -24,7 +24,7 @@ fi; if [ "$PBF_URL" != "" ]; then echo Downloading OSM extract from "$PBF_URL" - curl -L "$PBF_URL" --create-dirs -o $OSMFILE + curl -L "$PBF_URL" -C - --create-dirs -o $OSMFILE fi if [ "$PBF_PATH" != "" ]; then From 127c73a31ff202c40b0e6a0a333d7432eccb5192 Mon Sep 17 00:00:00 2001 From: Stefanic <4499284+Stefanic@users.noreply.github.com> Date: Mon, 27 Jun 2022 19:00:43 +0200 Subject: [PATCH 13/96] PBF download supports resume --- 4.0/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.0/init.sh b/4.0/init.sh index 96183af2..64d55cb3 100755 --- a/4.0/init.sh +++ b/4.0/init.sh @@ -41,7 +41,7 @@ fi if [ "$PBF_URL" != "" ]; then echo Downloading OSM extract from "$PBF_URL" - curl -L "$PBF_URL" --create-dirs -o $OSMFILE + curl -L "$PBF_URL" -C - --create-dirs -o $OSMFILE fi if [ "$PBF_PATH" != "" ]; then From 2555bc6d2c6eac899206d97f1fc6278efac42bcf Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Thu, 21 Jul 2022 09:35:22 +0200 Subject: [PATCH 14/96] Point documentation to issues, closes #245 [ci skip] --- 4.0/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/4.0/README.md b/4.0/README.md index d330eb7a..b827e46c 100644 --- a/4.0/README.md +++ b/4.0/README.md @@ -205,3 +205,8 @@ docker run -it --rm \ ## Docker Compose In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) +- [AWS hardware for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) From 79bcded98bd3bfb7b4b878b43ce0306ae5ae85da Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Tue, 26 Jul 2022 12:53:47 +0200 Subject: [PATCH 15/96] add new use case section to ToC [skip ci] (#347) --- 4.0/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/4.0/README.md b/4.0/README.md index b827e46c..b9c27b0f 100644 --- a/4.0/README.md +++ b/4.0/README.md @@ -15,6 +15,7 @@ - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) - [Development](#development) - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) *** From 2841cd4a34ba589d246745a905080f2e46f4cab6 Mon Sep 17 00:00:00 2001 From: Stefan Nicodem Date: Tue, 26 Jul 2022 18:33:27 +0200 Subject: [PATCH 16/96] Add planet instance documentation --- 3.7/README.md | 8 ++++++- 3.7/contrib/docker-compose-planet.yml | 33 +++++++++++++++++++++++++++ 4.0/README.md | 5 ++++ 4.0/contrib/docker-compose-planet.yml | 33 +++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 3.7/contrib/docker-compose-planet.yml create mode 100644 4.0/contrib/docker-compose-planet.yml diff --git a/3.7/README.md b/3.7/README.md index c7c6e0dc..7a7297a9 100644 --- a/3.7/README.md +++ b/3.7/README.md @@ -17,6 +17,8 @@ Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which yo If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur +Note: the --rm parameter means the container will be removed after stopping it (causing all imported data to be lost)! + ## Configuration ### General Parameters @@ -166,4 +168,8 @@ docker run -it --rm \ ## Docker Compose -In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + diff --git a/3.7/contrib/docker-compose-planet.yml b/3.7/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..94f1dad4 --- /dev/null +++ b/3.7/contrib/docker-compose-planet.yml @@ -0,0 +1,33 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:3.7 + restart: always + ports: + - "8080:8080" # Do not change the second port, only the first before the semicolon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: true + IMPORT_US_POSTCODES: true + IMPORT_GB_POSTCODES: true + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/12/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb + diff --git a/4.0/README.md b/4.0/README.md index b9c27b0f..10708fda 100644 --- a/4.0/README.md +++ b/4.0/README.md @@ -36,6 +36,8 @@ Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which yo If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur +Note: the --rm parameter means the container will be removed after stopping it (causing all imported data to be lost)! + ## Configuration ### General Parameters @@ -207,6 +209,9 @@ docker run -it --rm \ In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + ## Assorted use cases documented in issues - [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) diff --git a/4.0/contrib/docker-compose-planet.yml b/4.0/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..295f8e7b --- /dev/null +++ b/4.0/contrib/docker-compose-planet.yml @@ -0,0 +1,33 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.0 + restart: always + ports: + - "8080:8080" # Do not change the second port, only the first before the semicolon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: true + IMPORT_US_POSTCODES: true + IMPORT_GB_POSTCODES: true + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/12/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb + From 7ff024e0cdd8b7ab92c679ddb61c6675763037e8 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 27 Jul 2022 20:14:39 +0200 Subject: [PATCH 17/96] Update docker-compose-planet.yml - fix comment [skip ci] --- 4.0/contrib/docker-compose-planet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.0/contrib/docker-compose-planet.yml b/4.0/contrib/docker-compose-planet.yml index 295f8e7b..7c091bdd 100644 --- a/4.0/contrib/docker-compose-planet.yml +++ b/4.0/contrib/docker-compose-planet.yml @@ -13,7 +13,7 @@ services: image: mediagis/nominatim:4.0 restart: always ports: - - "8080:8080" # Do not change the second port, only the first before the semicolon + - "8080:8080" # Do not change the second port, only the first before the colon environment: PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ From 10a67239c242b0bb15b6767d4bee33c8e355f95d Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Tue, 23 Aug 2022 20:59:57 +0200 Subject: [PATCH 18/96] remove duplicated package --- 4.0/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.0/Dockerfile b/4.0/Dockerfile index b6b5088d..a0faf0a5 100644 --- a/4.0/Dockerfile +++ b/4.0/Dockerfile @@ -52,7 +52,7 @@ RUN true \ python3-psutil \ python3-jinja2 \ python3-datrie \ - python3-icu git \ + python3-icu \ python3-argparse-manpage \ # Misc. git \ From a4c4f05389a63653cb254ee1ac513a1b690daeae Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 24 Aug 2022 12:20:43 +0200 Subject: [PATCH 19/96] Nominatim 4.1.0 (#361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove duplicated package * support nominatim 4.1 * add NOMINATIM_TOKENIZER=icu to default env * updated missed references to old postgresql * test setting postgres_version in action workflows * increase sleep time * cleanup & references in /README.md * remove —rm parameter from README * README formatting [skip ci] * Update 4.1/contrib/docker-compose.yml Co-authored-by: Leonard Ehrenfried Co-authored-by: Leonard Ehrenfried --- .github/workflows/ci.yml | 28 ++-- 4.0/README.md | 14 +- 4.1/Dockerfile | 143 +++++++++++++++++ 4.1/README.md | 218 ++++++++++++++++++++++++++ 4.1/conf.d/apache.conf | 13 ++ 4.1/conf.d/env | 6 + 4.1/conf.d/postgres-import.conf | 2 + 4.1/conf.d/postgres-tuning.conf | 10 ++ 4.1/config.sh | 62 ++++++++ 4.1/contrib/docker-compose-planet.yml | 33 ++++ 4.1/contrib/docker-compose.yml | 20 +++ 4.1/init.sh | 111 +++++++++++++ 4.1/start.sh | 34 ++++ 4.1/startapache.sh | 5 + 4.1/startpostgres.sh | 4 + README.md | 3 +- 16 files changed, 684 insertions(+), 22 deletions(-) create mode 100644 4.1/Dockerfile create mode 100644 4.1/README.md create mode 100644 4.1/conf.d/apache.conf create mode 100644 4.1/conf.d/env create mode 100644 4.1/conf.d/postgres-import.conf create mode 100644 4.1/conf.d/postgres-tuning.conf create mode 100755 4.1/config.sh create mode 100644 4.1/contrib/docker-compose-planet.yml create mode 100644 4.1/contrib/docker-compose.yml create mode 100755 4.1/init.sh create mode 100755 4.1/start.sh create mode 100755 4.1/startapache.sh create mode 100755 4.1/startpostgres.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88f237bd..6bf4e43f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,10 @@ jobs: nominatim: - version: "4.0" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - - version: "3.7" + postgres_version: 12 + - version: "4.1" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + postgres_version: 14 runs-on: ubuntu-latest steps: @@ -42,7 +44,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -54,10 +56,10 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -v nominatim-data:/var/lib/postgresql/12/main \ + -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount @@ -66,11 +68,11 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -v /tmp/nominatim-data:/var/lib/postgresql/12/main \ + -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8003:8080 \ --name nominatim \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim @@ -82,7 +84,7 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -v /tmp/nominatim-data:/var/lib/postgresql/12/main \ + -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8004:8080 \ --name nominatim \ nominatim & @@ -101,7 +103,7 @@ jobs: -e IMPORT_STYLE=full \ -p 8005:8080 \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -113,7 +115,7 @@ jobs: -e IMPORT_STYLE=admin \ -p 8006:8080 \ nominatim & - sleep 35 + sleep 60 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -123,12 +125,12 @@ jobs: docker run -i --rm \ -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -v nominatim7-data:/var/lib/postgresql/12/main \ + -v nominatim7-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -v /tmp/data:/nominatim/data \ -p 8007:8080 \ --name nominatim \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -140,7 +142,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -p 8008:8080 \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -151,7 +153,7 @@ jobs: -e FREEZE="true" \ -p 8008:8080 \ nominatim & - sleep 35 + sleep 60 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check GB postcode import diff --git a/4.0/README.md b/4.0/README.md index 10708fda..6de92565 100644 --- a/4.0/README.md +++ b/4.0/README.md @@ -24,7 +24,7 @@ Download the required data, initialize the database and start nominatim in one go ``` -docker run -it --rm \ +docker run -it \ -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ @@ -36,8 +36,6 @@ Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which yo If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur -Note: the --rm parameter means the container will be removed after stopping it (causing all imported data to be lost)! - ## Configuration ### General Parameters @@ -107,7 +105,7 @@ If you want to keep your imported data across deletion and recreation of your co So if you want to be able to kill your container and start it up again with all the data still present use the following command: ``` -docker run -it --rm --shm-size=1g \ +docker run -it --shm-size=1g \ -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -e IMPORT_WIKIPEDIA=false \ @@ -132,7 +130,7 @@ The replication update can be performed only via HTTP. A sample of `PBF_PATH` variable usage is: ``` sh -docker run -it --rm \ +docker run -it \ -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ @@ -163,7 +161,7 @@ If there are no updates available this process will sleep for 15 minutes and try If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. ``` sh -docker run -it --rm \ +docker run -it \ -e PBF_PATH=/nominatim/data/merged.osm.pbf \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ @@ -177,7 +175,7 @@ where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mou Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: ``` sh -docker run -it --rm \ +docker run -it \ -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ -p 8080:8080 \ @@ -197,7 +195,7 @@ image and run the container with ``` docker build -t nominatim . && \ -docker run -it --rm \ +docker run -it \ -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ diff --git a/4.1/Dockerfile b/4.1/Dockerfile new file mode 100644 index 00000000..778083a4 --- /dev/null +++ b/4.1/Dockerfile @@ -0,0 +1,143 @@ +FROM ubuntu:jammy AS build + +ENV DEBIAN_FRONTEND noninteractive +ENV LANG C.UTF-8 + +WORKDIR /app + +RUN true \ + # Do not start daemons after installation. + && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. + build-essential \ + g++ \ + cmake \ + libpq-dev \ + zlib1g-dev \ + libbz2-dev \ + libproj-dev \ + libexpat1-dev \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + # PHP and Apache 2. + php \ + php-intl \ + php-pgsql \ + php-cgi \ + apache2 \ + libapache2-mod-php \ + # Python 3. + python3-dev \ + python3-pip \ + python3-tidylib \ + python3-psycopg2 \ + python3-setuptools \ + python3-dotenv \ + python3-psutil \ + python3-jinja2 \ + python3-datrie \ + python3-icu \ + python3-argparse-manpage \ + # Misc. + git \ + curl \ + sudo + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Osmium install to run continuous updates. +RUN pip3 install osmium + +# Nominatim install. +ENV NOMINATIM_VERSION 4.1.0 + +RUN true \ + && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && tar xf nominatim.tar.bz2 \ + && mkdir build \ + && cd build \ + && cmake ../Nominatim-$NOMINATIM_VERSION \ + && make -j`nproc` \ + && make install + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge \ + cpp-9 \ + gcc-9* \ + g++ \ + git \ + make \ + cmake* \ + llvm-10* \ + libc6-dev \ + linux-libc-dev \ + libclang-*-dev \ + build-essential \ + postgresql-server-dev-14 \ + && apt-get clean \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /app/src/.git \ + /var/lib/apt/lists/* \ + # Remove nominatim source and build directories + && rm /app/*.tar.bz2 \ + && rm -rf /app/build \ + && rm -rf /app/Nominatim-$NOMINATIM_VERSION + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/ +COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD qaIACxO6wMR3 +# how many threads should be use for importing +ENV THREADS=16 + +ENV PROJECT_DIR /nominatim + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD /app/start.sh diff --git a/4.1/README.md b/4.1/README.md new file mode 100644 index 00000000..343f675c --- /dev/null +++ b/4.1/README.md @@ -0,0 +1,218 @@ +# Nominatim Docker (Nominatim version 4.1) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.1 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. Check [https://download.geofabrik.de](https://download.geofabrik.de) +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: `16`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.1.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.1.0/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.1.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/14/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.1 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.1 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.1.0/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.1 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.1 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.1.0/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) +- [AWS hardware for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) diff --git a/4.1/conf.d/apache.conf b/4.1/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.1/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.1/conf.d/env b/4.1/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.1/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.1/conf.d/postgres-import.conf b/4.1/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.1/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.1/conf.d/postgres-tuning.conf b/4.1/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..610a9c7f --- /dev/null +++ b/4.1/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.1.0/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.1/config.sh b/4.1/config.sh new file mode 100755 index 00000000..483354c9 --- /dev/null +++ b/4.1/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.1/contrib/docker-compose-planet.yml b/4.1/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..d064c1ac --- /dev/null +++ b/4.1/contrib/docker-compose-planet.yml @@ -0,0 +1,33 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.1 + restart: always + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: true + IMPORT_US_POSTCODES: true + IMPORT_GB_POSTCODES: true + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/14/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb + diff --git a/4.1/contrib/docker-compose.yml b/4.1/contrib/docker-compose.yml new file mode 100644 index 00000000..26bea22b --- /dev/null +++ b/4.1/contrib/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.1 + restart: always + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.1#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/14/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.1/init.sh b/4.1/init.sh new file mode 100755 index 00000000..5197586c --- /dev/null +++ b/4.1/init.sh @@ -0,0 +1,111 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + curl https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + curl https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + curl https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + curl https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + curl -L "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then + chown postgres /var/lib/postgresql/14/main + sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main +fi + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} +sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +sudo -E -u nominatim nominatim admin --warm + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/14/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.1/start.sh b/4.1/start.sh new file mode 100755 index 00000000..6f0ee432 --- /dev/null +++ b/4.1/start.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex + +stopServices() { + service apache2 stop + service postgresql stop +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# fork a process and wait for it +tail -f /var/log/postgresql/postgresql-14-main.log & +wait diff --git a/4.1/startapache.sh b/4.1/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.1/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.1/startpostgres.sh b/4.1/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.1/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log diff --git a/README.md b/README.md index 15dcab15..7ddedcb3 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ See relevant installation instructions for each version in the /README. # Supported tags and respective `Dockerfile` links # +- [`4.1.0`, `4.1` (*4.1/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.1) - [`4.0.1`, `4.0` (*4.0/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.0) -- [`3.7.2`, `3.7` (*3.7/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.7) # Deprecated tags and respective `Dockerfile` links # +- [`3.7.2`, `3.7` (*3.7/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.7) - [`3.6.0`, `3.6` (*3.6/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.6) - [`3.5.2`, `3.5` (*3.5/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.5) - [`3.4.2`, `3.4` (*3.4/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.4) From be6332e577a4a8a545ea87bfb8ed3365edb357a5 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 24 Aug 2022 12:23:40 +0200 Subject: [PATCH 20/96] Update README.md [skip ci] --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 7ddedcb3..6829518a 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,7 @@ See relevant installation instructions for each version in the /README. - [`2.5.1`, `2.5` (*2.5/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/2.5) - -**Caution:** When you upgrade to another version (e.g. 3.4 to 3.5) there may be major version changes like Postgres and data incompatiblity. See the relevant instructions for each version. Also check the Nominatim migration guide [https://www.nominatim.org/release-docs/latest/admin/Migration/](https://www.nominatim.org/release-docs/latest/admin/Migration/). - -The latest version uses Ubuntu 20.04 and PostgreSQL 12. - - +**Caution:** When you upgrade to another version (e.g. 4.0 to 4.1) there may be major version changes like Postgres and data incompatiblity. See the relevant instructions for each version, a complete reimport is recommended. Also check the Nominatim migration guide [https://www.nominatim.org/release-docs/latest/admin/Migration/](https://www.nominatim.org/release-docs/latest/admin/Migration/). # Contributors From ff307400ad84bfc80cfd382c11d2c54a91bcf21d Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 24 Aug 2022 12:28:26 +0200 Subject: [PATCH 21/96] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6829518a..7bd83910 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ # How to use Clone repository - ``` - # git clone git@github.com:mediagis/nominatim-docker.git - # cd nominatim-docker/ + ```sh + git clone git@github.com:mediagis/nominatim-docker.git + cd nominatim-docker/ ``` See relevant installation instructions for each version in the /README.md From b5e76787c6cc0c3aa58cbbb6967908b929b82902 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Thu, 25 Aug 2022 14:41:41 +0200 Subject: [PATCH 22/96] backport graceful shutdowns from 4.1 (#363) * backport graceful shutdowns from 4.1 * Update ci.yml --- .github/workflows/ci.yml | 2 +- 4.0/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bf4e43f..9f4031d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,7 @@ jobs: -e IMPORT_GB_POSTCODES="true" \ -p 8009:8080 \ nominatim & - sleep 345 + sleep 400 ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=LE15+8TX" ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+3SY" ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+4ES" diff --git a/4.0/start.sh b/4.0/start.sh index 263e35e5..53e4e67b 100755 --- a/4.0/start.sh +++ b/4.0/start.sh @@ -4,7 +4,7 @@ stopServices() { service apache2 stop service postgresql stop } -trap stopServices TERM +trap stopServices SIGTERM TERM INT /app/config.sh From 8bb6c5b647a980e64285972c42679677d3a3a056 Mon Sep 17 00:00:00 2001 From: mtmail Date: Sat, 27 Aug 2022 19:32:02 +0200 Subject: [PATCH 23/96] After import do a second index step to avoid unindexed places in database (#366) --- 4.1/init.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/4.1/init.sh b/4.1/init.sh index 5197586c..7a665de8 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -75,6 +75,11 @@ if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz fi +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + sudo -E -u nominatim nominatim admin --check-database if [ "$REPLICATION_URL" != "" ]; then From b3ee196593c23f7dbc6d564467a7eb00b94a797e Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Sat, 27 Aug 2022 19:55:13 +0200 Subject: [PATCH 24/96] update contributors [skip ci] (#367) --- .all-contributorsrc | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 3dcea458..3878a364 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -98,6 +98,7 @@ "profile": "https://twitter.com/mtmthemovie", "contributions": [ "doc", + "code", "question", "review" ] diff --git a/README.md b/README.md index 7bd83910..8486ecc0 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Andrey Ruíz
📖
UntitleDude
💻
Jack McKernan
💻 -
mtmail
📖 💬 👀 +
mtmail
📖 💻 💬 👀
Andrey Oblivantsev
💻
Simone
💻 From 322deccac655584add9c4db08985323f506ee30f Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Sat, 27 Aug 2022 20:18:44 +0200 Subject: [PATCH 25/96] Update contributors [skip ci] (#369) * update contributors [skip ci] * add missing contributor [skip ci] --- .all-contributorsrc | 9 +++++++++ README.md | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 3878a364..19f5e1b4 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -400,6 +400,15 @@ "contributions": [ "code" ] + }, + { + "login": "anthropos9", + "name": "Sean Dean", + "avatar_url": "https://avatars.githubusercontent.com/u/3867685?v=4", + "profile": "https://github.com/anthropos9", + "contributions": [ + "doc" + ] } ], "contributorsPerLine": 6, diff --git a/README.md b/README.md index 8486ecc0..23fbf274 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ 100% working container for [Nominatim](https://github.com/openstreetmap/Nominatim). -![GitHub Workflow Status](https://img.shields.io/github/workflow/status/mediagis/nominatim-docker/CI?style=flat-square) ![Github All Contributors](https://img.shields.io/github/all-contributors/mediagis/nominatim-docker?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/mediagis/nominatim?style=flat-square) +![GitHub Workflow Status](https://img.shields.io/github/workflow/status/mediagis/nominatim-docker/CI?style=flat-square) ![Github All Contributors](https://img.shields.io/github/all-contributors/mediagis/nominatim-docker?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/mediagis/nominatim?style=flat-square) ![Docker Image Size with architecture (latest by date/latest semver)](https://img.shields.io/docker/image-size/mediagis/nominatim?style=flat-square) # How to use Clone repository @@ -95,6 +95,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Chris
💻 📖
iAlex97
💻 ⚠️
Mauricio Scheffer
💻 +
Sean Dean
📖 From 51f91a3fa7e3be01a54c4dd92229a1a8e2b6491a Mon Sep 17 00:00:00 2001 From: Philipp Gassmann Date: Tue, 20 Sep 2022 22:06:20 +0200 Subject: [PATCH 26/96] Continous replication, shutdown fix, output apache logs (#373) * fix command, fixes shutdown hook when command is set as string, docker will launch the script in /bin/sh and the actual script will run in a subshell and not receive the signals with ["/app/start.sh"] the script runs directly as PID 1 and properly execute the shutdown trap * start continuous replication process in start.sh also forward apache logs to docker * fix planet import example env vars need to be strings or numbers * remove unsafe import tuning for start after import * warm cache on start after init * UPDATE_MODE to configure replication * enable unsafe pg config override only for import * use ${PROJECT_DIR} * add check for UPDATE_MODE. Increase sleep to 90s 60s was very close to the finished import. * use different volumes/bindmounts for update test * fix github actions syntax * pbf data from 10 days ago is not available --- .github/workflows/ci.yml | 70 +++++++++++++++++++++------ 4.1/Dockerfile | 4 +- 4.1/README.md | 1 + 4.1/contrib/docker-compose-planet.yml | 7 ++- 4.1/init.sh | 3 ++ 4.1/start.sh | 33 ++++++++++++- 6 files changed, 97 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f4031d4..7cd19f7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: postgres_version: 12 - version: "4.1" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 + postgres_version: 14 runs-on: ubuntu-latest steps: @@ -44,13 +44,13 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" docker stop nominatim - - name: Check import with mount + - name: Check import with volume mount working-directory: .github/workflows run: |- docker run -i --rm \ @@ -59,27 +59,29 @@ jobs: -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount working-directory: .github/workflows run: |- + # get the data from four days ago to make sure there really are updates to apply + four_days_ago=`date --date="4 days ago" +%y%m%d` docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8003:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim - name: Check container restart and update with bind-mount working-directory: .github/workflows run: |- - # get the data from four days ago to make sure there really are updates to apply + # import to bind mount is done by previous step four_days_ago=`date --date="4 days ago" +%y%m%d` docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ @@ -94,6 +96,46 @@ jobs: ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" docker stop nominatim + - name: Check UPDATE_MODE=once with volume + if: matrix.nominatim.version != '4.0' + working-directory: .github/workflows + run: |- + # get the data from four days ago to make sure there really are updates to apply + four_days_ago=`date --date="4 days ago" +%y%m%d` + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e UPDATE_MODE=once \ + -v nominatim-update-volume:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ + -p 8004:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" + echo "check replication log for Update completed. Count:" + docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + docker stop nominatim + + - name: Check UPDATE_MODE=continuous with bind-mount + if: matrix.nominatim.version != '4.0' + working-directory: .github/workflows + run: |- + # get the data from few days ago to make sure there really are updates to apply + days_ago=`date --date="4 days ago" +%y%m%d` + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e UPDATE_MODE=continuous \ + -v /tmp/nominatim-update-bindmount:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ + -p 8004:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" + echo "check replication log for Update completed. Count:" + docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + docker stop nominatim + - name: Check import full style working-directory: .github/workflows run: |- @@ -103,7 +145,7 @@ jobs: -e IMPORT_STYLE=full \ -p 8005:8080 \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -115,7 +157,7 @@ jobs: -e IMPORT_STYLE=admin \ -p 8006:8080 \ nominatim & - sleep 60 + sleep 90 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -130,7 +172,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -142,7 +184,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -p 8008:8080 \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -153,7 +195,7 @@ jobs: -e FREEZE="true" \ -p 8008:8080 \ nominatim & - sleep 60 + sleep 90 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check GB postcode import @@ -180,6 +222,6 @@ jobs: if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} run: |- docker buildx build --platform linux/amd64,linux/arm64 --push \ - -t mediagis/nominatim:${{ matrix.nominatim.version }} \ - -t mediagis/nominatim:${{ matrix.nominatim.version }}-${{ github.sha }} . + -t mediagis/nominatim:${{ matrix.nominatim.version }} \ + -t mediagis/nominatim:${{ matrix.nominatim.version }}-${{ github.sha }} . working-directory: ${{ matrix.nominatim.version }} diff --git a/4.1/Dockerfile b/4.1/Dockerfile index 778083a4..8e4bf846 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -112,7 +112,7 @@ RUN true \ COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf # Postgres config overrides to improve import performance (but reduce crash recovery safety) -COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/ +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ COPY config.sh /app/config.sh @@ -140,4 +140,4 @@ EXPOSE 8080 COPY conf.d/env $PROJECT_DIR/.env -CMD /app/start.sh +CMD ["/app/start.sh"] diff --git a/4.1/README.md b/4.1/README.md index 343f675c..c6130c7f 100644 --- a/4.1/README.md +++ b/4.1/README.md @@ -47,6 +47,7 @@ The following environment variables are available for configuration: - `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. - `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.1.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) - `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) diff --git a/4.1/contrib/docker-compose-planet.yml b/4.1/contrib/docker-compose-planet.yml index d064c1ac..eacf003a 100644 --- a/4.1/contrib/docker-compose-planet.yml +++ b/4.1/contrib/docker-compose-planet.yml @@ -18,9 +18,9 @@ services: PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ NOMINATIM_PASSWORD: very_secure_password - IMPORT_WIKIPEDIA: true - IMPORT_US_POSTCODES: true - IMPORT_GB_POSTCODES: true + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" THREADS: 16 volumes: - type: bind @@ -30,4 +30,3 @@ services: source: /data/flatnode target: /nominatim/flatnode shm_size: 1gb - diff --git a/4.1/init.sh b/4.1/init.sh index 7a665de8..7815f04e 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -56,6 +56,9 @@ if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main fi +# temporarily enable unsafe import optimization config +cp /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled /etc/postgresql/14/main/conf.d/postgres-import.conf + sudo service postgresql start && \ sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ diff --git a/4.1/start.sh b/4.1/start.sh index 6f0ee432..3c00a3ea 100755 --- a/4.1/start.sh +++ b/4.1/start.sh @@ -1,8 +1,13 @@ #!/bin/bash -ex +tailpid=0 +replicationpid=0 + stopServices() { service apache2 stop service postgresql stop + kill $replicationpid + kill $tailpid } trap stopServices SIGTERM TERM INT @@ -29,6 +34,32 @@ cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --function service apache2 start +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + # fork a process and wait for it -tail -f /var/log/postgresql/postgresql-14-main.log & +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +echo "Warm database caches for search and reverse queries" +sudo -E -u nominatim nominatim admin --warm > /dev/null +echo "Warming finished" wait From 38a544be048e3d1666bfacc56c4df3b8ac1c6cb9 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Tue, 20 Sep 2022 22:10:35 +0200 Subject: [PATCH 27/96] add pgassmann as a contributor (#376) * update README.md [skip ci] * update .all-contributorsrc [skip ci] Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com> --- .all-contributorsrc | 11 +++++ README.md | 117 +++++++++++++++++++++++--------------------- 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 19f5e1b4..4fef30bb 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -409,6 +409,17 @@ "contributions": [ "doc" ] + }, + { + "login": "pgassmann", + "name": "Philipp Gassmann", + "avatar_url": "https://avatars.githubusercontent.com/u/460192?v=4", + "profile": "https://github.com/pgassmann", + "contributions": [ + "doc", + "code", + "test" + ] } ], "contributorsPerLine": 6, diff --git a/README.md b/README.md index 23fbf274..821f5c40 100644 --- a/README.md +++ b/README.md @@ -41,62 +41,67 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Andrew
💻 📖
Donato Lucia
💻
Georgios Markakis
📖
Philip Kozeny
🚇 💻 ⚠️ 👀 📖
Cezary Morga
💻
Thomas Nordquist
💻

Andrey Ruíz
📖
UntitleDude
💻
Jack McKernan
💻
mtmail
📖 💻 💬 👀
Andrey Oblivantsev
💻
Simone
💻

DuncanMackintosh
💻 📖
Iiro Alhonen
📖
Mikhail Snetkov
💻
FritschAuctores
💻
rebos
💻
Leonard Ehrenfried
🚇 💻 ⚠️ 👀 📖

Nicolas Roelandt
📖 💻 ⚠️
Sacerdoss
📖
Tobias Wich
📖 💻
Alex Chaplianka
📖
Idris Hayward
📖
Karl von Randow
📖

Marco Lechner
📖
Mateusz Gaweł
💻
Nicolas Ternisien
📖
oschlueter
💻
Tim Nonner
💻
thlor
💻 📖

Yun Wang
📖 💻
Stefanic
💻 📖
xpoinsard
📖 💻
Bartizan
💻 📖 ⚠️
galewis2
💻
Derick M.
💻 📖 ⚠️

norcis
💻
SClo
💻 📖
Chris
💻 📖
iAlex97
💻 ⚠️
Mauricio Scheffer
💻
Sean Dean
📖

Andrew
💻 📖
Donato Lucia
💻
Georgios Markakis
📖
Philip Kozeny
🚇 💻 ⚠️ 👀 📖
Cezary Morga
💻
Thomas Nordquist
💻

Andrey Ruíz
📖
UntitleDude
💻
Jack McKernan
💻
mtmail
📖 💻 💬 👀
Andrey Oblivantsev
💻
Simone
💻

DuncanMackintosh
💻 📖
Iiro Alhonen
📖
Mikhail Snetkov
💻
FritschAuctores
💻
rebos
💻
Leonard Ehrenfried
🚇 💻 ⚠️ 👀 📖

Nicolas Roelandt
📖 💻 ⚠️
Sacerdoss
📖
Tobias Wich
📖 💻
Alex Chaplianka
📖
Idris Hayward
📖
Karl von Randow
📖

Marco Lechner
📖
Mateusz Gaweł
💻
Nicolas Ternisien
📖
oschlueter
💻
Tim Nonner
💻
thlor
💻 📖

Yun Wang
📖 💻
Stefanic
💻 📖
xpoinsard
📖 💻
Bartizan
💻 📖 ⚠️
galewis2
💻
Derick M.
💻 📖 ⚠️

norcis
💻
SClo
💻 📖
Chris
💻 📖
iAlex97
💻 ⚠️
Mauricio Scheffer
💻
Sean Dean
📖

Philipp Gassmann
📖 💻 ⚠️
From 631bb5ce34aea8ee7c1480ee1dbd69c3d2018c5f Mon Sep 17 00:00:00 2001 From: chingan Date: Fri, 14 Oct 2022 09:02:59 +0800 Subject: [PATCH 28/96] Pass environment variables to more sudo commands --- 4.1/start.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/4.1/start.sh b/4.1/start.sh index 3c00a3ea..5acb9473 100755 --- a/4.1/start.sh +++ b/4.1/start.sh @@ -37,18 +37,18 @@ service apache2 start # start continous replication process if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then # run init in case replication settings changed - sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init if [ "$UPDATE_MODE" == "continuous" ]; then echo "starting continuous replication" - sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & replicationpid=${!} elif [ "$UPDATE_MODE" == "once" ]; then echo "starting replication once" - sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & replicationpid=${!} elif [ "$UPDATE_MODE" == "catch-up" ]; then echo "starting replication once in catch-up mode" - sudo -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & replicationpid=${!} else echo "skipping replication" From 5370a2c7ad3c475fe154b4d312a53102c320c4db Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Fri, 14 Oct 2022 09:41:45 +0200 Subject: [PATCH 29/96] Update Actions dependency versions --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cd19f7a..156b787c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,13 +21,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Build docker image run: docker build -t nominatim . From c0f7a5e85e3ac10894d078a588622aab40650e1e Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Fri, 14 Oct 2022 10:26:15 +0200 Subject: [PATCH 30/96] Add link for RDS instructions, closes #265 [ci skip] --- 4.1/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/4.1/README.md b/4.1/README.md index c6130c7f..575ab9fd 100644 --- a/4.1/README.md +++ b/4.1/README.md @@ -216,4 +216,5 @@ These files follow the naming convention of `docker-compose-*.yml` and contain c ## Assorted use cases documented in issues - [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) -- [AWS hardware for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) From 25eb1ed21446277d59e4590b4f09d96639949974 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Fri, 14 Oct 2022 12:09:26 +0200 Subject: [PATCH 31/96] Also update login-action --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 156b787c..1e11b4d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,7 +213,7 @@ jobs: - name: Login to DockerHub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} - uses: docker/login-action@v1 + uses: docker/login-action@v2.1.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 69eb4ace294fabae42b53a50eada02918e880378 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Thu, 24 Nov 2022 13:19:25 +0100 Subject: [PATCH 32/96] upgrade nominatim to 4.1.1 (#396) * upgrade nominatim to 4.1.1 * install missing lua packages for 4.1.1 --- 4.1/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/4.1/Dockerfile b/4.1/Dockerfile index 8e4bf846..909625f5 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -30,6 +30,7 @@ RUN true \ libboost-dev \ libboost-system-dev \ libboost-filesystem-dev \ + liblua5.4-dev \ # PostgreSQL. postgresql-contrib \ postgresql-server-dev-14 \ @@ -69,7 +70,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION 4.1.0 +ENV NOMINATIM_VERSION 4.1.1 RUN true \ && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ @@ -94,6 +95,7 @@ RUN true \ linux-libc-dev \ libclang-*-dev \ build-essential \ + liblua*-dev \ postgresql-server-dev-14 \ && apt-get clean \ # Clear temporary files and directories. From bccaf68de0cdc5bda0f6bad1a2bee99976d9e788 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Thu, 24 Nov 2022 13:19:58 +0100 Subject: [PATCH 33/96] update latest version in README.md [skip ci] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 821f5c40..9cd9d896 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ See relevant installation instructions for each version in the /README. # Supported tags and respective `Dockerfile` links # -- [`4.1.0`, `4.1` (*4.1/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.1) +- [`4.1.1`, `4.1` (*4.1/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.1) - [`4.0.1`, `4.0` (*4.0/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.0) # Deprecated tags and respective `Dockerfile` links # From 82af1988646befba10fc067c0fd3084104966325 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Tue, 29 Nov 2022 21:07:47 +0100 Subject: [PATCH 34/96] 4.2 upgrade (#399) * update urls to docs * add version 4.2 * fix ToC for 4.2 * Update postgres-tuning.conf * Update README.md remove 4.0 from the list of supported versions and deprecate it with the 4.2 release [skip ci] * Update ci.yml remove 4.0 from build process --- .github/workflows/ci.yml | 6 +- 4.1/README.md | 12 +- 4.2/Dockerfile | 145 +++++++++++++++++ 4.2/README.md | 220 ++++++++++++++++++++++++++ 4.2/conf.d/apache.conf | 13 ++ 4.2/conf.d/env | 6 + 4.2/conf.d/postgres-import.conf | 2 + 4.2/conf.d/postgres-tuning.conf | 10 ++ 4.2/config.sh | 62 ++++++++ 4.2/contrib/docker-compose-planet.yml | 32 ++++ 4.2/contrib/docker-compose.yml | 20 +++ 4.2/init.sh | 119 ++++++++++++++ 4.2/start.sh | 65 ++++++++ 4.2/startapache.sh | 5 + 4.2/startpostgres.sh | 4 + README.md | 3 +- 16 files changed, 714 insertions(+), 10 deletions(-) create mode 100644 4.2/Dockerfile create mode 100644 4.2/README.md create mode 100644 4.2/conf.d/apache.conf create mode 100644 4.2/conf.d/env create mode 100644 4.2/conf.d/postgres-import.conf create mode 100644 4.2/conf.d/postgres-tuning.conf create mode 100755 4.2/config.sh create mode 100644 4.2/contrib/docker-compose-planet.yml create mode 100644 4.2/contrib/docker-compose.yml create mode 100755 4.2/init.sh create mode 100755 4.2/start.sh create mode 100755 4.2/startapache.sh create mode 100755 4.2/startpostgres.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e11b4d9..7c1d092e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,12 +11,12 @@ jobs: strategy: matrix: nominatim: - - version: "4.0" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 12 - version: "4.1" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 + - version: "4.2" + update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + postgres_version: 14 runs-on: ubuntu-latest steps: diff --git a/4.1/README.md b/4.1/README.md index 575ab9fd..fb3e30ac 100644 --- a/4.1/README.md +++ b/4.1/README.md @@ -47,7 +47,7 @@ The following environment variables are available for configuration: - `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. - `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ -- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.1.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.1.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) - `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) @@ -74,7 +74,7 @@ The following environment variables are available to tune PostgreSQL: - `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) - `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) -See https://nominatim.org/release-docs/4.1.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. +See https://nominatim.org/release-docs/4.1.1/admin/Installation/#tuning-the-postgresql-database for more details on those settings. ### Import Style @@ -90,11 +90,11 @@ Available options are : - `full`: Default style that also includes points of interest. - `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. -See https://nominatim.org/release-docs/4.1.0/admin/Import/#filtering-imported-data for more details on those styles. +See https://nominatim.org/release-docs/4.1.1/admin/Import/#filtering-imported-data for more details on those styles. ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.1.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.1.1/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. ## Persistent container data @@ -144,7 +144,7 @@ where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that ## Updating the database -Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.1.0/admin/Update/). For a list of other methods see the output of: +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.1.1/admin/Update/). For a list of other methods see the output of: ```sh docker exec -it nominatim sudo -u nominatim nominatim replication --help @@ -189,7 +189,7 @@ docker run -it \ Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. -For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.1.0/customize/Tiger/). +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.1.1/customize/Tiger/). ## Development diff --git a/4.2/Dockerfile b/4.2/Dockerfile new file mode 100644 index 00000000..75a2e16b --- /dev/null +++ b/4.2/Dockerfile @@ -0,0 +1,145 @@ +FROM ubuntu:jammy AS build + +ENV DEBIAN_FRONTEND noninteractive +ENV LANG C.UTF-8 + +WORKDIR /app + +RUN true \ + # Do not start daemons after installation. + && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. + build-essential \ + g++ \ + cmake \ + libpq-dev \ + zlib1g-dev \ + libbz2-dev \ + libproj-dev \ + libexpat1-dev \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + liblua5.4-dev \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + # PHP and Apache 2. + php \ + php-intl \ + php-pgsql \ + php-cgi \ + apache2 \ + libapache2-mod-php \ + # Python 3. + python3-dev \ + python3-pip \ + python3-tidylib \ + python3-psycopg2 \ + python3-setuptools \ + python3-dotenv \ + python3-psutil \ + python3-jinja2 \ + python3-datrie \ + python3-icu \ + python3-argparse-manpage \ + # Misc. + git \ + curl \ + sudo + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Osmium install to run continuous updates. +RUN pip3 install osmium + +# Nominatim install. +ENV NOMINATIM_VERSION 4.2.0 + +RUN true \ + && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && tar xf nominatim.tar.bz2 \ + && mkdir build \ + && cd build \ + && cmake ../Nominatim-$NOMINATIM_VERSION \ + && make -j`nproc` \ + && make install + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge \ + cpp-9 \ + gcc-9* \ + g++ \ + git \ + make \ + cmake* \ + llvm-10* \ + libc6-dev \ + linux-libc-dev \ + libclang-*-dev \ + build-essential \ + liblua*-dev \ + postgresql-server-dev-14 \ + && apt-get clean \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /app/src/.git \ + /var/lib/apt/lists/* \ + # Remove nominatim source and build directories + && rm /app/*.tar.bz2 \ + && rm -rf /app/build \ + && rm -rf /app/Nominatim-$NOMINATIM_VERSION + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled +COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD qaIACxO6wMR3 +# how many threads should be use for importing +ENV THREADS=16 + +ENV PROJECT_DIR /nominatim + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD ["/app/start.sh"] diff --git a/4.2/README.md b/4.2/README.md new file mode 100644 index 00000000..4b1057fa --- /dev/null +++ b/4.2/README.md @@ -0,0 +1,220 @@ +# Nominatim Docker (Nominatim version 4.2) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.2 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. Check [https://download.geofabrik.de](https://download.geofabrik.de) +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.2.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: `16`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.2.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.2.0/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.2.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/14/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.2 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.2 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.2.0/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.2 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.2 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.2.0/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) diff --git a/4.2/conf.d/apache.conf b/4.2/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.2/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.2/conf.d/env b/4.2/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.2/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.2/conf.d/postgres-import.conf b/4.2/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.2/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.2/conf.d/postgres-tuning.conf b/4.2/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..23464952 --- /dev/null +++ b/4.2/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.2.0/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.2/config.sh b/4.2/config.sh new file mode 100755 index 00000000..483354c9 --- /dev/null +++ b/4.2/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.2/contrib/docker-compose-planet.yml b/4.2/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..95cda233 --- /dev/null +++ b/4.2/contrib/docker-compose-planet.yml @@ -0,0 +1,32 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.2 + restart: always + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/14/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb diff --git a/4.2/contrib/docker-compose.yml b/4.2/contrib/docker-compose.yml new file mode 100644 index 00000000..15237155 --- /dev/null +++ b/4.2/contrib/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.2 + restart: always + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.2#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/14/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.2/init.sh b/4.2/init.sh new file mode 100755 index 00000000..7815f04e --- /dev/null +++ b/4.2/init.sh @@ -0,0 +1,119 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + curl https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + curl https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + curl https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + curl https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + curl -L "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then + chown postgres /var/lib/postgresql/14/main + sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main +fi + +# temporarily enable unsafe import optimization config +cp /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled /etc/postgresql/14/main/conf.d/postgres-import.conf + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} +sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +sudo -E -u nominatim nominatim admin --warm + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/14/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.2/start.sh b/4.2/start.sh new file mode 100755 index 00000000..5acb9473 --- /dev/null +++ b/4.2/start.sh @@ -0,0 +1,65 @@ +#!/bin/bash -ex + +tailpid=0 +replicationpid=0 + +stopServices() { + service apache2 stop + service postgresql stop + kill $replicationpid + kill $tailpid +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + +# fork a process and wait for it +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +echo "Warm database caches for search and reverse queries" +sudo -E -u nominatim nominatim admin --warm > /dev/null +echo "Warming finished" +wait diff --git a/4.2/startapache.sh b/4.2/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.2/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.2/startpostgres.sh b/4.2/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.2/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log diff --git a/README.md b/README.md index 9cd9d896..2d85c13b 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ See relevant installation instructions for each version in the /README. # Supported tags and respective `Dockerfile` links # +- [`4.2.0`, `4.2` (*4.2/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.2) - [`4.1.1`, `4.1` (*4.1/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.1) -- [`4.0.1`, `4.0` (*4.0/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.0) # Deprecated tags and respective `Dockerfile` links # +- [`4.0.1`, `4.0` (*4.0/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.0) - [`3.7.2`, `3.7` (*3.7/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.7) - [`3.6.0`, `3.6` (*3.6/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.6) - [`3.5.2`, `3.5` (*3.5/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.5) From f6218e7ac5c725f2135e99754b206a0ee32d546f Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Tue, 29 Nov 2022 21:15:01 +0100 Subject: [PATCH 35/96] Update ci.yml add on: workflow_dispatch to the ci.yml so that we can manually trigger a build in the future --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c1d092e..242407f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: pull_request: + workflow_dispatch: jobs: docker: From c3e894d250bb307bf1c75127b7ffbadd11941456 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 30 Nov 2022 13:47:38 +0100 Subject: [PATCH 36/96] update README.md [skip ci] --- README.md | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2d85c13b..bd2c1b46 100644 --- a/README.md +++ b/README.md @@ -5,34 +5,29 @@ ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/mediagis/nominatim-docker/CI?style=flat-square) ![Github All Contributors](https://img.shields.io/github/all-contributors/mediagis/nominatim-docker?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/mediagis/nominatim?style=flat-square) ![Docker Image Size with architecture (latest by date/latest semver)](https://img.shields.io/docker/image-size/mediagis/nominatim?style=flat-square) # How to use -Clone repository - ```sh - git clone git@github.com:mediagis/nominatim-docker.git - cd nominatim-docker/ - ``` -See relevant installation instructions for each version in the /README.md +See relevant installation and usage instructions for each version in the ```/README.md``` file. -# Supported tags and respective `Dockerfile` links # +# Supported Nominatim versions # -- [`4.2.0`, `4.2` (*4.2/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.2) -- [`4.1.1`, `4.1` (*4.1/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.1) +- [`4.2.0`, `4.2`](https://github.com/mediagis/nominatim-docker/tree/master/4.2) +- [`4.1.1`, `4.1`](https://github.com/mediagis/nominatim-docker/tree/master/4.1) -# Deprecated tags and respective `Dockerfile` links # +# Deprecated Nominatim versions # -- [`4.0.1`, `4.0` (*4.0/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/4.0) -- [`3.7.2`, `3.7` (*3.7/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.7) -- [`3.6.0`, `3.6` (*3.6/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.6) -- [`3.5.2`, `3.5` (*3.5/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.5) -- [`3.4.2`, `3.4` (*3.4/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.4) -- [`3.3.1`, `3.3` (*3.3/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.3) -- [`3.2.1`, `3.2` (*3.2/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.2) -- [`3.1.0`, `3.1` (*3.1/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.1) -- [`3.0.1`, `3.0` (*3.0/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/3.0) -- [`2.5.1`, `2.5` (*2.5/Dockerfile*)](https://github.com/mediagis/nominatim-docker/tree/master/2.5) +- [`4.0.1`, `4.0`](https://github.com/mediagis/nominatim-docker/tree/master/4.0) +- [`3.7.2`, `3.7`](https://github.com/mediagis/nominatim-docker/tree/master/3.7) +- [`3.6.0`, `3.6`](https://github.com/mediagis/nominatim-docker/tree/master/3.6) +- [`3.5.2`, `3.5`](https://github.com/mediagis/nominatim-docker/tree/master/3.5) +- [`3.4.2`, `3.4`](https://github.com/mediagis/nominatim-docker/tree/master/3.4) +- [`3.3.1`, `3.3`](https://github.com/mediagis/nominatim-docker/tree/master/3.3) +- [`3.2.1`, `3.2`](https://github.com/mediagis/nominatim-docker/tree/master/3.2) +- [`3.1.0`, `3.1`](https://github.com/mediagis/nominatim-docker/tree/master/3.1) +- [`3.0.1`, `3.0`](https://github.com/mediagis/nominatim-docker/tree/master/3.0) +- [`2.5.1`, `2.5`](https://github.com/mediagis/nominatim-docker/tree/master/2.5) -**Caution:** When you upgrade to another version (e.g. 4.0 to 4.1) there may be major version changes like Postgres and data incompatiblity. See the relevant instructions for each version, a complete reimport is recommended. Also check the Nominatim migration guide [https://www.nominatim.org/release-docs/latest/admin/Migration/](https://www.nominatim.org/release-docs/latest/admin/Migration/). +**Caution:** Upgrading between Nominatim versions (e.g., 4.0 to 4.1) without reimporting your data may not be directly supported by the image. Always check the Nominatim migration guide [https://www.nominatim.org/release-docs/latest/admin/Migration/](https://www.nominatim.org/release-docs/latest/admin/Migration/). # Contributors From 44a2d1bc54fa40d62825489b622de11f08eae8c5 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Fri, 2 Dec 2022 13:10:35 +0100 Subject: [PATCH 37/96] add info that Nominatim is ready (#400) --- 4.1/start.sh | 3 +++ 4.2/start.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/4.1/start.sh b/4.1/start.sh index 5acb9473..c8911b76 100755 --- a/4.1/start.sh +++ b/4.1/start.sh @@ -62,4 +62,7 @@ tailpid=${!} echo "Warm database caches for search and reverse queries" sudo -E -u nominatim nominatim admin --warm > /dev/null echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + wait diff --git a/4.2/start.sh b/4.2/start.sh index 5acb9473..c8911b76 100755 --- a/4.2/start.sh +++ b/4.2/start.sh @@ -62,4 +62,7 @@ tailpid=${!} echo "Warm database caches for search and reverse queries" sudo -E -u nominatim nominatim admin --warm > /dev/null echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + wait From e921dffb693d50a59269a702372152f1d61f5778 Mon Sep 17 00:00:00 2001 From: saddfox <48035291+saddfox@users.noreply.github.com> Date: Sun, 4 Dec 2022 12:16:49 +0100 Subject: [PATCH 38/96] Add support for reverse-only imports (#402) * add option to pass --reverse-only * update documentation * fix db warm when using REVERSE_ONLY * update ci for REVERSE_ONLY * chmod ci test --- .github/workflows/assert-reverse-only | 29 +++++++++++++++++++++++++++ .github/workflows/ci.yml | 12 +++++++++++ 4.1/README.md | 1 + 4.1/init.sh | 13 ++++++++++-- 4.1/start.sh | 9 +++++++-- 4.2/README.md | 1 + 4.2/init.sh | 13 ++++++++++-- 4.2/start.sh | 9 +++++++-- 8 files changed, 79 insertions(+), 8 deletions(-) create mode 100755 .github/workflows/assert-reverse-only diff --git a/.github/workflows/assert-reverse-only b/.github/workflows/assert-reverse-only new file mode 100755 index 00000000..1d14168d --- /dev/null +++ b/.github/workflows/assert-reverse-only @@ -0,0 +1,29 @@ +#! /usr/bin/env python3 + +# This script sends a HTTP request to a URL and asserts that its +# response matches that of an instance running in reverse-only mode. +# +# It retries the HTTP request a number of times. + +import logging +import requests +import sys +import json + +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +logging.basicConfig(level=logging.WARNING) + +s = requests.Session() +retries = Retry(total=7, backoff_factor=2, status_forcelist=[502, 503, 504]) +s.mount('http://', HTTPAdapter(max_retries=retries)) + +resp = s.get(sys.argv[1]) +jsn = resp.json() + +if (resp.status_code == 404 and jsn["error"]["message"] == "Reverse-only import does not support forward searching."): + print("✅ Nominatim is running in reverse-only mode") +else: + print(f'Endpoint returned status code {resp.status_code}: {resp.content}') + sys.exit(1) \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 242407f8..7929cfcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,6 +212,18 @@ jobs: ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+3SY" ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+4ES" + - name: Check when using REVERSE_ONLY + working-directory: .github/workflows + run: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REVERSE_ONLY="true" \ + -p 8010:8080 \ + nominatim & + sleep 90 + ./assert-reverse-only "http://localhost:8010/search.php?q=avenue%20pasteur" + ./assert-non-empty-json "http://localhost:8010/reverse.php?lat=43.734&lon=7.42&format=jsonv2" + - name: Login to DockerHub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} uses: docker/login-action@v2.1.0 diff --git a/4.1/README.md b/4.1/README.md index fb3e30ac..3e667e12 100644 --- a/4.1/README.md +++ b/4.1/README.md @@ -49,6 +49,7 @@ The following environment variables are available for configuration: - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ - `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.1.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) - `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) - `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) diff --git a/4.1/init.sh b/4.1/init.sh index 7815f04e..c46664c6 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -71,7 +71,12 @@ sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" chown -R nominatim:nominatim ${PROJECT_DIR} cd ${PROJECT_DIR} -sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then echo "Importing Tiger address data" @@ -97,7 +102,11 @@ else fi fi -sudo -E -u nominatim nominatim admin --warm +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim admin --warm --reverse +else + sudo -E -u nominatim nominatim admin --warm +fi # gather statistics for query planner to potentially improve query performance # see, https://github.com/osm-search/Nominatim/issues/1023 diff --git a/4.1/start.sh b/4.1/start.sh index c8911b76..a18f40eb 100755 --- a/4.1/start.sh +++ b/4.1/start.sh @@ -59,8 +59,13 @@ fi tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & tailpid=${!} -echo "Warm database caches for search and reverse queries" -sudo -E -u nominatim nominatim admin --warm > /dev/null +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -E -u nominatim nominatim admin --warm > /dev/null +fi echo "Warming finished" echo "--> Nominatim is ready to accept requests" diff --git a/4.2/README.md b/4.2/README.md index 4b1057fa..cfc522d2 100644 --- a/4.2/README.md +++ b/4.2/README.md @@ -49,6 +49,7 @@ The following environment variables are available for configuration: - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ - `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.2.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) - `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) - `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) diff --git a/4.2/init.sh b/4.2/init.sh index 7815f04e..c46664c6 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -71,7 +71,12 @@ sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" chown -R nominatim:nominatim ${PROJECT_DIR} cd ${PROJECT_DIR} -sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then echo "Importing Tiger address data" @@ -97,7 +102,11 @@ else fi fi -sudo -E -u nominatim nominatim admin --warm +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim admin --warm --reverse +else + sudo -E -u nominatim nominatim admin --warm +fi # gather statistics for query planner to potentially improve query performance # see, https://github.com/osm-search/Nominatim/issues/1023 diff --git a/4.2/start.sh b/4.2/start.sh index c8911b76..a18f40eb 100755 --- a/4.2/start.sh +++ b/4.2/start.sh @@ -59,8 +59,13 @@ fi tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & tailpid=${!} -echo "Warm database caches for search and reverse queries" -sudo -E -u nominatim nominatim admin --warm > /dev/null +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -E -u nominatim nominatim admin --warm > /dev/null +fi echo "Warming finished" echo "--> Nominatim is ready to accept requests" From ea2df23c30dddb5df722206c7fe30e4b34b0edaa Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Sun, 4 Dec 2022 12:18:53 +0100 Subject: [PATCH 39/96] add saddfox as contributor [skip ci] --- .all-contributorsrc | 11 ++++++ README.md | 87 +++++++++++++++++++++++---------------------- 2 files changed, 55 insertions(+), 43 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 4fef30bb..14f7b306 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -420,6 +420,17 @@ "code", "test" ] + }, + { + "login": "saddfox", + "name": "saddfox", + "avatar_url": "https://avatars.githubusercontent.com/u/48035291?v=4", + "profile": "https://github.com/saddfox", + "contributions": [ + "doc", + "code", + "test" + ] } ], "contributorsPerLine": 6, diff --git a/README.md b/README.md index bd2c1b46..5c501179 100644 --- a/README.md +++ b/README.md @@ -39,63 +39,64 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - + +

Andrew
💻 📖
Donato Lucia
💻
Georgios Markakis
📖
Philip Kozeny
🚇 💻 ⚠️ 👀 📖
Cezary Morga
💻
Thomas Nordquist
💻		Andrew
Andrew
💻 📖		Donato Lucia
Donato Lucia
💻		Georgios Markakis
Georgios Markakis
📖		Philip Kozeny
Philip Kozeny
🚇 💻 ⚠️ 👀 📖		Cezary Morga
Cezary Morga
💻		Thomas Nordquist
Thomas Nordquist
💻

Andrey Ruíz
📖
UntitleDude
💻
Jack McKernan
💻
mtmail
📖 💻 💬 👀
Andrey Oblivantsev
💻
Simone
💻		Andrey Ruíz
Andrey Ruíz
📖		UntitleDude
UntitleDude
💻		Jack McKernan
Jack McKernan
💻		mtmail
mtmail
📖 💻 💬 👀		Andrey Oblivantsev
Andrey Oblivantsev
💻		Simone
Simone
💻

DuncanMackintosh
💻 📖
Iiro Alhonen
📖
Mikhail Snetkov
💻
FritschAuctores
💻
rebos
💻
Leonard Ehrenfried
🚇 💻 ⚠️ 👀 📖		DuncanMackintosh
DuncanMackintosh
💻 📖		Iiro Alhonen
Iiro Alhonen
📖		Mikhail Snetkov
Mikhail Snetkov
💻		FritschAuctores
FritschAuctores
💻		rebos
rebos
💻		Leonard Ehrenfried
Leonard Ehrenfried
🚇 💻 ⚠️ 👀 📖

Nicolas Roelandt
📖 💻 ⚠️
Sacerdoss
📖
Tobias Wich
📖 💻
Alex Chaplianka
📖
Idris Hayward
📖
Karl von Randow
📖		Nicolas Roelandt
Nicolas Roelandt
📖 💻 ⚠️		Sacerdoss
Sacerdoss
📖		Tobias Wich
Tobias Wich
📖 💻		Alex Chaplianka
Alex Chaplianka
📖		Idris Hayward
Idris Hayward
📖		Karl von Randow
Karl von Randow
📖

Marco Lechner
📖
Mateusz Gaweł
💻
Nicolas Ternisien
📖
oschlueter
💻
Tim Nonner
💻
thlor
💻 📖		Marco Lechner
Marco Lechner
📖		Mateusz Gaweł
Mateusz Gaweł
💻		Nicolas Ternisien
Nicolas Ternisien
📖		oschlueter
oschlueter
💻		Tim Nonner
Tim Nonner
💻		thlor
thlor
💻 📖

Yun Wang
📖 💻
Stefanic
💻 📖
xpoinsard
📖 💻
Bartizan
💻 📖 ⚠️
galewis2
💻
Derick M.
💻 📖 ⚠️		Yun Wang
Yun Wang
📖 💻		Stefanic
Stefanic
💻 📖		xpoinsard
xpoinsard
📖 💻		Bartizan
Bartizan
💻 📖 ⚠️		galewis2
galewis2
💻		Derick M.
Derick M.
💻 📖 ⚠️

norcis
💻
SClo
💻 📖
Chris
💻 📖
iAlex97
💻 ⚠️
Mauricio Scheffer
💻
Sean Dean
📖		norcis
norcis
💻		SClo
SClo
💻 📖		Chris
Chris
💻 📖		iAlex97
iAlex97
💻 ⚠️		Mauricio Scheffer
Mauricio Scheffer
💻		Sean Dean
Sean Dean
📖

Philipp Gassmann
📖 💻 ⚠️		Philipp Gassmann
Philipp Gassmann
📖 💻 ⚠️		saddfox
saddfox
📖 💻 ⚠️
From 590890262399821f895658de2f2b9a7db109cf01 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Sun, 4 Dec 2022 13:23:29 +0100 Subject: [PATCH 40/96] increase sleep times --- .github/workflows/ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7929cfcb..88b785d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -60,7 +60,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount @@ -75,7 +75,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim @@ -111,7 +111,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 180 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -131,7 +131,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 180 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -146,7 +146,7 @@ jobs: -e IMPORT_STYLE=full \ -p 8005:8080 \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -158,7 +158,7 @@ jobs: -e IMPORT_STYLE=admin \ -p 8006:8080 \ nominatim & - sleep 90 + sleep 120 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -173,7 +173,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -185,7 +185,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -p 8008:8080 \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -196,7 +196,7 @@ jobs: -e FREEZE="true" \ -p 8008:8080 \ nominatim & - sleep 90 + sleep 120 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check GB postcode import @@ -207,7 +207,7 @@ jobs: -e IMPORT_GB_POSTCODES="true" \ -p 8009:8080 \ nominatim & - sleep 400 + sleep 600 ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=LE15+8TX" ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+3SY" ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+4ES" @@ -220,7 +220,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8010:8080 \ nominatim & - sleep 90 + sleep 120 ./assert-reverse-only "http://localhost:8010/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8010/reverse.php?lat=43.734&lon=7.42&format=jsonv2" From e7bb82bf4463bda48904943d4133b99b06566884 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 4 Jan 2023 18:47:58 +0100 Subject: [PATCH 41/96] set custom user agent (#414) * set customer user agent https://github.com/mediagis/nominatim-docker/issues/413 * switch to arg for build time and add env into collapsed image * fix arg & env usage * trying to fix env variable --- 4.1/Dockerfile | 14 ++++++++------ 4.1/init.sh | 8 ++++---- 4.2/Dockerfile | 14 ++++++++------ 4.2/init.sh | 8 ++++---- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/4.1/Dockerfile b/4.1/Dockerfile index 909625f5..642f0b5d 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:jammy AS build -ENV DEBIAN_FRONTEND noninteractive -ENV LANG C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 WORKDIR /app @@ -70,10 +70,11 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION 4.1.1 +ENV NOMINATIM_VERSION=4.1.1 +ARG USER_AGENT="mediagis/nominatim-docker" RUN true \ - && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ && tar xf nominatim.tar.bz2 \ && mkdir build \ && cd build \ @@ -129,11 +130,12 @@ FROM scratch COPY --from=build / / # Please override this -ENV NOMINATIM_PASSWORD qaIACxO6wMR3 +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 # how many threads should be use for importing ENV THREADS=16 -ENV PROJECT_DIR /nominatim +ENV PROJECT_DIR=/nominatim +ENV USER_AGENT=$USER_AGENT WORKDIR /app diff --git a/4.1/init.sh b/4.1/init.sh index c46664c6..4bda4e6b 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -4,7 +4,7 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - curl https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + curl -A {$USER_AGENT} https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -13,7 +13,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - curl https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz + curl -A {$USER_AGENT} https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -22,7 +22,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - curl https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz + curl -A $USER_AGENT https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -31,7 +31,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - curl https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + curl -A $USER_AGENT https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz diff --git a/4.2/Dockerfile b/4.2/Dockerfile index 75a2e16b..f780b873 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:jammy AS build -ENV DEBIAN_FRONTEND noninteractive -ENV LANG C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 WORKDIR /app @@ -70,10 +70,11 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION 4.2.0 +ENV NOMINATIM_VERSION=4.2.0 +ARG USER_AGENT="mediagis/nominatim-docker" RUN true \ - && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ && tar xf nominatim.tar.bz2 \ && mkdir build \ && cd build \ @@ -129,11 +130,12 @@ FROM scratch COPY --from=build / / # Please override this -ENV NOMINATIM_PASSWORD qaIACxO6wMR3 +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 # how many threads should be use for importing ENV THREADS=16 -ENV PROJECT_DIR /nominatim +ENV PROJECT_DIR=/nominatim +ENV USER_AGENT=$USER_AGENT WORKDIR /app diff --git a/4.2/init.sh b/4.2/init.sh index c46664c6..4bda4e6b 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -4,7 +4,7 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - curl https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + curl -A {$USER_AGENT} https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -13,7 +13,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - curl https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz + curl -A {$USER_AGENT} https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -22,7 +22,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - curl https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz + curl -A $USER_AGENT https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -31,7 +31,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - curl https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + curl -A $USER_AGENT https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz From 0c1152e5510320d04823b6a5aa3dabd18e7bbcd6 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 9 Jan 2023 13:53:07 +0100 Subject: [PATCH 42/96] Fix user agent, fail download with error message (#417) * Fix user agent, fail download with error message * Add comment * Port CURL to 4.1 --- 4.1/Dockerfile | 6 ++++-- 4.1/init.sh | 11 ++++++----- 4.2/Dockerfile | 6 ++++-- 4.2/init.sh | 12 +++++++----- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/4.1/Dockerfile b/4.1/Dockerfile index 642f0b5d..c42920ab 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -71,7 +71,7 @@ RUN pip3 install osmium # Nominatim install. ENV NOMINATIM_VERSION=4.1.1 -ARG USER_AGENT="mediagis/nominatim-docker" +ARG USER_AGENT="mediagis/nominatim-docker:$NOMINATIM_VERSION" RUN true \ && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ @@ -134,8 +134,10 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 # how many threads should be use for importing ENV THREADS=16 +# i cannot figure out a way to remove this duplication, if anyone knows, please get in touch +ENV USER_AGENT="mediagis/nominatim-docker:4.1.0" + ENV PROJECT_DIR=/nominatim -ENV USER_AGENT=$USER_AGENT WORKDIR /app diff --git a/4.1/init.sh b/4.1/init.sh index 4bda4e6b..772e006c 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -1,10 +1,11 @@ #!/bin/bash -ex OSMFILE=${PROJECT_DIR}/data.osm.pbf +CURL="curl -L -A ${USER_AGENT} --fail-with-body" if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - curl -A {$USER_AGENT} https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + ${CURL} https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -13,7 +14,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - curl -A {$USER_AGENT} https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz + ${CURL} https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -22,7 +23,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - curl -A $USER_AGENT https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz + ${CURL} https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -31,7 +32,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - curl -A $USER_AGENT https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + ${CURL} https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz @@ -41,7 +42,7 @@ fi if [ "$PBF_URL" != "" ]; then echo Downloading OSM extract from "$PBF_URL" - curl -L "$PBF_URL" -C - --create-dirs -o $OSMFILE + ${CURL} "$PBF_URL" -C - --create-dirs -o $OSMFILE fi if [ "$PBF_PATH" != "" ]; then diff --git a/4.2/Dockerfile b/4.2/Dockerfile index f780b873..db7b277b 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -71,7 +71,7 @@ RUN pip3 install osmium # Nominatim install. ENV NOMINATIM_VERSION=4.2.0 -ARG USER_AGENT="mediagis/nominatim-docker" +ARG USER_AGENT="mediagis/nominatim-docker:${NOMINATIM_VERSION}" RUN true \ && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ @@ -135,7 +135,9 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 ENV THREADS=16 ENV PROJECT_DIR=/nominatim -ENV USER_AGENT=$USER_AGENT + +# i cannot figure out a way to remove this duplication, if anyone knows, please get in touch +ENV USER_AGENT="mediagis/nominatim-docker:4.2.0" WORKDIR /app diff --git a/4.2/init.sh b/4.2/init.sh index 4bda4e6b..a6e04423 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -2,9 +2,11 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf +CURL="curl -L -A ${USER_AGENT} --fail-with-body" + if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - curl -A {$USER_AGENT} https://nominatim.org/data/wikimedia-importance.sql.gz -L -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + ${CURL} https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -13,7 +15,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - curl -A {$USER_AGENT} https://nominatim.org/data/gb_postcodes.csv.gz -L -o ${PROJECT_DIR}/gb_postcodes.csv.gz + ${CURL} https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -22,7 +24,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - curl -A $USER_AGENT https://nominatim.org/data/us_postcodes.csv.gz -L -o ${PROJECT_DIR}/us_postcodes.csv.gz + ${CURL} https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -31,7 +33,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - curl -A $USER_AGENT https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -L -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + ${CURL} https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz @@ -41,7 +43,7 @@ fi if [ "$PBF_URL" != "" ]; then echo Downloading OSM extract from "$PBF_URL" - curl -L "$PBF_URL" -C - --create-dirs -o $OSMFILE + ${CURL} "$PBF_URL" -C - --create-dirs -o $OSMFILE fi if [ "$PBF_PATH" != "" ]; then From 786b76ffa02bf821d1a54e40920bb0e91f58ded1 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Mon, 20 Feb 2023 20:30:34 +0000 Subject: [PATCH 43/96] update Nominatim --- 3.7/Dockerfile | 2 +- 4.0/Dockerfile | 2 +- 4.1/Dockerfile | 2 +- 4.2/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/3.7/Dockerfile b/3.7/Dockerfile index 85c33ad3..8ceb9bb8 100644 --- a/3.7/Dockerfile +++ b/3.7/Dockerfile @@ -68,7 +68,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION 3.7.2 +ENV NOMINATIM_VERSION 3.7.3 RUN true \ && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ diff --git a/4.0/Dockerfile b/4.0/Dockerfile index a0faf0a5..e95e8944 100644 --- a/4.0/Dockerfile +++ b/4.0/Dockerfile @@ -69,7 +69,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION 4.0.1 +ENV NOMINATIM_VERSION 4.0.2 RUN true \ && curl https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ diff --git a/4.1/Dockerfile b/4.1/Dockerfile index c42920ab..b04026fe 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -70,7 +70,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.1.1 +ENV NOMINATIM_VERSION=4.1.2 ARG USER_AGENT="mediagis/nominatim-docker:$NOMINATIM_VERSION" RUN true \ diff --git a/4.2/Dockerfile b/4.2/Dockerfile index db7b277b..e9abec4a 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -70,7 +70,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.2.0 +ENV NOMINATIM_VERSION=4.2.1 ARG USER_AGENT="mediagis/nominatim-docker:${NOMINATIM_VERSION}" RUN true \ From 6276787e309f4a7c8053e6fa78c4acc2fea4d393 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 22 Feb 2023 07:38:35 +0000 Subject: [PATCH 44/96] update missing version references --- 4.1/Dockerfile | 2 +- 4.2/Dockerfile | 2 +- README.md | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/4.1/Dockerfile b/4.1/Dockerfile index b04026fe..11b0889a 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -135,7 +135,7 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 ENV THREADS=16 # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT="mediagis/nominatim-docker:4.1.0" +ENV USER_AGENT="mediagis/nominatim-docker:4.1.2" ENV PROJECT_DIR=/nominatim diff --git a/4.2/Dockerfile b/4.2/Dockerfile index e9abec4a..58b036fc 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -137,7 +137,7 @@ ENV THREADS=16 ENV PROJECT_DIR=/nominatim # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT="mediagis/nominatim-docker:4.2.0" +ENV USER_AGENT="mediagis/nominatim-docker:4.2.1" WORKDIR /app diff --git a/README.md b/README.md index 5c501179..036ab6f5 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,13 @@ See relevant installation and usage instructions for each version in the ``` Date: Tue, 7 Mar 2023 15:44:03 +0100 Subject: [PATCH 45/96] Add quotes around user agent to allow spaces in it (#427) --- 4.1/init.sh | 2 +- 4.2/init.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/4.1/init.sh b/4.1/init.sh index 772e006c..c1631d32 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -1,7 +1,7 @@ #!/bin/bash -ex OSMFILE=${PROJECT_DIR}/data.osm.pbf -CURL="curl -L -A ${USER_AGENT} --fail-with-body" +CURL="curl -L -A \"${USER_AGENT}\" --fail-with-body" if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" diff --git a/4.2/init.sh b/4.2/init.sh index a6e04423..846cd152 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -2,7 +2,7 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf -CURL="curl -L -A ${USER_AGENT} --fail-with-body" +CURL="curl -L -A \"${USER_AGENT}\" --fail-with-body" if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" From 39151ef553c5bf414f503783946ce10d1303498d Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Tue, 7 Mar 2023 16:04:36 +0100 Subject: [PATCH 46/96] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 036ab6f5..94d47b29 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ 100% working container for [Nominatim](https://github.com/openstreetmap/Nominatim). -![GitHub Workflow Status](https://img.shields.io/github/workflow/status/mediagis/nominatim-docker/CI?style=flat-square) ![Github All Contributors](https://img.shields.io/github/all-contributors/mediagis/nominatim-docker?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/mediagis/nominatim?style=flat-square) ![Docker Image Size with architecture (latest by date/latest semver)](https://img.shields.io/docker/image-size/mediagis/nominatim?style=flat-square) +![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/mediagis/nominatim-docker/ci.yml?branch=master&style=flat-square) ![Github All Contributors](https://img.shields.io/github/all-contributors/mediagis/nominatim-docker?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/mediagis/nominatim?style=flat-square) ![Docker Image Size with architecture (latest by date/latest semver)](https://img.shields.io/docker/image-size/mediagis/nominatim?style=flat-square) # How to use From 82c9080fa3fb4101db54587f9f38aa8a64e783c1 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Tue, 18 Apr 2023 12:16:54 +0200 Subject: [PATCH 47/96] 4.2.3 update & fix curl (#444) --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++--------- 4.1/Dockerfile | 4 ++-- 4.1/init.sh | 13 +++++++------ 4.2/Dockerfile | 6 +++--- 4.2/init.sh | 12 ++++++------ README.md | 2 +- 6 files changed, 43 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88b785d9..7257cd2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,11 @@ jobs: - version: "4.1" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 + user_agent: mediagis/nominatim-docker-action:4.1 - version: "4.2" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 + user_agent: mediagis/nominatim-docker-action:4.2 runs-on: ubuntu-latest steps: @@ -42,6 +44,7 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8001:8080 \ --name nominatim \ nominatim & @@ -57,6 +60,7 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & @@ -71,6 +75,7 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8003:8080 \ --name nominatim \ @@ -87,6 +92,7 @@ jobs: docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8004:8080 \ --name nominatim \ @@ -107,6 +113,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ -e UPDATE_MODE=once \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -v nominatim-update-volume:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8004:8080 \ --name nominatim \ @@ -127,6 +134,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-${days_ago}.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ -e UPDATE_MODE=continuous \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -v /tmp/nominatim-update-bindmount:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8004:8080 \ --name nominatim \ @@ -144,6 +152,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ -e IMPORT_STYLE=full \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8005:8080 \ nominatim & sleep 120 @@ -156,6 +165,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ -e IMPORT_STYLE=admin \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8006:8080 \ nominatim & sleep 120 @@ -168,6 +178,7 @@ jobs: docker run -i --rm \ -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -v nominatim7-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -v /tmp/data:/nominatim/data \ -p 8007:8080 \ @@ -183,6 +194,7 @@ jobs: run: |- docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8008:8080 \ nominatim & sleep 120 @@ -193,36 +205,39 @@ jobs: run: |- docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -e FREEZE="true" \ - -p 8008:8080 \ + -p 8009:8080 \ nominatim & sleep 120 - ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: Check GB postcode import working-directory: .github/workflows run: |- docker run -i --rm \ -e PBF_URL=https://download.geofabrik.de/europe/great-britain/england/rutland-latest.osm.pbf \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -e IMPORT_GB_POSTCODES="true" \ - -p 8009:8080 \ + -p 8010:8080 \ nominatim & sleep 600 - ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=LE15+8TX" - ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+3SY" - ./assert-non-empty-json "http://localhost:8009/search.php?postalcode=PE9+4ES" + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=LE15+8TX" + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+3SY" + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+4ES" - name: Check when using REVERSE_ONLY working-directory: .github/workflows run: |- docker run -i --rm \ -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -e REVERSE_ONLY="true" \ - -p 8010:8080 \ + -p 8011:8080 \ nominatim & sleep 120 - ./assert-reverse-only "http://localhost:8010/search.php?q=avenue%20pasteur" - ./assert-non-empty-json "http://localhost:8010/reverse.php?lat=43.734&lon=7.42&format=jsonv2" + ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" + ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" - name: Login to DockerHub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} diff --git a/4.1/Dockerfile b/4.1/Dockerfile index 11b0889a..91ceadd4 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -71,7 +71,7 @@ RUN pip3 install osmium # Nominatim install. ENV NOMINATIM_VERSION=4.1.2 -ARG USER_AGENT="mediagis/nominatim-docker:$NOMINATIM_VERSION" +ARG USER_AGENT=mediagis/nominatim-docker:$NOMINATIM_VERSION RUN true \ && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ @@ -135,7 +135,7 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 ENV THREADS=16 # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT="mediagis/nominatim-docker:4.1.2" +ENV USER_AGENT=mediagis/nominatim-docker:4.1.2 ENV PROJECT_DIR=/nominatim diff --git a/4.1/init.sh b/4.1/init.sh index c1631d32..55f53595 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -1,11 +1,12 @@ #!/bin/bash -ex OSMFILE=${PROJECT_DIR}/data.osm.pbf -CURL="curl -L -A \"${USER_AGENT}\" --fail-with-body" + +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - ${CURL} https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + "${CURL[@]}" https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -14,7 +15,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - ${CURL} https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz + "${CURL[@]}" https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -23,7 +24,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - ${CURL} https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz + "${CURL[@]}" https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -32,7 +33,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - ${CURL} https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + "${CURL[@]}" https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz @@ -42,7 +43,7 @@ fi if [ "$PBF_URL" != "" ]; then echo Downloading OSM extract from "$PBF_URL" - ${CURL} "$PBF_URL" -C - --create-dirs -o $OSMFILE + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE fi if [ "$PBF_PATH" != "" ]; then diff --git a/4.2/Dockerfile b/4.2/Dockerfile index 58b036fc..395ac604 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -70,8 +70,8 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.2.1 -ARG USER_AGENT="mediagis/nominatim-docker:${NOMINATIM_VERSION}" +ENV NOMINATIM_VERSION=4.2.3 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} RUN true \ && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ @@ -137,7 +137,7 @@ ENV THREADS=16 ENV PROJECT_DIR=/nominatim # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT="mediagis/nominatim-docker:4.2.1" +ENV USER_AGENT=mediagis/nominatim-docker:4.2.3 WORKDIR /app diff --git a/4.2/init.sh b/4.2/init.sh index 846cd152..55f53595 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -2,11 +2,11 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf -CURL="curl -L -A \"${USER_AGENT}\" --fail-with-body" +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - ${CURL} https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + "${CURL[@]}" https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -15,7 +15,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - ${CURL} https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz + "${CURL[@]}" https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -24,7 +24,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - ${CURL} https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz + "${CURL[@]}" https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -33,7 +33,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - ${CURL} https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + "${CURL[@]}" https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz @@ -43,7 +43,7 @@ fi if [ "$PBF_URL" != "" ]; then echo Downloading OSM extract from "$PBF_URL" - ${CURL} "$PBF_URL" -C - --create-dirs -o $OSMFILE + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE fi if [ "$PBF_PATH" != "" ]; then diff --git a/README.md b/README.md index 94d47b29..e97ea8d2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ See relevant installation and usage instructions for each version in the ``` Date: Wed, 21 Jun 2023 12:02:49 +0200 Subject: [PATCH 48/96] Use rehosted downloads from storage box --- 4.2/Dockerfile | 4 +++- 4.2/init.sh | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/4.2/Dockerfile b/4.2/Dockerfile index 395ac604..a408e133 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -58,7 +58,9 @@ RUN true \ # Misc. git \ curl \ - sudo + sudo \ + sshpass \ + openssh-client # Configure postgres. diff --git a/4.2/init.sh b/4.2/init.sh index 55f53595..3ae2bdb9 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -4,9 +4,11 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - "${CURL[@]}" https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -15,7 +17,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - "${CURL[@]}" https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -24,7 +26,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - "${CURL[@]}" https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -33,7 +35,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - "${CURL[@]}" https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz From 9e5c673ea34d05f07f44c9889768995bc22a12ff Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 21 Jun 2023 14:03:49 +0200 Subject: [PATCH 49/96] Also use scp in 4.1 --- 4.1/Dockerfile | 4 +++- 4.1/init.sh | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/4.1/Dockerfile b/4.1/Dockerfile index 91ceadd4..d6d9ee59 100644 --- a/4.1/Dockerfile +++ b/4.1/Dockerfile @@ -58,7 +58,9 @@ RUN true \ # Misc. git \ curl \ - sudo + sudo \ + sshpass \ + openssh-client # Configure postgres. diff --git a/4.1/init.sh b/4.1/init.sh index 55f53595..3ae2bdb9 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -4,9 +4,11 @@ OSMFILE=${PROJECT_DIR}/data.osm.pbf CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" - "${CURL[@]}" https://nominatim.org/data/wikimedia-importance.sql.gz -o ${PROJECT_DIR}/wikimedia-importance.sql.gz + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz elif [ -f "$IMPORT_WIKIPEDIA" ]; then # use local file if asked ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz @@ -15,7 +17,7 @@ else fi; if [ "$IMPORT_GB_POSTCODES" = "true" ]; then - "${CURL[@]}" https://nominatim.org/data/gb_postcodes.csv.gz -o ${PROJECT_DIR}/gb_postcodes.csv.gz + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz elif [ -f "$IMPORT_GB_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz @@ -24,7 +26,7 @@ else \ fi; if [ "$IMPORT_US_POSTCODES" = "true" ]; then - "${CURL[@]}" https://nominatim.org/data/us_postcodes.csv.gz -o ${PROJECT_DIR}/us_postcodes.csv.gz + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz elif [ -f "$IMPORT_US_POSTCODES" ]; then # use local file if asked ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz @@ -33,7 +35,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - "${CURL[@]}" https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz -o ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz From 928716612e7e285f0507f6290c2a32b0bf2e8d1a Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 21 Jun 2023 14:33:27 +0200 Subject: [PATCH 50/96] Increase retries --- .github/workflows/assert-non-empty-json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assert-non-empty-json b/.github/workflows/assert-non-empty-json index cd62b3b9..add85d91 100755 --- a/.github/workflows/assert-non-empty-json +++ b/.github/workflows/assert-non-empty-json @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=7, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) +retries = Retry(total=18, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) From d5de6d6b4db632f67fbebbec10f471d603aae0a6 Mon Sep 17 00:00:00 2001 From: ialex Date: Thu, 28 Sep 2023 13:51:45 +0300 Subject: [PATCH 51/96] add dependencies and test build/import of nominatim 4.3.0 --- .github/workflows/ci.yml | 8 +- 4.3/Dockerfile | 154 ++++++++++++++++++ 4.3/README.md | 221 ++++++++++++++++++++++++++ 4.3/conf.d/apache.conf | 13 ++ 4.3/conf.d/env | 6 + 4.3/conf.d/postgres-import.conf | 2 + 4.3/conf.d/postgres-tuning.conf | 10 ++ 4.3/config.sh | 62 ++++++++ 4.3/contrib/docker-compose-planet.yml | 32 ++++ 4.3/contrib/docker-compose.yml | 20 +++ 4.3/init.sh | 134 ++++++++++++++++ 4.3/start.sh | 75 +++++++++ 4.3/startapache.sh | 5 + 4.3/startpostgres.sh | 4 + README.md | 3 +- 15 files changed, 744 insertions(+), 5 deletions(-) create mode 100644 4.3/Dockerfile create mode 100644 4.3/README.md create mode 100644 4.3/conf.d/apache.conf create mode 100644 4.3/conf.d/env create mode 100644 4.3/conf.d/postgres-import.conf create mode 100644 4.3/conf.d/postgres-tuning.conf create mode 100755 4.3/config.sh create mode 100644 4.3/contrib/docker-compose-planet.yml create mode 100644 4.3/contrib/docker-compose.yml create mode 100755 4.3/init.sh create mode 100755 4.3/start.sh create mode 100755 4.3/startapache.sh create mode 100755 4.3/startpostgres.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7257cd2d..8c3e600e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,14 @@ jobs: strategy: matrix: nominatim: - - version: "4.1" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 - user_agent: mediagis/nominatim-docker-action:4.1 - version: "4.2" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 user_agent: mediagis/nominatim-docker-action:4.2 + - version: "4.3" + update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + postgres_version: 14 + user_agent: mediagis/nominatim-docker-action:4.3 runs-on: ubuntu-latest steps: diff --git a/4.3/Dockerfile b/4.3/Dockerfile new file mode 100644 index 00000000..e5cf7386 --- /dev/null +++ b/4.3/Dockerfile @@ -0,0 +1,154 @@ +FROM ubuntu:jammy AS build + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 + +WORKDIR /app + +RUN true \ + # Do not start daemons after installation. + && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. + build-essential \ + g++ \ + cmake \ + libpq-dev \ + zlib1g-dev \ + libbz2-dev \ + libproj-dev \ + libexpat1-dev \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + liblua5.4-dev \ + nlohmann-json3-dev \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + # PHP and Apache 2. + php \ + php-intl \ + php-pgsql \ + php-cgi \ + apache2 \ + libapache2-mod-php \ + # Python 3. + python3-dev \ + python3-pip \ + python3-tidylib \ + python3-psycopg2 \ + python3-setuptools \ + python3-dotenv \ + python3-psutil \ + python3-jinja2 \ + python3-sqlalchemy \ + python3-asyncpg \ + python3-datrie \ + python3-icu \ + python3-argparse-manpage \ + # Misc. + git \ + curl \ + sudo \ + sshpass \ + openssh-client + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Osmium install to run continuous updates. +RUN pip3 install osmium + +# Nominatim install. +ENV NOMINATIM_VERSION=4.3.0 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} + +RUN true \ + && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && tar xf nominatim.tar.bz2 \ + && mkdir build \ + && cd build \ + && cmake ../Nominatim-$NOMINATIM_VERSION \ + && make -j`nproc` \ + && make install + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge \ + cpp-9 \ + gcc-9* \ + g++ \ + git \ + make \ + cmake* \ + llvm-10* \ + libc6-dev \ + linux-libc-dev \ + libclang-*-dev \ + build-essential \ + liblua*-dev \ + postgresql-server-dev-14 \ + && apt-get clean \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /app/src/.git \ + /var/lib/apt/lists/* \ + # Remove nominatim source and build directories + && rm /app/*.tar.bz2 \ + && rm -rf /app/build \ + && rm -rf /app/Nominatim-$NOMINATIM_VERSION + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled +COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 +# how many threads should be use for importing +ENV THREADS=16 + +ENV PROJECT_DIR=/nominatim + +# i cannot figure out a way to remove this duplication, if anyone knows, please get in touch +ENV USER_AGENT=mediagis/nominatim-docker:4.3.0 + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD ["/app/start.sh"] diff --git a/4.3/README.md b/4.3/README.md new file mode 100644 index 00000000..5a8d9e6d --- /dev/null +++ b/4.3/README.md @@ -0,0 +1,221 @@ +# Nominatim Docker (Nominatim version 4.3) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. Check [https://download.geofabrik.de](https://download.geofabrik.de) +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.3.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: `16`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.3.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.3.0/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/14/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.3.0/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.3.0/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) diff --git a/4.3/conf.d/apache.conf b/4.3/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.3/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.3/conf.d/env b/4.3/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.3/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.3/conf.d/postgres-import.conf b/4.3/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.3/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.3/conf.d/postgres-tuning.conf b/4.3/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..ef71eaf8 --- /dev/null +++ b/4.3/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.3.0/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.3/config.sh b/4.3/config.sh new file mode 100755 index 00000000..483354c9 --- /dev/null +++ b/4.3/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.3/contrib/docker-compose-planet.yml b/4.3/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..b7634b47 --- /dev/null +++ b/4.3/contrib/docker-compose-planet.yml @@ -0,0 +1,32 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.3 + restart: always + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/14/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb diff --git a/4.3/contrib/docker-compose.yml b/4.3/contrib/docker-compose.yml new file mode 100644 index 00000000..81e89bbe --- /dev/null +++ b/4.3/contrib/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.3 + restart: always + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.3#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/14/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.3/init.sh b/4.3/init.sh new file mode 100755 index 00000000..e8bbe7c0 --- /dev/null +++ b/4.3/init.sh @@ -0,0 +1,134 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") + +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then + chown postgres /var/lib/postgresql/14/main + sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main +fi + +# temporarily enable unsafe import optimization config +cp /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled /etc/postgresql/14/main/conf.d/postgres-import.conf + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +sudo -H -E -u nominatim env + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -H -E -u nominatim nominatim admin --warm --reverse +else + sudo -H -E -u nominatim nominatim admin --warm +fi + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/14/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.3/start.sh b/4.3/start.sh new file mode 100755 index 00000000..5fa843d3 --- /dev/null +++ b/4.3/start.sh @@ -0,0 +1,75 @@ +#!/bin/bash -ex + +tailpid=0 +replicationpid=0 + +stopServices() { + service apache2 stop + service postgresql stop + kill $replicationpid + kill $tailpid +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + +# fork a process and wait for it +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +sudo -H -E -u nominatim env + +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -H -E -u nominatim nominatim admin --warm > /dev/null +fi +echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + +wait diff --git a/4.3/startapache.sh b/4.3/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.3/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.3/startpostgres.sh b/4.3/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.3/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log diff --git a/README.md b/README.md index e97ea8d2..792d98c9 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ See relevant installation and usage instructions for each version in the ``` Date: Thu, 28 Sep 2023 13:54:32 +0300 Subject: [PATCH 52/96] remove nlohmann-json3-dev after build as it is marked as dev dependency --- 4.3/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/4.3/Dockerfile b/4.3/Dockerfile index e5cf7386..d0afa7ef 100644 --- a/4.3/Dockerfile +++ b/4.3/Dockerfile @@ -103,6 +103,7 @@ RUN true \ build-essential \ liblua*-dev \ postgresql-server-dev-14 \ + nlohmann-json3-dev \ && apt-get clean \ # Clear temporary files and directories. && rm -rf \ From 0b4666b624a23f373cf1c45eb8871dda2918a0d1 Mon Sep 17 00:00:00 2001 From: ialex Date: Thu, 28 Sep 2023 14:29:40 +0300 Subject: [PATCH 53/96] remove print env when importing --- 4.3/init.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/4.3/init.sh b/4.3/init.sh index e8bbe7c0..6a958b02 100755 --- a/4.3/init.sh +++ b/4.3/init.sh @@ -106,8 +106,6 @@ else fi fi -sudo -H -E -u nominatim env - if [ "$REVERSE_ONLY" = "true" ]; then sudo -H -E -u nominatim nominatim admin --warm --reverse else From 8cea735356875c4d9460249c4482291d31cac485 Mon Sep 17 00:00:00 2001 From: ialex Date: Thu, 28 Sep 2023 14:31:12 +0300 Subject: [PATCH 54/96] remove restart policy --- 4.3/contrib/docker-compose-planet.yml | 1 - 4.3/contrib/docker-compose.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/4.3/contrib/docker-compose-planet.yml b/4.3/contrib/docker-compose-planet.yml index b7634b47..86455146 100644 --- a/4.3/contrib/docker-compose-planet.yml +++ b/4.3/contrib/docker-compose-planet.yml @@ -11,7 +11,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.3 - restart: always ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.3/contrib/docker-compose.yml b/4.3/contrib/docker-compose.yml index 81e89bbe..7f3bc6c8 100644 --- a/4.3/contrib/docker-compose.yml +++ b/4.3/contrib/docker-compose.yml @@ -4,7 +4,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.3 - restart: always ports: - "8080:8080" environment: From a6890565403f192b95e480d6cb2b01910ebe8cac Mon Sep 17 00:00:00 2001 From: ialex Date: Thu, 28 Sep 2023 17:19:38 +0300 Subject: [PATCH 55/96] use --search-only flag instead of --reverse when warming up --- 4.3/init.sh | 2 +- 4.3/start.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/4.3/init.sh b/4.3/init.sh index 6a958b02..ea75df89 100755 --- a/4.3/init.sh +++ b/4.3/init.sh @@ -107,7 +107,7 @@ else fi if [ "$REVERSE_ONLY" = "true" ]; then - sudo -H -E -u nominatim nominatim admin --warm --reverse + sudo -H -E -u nominatim nominatim admin --warm --search-only else sudo -H -E -u nominatim nominatim admin --warm fi diff --git a/4.3/start.sh b/4.3/start.sh index 5fa843d3..1b7602dd 100755 --- a/4.3/start.sh +++ b/4.3/start.sh @@ -59,11 +59,9 @@ fi tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & tailpid=${!} -sudo -H -E -u nominatim env - if [ "$REVERSE_ONLY" = "true" ]; then echo "Warm database caches for reverse queries" - sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null + sudo -H -E -u nominatim nominatim admin --warm --search-only > /dev/null else echo "Warm database caches for search and reverse queries" sudo -H -E -u nominatim nominatim admin --warm > /dev/null From b4ec7ddbe86b6190a2b0e835e862f5a38066ab12 Mon Sep 17 00:00:00 2001 From: ialex Date: Thu, 28 Sep 2023 18:26:44 +0300 Subject: [PATCH 56/96] add comment about warmup workaround --search-only --- 4.3/start.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/4.3/start.sh b/4.3/start.sh index 1b7602dd..32e3719a 100755 --- a/4.3/start.sh +++ b/4.3/start.sh @@ -61,6 +61,9 @@ tailpid=${!} if [ "$REVERSE_ONLY" = "true" ]; then echo "Warm database caches for reverse queries" + # --search-only is a workaround until https://github.com/osm-search/Nominatim/issues/3213 + # is merged and a new Nominatim version (probably 4.3.1) is released. + # Afterwards, we should revert back to using --reverse instead sudo -H -E -u nominatim nominatim admin --warm --search-only > /dev/null else echo "Warm database caches for search and reverse queries" From c06e9bbb2936f3121000a54a95ce3b176df67f2d Mon Sep 17 00:00:00 2001 From: ialex Date: Wed, 4 Oct 2023 16:05:56 +0300 Subject: [PATCH 57/96] temporarily increase query timeout to avoid failing planet import during database warmup --- 4.3/init.sh | 7 +++++++ 4.3/start.sh | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/4.3/init.sh b/4.3/init.sh index ea75df89..2fde6e05 100755 --- a/4.3/init.sh +++ b/4.3/init.sh @@ -106,11 +106,18 @@ else fi fi +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 if [ "$REVERSE_ONLY" = "true" ]; then + # --search-only is a workaround until https://github.com/osm-search/Nominatim/issues/3213 + # is merged and a new Nominatim version (probably 4.3.1) is released. + # Afterwards, we should revert back to using --reverse instead sudo -H -E -u nominatim nominatim admin --warm --search-only else sudo -H -E -u nominatim nominatim admin --warm fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 # gather statistics for query planner to potentially improve query performance # see, https://github.com/osm-search/Nominatim/issues/1023 diff --git a/4.3/start.sh b/4.3/start.sh index 32e3719a..cec4f342 100755 --- a/4.3/start.sh +++ b/4.3/start.sh @@ -59,6 +59,8 @@ fi tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & tailpid=${!} +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 if [ "$REVERSE_ONLY" = "true" ]; then echo "Warm database caches for reverse queries" # --search-only is a workaround until https://github.com/osm-search/Nominatim/issues/3213 @@ -69,6 +71,8 @@ else echo "Warm database caches for search and reverse queries" sudo -H -E -u nominatim nominatim admin --warm > /dev/null fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 echo "Warming finished" echo "--> Nominatim is ready to accept requests" From 746eb95b49dc2587f36415d8a8f59c8ae00a0a32 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Sat, 7 Oct 2023 15:08:04 +0200 Subject: [PATCH 58/96] update to 4.3.1 --- 4.3/Dockerfile | 4 ++-- 4.3/README.md | 12 ++++++------ 4.3/conf.d/postgres-tuning.conf | 2 +- 4.3/init.sh | 5 +---- 4.3/start.sh | 5 +---- README.md | 2 +- 6 files changed, 12 insertions(+), 18 deletions(-) diff --git a/4.3/Dockerfile b/4.3/Dockerfile index d0afa7ef..ec4d730a 100644 --- a/4.3/Dockerfile +++ b/4.3/Dockerfile @@ -75,7 +75,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.3.0 +ENV NOMINATIM_VERSION=4.3.1 ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} RUN true \ @@ -143,7 +143,7 @@ ENV THREADS=16 ENV PROJECT_DIR=/nominatim # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT=mediagis/nominatim-docker:4.3.0 +ENV USER_AGENT=mediagis/nominatim-docker:4.3.1 WORKDIR /app diff --git a/4.3/README.md b/4.3/README.md index 5a8d9e6d..319e943b 100644 --- a/4.3/README.md +++ b/4.3/README.md @@ -47,7 +47,7 @@ The following environment variables are available for configuration: - `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. - `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ -- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.3.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.3.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) - `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) @@ -75,7 +75,7 @@ The following environment variables are available to tune PostgreSQL: - `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) - `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) -See https://nominatim.org/release-docs/4.3.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. +See https://nominatim.org/release-docs/4.3.1/admin/Installation/#tuning-the-postgresql-database for more details on those settings. ### Import Style @@ -91,11 +91,11 @@ Available options are : - `full`: Default style that also includes points of interest. - `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. -See https://nominatim.org/release-docs/4.3.0/admin/Import/#filtering-imported-data for more details on those styles. +See https://nominatim.org/release-docs/4.3.1/admin/Import/#filtering-imported-data for more details on those styles. ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.1/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. ## Persistent container data @@ -145,7 +145,7 @@ where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that ## Updating the database -Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.3.0/admin/Update/). For a list of other methods see the output of: +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.3.1/admin/Update/). For a list of other methods see the output of: ```sh docker exec -it nominatim sudo -u nominatim nominatim replication --help @@ -190,7 +190,7 @@ docker run -it \ Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. -For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.3.0/customize/Tiger/). +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.3.1/customize/Tiger/). ## Development diff --git a/4.3/conf.d/postgres-tuning.conf b/4.3/conf.d/postgres-tuning.conf index ef71eaf8..21e1a094 100644 --- a/4.3/conf.d/postgres-tuning.conf +++ b/4.3/conf.d/postgres-tuning.conf @@ -1,4 +1,4 @@ -# See https://nominatim.org/release-docs/4.3.0/admin/Installation/#tuning-the-postgresql-database +# See https://nominatim.org/release-docs/4.3.1/admin/Installation/#tuning-the-postgresql-database shared_buffers = 2GB maintenance_work_mem = 10GB autovacuum_work_mem = 2GB diff --git a/4.3/init.sh b/4.3/init.sh index 2fde6e05..93e2da27 100755 --- a/4.3/init.sh +++ b/4.3/init.sh @@ -109,10 +109,7 @@ fi export NOMINATIM_QUERY_TIMEOUT=600 export NOMINATIM_REQUEST_TIMEOUT=3600 if [ "$REVERSE_ONLY" = "true" ]; then - # --search-only is a workaround until https://github.com/osm-search/Nominatim/issues/3213 - # is merged and a new Nominatim version (probably 4.3.1) is released. - # Afterwards, we should revert back to using --reverse instead - sudo -H -E -u nominatim nominatim admin --warm --search-only + sudo -H -E -u nominatim nominatim admin --warm --reverse else sudo -H -E -u nominatim nominatim admin --warm fi diff --git a/4.3/start.sh b/4.3/start.sh index cec4f342..c59a4c85 100755 --- a/4.3/start.sh +++ b/4.3/start.sh @@ -63,10 +63,7 @@ export NOMINATIM_QUERY_TIMEOUT=600 export NOMINATIM_REQUEST_TIMEOUT=3600 if [ "$REVERSE_ONLY" = "true" ]; then echo "Warm database caches for reverse queries" - # --search-only is a workaround until https://github.com/osm-search/Nominatim/issues/3213 - # is merged and a new Nominatim version (probably 4.3.1) is released. - # Afterwards, we should revert back to using --reverse instead - sudo -H -E -u nominatim nominatim admin --warm --search-only > /dev/null + sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null else echo "Warm database caches for search and reverse queries" sudo -H -E -u nominatim nominatim admin --warm > /dev/null diff --git a/README.md b/README.md index 792d98c9..be859a81 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ See relevant installation and usage instructions for each version in the ``` Date: Thu, 12 Oct 2023 15:10:17 +0200 Subject: [PATCH 59/96] use nproc to set sensible threads default --- 4.2/Dockerfile | 2 -- 4.2/README.md | 2 +- 4.2/contrib/docker-compose-planet.yml | 1 - 4.2/init.sh | 8 ++++++++ 4.3/Dockerfile | 2 -- 4.3/README.md | 2 +- 4.3/init.sh | 8 ++++++++ 7 files changed, 18 insertions(+), 7 deletions(-) diff --git a/4.2/Dockerfile b/4.2/Dockerfile index a408e133..354bd806 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -133,8 +133,6 @@ COPY --from=build / / # Please override this ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 -# how many threads should be use for importing -ENV THREADS=16 ENV PROJECT_DIR=/nominatim diff --git a/4.2/README.md b/4.2/README.md index cfc522d2..f0f8aa39 100644 --- a/4.2/README.md +++ b/4.2/README.md @@ -54,7 +54,7 @@ The following environment variables are available for configuration: - `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) - `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) - `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) -- `THREADS`: How many threads should be used to import (default: `16`) +- `THREADS`: How many threads should be used to import (default: number of processing units available to the current process via `nproc`) - `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) The following run parameters are available for configuration: diff --git a/4.2/contrib/docker-compose-planet.yml b/4.2/contrib/docker-compose-planet.yml index 95cda233..b7b09883 100644 --- a/4.2/contrib/docker-compose-planet.yml +++ b/4.2/contrib/docker-compose-planet.yml @@ -11,7 +11,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.2 - restart: always ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.2/init.sh b/4.2/init.sh index 3ae2bdb9..a18c8db2 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -6,6 +6,14 @@ CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' +# Check if THREADS is not set or is empty +if [ -z "$THREADS" ]; then + THREADS=$(nproc) +fi + +# Export so it's available for any process run after this script +export THREADS + if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz diff --git a/4.3/Dockerfile b/4.3/Dockerfile index d0afa7ef..cf36bf4b 100644 --- a/4.3/Dockerfile +++ b/4.3/Dockerfile @@ -137,8 +137,6 @@ COPY --from=build / / # Please override this ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 -# how many threads should be use for importing -ENV THREADS=16 ENV PROJECT_DIR=/nominatim diff --git a/4.3/README.md b/4.3/README.md index 5a8d9e6d..fa6e4d89 100644 --- a/4.3/README.md +++ b/4.3/README.md @@ -54,7 +54,7 @@ The following environment variables are available for configuration: - `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) - `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) - `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) -- `THREADS`: How many threads should be used to import (default: `16`) +- `THREADS`: How many threads should be used to import (default: number of processing units available to the current process via `nproc`) - `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) The following run parameters are available for configuration: diff --git a/4.3/init.sh b/4.3/init.sh index 2fde6e05..5ab9ed57 100755 --- a/4.3/init.sh +++ b/4.3/init.sh @@ -6,6 +6,14 @@ CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' +# Check if THREADS is not set or is empty +if [ -z "$THREADS" ]; then + THREADS=$(nproc) +fi + +# Export so it's available for any process run after this script +export THREADS + if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz From e9e9dad643173dac14c49ce0e28a7c40f1cc9ff6 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Thu, 12 Oct 2023 15:58:47 +0200 Subject: [PATCH 60/96] remove export threads since everything is handled in init.sh --- 4.2/init.sh | 3 --- 4.3/init.sh | 3 --- 2 files changed, 6 deletions(-) diff --git a/4.2/init.sh b/4.2/init.sh index a18c8db2..b71cd262 100755 --- a/4.2/init.sh +++ b/4.2/init.sh @@ -11,9 +11,6 @@ if [ -z "$THREADS" ]; then THREADS=$(nproc) fi -# Export so it's available for any process run after this script -export THREADS - if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz diff --git a/4.3/init.sh b/4.3/init.sh index 5ab9ed57..1aefd92a 100755 --- a/4.3/init.sh +++ b/4.3/init.sh @@ -11,9 +11,6 @@ if [ -z "$THREADS" ]; then THREADS=$(nproc) fi -# Export so it's available for any process run after this script -export THREADS - if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz From 35eb52f01635702a04ba1e04b906218072dc3e3b Mon Sep 17 00:00:00 2001 From: gsg-git <92863111+gsg-git@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:12:55 +0100 Subject: [PATCH 61/96] Documentation improvement [ci skip] (#493) --- 4.3/README.md | 31 ++++++++++++++++++-- 4.3/example.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 4.3/example.md diff --git a/4.3/README.md b/4.3/README.md index f6c0c3be..db0d97e5 100644 --- a/4.3/README.md +++ b/4.3/README.md @@ -8,6 +8,7 @@ - [PostgreSQL Tuning](#postgresql-tuning) - [Import Style](#import-style) - [Flatnode files](#flatnode-files) + - [Configuration Example](#configuration-example) - [Persistent container data](#persistent-container-data) - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) - [Updating the database](#updating-the-database) @@ -42,9 +43,15 @@ If you want to check that your data import was successful, you can use the API w The following environment variables are available for configuration: -- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. Check [https://download.geofabrik.de](https://download.geofabrik.de) +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. + Check [https://download.geofabrik.de](https://download.geofabrik.de) + Since the download speed is restricted at Geofabrik, there is a recommended list of mirrors for importing the full planet at [OSM Wiki](https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors). + At the mirror sites you can find the folder /planet which contains the planet-latest.osm.pbf + and often a `/replication` folder for the `REPLICATION_URL`. - `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. -- `REPLICATION_URL`: Where to get updates from. Also available from Geofabrik. +- `REPLICATION_URL`: Where to get updates from. For example Geofabrik's update for the Europe extract are available at `https://download.geofabrik.de/europe-updates/` +Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CONTINENT/$COUNTRY-updates/` + - `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ - `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.3.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) @@ -95,7 +102,22 @@ See https://nominatim.org/release-docs/4.3.1/admin/Import/#filtering-imported-da ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.1/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +```sh +docker run -it \ + -v nominatim-flatnode:/nominatim/flatnode \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +### Configuration Example + +Here you can find a [configuration example](example.md) for all flags you can use for the container creation. + ## Persistent container data @@ -219,3 +241,6 @@ These files follow the naming convention of `docker-compose-*.yml` and contain c - [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) - [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) +- [Upgrading Nominatim](https://github.com/mediagis/nominatim-docker/discussions/317) +- [Using Nominatim UI](https://github.com/mediagis/nominatim-docker/discussions/486#discussioncomment-7239861) + diff --git a/4.3/example.md b/4.3/example.md new file mode 100644 index 00000000..bdf4f108 --- /dev/null +++ b/4.3/example.md @@ -0,0 +1,79 @@ +# Configuration Example + +```sh +docker run -it \ + #Sets the flatnode file, which is to reduce the load on the database when you plan to use multiple countrys together bigger than 6GB + #and highly recommended if you want to import the World! + -v nominatim-flatnode:/nominatim/flatnode \ + + #PostgreSQL Tuning, without the need to edit the .conf after the setup (Nominatim default recommended values) + -e POSTGRES_SHARED_BUFFERS=2GB \ + -e POSTGRES_MAINTAINENCE_WORK_MEM=10GB \ + -e POSTGRES_AUTOVACUUM_WORK_MEM=2GB \ + -e POSTGRES_WORK_MEM=50MB \ + -e POSTGRES_EFFECTIVE_CACHE_SIZE=24GB \ + -e POSTGRES_SYNCHRONOUS_COMMIT=off \ + -e POSTGRES_MAX_WAL_SIZE=1GB \ + -e POSTGRES_CHECKPOINT_TIMEOUT=10min \ + -e POSTGRES_CHECKPOINT_COMPLETITION_TARGET=0.9 \ + + #Sets the target for the initial file for the import. If the file is already on the local system you use: + #-e PBF_PATH=/path/to/your/planet-latest.osm.pbf PBF_URL cannot be used together with PBF_PATH! + -e PBF_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf \ + + #Sets the Path, where Nominatim gets the map updates - the REPLICATION_URL is never a file. + -e REPLICATION_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ \ + + #How often upstream publishes diffs (in seconds, default: 86400). Requires REPLICATION_URL to be set. + -e REPLICATION_UPDATE_INTERVAL=43200 + + #How long to sleep if no update found yet (in seconds, default: 900). Requires REPLICATION_URL to be set. + -e REPLICATION_RECHECK_INTERVAL=450 + + #Configures the way the map files will be updated (default: none) + -e UPDATE_MODE=continuous/once/catch-up/none + + #Disables the updates to save space for example (default: false) + -e FREEZE=true/false + + #If you only want to use the Nominatim database for reverse lookups. (default: false) + -e REVERSE_ONLY=true/false + + #When enabled additional Wikipedia Data will be loaded (default off) + -e IMPORT_WIKIPEDIA=true/false + + #Whether to download and import the US postcode dump (true) or path to US postcode dump in the container. (default: false) + -e IMPORT_US_POSTCODES=true/false/path + + #Whether to download and import the GB postcode dump (true) or path to GB postcode dump in the container. (default: false) + -e IMPORT_GB_POSTCODES=true/false/path + + #Sets either an importfilter for a reduced data import or the full set and the full set with additional data (default: full): + #admin: Only import administrative boundaries and places. + #street: Like the admin style but also adds streets. + #address: Import all data necessary to compute addresses down to house number level. + #full: Default style that also includes points of interest. + #extratags: Like the full style but also adds most of the OSM tags into the extratags column. + -e IMPORT_STYLE=admin/street/address/full/extratags + + #Whether to download and import the Tiger address data (true) or path to a preprocessed Tiger address set in the container. (default: false) + -e IMPORT_TIGER_ADDRESSES=true/false/path + + #Sets the used threads at the import (default 16) + -e THREADS=10 \ + + #Sets the Docker tmpfs. Highly recommended for bigger imports like Europe. At least 1GB - ideally half of the available RAM. + --shm-size=60g \ + + #The password to connect to the database with (default: qaIACxO6wMR3) + -e NOMINATIM_PASSWORD=supersafepassword + + #Sets the ports of the container guest:host + -p 8080:8080 \ + + #Sets the name of the container + --name nominatim \ + + #Here you choose the Docker image and version + mediagis/nominatim:4.3 +``` From 037281b55cb44cfc9bcf669cb4b5d82c87f09ce8 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:20:16 +0000 Subject: [PATCH 62/96] update README.md [skip ci] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index be859a81..9a9795ca 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Philipp Gassmann
Philipp Gassmann
📖 💻 ⚠️ saddfox
saddfox
📖 💻 ⚠️ + gsg-git
gsg-git
📖 From e596cde06625174f9f786379538ac99f1c079117 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:20:21 +0000 Subject: [PATCH 63/96] update .all-contributorsrc [skip ci] --- .all-contributorsrc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 14f7b306..9990b1ed 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -431,6 +431,15 @@ "code", "test" ] + }, + { + "login": "gsg-git", + "name": "gsg-git", + "avatar_url": "https://avatars.githubusercontent.com/u/92863111?v=4", + "profile": "https://github.com/gsg-git", + "contributions": [ + "doc" + ] } ], "contributorsPerLine": 6, @@ -439,5 +448,6 @@ "repoType": "github", "repoHost": "https://github.com", "skipCi": true, - "commitConvention": "none" + "commitConvention": "none", + "commitType": "docs" } From e109f8f502fe47fa06020760249dfe7397199597 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Fri, 17 Nov 2023 17:43:36 +0100 Subject: [PATCH 64/96] Update Nominatim version to 4.3.2 --- 4.3/Dockerfile | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/4.3/Dockerfile b/4.3/Dockerfile index f66983c8..76bb0769 100644 --- a/4.3/Dockerfile +++ b/4.3/Dockerfile @@ -75,7 +75,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.3.1 +ENV NOMINATIM_VERSION=4.3.2 ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} RUN true \ @@ -141,7 +141,7 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 ENV PROJECT_DIR=/nominatim # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT=mediagis/nominatim-docker:4.3.1 +ENV USER_AGENT=mediagis/nominatim-docker:4.3.2 WORKDIR /app diff --git a/README.md b/README.md index 9a9795ca..8ccb7515 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ See relevant installation and usage instructions for each version in the ``` Date: Fri, 17 Nov 2023 17:42:09 +0100 Subject: [PATCH 65/96] Update Nominatim version to 4.2.4 --- 4.2/Dockerfile | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/4.2/Dockerfile b/4.2/Dockerfile index 354bd806..3760098a 100644 --- a/4.2/Dockerfile +++ b/4.2/Dockerfile @@ -72,7 +72,7 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.2.3 +ENV NOMINATIM_VERSION=4.2.4 ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} RUN true \ @@ -137,7 +137,7 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 ENV PROJECT_DIR=/nominatim # i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT=mediagis/nominatim-docker:4.2.3 +ENV USER_AGENT=mediagis/nominatim-docker:4.2.4 WORKDIR /app diff --git a/README.md b/README.md index 9a9795ca..d8644a93 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ See relevant installation and usage instructions for each version in the ``` Date: Mon, 25 Dec 2023 12:29:04 +0100 Subject: [PATCH 66/96] deduplicate USER_AGENT specification in Dockerfile Testing: ```bash docker build ./4.3/ -t some:name && \ docker run --entrypoint /bin/bash some:name \ -c 'echo user agent is: $USER_AGENT' ``` --- 4.3/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/4.3/Dockerfile b/4.3/Dockerfile index 76bb0769..8afb61b0 100644 --- a/4.3/Dockerfile +++ b/4.3/Dockerfile @@ -1,3 +1,6 @@ +ARG NOMINATIM_VERSION=4.3.2 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} + FROM ubuntu:jammy AS build ENV DEBIAN_FRONTEND=noninteractive @@ -75,8 +78,8 @@ RUN true \ RUN pip3 install osmium # Nominatim install. -ENV NOMINATIM_VERSION=4.3.2 -ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} +ARG NOMINATIM_VERSION +ARG USER_AGENT RUN true \ && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ @@ -140,8 +143,8 @@ ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 ENV PROJECT_DIR=/nominatim -# i cannot figure out a way to remove this duplication, if anyone knows, please get in touch -ENV USER_AGENT=mediagis/nominatim-docker:4.3.2 +ARG USER_AGENT +ENV USER_AGENT=${USER_AGENT} WORKDIR /app From f3d44ad7e5aa6b86aecfba519a3fed7d39e55da5 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 05:47:59 +0000 Subject: [PATCH 67/96] update README.md [skip ci] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 042219d7..22ab0ea6 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Philipp Gassmann
Philipp Gassmann
📖 💻 ⚠️ saddfox
saddfox
📖 💻 ⚠️ gsg-git
gsg-git
📖 + Stefan Breunig
Stefan Breunig
💻 From 3fdd12e83a4a9607ec9d3bf7a793259fb917fe2d Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 05:48:00 +0000 Subject: [PATCH 68/96] update .all-contributorsrc [skip ci] --- .all-contributorsrc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 9990b1ed..31864f4a 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -440,6 +440,15 @@ "contributions": [ "doc" ] + }, + { + "login": "breunigs", + "name": "Stefan Breunig", + "avatar_url": "https://avatars.githubusercontent.com/u/307954?v=4", + "profile": "https://github.com/breunigs", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 6, From 48cd8d621e2c0b79d593233f3b33b99a5718dc64 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 21 Feb 2024 16:54:05 +0100 Subject: [PATCH 69/96] Use date-based tag rather than git SHA --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c3e600e..523da39b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,10 +246,14 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set container date tag + run: | + echo "DATE_TAG=$(date +%Y-%m-%dT%H-%M)" >> $GITHUB_ENV + - name: Build and push production docker image to Dockerhub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} run: |- docker buildx build --platform linux/amd64,linux/arm64 --push \ -t mediagis/nominatim:${{ matrix.nominatim.version }} \ - -t mediagis/nominatim:${{ matrix.nominatim.version }}-${{ github.sha }} . + -t mediagis/nominatim:${{ matrix.nominatim.version }}-${DATE_TAG} . working-directory: ${{ matrix.nominatim.version }} From 107c3e28c1a8b5dc6ce3972d23828116037682ee Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 12:07:11 +0100 Subject: [PATCH 70/96] Update to Nominatim 4.4 --- 4.4/Dockerfile | 156 ++++++++++++++++ 4.4/README.md | 246 ++++++++++++++++++++++++++ 4.4/conf.d/apache.conf | 13 ++ 4.4/conf.d/env | 6 + 4.4/conf.d/postgres-import.conf | 2 + 4.4/conf.d/postgres-tuning.conf | 10 ++ 4.4/config.sh | 62 +++++++ 4.4/contrib/docker-compose-planet.yml | 31 ++++ 4.4/contrib/docker-compose.yml | 19 ++ 4.4/example.md | 79 +++++++++ 4.4/init.sh | 141 +++++++++++++++ 4.4/start.sh | 77 ++++++++ 4.4/startapache.sh | 5 + 4.4/startpostgres.sh | 4 + 14 files changed, 851 insertions(+) create mode 100644 4.4/Dockerfile create mode 100644 4.4/README.md create mode 100644 4.4/conf.d/apache.conf create mode 100644 4.4/conf.d/env create mode 100644 4.4/conf.d/postgres-import.conf create mode 100644 4.4/conf.d/postgres-tuning.conf create mode 100755 4.4/config.sh create mode 100644 4.4/contrib/docker-compose-planet.yml create mode 100644 4.4/contrib/docker-compose.yml create mode 100644 4.4/example.md create mode 100755 4.4/init.sh create mode 100755 4.4/start.sh create mode 100755 4.4/startapache.sh create mode 100755 4.4/startpostgres.sh diff --git a/4.4/Dockerfile b/4.4/Dockerfile new file mode 100644 index 00000000..fc11891f --- /dev/null +++ b/4.4/Dockerfile @@ -0,0 +1,156 @@ +ARG NOMINATIM_VERSION=4.4.0 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} + +FROM ubuntu:jammy AS build + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 + +WORKDIR /app + +RUN true \ + # Do not start daemons after installation. + && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. + build-essential \ + g++ \ + cmake \ + libpq-dev \ + zlib1g-dev \ + libbz2-dev \ + libproj-dev \ + libexpat1-dev \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + liblua5.4-dev \ + nlohmann-json3-dev \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + # PHP and Apache 2. + php \ + php-intl \ + php-pgsql \ + php-cgi \ + apache2 \ + libapache2-mod-php \ + # Python 3. + python3-dev \ + python3-pip \ + python3-tidylib \ + python3-psycopg2 \ + python3-setuptools \ + python3-dotenv \ + python3-psutil \ + python3-jinja2 \ + python3-sqlalchemy \ + python3-asyncpg \ + python3-datrie \ + python3-icu \ + python3-argparse-manpage \ + # Misc. + git \ + curl \ + sudo \ + sshpass \ + openssh-client + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Osmium install to run continuous updates. +RUN pip3 install osmium + +# Nominatim install. +ARG NOMINATIM_VERSION +ARG USER_AGENT + +RUN true \ + && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && tar xf nominatim.tar.bz2 \ + && mkdir build \ + && cd build \ + && cmake ../Nominatim-$NOMINATIM_VERSION \ + && make -j`nproc` \ + && make install + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge \ + cpp-9 \ + gcc-9* \ + g++ \ + git \ + make \ + cmake* \ + llvm-10* \ + libc6-dev \ + linux-libc-dev \ + libclang-*-dev \ + build-essential \ + liblua*-dev \ + postgresql-server-dev-14 \ + nlohmann-json3-dev \ + && apt-get clean \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /app/src/.git \ + /var/lib/apt/lists/* \ + # Remove nominatim source and build directories + && rm /app/*.tar.bz2 \ + && rm -rf /app/build \ + && rm -rf /app/Nominatim-$NOMINATIM_VERSION + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled +COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 + +ENV PROJECT_DIR=/nominatim + +ARG USER_AGENT +ENV USER_AGENT=${USER_AGENT} + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD ["/app/start.sh"] diff --git a/4.4/README.md b/4.4/README.md new file mode 100644 index 00000000..ac5f4d7e --- /dev/null +++ b/4.4/README.md @@ -0,0 +1,246 @@ +# Nominatim Docker (Nominatim version 4.3) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Configuration Example](#configuration-example) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. + Check [https://download.geofabrik.de](https://download.geofabrik.de) + Since the download speed is restricted at Geofabrik, there is a recommended list of mirrors for importing the full planet at [OSM Wiki](https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors). + At the mirror sites you can find the folder /planet which contains the planet-latest.osm.pbf + and often a `/replication` folder for the `REPLICATION_URL`. +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. For example Geofabrik's update for the Europe extract are available at `https://download.geofabrik.de/europe-updates/` +Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CONTINENT/$COUNTRY-updates/` + +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: number of processing units available to the current process via `nproc`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.4.0/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +```sh +docker run -it \ + -v nominatim-flatnode:/nominatim/flatnode \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +### Configuration Example + +Here you can find a [configuration example](example.md) for all flags you can use for the container creation. + + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/14/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.0/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.0/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) +- [Upgrading Nominatim](https://github.com/mediagis/nominatim-docker/discussions/317) +- [Using Nominatim UI](https://github.com/mediagis/nominatim-docker/discussions/486#discussioncomment-7239861) + diff --git a/4.4/conf.d/apache.conf b/4.4/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.4/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.4/conf.d/env b/4.4/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.4/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.4/conf.d/postgres-import.conf b/4.4/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.4/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.4/conf.d/postgres-tuning.conf b/4.4/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..2b3c97f5 --- /dev/null +++ b/4.4/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.4/config.sh b/4.4/config.sh new file mode 100755 index 00000000..483354c9 --- /dev/null +++ b/4.4/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.4/contrib/docker-compose-planet.yml b/4.4/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..86455146 --- /dev/null +++ b/4.4/contrib/docker-compose-planet.yml @@ -0,0 +1,31 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.3 + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/14/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb diff --git a/4.4/contrib/docker-compose.yml b/4.4/contrib/docker-compose.yml new file mode 100644 index 00000000..7f3bc6c8 --- /dev/null +++ b/4.4/contrib/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.3 + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.3#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/14/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.4/example.md b/4.4/example.md new file mode 100644 index 00000000..bdf4f108 --- /dev/null +++ b/4.4/example.md @@ -0,0 +1,79 @@ +# Configuration Example + +```sh +docker run -it \ + #Sets the flatnode file, which is to reduce the load on the database when you plan to use multiple countrys together bigger than 6GB + #and highly recommended if you want to import the World! + -v nominatim-flatnode:/nominatim/flatnode \ + + #PostgreSQL Tuning, without the need to edit the .conf after the setup (Nominatim default recommended values) + -e POSTGRES_SHARED_BUFFERS=2GB \ + -e POSTGRES_MAINTAINENCE_WORK_MEM=10GB \ + -e POSTGRES_AUTOVACUUM_WORK_MEM=2GB \ + -e POSTGRES_WORK_MEM=50MB \ + -e POSTGRES_EFFECTIVE_CACHE_SIZE=24GB \ + -e POSTGRES_SYNCHRONOUS_COMMIT=off \ + -e POSTGRES_MAX_WAL_SIZE=1GB \ + -e POSTGRES_CHECKPOINT_TIMEOUT=10min \ + -e POSTGRES_CHECKPOINT_COMPLETITION_TARGET=0.9 \ + + #Sets the target for the initial file for the import. If the file is already on the local system you use: + #-e PBF_PATH=/path/to/your/planet-latest.osm.pbf PBF_URL cannot be used together with PBF_PATH! + -e PBF_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf \ + + #Sets the Path, where Nominatim gets the map updates - the REPLICATION_URL is never a file. + -e REPLICATION_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ \ + + #How often upstream publishes diffs (in seconds, default: 86400). Requires REPLICATION_URL to be set. + -e REPLICATION_UPDATE_INTERVAL=43200 + + #How long to sleep if no update found yet (in seconds, default: 900). Requires REPLICATION_URL to be set. + -e REPLICATION_RECHECK_INTERVAL=450 + + #Configures the way the map files will be updated (default: none) + -e UPDATE_MODE=continuous/once/catch-up/none + + #Disables the updates to save space for example (default: false) + -e FREEZE=true/false + + #If you only want to use the Nominatim database for reverse lookups. (default: false) + -e REVERSE_ONLY=true/false + + #When enabled additional Wikipedia Data will be loaded (default off) + -e IMPORT_WIKIPEDIA=true/false + + #Whether to download and import the US postcode dump (true) or path to US postcode dump in the container. (default: false) + -e IMPORT_US_POSTCODES=true/false/path + + #Whether to download and import the GB postcode dump (true) or path to GB postcode dump in the container. (default: false) + -e IMPORT_GB_POSTCODES=true/false/path + + #Sets either an importfilter for a reduced data import or the full set and the full set with additional data (default: full): + #admin: Only import administrative boundaries and places. + #street: Like the admin style but also adds streets. + #address: Import all data necessary to compute addresses down to house number level. + #full: Default style that also includes points of interest. + #extratags: Like the full style but also adds most of the OSM tags into the extratags column. + -e IMPORT_STYLE=admin/street/address/full/extratags + + #Whether to download and import the Tiger address data (true) or path to a preprocessed Tiger address set in the container. (default: false) + -e IMPORT_TIGER_ADDRESSES=true/false/path + + #Sets the used threads at the import (default 16) + -e THREADS=10 \ + + #Sets the Docker tmpfs. Highly recommended for bigger imports like Europe. At least 1GB - ideally half of the available RAM. + --shm-size=60g \ + + #The password to connect to the database with (default: qaIACxO6wMR3) + -e NOMINATIM_PASSWORD=supersafepassword + + #Sets the ports of the container guest:host + -p 8080:8080 \ + + #Sets the name of the container + --name nominatim \ + + #Here you choose the Docker image and version + mediagis/nominatim:4.3 +``` diff --git a/4.4/init.sh b/4.4/init.sh new file mode 100755 index 00000000..e3e32f30 --- /dev/null +++ b/4.4/init.sh @@ -0,0 +1,141 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") + +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + +# Check if THREADS is not set or is empty +if [ -z "$THREADS" ]; then + THREADS=$(nproc) +fi + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then + chown postgres /var/lib/postgresql/14/main + sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main +fi + +# temporarily enable unsafe import optimization config +cp /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled /etc/postgresql/14/main/conf.d/postgres-import.conf + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -H -E -u nominatim nominatim admin --warm --reverse +else + sudo -H -E -u nominatim nominatim admin --warm +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/14/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.4/start.sh b/4.4/start.sh new file mode 100755 index 00000000..c59a4c85 --- /dev/null +++ b/4.4/start.sh @@ -0,0 +1,77 @@ +#!/bin/bash -ex + +tailpid=0 +replicationpid=0 + +stopServices() { + service apache2 stop + service postgresql stop + kill $replicationpid + kill $tailpid +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + +# fork a process and wait for it +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -H -E -u nominatim nominatim admin --warm > /dev/null +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 +echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + +wait diff --git a/4.4/startapache.sh b/4.4/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.4/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.4/startpostgres.sh b/4.4/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.4/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log From 3603d20a86954953975b0d5b6bf41ff32d27a6bf Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 12:08:04 +0100 Subject: [PATCH 71/96] Upgrade versions in ci config --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 523da39b..011262f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,14 @@ jobs: strategy: matrix: nominatim: - - version: "4.2" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 - user_agent: mediagis/nominatim-docker-action:4.2 - version: "4.3" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 user_agent: mediagis/nominatim-docker-action:4.3 + - version: "4.4" + update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + postgres_version: 14 + user_agent: mediagis/nominatim-docker-action:4.4 runs-on: ubuntu-latest steps: From 3cf67a80bd8451f016b341ec055cb28561a30050 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 12:11:35 +0100 Subject: [PATCH 72/96] Add 4.4.0 to overview --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 22ab0ea6..d8dc3eda 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ See relevant installation and usage instructions for each version in the ``` Date: Mon, 18 Mar 2024 12:23:59 +0100 Subject: [PATCH 73/96] Reduce sleep time a bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 011262f3..7136f964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 180 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -139,7 +139,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 180 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log From 576c1efca63f0baae582d899696578607e4cc230 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 13:42:29 +0100 Subject: [PATCH 74/96] Lower sleep even more --- .github/workflows/assert-empty-json | 2 +- .github/workflows/assert-reverse-only | 2 +- .github/workflows/ci.yml | 24 +++++++++++------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/assert-empty-json b/.github/workflows/assert-empty-json index a0b810f6..a43cbeb2 100755 --- a/.github/workflows/assert-empty-json +++ b/.github/workflows/assert-empty-json @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=7, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) +retries = Retry(total=18, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/assert-reverse-only b/.github/workflows/assert-reverse-only index 1d14168d..48876aae 100755 --- a/.github/workflows/assert-reverse-only +++ b/.github/workflows/assert-reverse-only @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=7, backoff_factor=2, status_forcelist=[502, 503, 504]) +retries = Retry(total=18, backoff_factor=2, status_forcelist=[502, 503, 504]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7136f964..fd7af116 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -64,7 +64,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount @@ -80,7 +80,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim @@ -104,7 +104,6 @@ jobs: docker stop nominatim - name: Check UPDATE_MODE=once with volume - if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from four days ago to make sure there really are updates to apply @@ -118,14 +117,13 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log docker stop nominatim - name: Check UPDATE_MODE=continuous with bind-mount - if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from few days ago to make sure there really are updates to apply @@ -139,7 +137,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -155,7 +153,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8005:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -168,7 +166,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8006:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -184,7 +182,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -197,7 +195,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8008:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -209,7 +207,7 @@ jobs: -e FREEZE="true" \ -p 8009:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: Check GB postcode import @@ -235,7 +233,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8011:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" From 1a0c50174905355713eb901d4a5c6ecb3bb9b45f Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 13:45:23 +0100 Subject: [PATCH 75/96] Update documenation --- 4.4/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/4.4/README.md b/4.4/README.md index ac5f4d7e..5b8123c5 100644 --- a/4.4/README.md +++ b/4.4/README.md @@ -30,7 +30,7 @@ docker run -it \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. @@ -102,7 +102,7 @@ See https://nominatim.org/release-docs/4.4.0/admin/Import/#filtering-imported-da ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. ```sh docker run -it \ @@ -111,7 +111,7 @@ docker run -it \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` ### Configuration Example @@ -137,7 +137,7 @@ docker run -it --shm-size=1g \ -v nominatim-data:/var/lib/postgresql/14/main \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` ## OpenStreetMap Data Extracts @@ -160,7 +160,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ @@ -191,7 +191,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ @@ -207,7 +207,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/extras:/nominatim/extras \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. From eb0b99115be931330e7b6da0927cccb3bbfa3d5a Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 13:51:13 +0100 Subject: [PATCH 76/96] Apply suggestions from code review Co-authored-by: Philip Kozeny --- 4.4/README.md | 2 +- 4.4/contrib/docker-compose-planet.yml | 2 +- 4.4/contrib/docker-compose.yml | 2 +- 4.4/example.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/4.4/README.md b/4.4/README.md index 5b8123c5..0ac5b5bc 100644 --- a/4.4/README.md +++ b/4.4/README.md @@ -1,4 +1,4 @@ -# Nominatim Docker (Nominatim version 4.3) +# Nominatim Docker (Nominatim version 4.4) ## Table of contents diff --git a/4.4/contrib/docker-compose-planet.yml b/4.4/contrib/docker-compose-planet.yml index 86455146..8603adbd 100644 --- a/4.4/contrib/docker-compose-planet.yml +++ b/4.4/contrib/docker-compose-planet.yml @@ -10,7 +10,7 @@ version: "3" services: nominatim: container_name: nominatim - image: mediagis/nominatim:4.3 + image: mediagis/nominatim:4.4 ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.4/contrib/docker-compose.yml b/4.4/contrib/docker-compose.yml index 7f3bc6c8..cf648e18 100644 --- a/4.4/contrib/docker-compose.yml +++ b/4.4/contrib/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: nominatim: container_name: nominatim - image: mediagis/nominatim:4.3 + image: mediagis/nominatim:4.4 ports: - "8080:8080" environment: diff --git a/4.4/example.md b/4.4/example.md index bdf4f108..370b64a7 100644 --- a/4.4/example.md +++ b/4.4/example.md @@ -75,5 +75,5 @@ docker run -it \ --name nominatim \ #Here you choose the Docker image and version - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` From e6de6db37d4205dd2344c6fe7843b48d01365289 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 14:43:33 +0100 Subject: [PATCH 77/96] Revert "Lower sleep even more" This reverts commit 576c1efca63f0baae582d899696578607e4cc230. --- .github/workflows/assert-empty-json | 2 +- .github/workflows/assert-reverse-only | 2 +- .github/workflows/ci.yml | 24 +++++++++++++----------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/assert-empty-json b/.github/workflows/assert-empty-json index a43cbeb2..a0b810f6 100755 --- a/.github/workflows/assert-empty-json +++ b/.github/workflows/assert-empty-json @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=18, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) +retries = Retry(total=7, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/assert-reverse-only b/.github/workflows/assert-reverse-only index 48876aae..1d14168d 100755 --- a/.github/workflows/assert-reverse-only +++ b/.github/workflows/assert-reverse-only @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=18, backoff_factor=2, status_forcelist=[502, 503, 504]) +retries = Retry(total=7, backoff_factor=2, status_forcelist=[502, 503, 504]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd7af116..7136f964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -64,7 +64,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount @@ -80,7 +80,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim @@ -104,6 +104,7 @@ jobs: docker stop nominatim - name: Check UPDATE_MODE=once with volume + if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from four days ago to make sure there really are updates to apply @@ -117,13 +118,14 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log docker stop nominatim - name: Check UPDATE_MODE=continuous with bind-mount + if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from few days ago to make sure there really are updates to apply @@ -137,7 +139,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -153,7 +155,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8005:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -166,7 +168,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8006:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -182,7 +184,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -195,7 +197,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8008:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -207,7 +209,7 @@ jobs: -e FREEZE="true" \ -p 8009:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: Check GB postcode import @@ -233,7 +235,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8011:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" From 9c1786ca29c3b75fca39cd046b5768c7c4021ddb Mon Sep 17 00:00:00 2001 From: fredericks1982 <32093333+fredericks1982@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:55:57 +0100 Subject: [PATCH 78/96] Update docker-compose.yml Minor change: the URL in the comment referred to version 4.3 instead of 4.4 --- 4.4/contrib/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.4/contrib/docker-compose.yml b/4.4/contrib/docker-compose.yml index cf648e18..c8470b55 100644 --- a/4.4/contrib/docker-compose.yml +++ b/4.4/contrib/docker-compose.yml @@ -7,7 +7,7 @@ services: ports: - "8080:8080" environment: - # see https://github.com/mediagis/nominatim-docker/tree/master/4.3#configuration for more options + # see https://github.com/mediagis/nominatim-docker/tree/master/4.4#configuration for more options PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ NOMINATIM_PASSWORD: very_secure_password From a53bc02414ea4fabae92e6f4169c78b8f6ad46d0 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 20 Mar 2024 10:33:35 +0100 Subject: [PATCH 79/96] Add comment about rehosting files [ci skip] --- 4.4/init.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/4.4/init.sh b/4.4/init.sh index e3e32f30..ad8165da 100755 --- a/4.4/init.sh +++ b/4.4/init.sh @@ -11,6 +11,9 @@ if [ -z "$THREADS" ]; then THREADS=$(nproc) fi +# we re-host the files because inconsiderate users eat up all of nominatim.org's bandwidth +# https://github.com/mediagis/nominatim-docker/issues/416 + if [ "$IMPORT_WIKIPEDIA" = "true" ]; then echo "Downloading Wikipedia importance dump" ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz From e846fc5eb5c896c1feb090d42dd839bc2808c91e Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 20 Mar 2024 11:00:44 +0100 Subject: [PATCH 80/96] Upgrade to Tiger 2023 --- 4.4/init.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/4.4/init.sh b/4.4/init.sh index ad8165da..b4a351f3 100755 --- a/4.4/init.sh +++ b/4.4/init.sh @@ -11,7 +11,8 @@ if [ -z "$THREADS" ]; then THREADS=$(nproc) fi -# we re-host the files because inconsiderate users eat up all of nominatim.org's bandwidth +# we re-host the files on a Hetzner storage box because inconsiderate users eat up all of +# nominatim.org's bandwidth # https://github.com/mediagis/nominatim-docker/issues/416 if [ "$IMPORT_WIKIPEDIA" = "true" ]; then @@ -43,7 +44,7 @@ else fi; if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then - ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + ${SCP}:tiger2023-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then # use local file if asked ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz From 581d762b49bd776f1e38c8c58b0e5a4c553be11d Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 20 Mar 2024 11:24:40 +0100 Subject: [PATCH 81/96] Increase sleep --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7136f964..bee1dfc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 5 + sleep 25 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" From 9b1b085930faa62f4234edb470c4a82078220ebf Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 11:05:27 +0200 Subject: [PATCH 82/96] Separate build, test and deploy stages in GitHub CI actions. Execute test steps in parallel. --- .github/workflows/ci.yml | 432 ++++++++++++++++++++------------------- 1 file changed, 221 insertions(+), 211 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bee1dfc8..537faeba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,238 +6,248 @@ on: workflow_dispatch: jobs: - docker: - name: Build and test docker image + build: + name: Build docker image strategy: matrix: nominatim: - version: "4.3" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 - user_agent: mediagis/nominatim-docker-action:4.3 - version: "4.4" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 - user_agent: mediagis/nominatim-docker-action:4.4 - + runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - - name: Build docker image - run: docker build -t nominatim . - working-directory: ${{ matrix.nominatim.version }} - - - name: Check import with PBF_URL and update - working-directory: .github/workflows - run: |- - # get the data from four days ago to make sure there really are updates to apply - four_days_ago=`date --date="4 days ago" +%y%m%d` - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -p 8001:8080 \ - --name nominatim \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" - ${{ matrix.nominatim.update_command }} - ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" - docker stop nominatim - - - name: Check import with volume mount - working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ - -p 8002:8080 \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - - - name: Check import with bind-mount - working-directory: .github/workflows - run: |- - # get the data from four days ago to make sure there really are updates to apply - four_days_ago=`date --date="4 days ago" +%y%m%d` - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ - -p 8003:8080 \ - --name nominatim \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" - docker stop nominatim + - name: Build and export image + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.nominatim.version }} + tags: nominatim:latest + outputs: type=docker,dest=/tmp/nominatim.tar + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: nominatim-${{ matrix.nominatim.version }} + path: /tmp/nominatim.tar - - name: Check container restart and update with bind-mount - working-directory: .github/workflows - run: |- - # import to bind mount is done by previous step - four_days_ago=`date --date="4 days ago" +%y%m%d` - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -v /tmp/nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ - -p 8004:8080 \ - --name nominatim \ - nominatim & - sleep 25 - ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" - ${{ matrix.nominatim.update_command }} - ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" - docker stop nominatim + test: + name: Test docker image + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + nominatim: + - version: "4.3" + - version: "4.4" + + test_scenario: + - name: Import with PBF_URL and update + commands: |- + # get the data from four days ago to make sure there really are updates to apply + four_days_ago=`date --date="4 days ago" +%y%m%d` + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -p 8001:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" + docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" + + - name: Import with volume mount + commands: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8002:8080 \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" + + - name: Import with bind-mount, container restart & update + commands: |- + # get the data from four days ago to make sure there really are updates to apply + four_days_ago=`date --date="4 days ago" +%y%m%d` + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -v /tmp/nominatim-data:/var/lib/postgresql/14/main \ + -p 8003:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" + + # Stop container + docker stop nominatim - - name: Check UPDATE_MODE=once with volume - if: matrix.nominatim.version != '4.0' - working-directory: .github/workflows - run: |- - # get the data from four days ago to make sure there really are updates to apply - four_days_ago=`date --date="4 days ago" +%y%m%d` - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e UPDATE_MODE=once \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -v nominatim-update-volume:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ - -p 8004:8080 \ - --name nominatim \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" - echo "check replication log for Update completed. Count:" - docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log - docker stop nominatim + # import to bind mount is done previously + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -v /tmp/nominatim-data:/var/lib/postgresql/14/main \ + -p 8004:8080 \ + --name nominatim \ + nominatim & + sleep 25 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" + docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" - - name: Check UPDATE_MODE=continuous with bind-mount - if: matrix.nominatim.version != '4.0' - working-directory: .github/workflows - run: |- - # get the data from few days ago to make sure there really are updates to apply - days_ago=`date --date="4 days ago" +%y%m%d` - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-${days_ago}.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e UPDATE_MODE=continuous \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -v /tmp/nominatim-update-bindmount:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ - -p 8004:8080 \ - --name nominatim \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" - echo "check replication log for Update completed. Count:" - docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log - docker stop nominatim + - name: UPDATE_MODE=once with volume + commands: |- + # get the data from four days ago to make sure there really are updates to apply + four_days_ago=`date --date="4 days ago" +%y%m%d` + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${four_days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e UPDATE_MODE=once \ + -v nominatim-update-volume:/var/lib/postgresql/14/main \ + -p 8004:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" + echo -n "check replication log for Update completed. Count:" + docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log - - name: Check import full style - working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e IMPORT_STYLE=full \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -p 8005:8080 \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" + - name: UPDATE_MODE=continuous with bind-mount + commands: |- + # get the data from few days ago to make sure there really are updates to apply + days_ago=`date --date="4 days ago" +%y%m%d` + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-${days_ago}.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e UPDATE_MODE=continuous \ + -v /tmp/nominatim-update-bindmount:/var/lib/postgresql/14/main \ + -p 8004:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" + echo -n "check replication log for Update completed. Count:" + docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + + - name: Import full style + commands: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_STYLE=full \ + -p 8005:8080 \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" + + - name: Import admin style + commands: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_STYLE=admin \ + -p 8006:8080 \ + nominatim & + sleep 120 + ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - - name: Check import admin style - working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e IMPORT_STYLE=admin \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -p 8006:8080 \ - nominatim & - sleep 120 - ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" + - name: Import with PBF_PATH + commands: |- + wget --cut-dirs=1 -nH -xP /tmp/data http://download.geofabrik.de/europe/monaco-latest.osm.pbf + docker run -i --rm \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -v nominatim7-data:/var/lib/postgresql/14/main \ + -v /tmp/data:/nominatim/data \ + -p 8007:8080 \ + --name nominatim \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" - - name: Check import with PBF_PATH - working-directory: .github/workflows - run: |- - wget --cut-dirs=1 -nH -xP /tmp/data http://download.geofabrik.de/europe/monaco-latest.osm.pbf - docker run -i --rm \ - -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ - -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -v nominatim7-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ - -v /tmp/data:/nominatim/data \ - -p 8007:8080 \ - --name nominatim \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" - docker stop nominatim - docker volume rm nominatim7-data + - name: REPLICATION_URL is blank + commands: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -p 8008:8080 \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - - name: Check when REPLICATION_URL is blank - working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -p 8008:8080 \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" + - name: FREEZE + commands: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e FREEZE="true" \ + -p 8009:8080 \ + nominatim & + sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - - name: Check when using FREEZE - working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -e FREEZE="true" \ - -p 8009:8080 \ - nominatim & - sleep 120 - ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" + - name: GB postcode import + commands: |- + docker run -i --rm \ + -e PBF_URL=https://download.geofabrik.de/europe/great-britain/england/rutland-latest.osm.pbf \ + -e IMPORT_GB_POSTCODES="true" \ + -p 8010:8080 \ + nominatim & + sleep 600 + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=LE15+8TX" + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+3SY" + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+4ES" + + - name: REVERSE_ONLY + commands: |- + docker run -i --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REVERSE_ONLY="true" \ + -p 8011:8080 \ + nominatim & + sleep 120 + ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" + ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" - - name: Check GB postcode import - working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=https://download.geofabrik.de/europe/great-britain/england/rutland-latest.osm.pbf \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -e IMPORT_GB_POSTCODES="true" \ - -p 8010:8080 \ - nominatim & - sleep 600 - ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=LE15+8TX" - ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+3SY" - ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+4ES" + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: nominatim-${{ matrix.nominatim.version }} + path: /tmp + - name: Load image + run: | + docker load --input /tmp/nominatim.tar + docker image ls -a + - name: Checkout + uses: actions/checkout@v4 - - name: Check when using REVERSE_ONLY + # Excute the test scenario + - name: ${{ matrix.test_scenario.name }} working-directory: .github/workflows - run: |- - docker run -i --rm \ - -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ - -e USER_AGENT=${{matrix.nominatim.user_agent}} \ - -e REVERSE_ONLY="true" \ - -p 8011:8080 \ - nominatim & - sleep 120 - ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" - ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" + run: ${{ matrix.test_scenario.commands }} + + publish: + name: Publish docker image + needs: test + runs-on: ubuntu-latest + + strategy: + matrix: + nominatim: + - version: "4.3" + - version: "4.4" + + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} @@ -245,7 +255,7 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Set container date tag run: | echo "DATE_TAG=$(date +%Y-%m-%dT%H-%M)" >> $GITHUB_ENV @@ -256,4 +266,4 @@ jobs: docker buildx build --platform linux/amd64,linux/arm64 --push \ -t mediagis/nominatim:${{ matrix.nominatim.version }} \ -t mediagis/nominatim:${{ matrix.nominatim.version }}-${DATE_TAG} . - working-directory: ${{ matrix.nominatim.version }} + working-directory: ${{ matrix.nominatim.version }} \ No newline at end of file From a70da36a62a726b5fcb85002930ed8306ad37e2c Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 17 Apr 2024 10:05:08 +0200 Subject: [PATCH 83/96] Add checkout step to publish --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 537faeba..c5683569 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,6 +246,9 @@ jobs: - version: "4.4" steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From c6b05ffd0dc9dac60f33cc00243463373f896192 Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 08:37:51 +0200 Subject: [PATCH 84/96] bugfix: avoid killing replication process on shutdown if it was not started --- .github/workflows/ci.yml | 17 ++++++++++++++++- 4.4/start.sh | 5 ++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5683569..4847209c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,7 +215,22 @@ jobs: sleep 120 ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" - + + - name: Check for clean shutdown + commands: |- + docker run --detach --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -p 8009:8080 \ + --name nominatim \ + nominatim + sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" + # Shutdown the container + docker stop -t 60 nominatim + # Verify that the exit code is zero + CONTAINER_EXIT_CODE=$(docker inspect nominatim --format='{{.State.ExitCode}}') + test "$CONTAINER_EXIT_CODE" -eq 0 + steps: - name: Download artifact uses: actions/download-artifact@v4 diff --git a/4.4/start.sh b/4.4/start.sh index c59a4c85..dedab409 100755 --- a/4.4/start.sh +++ b/4.4/start.sh @@ -6,7 +6,10 @@ replicationpid=0 stopServices() { service apache2 stop service postgresql stop - kill $replicationpid + if [ $replicationpid -ne 0 ]; then + echo "Shutting down replication process" + kill $replicationpid + fi kill $tailpid } trap stopServices SIGTERM TERM INT From 227361b027c45489e5e5dc80c933474a8779d0ff Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 10:19:14 +0200 Subject: [PATCH 85/96] Dockerfile: use cache for apt and pip install start.sh: Add comments in stopServices() and exit code 0 --- 4.4/Dockerfile | 10 +++++----- 4.4/start.sh | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/4.4/Dockerfile b/4.4/Dockerfile index fc11891f..80494fe1 100644 --- a/4.4/Dockerfile +++ b/4.4/Dockerfile @@ -8,9 +8,10 @@ ENV LANG=C.UTF-8 WORKDIR /app -RUN true \ +RUN \ + --mount=type=cache,target=/var/cache/apt \ # Do not start daemons after installation. - && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ && chmod +x /usr/sbin/policy-rc.d \ # Install all required packages. && apt-get -y update -qq \ @@ -75,7 +76,8 @@ RUN true \ && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf # Osmium install to run continuous updates. -RUN pip3 install osmium +RUN --mount=type=cache,target=/root/.cache/pip \ + pip3 install osmium # Nominatim install. ARG NOMINATIM_VERSION @@ -107,12 +109,10 @@ RUN true \ liblua*-dev \ postgresql-server-dev-14 \ nlohmann-json3-dev \ - && apt-get clean \ # Clear temporary files and directories. && rm -rf \ /tmp/* \ /var/tmp/* \ - /root/.cache \ /app/src/.git \ /var/lib/apt/lists/* \ # Remove nominatim source and build directories diff --git a/4.4/start.sh b/4.4/start.sh index dedab409..0fdd86cf 100755 --- a/4.4/start.sh +++ b/4.4/start.sh @@ -6,11 +6,14 @@ replicationpid=0 stopServices() { service apache2 stop service postgresql stop + # Check if the replication process is active if [ $replicationpid -ne 0 ]; then echo "Shutting down replication process" kill $replicationpid fi kill $tailpid + # Force exit code 0 to signal a succesfull shutdown to Docker + exit 0 } trap stopServices SIGTERM TERM INT From 1992f00038fdd1c42bb1a2e7ca699df871946d0b Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 11:03:30 +0200 Subject: [PATCH 86/96] Remove forced sleep in CI --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4847209c..160cb932 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -70,7 +70,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/14/main \ -p 8002:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Import with bind-mount, container restart & update @@ -84,7 +84,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" # Stop container @@ -98,7 +98,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 25 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once @@ -116,7 +116,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -133,7 +133,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -146,7 +146,7 @@ jobs: -e IMPORT_STYLE=full \ -p 8005:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Import admin style @@ -157,7 +157,7 @@ jobs: -e IMPORT_STYLE=admin \ -p 8006:8080 \ nominatim & - sleep 120 + ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Import with PBF_PATH @@ -171,7 +171,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" - name: REPLICATION_URL is blank @@ -180,7 +180,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -p 8008:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: FREEZE @@ -190,7 +190,7 @@ jobs: -e FREEZE="true" \ -p 8009:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: GB postcode import @@ -200,7 +200,7 @@ jobs: -e IMPORT_GB_POSTCODES="true" \ -p 8010:8080 \ nominatim & - sleep 600 + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=LE15+8TX" ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+3SY" ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+4ES" @@ -212,7 +212,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8011:8080 \ nominatim & - sleep 120 + ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" @@ -223,14 +223,14 @@ jobs: -p 8009:8080 \ --name nominatim \ nominatim - sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" # Shutdown the container docker stop -t 60 nominatim # Verify that the exit code is zero CONTAINER_EXIT_CODE=$(docker inspect nominatim --format='{{.State.ExitCode}}') test "$CONTAINER_EXIT_CODE" -eq 0 - + steps: - name: Download artifact uses: actions/download-artifact@v4 From 04466a2b1d754fa3c0bf6f1e991d3741ec240800 Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 11:19:41 +0200 Subject: [PATCH 87/96] ci: use tail with grep apply fix also to version 4.3 --- .github/workflows/ci.yml | 4 ++-- 4.3/start.sh | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 160cb932..d1c326c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,7 @@ jobs: ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" - docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + docker exec nominatim /bin/bash -c 'tail -f /var/log/replication.log | grep -m 1 -c "Update completed."' - name: UPDATE_MODE=continuous with bind-mount commands: |- @@ -136,7 +136,7 @@ jobs: ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" - docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + docker exec nominatim /bin/bash -c 'tail -f /var/log/replication.log | grep -m 1 -c "Update completed."' - name: Import full style commands: |- diff --git a/4.3/start.sh b/4.3/start.sh index c59a4c85..0fdd86cf 100755 --- a/4.3/start.sh +++ b/4.3/start.sh @@ -6,8 +6,14 @@ replicationpid=0 stopServices() { service apache2 stop service postgresql stop - kill $replicationpid + # Check if the replication process is active + if [ $replicationpid -ne 0 ]; then + echo "Shutting down replication process" + kill $replicationpid + fi kill $tailpid + # Force exit code 0 to signal a succesfull shutdown to Docker + exit 0 } trap stopServices SIGTERM TERM INT From ba5ae37826b7ed0ac1553d4dd5a44ee35e9ad78a Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:35:35 +0000 Subject: [PATCH 88/96] update README.md [skip ci] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d8dc3eda..ccd68067 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d saddfox
saddfox
📖 💻 ⚠️ gsg-git
gsg-git
📖 Stefan Breunig
Stefan Breunig
💻 + carlomion
carlomion
💻 🚇 From a2e5109e3f98950f04df29f29d40f5ec82d5f60d Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:35:36 +0000 Subject: [PATCH 89/96] update .all-contributorsrc [skip ci] --- .all-contributorsrc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 31864f4a..e8af99cb 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -449,6 +449,16 @@ "contributions": [ "code" ] + }, + { + "login": "carlomion", + "name": "carlomion", + "avatar_url": "https://avatars.githubusercontent.com/u/161817799?v=4", + "profile": "https://github.com/carlomion", + "contributions": [ + "code", + "infra" + ] } ], "contributorsPerLine": 6, From a714cf5e985b92557bf968f86045fa61f290f3eb Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Thu, 18 Apr 2024 08:58:25 +0200 Subject: [PATCH 90/96] Dockerfile: use private RUN cache to avoid conflicting builds --- 4.4/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/4.4/Dockerfile b/4.4/Dockerfile index 80494fe1..ef4f58ec 100644 --- a/4.4/Dockerfile +++ b/4.4/Dockerfile @@ -8,8 +8,13 @@ ENV LANG=C.UTF-8 WORKDIR /app -RUN \ - --mount=type=cache,target=/var/cache/apt \ +# Inspired by https://github.com/reproducible-containers/buildkit-cache-dance?tab=readme-ov-file#apt-get-github-actions +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + # Keep downloaded APT packages in the docker build cache + rm -f /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \ # Do not start daemons after installation. echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ && chmod +x /usr/sbin/policy-rc.d \ @@ -76,7 +81,7 @@ RUN true \ && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf # Osmium install to run continuous updates. -RUN --mount=type=cache,target=/root/.cache/pip \ +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ pip3 install osmium # Nominatim install. @@ -114,7 +119,6 @@ RUN true \ /tmp/* \ /var/tmp/* \ /app/src/.git \ - /var/lib/apt/lists/* \ # Remove nominatim source and build directories && rm /app/*.tar.bz2 \ && rm -rf /app/build \ From bf7f3a01ee91f45958555a3cb3819e603023f62c Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Thu, 18 Apr 2024 09:20:28 +0200 Subject: [PATCH 91/96] ci: use docker/metadata-action to set tags and labels --- .github/workflows/ci.yml | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1c326c6..00e9140e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,25 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - + + # Compute the Docker tags and labels to apply to the built image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # List of Docker images to use as base name for tags + images: | + mediagis/nominatim + + # Disable generation of latest tag + flavor: | + latest=false + prefix=${{ matrix.nominatim.version }} + + # Customize the date format + tags: | + type=schedule,pattern={{date 'YYYY-MM-DDTHH-mm'}},prefix=- + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -274,14 +292,11 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set container date tag - run: | - echo "DATE_TAG=$(date +%Y-%m-%dT%H-%M)" >> $GITHUB_ENV - - - name: Build and push production docker image to Dockerhub - if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} - run: |- - docker buildx build --platform linux/amd64,linux/arm64 --push \ - -t mediagis/nominatim:${{ matrix.nominatim.version }} \ - -t mediagis/nominatim:${{ matrix.nominatim.version }}-${DATE_TAG} . - working-directory: ${{ matrix.nominatim.version }} \ No newline at end of file + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.nominatim.version }} + platforms: linux/amd64,linux/arm64 + push: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'mediagis' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 882bfc9ec874a23f679f36be68665645d0dd7080 Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Thu, 18 Apr 2024 09:43:38 +0200 Subject: [PATCH 92/96] Dockerfile: avoid removing specific packages after compilation, instead use `apt-get --autoremove`. --- 4.4/Dockerfile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/4.4/Dockerfile b/4.4/Dockerfile index ef4f58ec..23d79a93 100644 --- a/4.4/Dockerfile +++ b/4.4/Dockerfile @@ -29,7 +29,6 @@ RUN \ -o APT::Install-Suggests="false" \ # Build tools from sources. build-essential \ - g++ \ cmake \ libpq-dev \ zlib1g-dev \ @@ -99,18 +98,13 @@ RUN true \ RUN true \ # Remove development and unused packages. - && apt-get -y remove --purge \ - cpp-9 \ - gcc-9* \ - g++ \ + && apt-get -y remove --purge --auto-remove \ + build-essential \ + cmake \ git \ - make \ - cmake* \ llvm-10* \ - libc6-dev \ linux-libc-dev \ libclang-*-dev \ - build-essential \ liblua*-dev \ postgresql-server-dev-14 \ nlohmann-json3-dev \ From 101a702f6126dcdf25d66eac999380dd4239a823 Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Thu, 18 Apr 2024 10:23:10 +0200 Subject: [PATCH 93/96] Fix generation of custom date tag --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00e9140e..be602e2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -276,11 +276,11 @@ jobs: # Disable generation of latest tag flavor: | latest=false - prefix=${{ matrix.nominatim.version }} - # Customize the date format + # Add simple tag with nominatim version + tag with custom date format tags: | - type=schedule,pattern={{date 'YYYY-MM-DDTHH-mm'}},prefix=- + type=raw,value=${{ matrix.nominatim.version }} + type=raw,value=${{ matrix.nominatim.version }}-{{date 'YYYY-MM-DDTHH-mm'}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 4fd346d78a082666402d08c48e888b7943fd080f Mon Sep 17 00:00:00 2001 From: Adrian Young Date: Mon, 12 Aug 2024 14:12:44 +0100 Subject: [PATCH 94/96] Mounted volume group ownership fix When attempting to host this image in K8S with a persistent volume, the permission on the postgres directory were not correct. ``` Error: The cluster is owned by group id *** which does not exist ``` --- 4.4/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.4/init.sh b/4.4/init.sh index b4a351f3..4cacfae0 100755 --- a/4.4/init.sh +++ b/4.4/init.sh @@ -65,7 +65,7 @@ fi # if we use a bind mount then the PG directory is empty and we have to create it if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then - chown postgres /var/lib/postgresql/14/main + chown postgres:postgres /var/lib/postgresql/14/main sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main fi From 45b76b8240af0b57a68691fee958cf79e5e7f274 Mon Sep 17 00:00:00 2001 From: Philip Kozeny Date: Wed, 28 Aug 2024 14:07:44 +0200 Subject: [PATCH 95/96] upgrade to nominatim 4.4.1 --- 4.4/Dockerfile | 2 +- 4.4/README.md | 12 ++++++------ 4.4/conf.d/postgres-tuning.conf | 2 +- README.md | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/4.4/Dockerfile b/4.4/Dockerfile index 23d79a93..e791d6ba 100644 --- a/4.4/Dockerfile +++ b/4.4/Dockerfile @@ -1,4 +1,4 @@ -ARG NOMINATIM_VERSION=4.4.0 +ARG NOMINATIM_VERSION=4.4.1 ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} FROM ubuntu:jammy AS build diff --git a/4.4/README.md b/4.4/README.md index 0ac5b5bc..f661c652 100644 --- a/4.4/README.md +++ b/4.4/README.md @@ -54,7 +54,7 @@ Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CON - `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ -- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) - `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) @@ -82,7 +82,7 @@ The following environment variables are available to tune PostgreSQL: - `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) - `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) -See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. +See https://nominatim.org/release-docs/4.4.1/admin/Installation/#tuning-the-postgresql-database for more details on those settings. ### Import Style @@ -98,11 +98,11 @@ Available options are : - `full`: Default style that also includes points of interest. - `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. -See https://nominatim.org/release-docs/4.4.0/admin/Import/#filtering-imported-data for more details on those styles. +See https://nominatim.org/release-docs/4.4.1/admin/Import/#filtering-imported-data for more details on those styles. ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.1/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. ```sh docker run -it \ @@ -167,7 +167,7 @@ where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that ## Updating the database -Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.0/admin/Update/). For a list of other methods see the output of: +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.1/admin/Update/). For a list of other methods see the output of: ```sh docker exec -it nominatim sudo -u nominatim nominatim replication --help @@ -212,7 +212,7 @@ docker run -it \ Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. -For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.0/customize/Tiger/). +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.1/customize/Tiger/). ## Development diff --git a/4.4/conf.d/postgres-tuning.conf b/4.4/conf.d/postgres-tuning.conf index 2b3c97f5..4bf5b4b1 100644 --- a/4.4/conf.d/postgres-tuning.conf +++ b/4.4/conf.d/postgres-tuning.conf @@ -1,4 +1,4 @@ -# See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database +# See https://nominatim.org/release-docs/4.4.1/admin/Installation/#tuning-the-postgresql-database shared_buffers = 2GB maintenance_work_mem = 10GB autovacuum_work_mem = 2GB diff --git a/README.md b/README.md index ccd68067..5485d5fb 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ See relevant installation and usage instructions for each version in the ``` Date: Thu, 29 Aug 2024 08:55:30 +0200 Subject: [PATCH 96/96] remove restart policy from docker contrib --- 3.6/contrib/docker-compose.yml | 1 - 3.7/contrib/docker-compose-planet.yml | 1 - 3.7/contrib/docker-compose.yml | 1 - 4.0/contrib/docker-compose-planet.yml | 1 - 4.0/contrib/docker-compose.yml | 1 - 4.1/contrib/docker-compose-planet.yml | 1 - 4.1/contrib/docker-compose.yml | 1 - 4.2/contrib/docker-compose.yml | 1 - 8 files changed, 8 deletions(-) diff --git a/3.6/contrib/docker-compose.yml b/3.6/contrib/docker-compose.yml index 3657237f..171b659c 100644 --- a/3.6/contrib/docker-compose.yml +++ b/3.6/contrib/docker-compose.yml @@ -4,7 +4,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:3.6 - restart: always ports: - "8080:8080" environment: diff --git a/3.7/contrib/docker-compose-planet.yml b/3.7/contrib/docker-compose-planet.yml index 94f1dad4..11d0f04f 100644 --- a/3.7/contrib/docker-compose-planet.yml +++ b/3.7/contrib/docker-compose-planet.yml @@ -11,7 +11,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:3.7 - restart: always ports: - "8080:8080" # Do not change the second port, only the first before the semicolon environment: diff --git a/3.7/contrib/docker-compose.yml b/3.7/contrib/docker-compose.yml index 595ea08e..fbe27849 100644 --- a/3.7/contrib/docker-compose.yml +++ b/3.7/contrib/docker-compose.yml @@ -4,7 +4,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:3.7 - restart: always ports: - "8080:8080" environment: diff --git a/4.0/contrib/docker-compose-planet.yml b/4.0/contrib/docker-compose-planet.yml index 7c091bdd..95deeb25 100644 --- a/4.0/contrib/docker-compose-planet.yml +++ b/4.0/contrib/docker-compose-planet.yml @@ -11,7 +11,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.0 - restart: always ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.0/contrib/docker-compose.yml b/4.0/contrib/docker-compose.yml index d7ba5f58..3078718a 100644 --- a/4.0/contrib/docker-compose.yml +++ b/4.0/contrib/docker-compose.yml @@ -4,7 +4,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.0 - restart: always ports: - "8080:8080" environment: diff --git a/4.1/contrib/docker-compose-planet.yml b/4.1/contrib/docker-compose-planet.yml index eacf003a..5f0bf7c8 100644 --- a/4.1/contrib/docker-compose-planet.yml +++ b/4.1/contrib/docker-compose-planet.yml @@ -11,7 +11,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.1 - restart: always ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.1/contrib/docker-compose.yml b/4.1/contrib/docker-compose.yml index 26bea22b..88d2522a 100644 --- a/4.1/contrib/docker-compose.yml +++ b/4.1/contrib/docker-compose.yml @@ -4,7 +4,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.1 - restart: always ports: - "8080:8080" environment: diff --git a/4.2/contrib/docker-compose.yml b/4.2/contrib/docker-compose.yml index 15237155..9edd4bb0 100644 --- a/4.2/contrib/docker-compose.yml +++ b/4.2/contrib/docker-compose.yml @@ -4,7 +4,6 @@ services: nominatim: container_name: nominatim image: mediagis/nominatim:4.2 - restart: always ports: - "8080:8080" environment: