From f039ddcb1a5bc2cdbab9bac970b3eb4cb5c61136 Mon Sep 17 00:00:00 2001 From: Ellen Poe Date: Thu, 23 Feb 2023 08:13:26 -0800 Subject: [PATCH] Add address interpolation for pelias search endpoint --- Earthfile | 45 +++++++++++++++++----- docker-compose.yaml | 23 +++++++++++ services/pelias/docker-compose-import.yaml | 6 +++ services/pelias/init_interpolation.sh | 21 ++++++++++ services/pelias/pelias.json.template | 3 +- 5 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 services/pelias/init_interpolation.sh diff --git a/Earthfile b/Earthfile index a9dd735e7..e8db75997 100644 --- a/Earthfile +++ b/Earthfile @@ -40,6 +40,7 @@ save: BUILD +save-valhalla --area=${area} BUILD +save-elasticsearch --area=${area} --countries=${countries} BUILD +save-placeholder --area=${area} --countries=${countries} + BUILD +save-interpolation --area=${area} --countries=${countries} BUILD +save-pelias-config --area=${area} --countries=${countries} BUILD +save-tileserver-natural-earth @@ -101,10 +102,18 @@ save-placeholder: FROM +save-base ARG --required area ARG countries - COPY (+pelias-prepare-placeholder/placeholder --area=${area} --countries=${countries}) /placeholder - RUN tar --zstd -cf /placeholder.tar.zst -C /placeholder . + COPY (+pelias-prepare-interpolation/data --area=${area} --countries=${countries}) /data + RUN tar --zstd -cf /placeholder.tar.zst -C /data/placeholder . SAVE ARTIFACT /placeholder.tar.zst AS LOCAL ./data/${area}.placeholder.tar.zst +save-interpolation: + FROM +save-base + ARG --required area + ARG countries + COPY (+pelias-prepare-interpolation/data --area=${area} --countries=${countries}) /data + RUN tar --zstd -cf /interpolation.tar.zst -C /data/interpolation . + SAVE ARTIFACT /interpolation.tar.zst AS LOCAL ./data/${area}.interpolation.tar.zst + save-pelias-config: FROM +save-base ARG --required area @@ -222,39 +231,51 @@ pelias-download-wof: --service pelias_whosonfirst RUN docker-compose run -T 'pelias_whosonfirst' bash ./bin/download END - SAVE ARTIFACT /data/whosonfirst /whosonfirst + SAVE ARTIFACT /data pelias-prepare-polylines: ARG --required area ARG countries FROM +pelias-import-base + COPY (+pelias-download-wof/data --countries=${countries}) /data RUN chmod -R 777 /data # FIXME: not everything should have execute permissions! RUN mkdir -p /data/polylines COPY (+valhalla-build-polylines/polylines.0sv --area=${area}) /data/polylines/extract.0sv - SAVE ARTIFACT /data/polylines /polylines + SAVE ARTIFACT /data pelias-prepare-placeholder: ARG --required area ARG countries FROM +pelias-import-base - COPY (+pelias-download-wof/whosonfirst --countries=${countries}) /data/whosonfirst + COPY (+pelias-prepare-polylines/data --area=${area} --countries=${countries}) /data RUN chmod -R 777 /data # FIXME: not everything should have execute permissions! WITH DOCKER \ --compose compose.yaml \ --service pelias_placeholder RUN docker-compose run -T 'pelias_placeholder' bash -c "./cmd/extract.sh && ./cmd/build.sh" END - SAVE ARTIFACT /data/placeholder /placeholder + SAVE ARTIFACT /data + +pelias-prepare-interpolation: + ARG --required area + ARG countries + FROM +pelias-import-base + COPY (+pelias-prepare-placeholder/data --area=${area} --countries=${countries}) /data + RUN chmod -R 777 /data # FIXME: not everything should have execute permissions! + WITH DOCKER \ + --compose compose.yaml \ + --service pelias_interpolation + RUN docker-compose run -T 'pelias_interpolation' bash -c "./docker_build.sh" + END + SAVE ARTIFACT /data pelias-import: ARG --required area ARG countries FROM +pelias-import-base - COPY (+pelias-download-wof/whosonfirst --countries=${countries}) /data/whosonfirst - COPY (+pelias-prepare-polylines/polylines --area=${area} --countries=${countries}) /data/polylines + COPY (+pelias-prepare-interpolation/data --area=${area} --countries=${countries}) /data RUN mkdir tools COPY services/pelias/wait.sh ./tools/wait.sh - RUN mkdir /data/elasticsearch RUN chmod -R 777 /data # FIXME: not everything should have execute permissions! WITH DOCKER --compose compose.yaml --service pelias_schema @@ -517,8 +538,11 @@ valhalla-build: ARG --required area + USER valhalla USER root RUN mkdir -p /data/osm + RUN mkdir -p /data/valhalla + RUN chown -R valhalla /data COPY (+extract/data.osm.pbf --area=${area}) /data/osm/data.osm.pbf USER valhalla @@ -529,6 +553,9 @@ valhalla-build: valhalla-build-polylines: FROM +valhalla-build + ARG --required area + COPY (+valhalla-build/tiles --area=${area}) /data/tiles + RUN tar -cf /data/valhalla/tiles.tar -C /data/tiles . RUN valhalla_export_edges valhalla.json > /tiles/polylines.0sv SAVE ARTIFACT /tiles/polylines.0sv diff --git a/docker-compose.yaml b/docker-compose.yaml index 4329e272d..7e858c363 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -101,6 +101,15 @@ services: volumes: - "./data/:/bootstrap/:ro" - "pelias_elasticsearch_data:/usr/share/elasticsearch/data" + pelias-interpolation-init: + image: ghcr.io/headwaymaps/pelias-init:latest + env_file: .env + environment: + INTERPOLATION_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.interpolation.tar.zst + command: [ "/bin/bash", "/app/init_interpolation.sh" ] + volumes: + - "./data/:/bootstrap/:ro" + - "pelias_interpolation_data:/data/interpolation" pelias-placeholder-init: image: ghcr.io/headwaymaps/pelias-init:latest env_file: .env @@ -115,6 +124,19 @@ services: restart: always networks: - pelias_backend + pelias-interpolation: + image: pelias/interpolation + restart: always + networks: + - pelias_backend + volumes: + - "pelias_config_data:/config:ro" + - "pelias_interpolation_data:/data/interpolation" + depends_on: + pelias-config-init: + condition: service_completed_successfully + pelias-interpolation-init: + condition: service_completed_successfully pelias-api: image: pelias/api:master restart: always @@ -177,6 +199,7 @@ networks: valhalla_frontend: volumes: pelias_config_data: + pelias_interpolation_data: pelias_placeholder_data: pelias_elasticsearch_data: tileserver_data: diff --git a/services/pelias/docker-compose-import.yaml b/services/pelias/docker-compose-import.yaml index fabb187e0..69923d051 100644 --- a/services/pelias/docker-compose-import.yaml +++ b/services/pelias/docker-compose-import.yaml @@ -32,6 +32,12 @@ services: - "${DATA_DIR}:/data" depends_on: - pelias_elasticsearch + pelias_interpolation: + image: pelias/interpolation:master + container_name: pelias_interpolation + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" pelias_placeholder: image: pelias/placeholder:master container_name: pelias_placeholder diff --git a/services/pelias/init_interpolation.sh b/services/pelias/init_interpolation.sh new file mode 100644 index 000000000..4f6aefd2b --- /dev/null +++ b/services/pelias/init_interpolation.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -xe +set -o pipefail + +if [ ! -z "$(ls -A /data/interpolation)" ]; then + echo "Nothing to do, already have interpolation data" +elif [ -f "${INTERPOLATION_ARTIFACT_SOURCE_PATH}" ]; then + echo "Extracting artifact." + mkdir -p /data/interpolation + tar --zstd -xf "${INTERPOLATION_ARTIFACT_SOURCE_PATH}" -C /data/interpolation +elif [ ! -z "${INTERPOLATION_ARTIFACT_URL}" ]; then + echo "Downloading and extracting artifact." + rm -fr /tmp/interpolation.download + mkdir -p /tmp/interpolation.download + wget --tries=100 -O- "${INTERPOLATION_ARTIFACT_URL}" | tar --zstd -x -C /tmp/interpolation.download + mv /tmp/interpolation.download /data/interpolation +else + echo "No interpolation artifact available." + exit 1 +fi diff --git a/services/pelias/pelias.json.template b/services/pelias/pelias.json.template index 062e3ac36..eed91c571 100644 --- a/services/pelias/pelias.json.template +++ b/services/pelias/pelias.json.template @@ -22,7 +22,8 @@ "services": { "placeholder": { "url": "http://pelias-placeholder:4100" }, "libpostal": { "url": "http://pelias-libpostal:4400" }, - "pip": { "url": "http://pelias-pip:4400" } + "pip": { "url": "http://pelias-pip:4400" }, + "interpolation": { "url": "http://pelias-interpolation:3000" } } }, "imports": {