Add address interpolation for pelias search endpoint

Earthfile

@@ -40,6 +40,7 @@ save:
     BUILD +save-valhalla --area=${area}
     BUILD +save-elasticsearch --area=${area} --countries=${countries}
     BUILD +save-placeholder --area=${area} --countries=${countries}
+    BUILD +save-interpolation --area=${area} --countries=${countries}
     BUILD +save-pelias-config --area=${area} --countries=${countries}
     BUILD +save-tileserver-natural-earth
 
@@ -101,10 +102,18 @@ save-placeholder:
     FROM +save-base
     ARG --required area
     ARG countries
-    COPY (+pelias-prepare-placeholder/placeholder --area=${area} --countries=${countries}) /placeholder
-    RUN tar --zstd -cf /placeholder.tar.zst -C /placeholder .
+    COPY (+pelias-prepare-interpolation/data --area=${area} --countries=${countries}) /data
+    RUN tar --zstd -cf /placeholder.tar.zst -C /data/placeholder .
     SAVE ARTIFACT /placeholder.tar.zst AS LOCAL ./data/${area}.placeholder.tar.zst
 
+save-interpolation:
+    FROM +save-base
+    ARG --required area
+    ARG countries
+    COPY (+pelias-prepare-interpolation/data --area=${area} --countries=${countries}) /data
+    RUN tar --zstd -cf /interpolation.tar.zst -C /data/interpolation .
+    SAVE ARTIFACT /interpolation.tar.zst AS LOCAL ./data/${area}.interpolation.tar.zst
+
 save-pelias-config:
     FROM +save-base
     ARG --required area
@@ -222,39 +231,51 @@ pelias-download-wof:
             --service pelias_whosonfirst
         RUN docker-compose run -T 'pelias_whosonfirst' bash ./bin/download
     END
-    SAVE ARTIFACT /data/whosonfirst /whosonfirst
+    SAVE ARTIFACT /data
 
 pelias-prepare-polylines:
     ARG --required area
     ARG countries
     FROM +pelias-import-base
+    COPY (+pelias-download-wof/data --countries=${countries}) /data
     RUN chmod -R 777 /data # FIXME: not everything should have execute permissions!
     RUN mkdir -p /data/polylines
     COPY (+valhalla-build-polylines/polylines.0sv --area=${area}) /data/polylines/extract.0sv
-    SAVE ARTIFACT /data/polylines /polylines
+    SAVE ARTIFACT /data
 
 pelias-prepare-placeholder:
     ARG --required area
     ARG countries
     FROM +pelias-import-base
-    COPY (+pelias-download-wof/whosonfirst --countries=${countries}) /data/whosonfirst
+    COPY (+pelias-prepare-polylines/data --area=${area} --countries=${countries}) /data
     RUN chmod -R 777 /data # FIXME: not everything should have execute permissions!
     WITH DOCKER \
             --compose compose.yaml \
             --service pelias_placeholder
         RUN docker-compose run -T 'pelias_placeholder' bash -c "./cmd/extract.sh && ./cmd/build.sh"
     END
-    SAVE ARTIFACT /data/placeholder /placeholder
+    SAVE ARTIFACT /data
+
+pelias-prepare-interpolation:
+    ARG --required area
+    ARG countries
+    FROM +pelias-import-base
+    COPY (+pelias-prepare-placeholder/data --area=${area} --countries=${countries}) /data
+    RUN chmod -R 777 /data # FIXME: not everything should have execute permissions!
+    WITH DOCKER \
+            --compose compose.yaml \
+            --service pelias_interpolation
+        RUN docker-compose run -T 'pelias_interpolation' bash -c "./docker_build.sh"
+    END
+    SAVE ARTIFACT /data
 
 pelias-import:
     ARG --required area
     ARG countries
     FROM +pelias-import-base
-    COPY (+pelias-download-wof/whosonfirst --countries=${countries}) /data/whosonfirst
-    COPY (+pelias-prepare-polylines/polylines --area=${area} --countries=${countries}) /data/polylines
+    COPY (+pelias-prepare-interpolation/data --area=${area} --countries=${countries}) /data
     RUN mkdir tools
     COPY services/pelias/wait.sh ./tools/wait.sh
-    RUN mkdir /data/elasticsearch
     RUN chmod -R 777 /data # FIXME: not everything should have execute permissions!
 
     WITH DOCKER --compose compose.yaml --service pelias_schema
@@ -517,8 +538,11 @@ valhalla-build:
 
     ARG --required area
 
+    USER valhalla
     USER root
     RUN mkdir -p /data/osm
+    RUN mkdir -p /data/valhalla
+    RUN chown -R valhalla /data
     COPY (+extract/data.osm.pbf --area=${area}) /data/osm/data.osm.pbf
 
     USER valhalla
@@ -529,6 +553,9 @@ valhalla-build:
 valhalla-build-polylines:
     FROM +valhalla-build
 
+    ARG --required area
+    COPY (+valhalla-build/tiles --area=${area}) /data/tiles
+    RUN tar -cf /data/valhalla/tiles.tar -C /data/tiles .
     RUN valhalla_export_edges valhalla.json > /tiles/polylines.0sv
 
     SAVE ARTIFACT /tiles/polylines.0sv

docker-compose.yaml

@@ -101,6 +101,15 @@ services:
     volumes:
       - "./data/:/bootstrap/:ro"
       - "pelias_elasticsearch_data:/usr/share/elasticsearch/data"
+  pelias-interpolation-init:
+    image: ghcr.io/headwaymaps/pelias-init:latest
+    env_file: .env
+    environment:
+      INTERPOLATION_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.interpolation.tar.zst
+    command: [ "/bin/bash", "/app/init_interpolation.sh" ]
+    volumes:
+      - "./data/:/bootstrap/:ro"
+      - "pelias_interpolation_data:/data/interpolation"
   pelias-placeholder-init:
     image: ghcr.io/headwaymaps/pelias-init:latest
     env_file: .env
@@ -115,6 +124,19 @@ services:
     restart: always
     networks:
       - pelias_backend
+  pelias-interpolation:
+    image: pelias/interpolation
+    restart: always
+    networks:
+      - pelias_backend
+    volumes:
+      - "pelias_config_data:/config:ro"
+      - "pelias_interpolation_data:/data/interpolation"
+    depends_on:
+      pelias-config-init:
+        condition: service_completed_successfully
+      pelias-interpolation-init:
+        condition: service_completed_successfully
   pelias-api:
     image: pelias/api:master
     restart: always
@@ -177,6 +199,7 @@ networks:
   valhalla_frontend:
 volumes:
   pelias_config_data:
+  pelias_interpolation_data:
   pelias_placeholder_data:
   pelias_elasticsearch_data:
   tileserver_data:

services/pelias/docker-compose-import.yaml

@@ -32,6 +32,12 @@ services:
       - "${DATA_DIR}:/data"
     depends_on:
       - pelias_elasticsearch
+  pelias_interpolation:
+    image: pelias/interpolation:master
+    container_name: pelias_interpolation
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
   pelias_placeholder:
     image: pelias/placeholder:master
     container_name: pelias_placeholder

services/pelias/init_interpolation.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -xe
+set -o pipefail
+
+if [ ! -z "$(ls -A /data/interpolation)" ]; then
+    echo "Nothing to do, already have interpolation data"
+elif [ -f "${INTERPOLATION_ARTIFACT_SOURCE_PATH}" ]; then
+    echo "Extracting artifact."
+    mkdir -p /data/interpolation
+    tar --zstd -xf "${INTERPOLATION_ARTIFACT_SOURCE_PATH}" -C /data/interpolation
+elif [ ! -z "${INTERPOLATION_ARTIFACT_URL}" ]; then
+    echo "Downloading and extracting artifact."
+    rm -fr /tmp/interpolation.download
+    mkdir -p /tmp/interpolation.download
+    wget --tries=100 -O- "${INTERPOLATION_ARTIFACT_URL}" | tar --zstd -x -C /tmp/interpolation.download
+    mv /tmp/interpolation.download /data/interpolation
+else
+    echo "No interpolation artifact available."
+    exit 1
+fi

services/pelias/pelias.json.template

@@ -22,7 +22,8 @@
     "services": {
       "placeholder": { "url": "http://pelias-placeholder:4100" },
       "libpostal": { "url": "http://pelias-libpostal:4400" },
-      "pip": { "url": "http://pelias-pip:4400" }
+      "pip": { "url": "http://pelias-pip:4400" },
+      "interpolation": { "url": "http://pelias-interpolation:3000" }
     }
   },
   "imports": {