Improve export task and fix tileset

apache · Nov 25, 2023 · 2fc612d · 2fc612d
1 parent 4ec3f6d
commit 2fc612d
Show file tree

Hide file tree

Showing 13 changed files with 317 additions and 75 deletions.
diff --git a/.run/basemap-export-mbtiles.run.xml b/.run/basemap-export-mbtiles.run.xml
@@ -0,0 +1,22 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="basemap-export-mbtiles" type="Application" factoryName="Application">
+    <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+    <module name="baremaps-cli" />
+    <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.js --repository tiles.mbtiles --format mbtiles" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
+    <extension name="coverage">
+      <pattern>
+        <option name="PATTERN" value="org.apache.baremaps.server.ogcapi.*" />
+        <option name="ENABLED" value="true" />
+      </pattern>
+    </extension>
+    <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+      <option name="credential" />
+      <option name="region" />
+      <option name="useCurrentConnection" value="false" />
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>
diff --git a/.run/basemap-export-pmtiles.run.xml b/.run/basemap-export-pmtiles.run.xml
@@ -0,0 +1,22 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="basemap-export-pmtiles" type="Application" factoryName="Application">
+    <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+    <module name="baremaps-cli" />
+    <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.js --repository tiles.mbtiles --format pmtiles" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
+    <extension name="coverage">
+      <pattern>
+        <option name="PATTERN" value="org.apache.baremaps.server.ogcapi.*" />
+        <option name="ENABLED" value="true" />
+      </pattern>
+    </extension>
+    <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+      <option name="credential" />
+      <option name="region" />
+      <option name="useCurrentConnection" value="false" />
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>
diff --git a/.run/basemap-workflow.run.xml b/.run/basemap-workflow.run.xml
@@ -2,8 +2,13 @@
   <configuration default="false" name="basemap-workflow" type="Application" factoryName="Application">
     <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
     <module name="baremaps-cli" />
-    <option name="PROGRAM_PARAMETERS" value="workflow execute --file workflow.js" />
+    <option name="PROGRAM_PARAMETERS" value="workflow execute --file import.js" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
+    <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+      <option name="credential" />
+      <option name="region" />
+      <option name="useCurrentConnection" value="false" />
+    </extension>
     <method v="2">
       <option name="Make" enabled="true" />
     </method>

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
@@ -37,4 +37,14 @@ public TileStoreException(String message) {
   public TileStoreException(Throwable cause) {
     super(cause);
   }
+
+  /**
+   * Constructs a {@code BlobStoreException} with the specified detail message and cause.
+   *
+   * @param message the message
+   * @param cause the cause
+   */
+  public TileStoreException(String message, Throwable cause) {
+    super(message, cause);
+  }
 }
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java
@@ -22,32 +22,43 @@
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+import java.util.Optional;
 import org.apache.baremaps.tilestore.TileCoord;
 import org.apache.baremaps.tilestore.TileStore;
 import org.apache.baremaps.tilestore.TileStoreException;
+import org.apache.baremaps.vectortile.tileset.Tileset;
 
 public class PMTilesStore implements TileStore {
 
   private final PMTilesWriter writer;
 
-  public PMTilesStore(Path path) {
+  public PMTilesStore(Path path, Tileset tileset) {
     try {
-      this.writer = new PMTilesWriter(path);
-
       var metadata = new HashMap<String, Object>();
-      metadata.put("name", "baremaps");
+      metadata.put("name", tileset.getName());
       metadata.put("type", "baselayer");
-      metadata.put("version", "0.0.1");
-      metadata.put("description", "PMTiles generated by Baremaps");
-      metadata.put("attribution", "OpenStreetMap contributors");
-      metadata.put("vector_layers", List.of(
-          "aerialway", "aeroway", "amenity", "attraction",
-          "barrier", "boundary", "building", "highway", "landuse",
-          "leisure", "man_made", "natural", "ocean", "point",
-          "power", "railway", "route", "waterway")
-          .stream().map(s -> Map.of("id", s)).toList());
-      this.writer.writeMetadata(metadata);
+      metadata.put("version", tileset.getVersion());
+      metadata.put("description", tileset.getDescription());
+      metadata.put("attribution", tileset.getAttribution());
+      metadata.put("vector_layers", tileset.getVectorLayers());
+
+      var minZoom = Optional.ofNullable(tileset.getMinzoom()).orElse(0);
+      var maxZoom = Optional.ofNullable(tileset.getMaxzoom()).orElse(14);
+      var bounds = Optional.ofNullable(tileset.getBounds()).orElse(List.of(-180d, -90d, 180d, 90d));
+      var center = Optional.ofNullable(tileset.getCenter()).orElse(List.of(0d, 0d, 3d));
+
+      writer = new PMTilesWriter(path);
+      writer.setMetadata(metadata);
+      writer.setMinZoom(minZoom);
+      writer.setMaxZoom(maxZoom);
+      writer.setMinLon(bounds.get(0));
+      writer.setMinLat(bounds.get(1));
+      writer.setMaxLon(bounds.get(2));
+      writer.setMaxLat(bounds.get(3));
+      writer.setCenterLon(center.get(0));
+      writer.setCenterLat(center.get(1));
+      writer.setMinZoom(tileset.getMinzoom());
+      writer.setMaxZoom(tileset.getMaxzoom());
 
     } catch (IOException e) {
       throw new RuntimeException(e);
@@ -62,7 +73,7 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
   @Override
   public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
     try {
-      writer.writeTile(tileCoord.z(), tileCoord.x(), tileCoord.y(), blob.array());
+      writer.setTile(tileCoord.z(), tileCoord.x(), tileCoord.y(), blob.array());
     } catch (IOException e) {
       throw new TileStoreException(e);
     }
@@ -76,7 +87,7 @@ public void delete(TileCoord tileCoord) throws TileStoreException {
   @Override
   public void close() throws TileStoreException {
     try {
-      writer.finalize();
+      writer.write();
     } catch (IOException e) {
       throw new TileStoreException(e);
     }

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java
@@ -28,20 +28,38 @@
 
 public class PMTilesWriter {
 
-  private final Path path;
+  private Path path;
 
   private Map<String, Object> metadata = new HashMap<>();
 
-  private final List<Entry> entries;
+  private List<Entry> entries;
 
-  private final Map<Long, Long> tileHashToOffset;
+  private Map<Long, Long> tileHashToOffset;
 
   private Long lastTileHash = null;
 
   private Path tilePath;
 
   private boolean clustered = true;
 
+  private int minZoom = 0;
+
+  private int maxZoom = 14;
+
+  private double minLon = -180;
+
+  private double minLat = -90;
+
+  private double maxLon = 180;
+
+  private double maxLat = 90;
+
+  private int centerZoom = 3;
+
+  private double centerLat = 0;
+
+  private double centerLon = 0;
+
   public PMTilesWriter(Path path) throws IOException {
     this(path, new ArrayList<>(), new HashMap<>());
   }
@@ -51,14 +69,14 @@ public PMTilesWriter(Path path, List<Entry> entries, Map<Long, Long> tileHashToO
     this.path = path;
     this.entries = entries;
     this.tileHashToOffset = tileHashToOffset;
-    this.tilePath = Files.createTempFile(path.getParent(), "tiles", ".tmp");
+    this.tilePath = Files.createTempFile(path.getParent(), "tiles_", ".tmp");
   }
 
-  public void writeMetadata(Map<String, Object> metadata) {
+  public void setMetadata(Map<String, Object> metadata) {
     this.metadata = metadata;
   }
 
-  public void writeTile(int z, int x, int y, byte[] bytes) throws IOException {
+  public void setTile(int z, int x, int y, byte[] bytes) throws IOException {
     // Write the tile
     var tileId = PMTiles.zxyToTileId(z, x, y);
     var tileLength = bytes.length;
@@ -93,7 +111,43 @@ else if (tileHashToOffset.containsKey(tileHash)) {
     }
   }
 
-  public void finalize() throws IOException {
+  public void setMinZoom(int minZoom) {
+    this.minZoom = minZoom;
+  }
+
+  public void setMaxZoom(int maxZoom) {
+    this.maxZoom = maxZoom;
+  }
+
+  public void setMinLon(double minLon) {
+    this.minLon = minLon;
+  }
+
+  public void setMinLat(double minLat) {
+    this.minLat = minLat;
+  }
+
+  public void setMaxLon(double maxLon) {
+    this.maxLon = maxLon;
+  }
+
+  public void setMaxLat(double maxLat) {
+    this.maxLat = maxLat;
+  }
+
+  public void setCenterZoom(int centerZoom) {
+    this.centerZoom = centerZoom;
+  }
+
+  public void setCenterLat(double centerLat) {
+    this.centerLat = centerLat;
+  }
+
+  public void setCenterLon(double centerLon) {
+    this.centerLon = centerLon;
+  }
+
+  public void write() throws IOException {
     // Sort the entries by tile id
     if (!clustered) {
       entries.sort(Comparator.comparingLong(Entry::getTileId));
@@ -130,15 +184,15 @@ public void finalize() throws IOException {
     header.setTilesOffset(tilesOffset);
     header.setTilesLength(tilesLength);
 
-    header.setMinZoom(1);
-    header.setMaxZoom(14);
-    header.setMinLon(-180);
-    header.setMinLat(-90);
-    header.setMaxLon(180);
-    header.setMaxLat(90);
-    header.setCenterZoom(14);
-    header.setCenterLat(46.5197);
-    header.setCenterLon(6.6323);
+    header.setMinZoom(minZoom);
+    header.setMaxZoom(maxZoom);
+    header.setMinLon(minLon);
+    header.setMinLat(minLat);
+    header.setMaxLon(maxLon);
+    header.setMaxLat(maxLat);
+    header.setCenterZoom(centerZoom);
+    header.setCenterLat(centerLat);
+    header.setCenterLon(centerLon);
 
     try (var output = new LittleEndianDataOutputStream(new FileOutputStream(path.toFile()))) {
       PMTiles.serializeHeader(output, header);

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -99,6 +99,9 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
           byte[] bytes = resultSet.getBytes(1);
           gzip.write(bytes);
         }
+      } catch (Exception e) {
+        throw new TileStoreException(String.format("Failed to execute statement: %s", statement),
+            e);
       }
 
       // Log slow queries (> 10s)
@@ -154,7 +157,8 @@ protected static Query prepareQuery(Tileset tileset, int zoom) {
           }
 
           // Add the sql to the layer sql
-          var querySql = query.getSql()
+          var querySql = query.getSql().trim()
+              .replaceAll("\\s+", " ")
               .replace(";", "")
               .replace("?", "??")
               .replace("$zoom", String.valueOf(zoom));

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -82,13 +82,15 @@ public void execute(WorkflowContext context) throws Exception {
           TileCoord.iterator(envelope, tileset.getMinzoom(), tileset.getMaxzoom());
       var tileCoordStream =
           StreamUtils.stream(tileCoordIterator).peek(new ProgressLogger<>(count, 5000));
+
       var bufferedTileEntryStream = StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> {
         try {
           return new TileEntry(tile, sourceTileStore.read(tile));
         } catch (TileStoreException e) {
           throw new RuntimeException(e);
         }
       }, 1000);
+
       var partitionedTileEntryStream = StreamUtils.partition(bufferedTileEntryStream, 1000);
       partitionedTileEntryStream.forEach(batch -> {
         try {
@@ -120,7 +122,7 @@ private TileStore targetTileStore(Tileset source) throws TileStoreException, IOE
         return tilesStore;
       case pmtiles:
         Files.deleteIfExists(repository);
-        var tileStore = new PMTilesStore(repository);
+        var tileStore = new PMTilesStore(repository, source);
         return tileStore;
       default:
         throw new IllegalArgumentException("Unsupported format");

diff --git a/basemap/import.js b/basemap/import.js
@@ -18,6 +18,22 @@ import config from "./config.js";
 
 export default {
   "steps": [
+    {
+      "id": "initialize-database",
+        "needs": [],
+        "tasks": [
+          {
+            "type": "ExecuteSqlScript",
+            "file": "queries/assertions.sql",
+            "database": config.database,
+          },
+          {
+            "type": "ExecuteSqlScript",
+            "file": "queries/functions.sql",
+            "database": config.database,
+          },
+        ]
+    },
     {
       "id": "natural-earth",
       "needs": [],

diff --git a/basemap/layers/building/extrusion.js b/basemap/layers/building/extrusion.js
@@ -45,15 +45,6 @@ export default {
             16,
             ['get', "extrusion:height"]
         ],
-        "fill-extrusion-opacity": [
-            'interpolate',
-            ['linear'],
-            ['zoom'],
-            15,
-            0,
-            16,
-            0.8
-        ],
         "fill-extrusion-color": theme.buildingShapeFillColor,
         // Having muliple colors for building parts results in z-fighting
         // https://github.com/maplibre/maplibre-gl-js/issues/3157