Permit post-process merging in custommap schemas (#626)

planetiler-custommap/README.md

@@ -192,6 +192,8 @@ A layer contains a thematically-related set of features from one or more input s
 
 - `id` - Unique name of this layer
 - `features` - A list of features contained in this layer. See [Layer Features](#layer-feature)
+- `tile_post_process` - Optional processing operations to merge features with the same attributes in a rendered tile.
+  See [Tile Post Process](#tile-post-process)
 
 For example:
 
@@ -201,6 +203,11 @@ layers:
     features:
       - { ... }
       - { ... }
+    tile_post_process:
+      merge_line_strings:
+        min_length: 1
+        tolerance: 1
+        buffer: 5
 ```
 
 ## Layer Feature
@@ -279,6 +286,34 @@ tag_value: voltage
 type: integer
 ```
 
+## Tile Post Process
+
+Specific tile post processing operations for merging features may be defined:
+
+- `merge_line_strings` - Combines linestrings with the same set of attributes into a multilinestring where segments with
+  touching endpoints are merged.
+- `merge_polygons` - Combines polygons with the same set of attributes into a multipolygon where overlapping/touching polygons
+  are combined into fewer polygons covering the same area.
+
+The follow attributes for `merge_line_strings` may be set:
+- `min_length` - Minimum tile pixel length of features to emit, or 0 to emit all merged linestrings.
+- `tolerance` - After merging, simplify linestrings using this pixel tolerance, or -1 to skip simplification step.
+- `buffer` - Number of pixels outside the visible tile area to include detail for, or -1 to skip clipping step.
+
+The follow attribute for `merge_polygons` may be set:
+- `min_area` - Minimum area in square tile pixels of polygons to emit.
+
+For example:
+
+```yaml
+merge_line_strings:
+  min_length: 1
+  tolerance: 1
+  buffer: 5
+merge_polygons:
+  min_area: 1
+```
+
 ## Data Type
 
 A string enum that defines how to map from an input. Allowed values:

planetiler-custommap/planetiler.schema.json

@@ -109,6 +109,10 @@
             "items": {
               "$ref": "#/$defs/feature"
             }
+          },
+          "tile_post_process": {
+            "description": "Optional processing operations to merge features with the same attributes in a rendered tile",
+            "$ref": "#/$defs/tile_post_process"
           }
         }
       }
@@ -399,6 +403,39 @@
         }
       }
     },
+    "tile_post_process": {
+      "type": "object",
+      "properties": {
+        "merge_line_strings": {
+          "description": "Combines linestrings with the same set of attributes into a multilinestring where segments with touching endpoints are merged",
+          "type": "object",
+          "properties": {
+            "min_length": {
+              "description": "Minimum tile pixel length of features to emit, or 0 to emit all merged linestrings",
+              "type": "number"
+            },
+            "tolerance": {
+              "description": "After merging, simplify linestrings using this pixel tolerance, or -1 to skip simplification step",
+              "type": "number"
+            },
+            "buffer": {
+              "description": "Number of pixels outside the visible tile area to include detail for, or -1 to skip clipping step",
+              "type": "number"
+            }
+          }
+        },
+        "merge_polygons": {
+          "description": "Combines polygons with the same set of attributes into a multipolygon where overlapping/touching polygons are combined into fewer polygons covering the same area",
+          "type": "object",
+          "properties": {
+            "min_area": {
+              "description": "Minimum area in square tile pixels of polygons to emit",
+              "type": "number"
+            }
+          }
+        }
+      }
+    },
     "zoom_level": {
       "type": "integer",
       "minimum": 0,

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java

@@ -4,11 +4,14 @@ import static com.onthegomap.planetiler.expression.MultiExpression.Entry;
 import static java.util.Map.entry;
 
 import com.onthegomap.planetiler.FeatureCollector;
+import com.onthegomap.planetiler.FeatureMerge;
 import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.custommap.configschema.FeatureLayer;
 import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
 import com.onthegomap.planetiler.expression.MultiExpression;
 import com.onthegomap.planetiler.expression.MultiExpression.Index;
+import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.reader.SourceFeature;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -25,6 +28,8 @@ public class ConfiguredProfile implements Profile {
 
   private final SchemaConfig schema;
 
+  private final Collection<FeatureLayer> layers;
+  private final Map<String, FeatureLayer> layersById = new HashMap<>();
   private final Map<String, Index<ConfiguredFeature>> featureLayerMatcher;
   private final TagValueProducer tagValueProducer;
   private final Contexts.Root rootContext;
@@ -33,7 +38,7 @@ public class ConfiguredProfile implements Profile {
     this.schema = schema;
     this.rootContext = rootContext;
 
-    Collection<FeatureLayer> layers = schema.layers();
+    layers = schema.layers();
     if (layers == null || layers.isEmpty()) {
       throw new IllegalArgumentException("No layers defined");
     }
@@ -44,6 +49,7 @@ public class ConfiguredProfile implements Profile {
 
     for (var layer : layers) {
       String layerId = layer.id();
+      layersById.put(layerId, layer);
       for (var feature : layer.features()) {
         var configuredFeature = new ConfiguredFeature(layerId, tagValueProducer, feature, rootContext);
         var entry = new Entry<>(configuredFeature, configuredFeature.matchExpression());
@@ -84,6 +90,36 @@ public class ConfiguredProfile implements Profile {
     }
   }
 
+  @Override
+  public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
+    List<VectorTile.Feature> items) throws GeometryException {
+    FeatureLayer featureLayer = findFeatureLayer(layer);
+
+    if (featureLayer.postProcess() == null) {
+      return items;
+    }
+
+    if (featureLayer.postProcess().mergeLineStrings() != null) {
+      var merge = featureLayer.postProcess().mergeLineStrings();
+
+      items = FeatureMerge.mergeLineStrings(items,
+        merge.minLength(), // after merging, remove lines that are still less than {minLength}px long
+        merge.tolerance(), // simplify output linestrings using a {tolerance}px tolerance
+        merge.buffer() // remove any detail more than {buffer}px outside the tile boundary
+      );
+    }
+
+    if (featureLayer.postProcess().mergePolygons() != null) {
+      var merge = featureLayer.postProcess().mergePolygons();
+
+      items = FeatureMerge.mergeOverlappingPolygons(items,
+        merge.minArea() // after merging, remove polygons that are still less than {minArea} in square tile pixels
+      );
+    }
+
+    return items;
+  }
+
   @Override
   public String description() {
     return schema.schemaDescription();
@@ -98,4 +134,8 @@ public class ConfiguredProfile implements Profile {
     });
     return sources;
   }
+
+  public FeatureLayer findFeatureLayer(String layerId) {
+    return layersById.get(layerId);
+  }
 }

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureLayer.java

@@ -1,8 +1,10 @@
 package com.onthegomap.planetiler.custommap.configschema;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Collection;
 
 public record FeatureLayer(
   String id,
-  Collection<FeatureItem> features
+  Collection<FeatureItem> features,
+  @JsonProperty("tile_post_process") PostProcess postProcess
 ) {}

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/MergeLineStrings.java

@@ -0,0 +1,9 @@
+package com.onthegomap.planetiler.custommap.configschema;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record MergeLineStrings(
+  @JsonProperty("min_length") double minLength,
+  @JsonProperty("tolerance") double tolerance,
+  @JsonProperty("buffer") double buffer
+) {}

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/MergePolygons.java

@@ -0,0 +1,7 @@
+package com.onthegomap.planetiler.custommap.configschema;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record MergePolygons(
+  @JsonProperty("min_area") double minArea
+) {}

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/PostProcess.java

@@ -0,0 +1,8 @@
+package com.onthegomap.planetiler.custommap.configschema;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record PostProcess(
+  @JsonProperty("merge_line_strings") MergeLineStrings mergeLineStrings,
+  @JsonProperty("merge_polygons") MergePolygons mergePolygons
+) {}

planetiler-custommap/src/main/resources/samples/railways.yml

@@ -0,0 +1,43 @@
+schema_name: Railways
+schema_description: Railways (layers outputting merged & un-merged lines)
+attribution: <a href="https://www.openstreetmap.org/copyright" target="_blank">&copy; OpenStreetMap contributors</a>
+args:
+  area:
+    description: Geofabrik area to download
+    default: greater-london
+  osm_url:
+    description: OSM URL to download
+    default: '${ args.area == "planet" ? "aws:latest" : ("geofabrik:" + args.area) }'
+sources:
+  osm:
+    type: osm
+    url: '${ args.osm_url }'
+layers:
+- id: railways_merged
+  features:
+  - source: osm
+    geometry: line
+    min_zoom: 4
+    min_size: 0
+    include_when:
+      __all__:
+      - railway: rail
+      - usage: main
+    exclude_when:
+      service: __any__
+  tile_post_process:
+    merge_line_strings:
+      min_length: 0
+      tolerance: -1
+      buffer: -1
+- id: railways_unmerged
+  features:
+  - source: osm
+    geometry: line
+    min_zoom: 4
+    include_when:
+      __all__:
+      - railway: rail
+      - usage: main
+    exclude_when:
+      service: __any__

planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java

@@ -9,11 +9,17 @@ import static org.junit.jupiter.api.Assertions.*;
 import com.onthegomap.planetiler.FeatureCollector;
 import com.onthegomap.planetiler.FeatureCollector.Feature;
 import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.custommap.configschema.DataSourceType;
+import com.onthegomap.planetiler.custommap.configschema.MergeLineStrings;
+import com.onthegomap.planetiler.custommap.configschema.MergePolygons;
+import com.onthegomap.planetiler.custommap.configschema.PostProcess;
 import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
 import com.onthegomap.planetiler.custommap.util.TestConfigurableUtils;
+import com.onthegomap.planetiler.geo.GeoUtils;
+import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.reader.SimpleFeature;
 import com.onthegomap.planetiler.reader.SourceFeature;
 import com.onthegomap.planetiler.stats.Stats;
@@ -157,6 +163,89 @@ class ConfiguredFeatureTest {
     testFeature(pathFunction, schemaFilename, sf, test, expectedMatchCount);
   }
 
+  @Test
+  void testFeaturePostProcessorNoop() throws GeometryException {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: point
+      """;
+    var profile = loadConfig(config);
+
+    VectorTile.Feature feature = new VectorTile.Feature(
+      "testLayer",
+      1,
+      VectorTile.encodeGeometry(GeoUtils.point(0, 0)),
+      Map.of()
+    );
+    assertEquals(List.of(feature), profile.postProcessLayerFeatures("testLayer", 0, List.of(feature)));
+  }
+
+  @Test
+  void testFeaturePostProcessorMergeLineStrings() throws GeometryException {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: point
+        tile_post_process:
+          merge_line_strings:
+            min_length: 1
+            tolerance: 5
+            buffer: 10
+      """;
+    var profile = loadConfig(config);
+
+    VectorTile.Feature feature = new VectorTile.Feature(
+      "testLayer",
+      1,
+      VectorTile.encodeGeometry(GeoUtils.point(0, 0)),
+      Map.of()
+    );
+    assertEquals(List.of(feature), profile.postProcessLayerFeatures("testLayer", 0, List.of(feature)));
+  }
+
+  @Test
+  void testFeaturePostProcessorMergePolygons() throws GeometryException {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: point
+        tile_post_process:
+          merge_polygons:
+            min_area: 3
+      """;
+    var profile = loadConfig(config);
+
+    VectorTile.Feature feature = new VectorTile.Feature(
+      "testLayer",
+      1,
+      VectorTile.encodeGeometry(GeoUtils.point(0, 0)),
+      Map.of()
+    );
+    assertEquals(List.of(feature), profile.postProcessLayerFeatures("testLayer", 0, List.of(feature)));
+  }
+
   @Test
   void testStaticAttributeTest() {
     testPolygon(TEST_RESOURCE, "static_attribute.yml", waterTags, f -> {
@@ -786,6 +875,7 @@ class ConfiguredFeatureTest {
   void testInvalidSchemas() {
     testInvalidSchema("bad_geometry_type.yml", "Profile defined with invalid geometry type");
     testInvalidSchema("no_layers.yml", "Profile defined with no layers");
+    testInvalidSchema("invalid_post_process.yml", "Profile defined with invalid post process element");
   }
 
   private void testInvalidSchema(String filename, String message) {
@@ -1047,4 +1137,78 @@ class ConfiguredFeatureTest {
       assertEquals(output, feature.getMinPixelSizeAtZoom(11));
     }, 1);
   }
+
+  @Test
+  void testSchemaEmptyPostProcess() {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: point
+      """;
+    this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+    assertNull(loadConfig(config).findFeatureLayer("testLayer").postProcess());
+  }
+
+  @Test
+  void testSchemaPostProcessWithMergeLineStrings() {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: point
+        tile_post_process:
+          merge_line_strings:
+            min_length: 1
+            tolerance: 5
+            buffer: 10
+      """;
+    this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+    assertEquals(new PostProcess(
+      new MergeLineStrings(
+        1,
+        5,
+        10
+      ),
+      null
+    ), loadConfig(config).findFeatureLayer("testLayer").postProcess());
+  }
+
+  @Test
+  void testSchemaPostProcessWithMergePolygons() {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: point
+        tile_post_process:
+          merge_polygons:
+            min_area: 3
+      """;
+    this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+    assertEquals(new PostProcess(
+      null,
+      new MergePolygons(
+        3
+      )
+    ), loadConfig(config).findFeatureLayer("testLayer").postProcess());
+  }
 }

planetiler-custommap/src/test/resources/invalidSchema/invalid_post_process.yml

@@ -0,0 +1,21 @@
+schema_name: Test Case Schema
+schema_description: Test case tile schema
+attribution: Test attribution
+sources:
+  osm:
+    type: osm
+    url: geofabrik:rhode-island
+layers:
+- id: testLayer
+  features:
+  - source:
+    - osm
+    geometry: line
+    include_when:
+      natural: water
+    attributes:
+    - key: water
+    - value: wet
+  tile_post_process:
+    merge_everything:
+      min_length: 1