From da12fef79f04a3eb7c939eb2cd03f03ac3bb4042 Mon Sep 17 00:00:00 2001 From: Brian Sperlongano Date: Tue, 7 Jun 2022 23:34:21 +0200 Subject: [PATCH] Declarative schema from configuration file (#160) --- planetiler-basemap/pom.xml | 1 - .../planetiler/config/Arguments.java | 26 +- .../planetiler/expression/Expression.java | 22 ++ .../planetiler/geo/GeometryType.java | 47 ++- .../com/onthegomap/planetiler/TestUtils.java | 12 + .../planetiler/expression/ExpressionTest.java | 45 ++- .../expression/ExpressionTestUtil.java | 18 ++ .../expression/MultiExpressionTest.java | 2 +- .../planetiler/geo/GeometryTypeTest.java | 41 +++ planetiler-custommap/.gitignore | 1 + planetiler-custommap/README.md | 113 +++++++ planetiler-custommap/pom.xml | 55 ++++ .../custommap/ConfiguredFeature.java | 281 ++++++++++++++++ .../custommap/ConfiguredMapMain.java | 92 ++++++ .../custommap/ConfiguredProfile.java | 69 ++++ .../planetiler/custommap/TagCriteria.java | 55 ++++ .../custommap/TagValueProducer.java | 99 ++++++ .../configschema/AttributeDefinition.java | 15 + .../custommap/configschema/DataSource.java | 10 + .../configschema/DataSourceType.java | 10 + .../custommap/configschema/FeatureItem.java | 17 + .../custommap/configschema/FeatureLayer.java | 8 + .../custommap/configschema/SchemaConfig.java | 27 ++ .../custommap/configschema/ZoomOverride.java | 12 + .../main/resources/samples/highway_areas.yml | 37 +++ .../src/main/resources/samples/manholes.yml | 22 ++ .../src/main/resources/samples/owg_simple.yml | 120 +++++++ .../src/main/resources/samples/power.yml | 35 ++ .../custommap/ConfiguredFeatureTest.java | 299 ++++++++++++++++++ .../custommap/ConfiguredMapTest.java | 103 ++++++ .../custommap/SchemaYAMLLoadTest.java | 38 +++ .../custommap/util/TestConfigurableUtils.java | 22 ++ .../custommap/util/VerifyMonaco.java | 44 +++ .../invalidSchema/bad_geometry_type.yml | 18 ++ .../resources/invalidSchema/no_layers.yml | 7 + .../validSchema/data_type_attributes.yml | 31 ++ .../test/resources/validSchema/local_path.yml | 18 ++ .../resources/validSchema/road_motorway.yml | 39 +++ .../validSchema/static_attribute.yml | 18 ++ .../resources/validSchema/tag_attribute.yml | 17 + .../resources/validSchema/tag_include.yml | 27 ++ .../resources/validSchema/zoom_filter.yml | 37 +++ planetiler-dist/pom.xml | 5 + .../java/com/onthegomap/planetiler/Main.java | 2 + pom.xml | 11 + 45 files changed, 2007 insertions(+), 21 deletions(-) create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTestUtil.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java create mode 100644 planetiler-custommap/.gitignore create mode 100644 planetiler-custommap/README.md create mode 100644 planetiler-custommap/pom.xml create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagCriteria.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagValueProducer.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureLayer.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java create mode 100644 planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/ZoomOverride.java create mode 100644 planetiler-custommap/src/main/resources/samples/highway_areas.yml create mode 100644 planetiler-custommap/src/main/resources/samples/manholes.yml create mode 100644 planetiler-custommap/src/main/resources/samples/owg_simple.yml create mode 100644 planetiler-custommap/src/main/resources/samples/power.yml create mode 100644 planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java create mode 100644 planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java create mode 100644 planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java create mode 100644 planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/TestConfigurableUtils.java create mode 100644 planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/VerifyMonaco.java create mode 100644 planetiler-custommap/src/test/resources/invalidSchema/bad_geometry_type.yml create mode 100644 planetiler-custommap/src/test/resources/invalidSchema/no_layers.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/data_type_attributes.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/local_path.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/road_motorway.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/static_attribute.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/tag_attribute.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/tag_include.yml create mode 100644 planetiler-custommap/src/test/resources/validSchema/zoom_filter.yml diff --git a/planetiler-basemap/pom.xml b/planetiler-basemap/pom.xml index 98baa52c..c1256177 100644 --- a/planetiler-basemap/pom.xml +++ b/planetiler-basemap/pom.xml @@ -21,7 +21,6 @@ org.yaml snakeyaml - 1.30 org.commonmark diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java index 910682b4..1ded6737 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java @@ -200,7 +200,7 @@ public class Arguments { return value; } - /** Returns a {@link Path} parsed from {@code key} argument which may or may not exist. */ + /** Returns a {@link Path} parsed from {@code key} argument, or fall back to a default if the argument is not set. */ public Path file(String key, String description, Path defaultValue) { String value = getArg(key); Path file = value == null ? defaultValue : Path.of(value); @@ -208,6 +208,17 @@ public class Arguments { return file; } + /** Returns a {@link Path} parsed from {@code key} argument which may or may not exist. */ + public Path file(String key, String description) { + String value = getArg(key); + if (value == null) { + throw new IllegalArgumentException("Missing required parameter: " + key + " (" + description + ")"); + } + Path file = Path.of(value); + logArgValue(key, description, file); + return file; + } + /** * Returns a {@link Path} parsed from {@code key} argument which must exist for the program to function. * @@ -221,6 +232,19 @@ public class Arguments { return path; } + /** + * Returns a {@link Path} parsed from a required {@code key} argument which must exist for the program to function. + * + * @throws IllegalArgumentException if the file does not exist or if the parameter is not provided. + */ + public Path inputFile(String key, String description) { + Path path = file(key, description); + if (!Files.exists(path)) { + throw new IllegalArgumentException(path + " does not exist"); + } + return path; + } + /** Returns a boolean parsed from {@code key} argument where {@code "true"} is true and anything else is false. */ public boolean getBoolean(String key, String description, boolean defaultValue) { boolean value = "true".equalsIgnoreCase(getArg(key, Boolean.toString(defaultValue))); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java index 082afb67..e90ac024 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java @@ -41,6 +41,8 @@ public interface Expression { Expression FALSE = new Constant(false, "FALSE"); BiFunction GET_TAG = WithTags::getTag; + List dummyList = new NoopList<>(); + static And and(Expression... children) { return and(List.of(children)); } @@ -247,6 +249,26 @@ public interface Expression { */ boolean evaluate(WithTags input, List matchKeys); + //A list that silently drops all additions + class NoopList extends ArrayList { + private static final long serialVersionUID = 1L; + + @Override + public boolean add(T t) { + return true; + } + } + + /** + * Returns true if this expression matches an input element. + * + * @param input the input element + * @return true if this expression matches the input element + */ + default boolean evaluate(WithTags input) { + return evaluate(input, dummyList); + } + /** Returns Java code that can be used to reconstruct this expression. */ String generateJavaCode(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java index 7d34eb97..e34bb2a6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java @@ -1,5 +1,11 @@ package com.onthegomap.planetiler.geo; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.FeatureCollector.Feature; +import com.onthegomap.planetiler.expression.Expression; +import java.util.function.BiFunction; +import java.util.function.Function; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Lineal; import org.locationtech.jts.geom.Polygonal; @@ -7,17 +13,27 @@ import org.locationtech.jts.geom.Puntal; import vector_tile.VectorTileProto; public enum GeometryType { - UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0), - POINT(VectorTileProto.Tile.GeomType.POINT, 1), - LINE(VectorTileProto.Tile.GeomType.LINESTRING, 2), - POLYGON(VectorTileProto.Tile.GeomType.POLYGON, 4); + UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0, (f, l) -> { + throw new UnsupportedOperationException(); + }, "unknown"), + @JsonProperty("point") + POINT(VectorTileProto.Tile.GeomType.POINT, 1, FeatureCollector::point, "point"), + @JsonProperty("line") + LINE(VectorTileProto.Tile.GeomType.LINESTRING, 2, FeatureCollector::line, "linestring"), + @JsonProperty("polygon") + POLYGON(VectorTileProto.Tile.GeomType.POLYGON, 4, FeatureCollector::polygon, "polygon"); private final VectorTileProto.Tile.GeomType protobufType; private final int minPoints; + private final BiFunction geometryFactory; + private final String matchTypeString; - GeometryType(VectorTileProto.Tile.GeomType protobufType, int minPoints) { + GeometryType(VectorTileProto.Tile.GeomType protobufType, int minPoints, + BiFunction geometryFactory, String matchTypeString) { this.protobufType = protobufType; this.minPoints = minPoints; + this.geometryFactory = geometryFactory; + this.matchTypeString = matchTypeString; } public static GeometryType valueOf(Geometry geom) { @@ -49,4 +65,25 @@ public enum GeometryType { public int minPoints() { return minPoints; } + + /** + * Generates a factory method which creates a {@link Feature} from a {@link FeatureCollector} of the appropriate + * geometry type. + * + * @param layerName - name of the layer + * @return geometry factory method + */ + public Function geometryFactory(String layerName) { + return features -> geometryFactory.apply(features, layerName); + } + + /** + * Generates a test for whether a source feature is of the correct geometry to be included in the tile. + * + * @return geometry test method + */ + public Expression featureTest() { + return Expression.matchType(matchTypeString); + } + } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 7ec51106..8bcfb8ad 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -601,6 +601,18 @@ public class TestUtils { } } + public static void assertMinFeatureCount(Mbtiles db, String layer, int zoom, Map attrs, + Envelope envelope, int expected, Class clazz) { + try { + int num = Verify.getNumFeatures(db, layer, zoom, attrs, envelope, clazz); + + assertTrue(expected < num, + "z%d features in %s, expected at least %d got %d".formatted(zoom, layer, expected, num)); + } catch (GeometryException e) { + fail(e); + } + } + public static void assertFeatureNear(Mbtiles db, String layer, Map attrs, double lng, double lat, int minzoom, int maxzoom) { try { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java index 01366c7f..bb90f1e8 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java @@ -1,18 +1,14 @@ package com.onthegomap.planetiler.expression; -import static com.onthegomap.planetiler.TestUtils.newPoint; import static com.onthegomap.planetiler.expression.Expression.*; +import static com.onthegomap.planetiler.expression.ExpressionTestUtil.featureWithTags; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.onthegomap.planetiler.reader.SimpleFeature; -import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.WithTags; import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; @@ -22,14 +18,6 @@ class ExpressionTest { public static final Expression.MatchAny matchCD = matchAny("c", "d"); public static final Expression.MatchAny matchBC = matchAny("b", "c"); - static SourceFeature featureWithTags(String... tags) { - Map map = new HashMap<>(); - for (int i = 0; i < tags.length; i += 2) { - map.put(tags[i], tags[i + 1]); - } - return SimpleFeature.create(newPoint(0, 0), map); - } - @Test void testSimplify() { assertEquals(matchAB, matchAB.simplify()); @@ -159,4 +147,35 @@ class ExpressionTest { var expression = matchAnyTyped("key", WithTags::getDirection, 1); assertThrows(UnsupportedOperationException.class, expression::generateJavaCode); } + + @Test + void testEvaluate() { + WithTags feature = featureWithTags("key1", "value1", "key2", "value2"); + + //And + assertTrue(and(matchAny("key1", "value1"), matchAny("key2", "value2")).evaluate(feature)); + assertFalse(and(matchAny("key1", "value1"), matchAny("key2", "wrong")).evaluate(feature)); + assertFalse(and(matchAny("key1", "wrong"), matchAny("key2", "value2")).evaluate(feature)); + assertFalse(and(matchAny("key1", "wrong"), matchAny("key2", "wrong")).evaluate(feature)); + + //Or + assertTrue(or(matchAny("key1", "value1"), matchAny("key2", "value2")).evaluate(feature)); + assertTrue(or(matchAny("key1", "value1"), matchAny("key2", "wrong")).evaluate(feature)); + assertTrue(or(matchAny("key1", "wrong"), matchAny("key2", "value2")).evaluate(feature)); + assertFalse(or(matchAny("key1", "wrong"), matchAny("key2", "wrong")).evaluate(feature)); + + //Not + assertFalse(not(matchAny("key1", "value1")).evaluate(feature)); + assertTrue(not(matchAny("key1", "wrong")).evaluate(feature)); + + //MatchField + assertTrue(matchField("key1").evaluate(feature)); + assertFalse(matchField("wrong").evaluate(feature)); + assertTrue(not(matchAny("key1", "")).evaluate(feature)); + assertTrue(matchAny("wrong", "").evaluate(feature)); + + //Constants + assertTrue(TRUE.evaluate(feature)); + assertFalse(FALSE.evaluate(feature)); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTestUtil.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTestUtil.java new file mode 100644 index 00000000..7432950b --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTestUtil.java @@ -0,0 +1,18 @@ +package com.onthegomap.planetiler.expression; + +import static com.onthegomap.planetiler.TestUtils.newPoint; + +import com.onthegomap.planetiler.reader.SimpleFeature; +import com.onthegomap.planetiler.reader.SourceFeature; +import java.util.HashMap; +import java.util.Map; + +public class ExpressionTestUtil { + static SourceFeature featureWithTags(String... tags) { + Map map = new HashMap<>(); + for (int i = 0; i < tags.length; i += 2) { + map.put(tags[i], tags[i + 1]); + } + return SimpleFeature.create(newPoint(0, 0), map); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java index 54c14a22..b0064570 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java @@ -4,7 +4,7 @@ import static com.onthegomap.planetiler.TestUtils.newLineString; import static com.onthegomap.planetiler.TestUtils.newPoint; import static com.onthegomap.planetiler.TestUtils.rectangle; import static com.onthegomap.planetiler.expression.Expression.*; -import static com.onthegomap.planetiler.expression.ExpressionTest.featureWithTags; +import static com.onthegomap.planetiler.expression.ExpressionTestUtil.featureWithTags; import static com.onthegomap.planetiler.expression.MultiExpression.entry; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java new file mode 100644 index 00000000..db1c6403 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java @@ -0,0 +1,41 @@ +package com.onthegomap.planetiler.geo; + +import static java.util.Collections.emptyList; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.reader.SimpleFeature; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class GeometryTypeTest { + + @Test + void testGeometryFactory() throws Exception { + Map tags = Map.of("key1", "value1"); + + var line = + SimpleFeature.createFakeOsmFeature(TestUtils.newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList()); + var point = + SimpleFeature.createFakeOsmFeature(TestUtils.newPoint(0, 0), tags, "osm", null, 1, emptyList()); + var poly = + SimpleFeature.createFakeOsmFeature(TestUtils.newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, + emptyList()); + + Assertions.assertTrue(GeometryType.LINE.featureTest().evaluate(line)); + Assertions.assertFalse(GeometryType.LINE.featureTest().evaluate(point)); + Assertions.assertFalse(GeometryType.LINE.featureTest().evaluate(poly)); + + Assertions.assertFalse(GeometryType.POINT.featureTest().evaluate(line)); + Assertions.assertTrue(GeometryType.POINT.featureTest().evaluate(point)); + Assertions.assertFalse(GeometryType.POINT.featureTest().evaluate(poly)); + + Assertions.assertFalse(GeometryType.POLYGON.featureTest().evaluate(line)); + Assertions.assertFalse(GeometryType.POLYGON.featureTest().evaluate(point)); + Assertions.assertTrue(GeometryType.POLYGON.featureTest().evaluate(poly)); + + Assertions.assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(point)); + Assertions.assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(line)); + Assertions.assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(poly)); + } +} diff --git a/planetiler-custommap/.gitignore b/planetiler-custommap/.gitignore new file mode 100644 index 00000000..da5ce991 --- /dev/null +++ b/planetiler-custommap/.gitignore @@ -0,0 +1 @@ +*.mbtiles diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md new file mode 100644 index 00000000..0230cadc --- /dev/null +++ b/planetiler-custommap/README.md @@ -0,0 +1,113 @@ +# Configurable Planetiler Schema + +It is possible to customize planetiler's output from configuration files. This is done using the parameter: +`--schema=schema_file.yml` + +The schema file provides information to planetiler about how to construct the tiles and which layers, features, and +attributes will be posted to the file. Schema files are in [YAML](https://yaml.org) format. + +NOTE: The configuration schema is under active development so the format may change between releases. Feedback is +welcome to help shape the final product! + +For examples, see [samples](src/main/resources/samples) or [test cases](src/test/resources/validSchema). + +## Schema file definition + +The root of the schema has the following attributes: + +* `schema_name` - A descriptive name for the schema +* `schema_description` - A longer description of the schema +* `attribution` - An attribution statement, which may include HTML such as links +* `sources` - A list of sources from which features should be extracted, specified as a list of names. + See [Tag Mappings](#tag-mappings). +* `dataTypes` - A map of tag keys that should be treated as a certain data type, with strings being the default. + See [Tag Mappings](#tag-mappings). +* `layers` - A list of vector tile layers and their definitions. See [Layers](#layers) + +### Data Sources + +A data source contains geospatial objects with tags that are consumed by planetiler. The configured data sources in the +schema provide complete information on how to access those data sources. + +* `type` - Either `shapefile` or `osm` +* `url` - Location to download the shapefile from. For geofabrik named areas, use `geofabrik:` prefixes, for + example `geofabrik:rhode-island` + +### Layers + +A layer contains a thematically-related set of features. + +* `name` - Name of this layer +* `features` - A list of features contained in this layer. See [Features](#features) + +### Features + +A feature is a defined set of objects that meet specified filter criteria. + +* `geometry` - Include objects of a certain geometry type. Options are `polygon`, `line`, or `point`. +* `min_tile_cover_size` - include objects of a certain geometry size, where 1.0 means "is the same size as a tile at + this zoom". +* `include_when` - A tag specification which determines which features to include. If unspecified, all features from the + specified sources are included. See [Tag Filters](#tag-filters) +* `exclude_when` - A tag specification which determines which features to exclude. This rule is applied + after `includeWhen`. If unspecified, no exclusion filter is applied. See [Tag Filters](#tag-filters) +* `min_zoom` - Minimum zoom to show the feature that matches the filter specifications. +* `zoom_override` - List of rules that overrides the `min_zoom` for this feature if certain tags are present. If + multiple rules match, the first matching rule will be applied. See [Feature Zoom Overrides](#feature-zoom-override) +* `attributes` - Specifies the attributes that should be rendered into the tiles for this feature, and how they are + constructed. See [Attributes](#attributes) + +### Tag Mappings + +Specifies that certain tag key should have their values treated as being a certain data type. + +* `: data_type` - A key, along with one of `boolean`, `string`, `direction`, or `long` +* `: mapping` - A mapping which produces a new attribute by retrieving from a different key. + See [Tag Input and Output Mappings](#tag-input-and-output-mappings) + +### Tag Input and Output Mappings + +* `type`: One of `boolean`, `string`, `direction`, or `long` +* `output`: The name of the typed key that will be presented to the attribute logic + +### Feature Zoom Override + +Specifies a zoom-based inclusion rules for this feature. + +* `min` - Minimum zoom to render a feature matching this rule +* `tag` - List of tags for which this rule applies. Tags are specified as a list of key/value pairs + +### Attributes + +* `key` - Name of this attribute in the tile. +* `constant_value` - Value of the attribute in the tile, as a constant +* `tag_value` - Value of the attribute in the tile, as copied from the value of the specified tag key. If neither + constantValue nor tagValue are specified, the default behavior is to set the tag value equal to the input value ( + pass-through) +* `include_when` - A filter specification which determines whether to include this attribute. If unspecified, the + attribute will be included unless excluded by `excludeWhen`. See [Tag Filters](#tag-filters) +* `exclude_when` - A filter specification which determines whether to exclude this attribute. This rule is applied + after `includeWhen`. If unspecified, no exclusion filter is applied. See [Tag Filters](#tag-filters) +* `min_zoom` - The minimum zoom at which to render this attribute. +* `min_zoom_by_value` - Minimum zoom to render this attribute depending on the value. Contains a map of `value: zoom` + entries that indicate the minimum zoom for each possible value. + +### Tag Filters + +A tag filter matches an object based on its tagging. Multiple key entries may be specified: + +* `:` - Match objects that contain this key. +* ` ` - A single value or a list of values. Match objects in the specified key that contains one of these + values. If no values are specified, this will match any value tagged with the specified key. + +Example: match all `natural=water`: + + natural: water + +Example: match residential, commercial, and industrial land use: + + landuse: + - residential + - commercial + - industrial + diff --git a/planetiler-custommap/pom.xml b/planetiler-custommap/pom.xml new file mode 100644 index 00000000..3d4febca --- /dev/null +++ b/planetiler-custommap/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + + planetiler-custommap + + + com.onthegomap.planetiler + planetiler-parent + 0.5-SNAPSHOT + + + + + com.onthegomap.planetiler + planetiler-core + ${project.parent.version} + + + org.yaml + snakeyaml + + + org.commonmark + commonmark + + + + com.onthegomap.planetiler + planetiler-core + ${project.parent.version} + test-jar + test + + + + + + + io.github.zlika + reproducible-build-maven-plugin + + + + maven-deploy-plugin + + + true + + + + + diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java new file mode 100644 index 00000000..8e834aea --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java @@ -0,0 +1,281 @@ +package com.onthegomap.planetiler.custommap; + +import static com.onthegomap.planetiler.custommap.TagCriteria.matcher; +import static com.onthegomap.planetiler.expression.Expression.not; + +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.FeatureCollector.Feature; +import com.onthegomap.planetiler.custommap.configschema.AttributeDefinition; +import com.onthegomap.planetiler.custommap.configschema.FeatureItem; +import com.onthegomap.planetiler.custommap.configschema.ZoomOverride; +import com.onthegomap.planetiler.expression.Expression; +import com.onthegomap.planetiler.expression.MultiExpression; +import com.onthegomap.planetiler.expression.MultiExpression.Entry; +import com.onthegomap.planetiler.expression.MultiExpression.Index; +import com.onthegomap.planetiler.geo.GeometryException; +import com.onthegomap.planetiler.geo.GeometryType; +import com.onthegomap.planetiler.reader.SourceFeature; +import com.onthegomap.planetiler.reader.WithTags; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.ToIntFunction; + +/** + * A map feature, configured from a YML configuration file. + * + * {@link #matchExpression()} returns a filtering expression to limit input elements to ones this feature cares about, + * and {@link #processFeature(SourceFeature, FeatureCollector)} processes matching elements. + */ +public class ConfiguredFeature { + + private final Set sources; + private final Expression geometryTest; + private final Function geometryFactory; + private final Expression tagTest; + private final Index zoomOverride; + private final Integer featureMinZoom; + private final Integer featureMaxZoom; + private final TagValueProducer tagValueProducer; + + private static final double LOG4 = Math.log(4); + private static final Index NO_ZOOM_OVERRIDE = MultiExpression.of(List.of()).index(); + private static final Integer DEFAULT_MAX_ZOOM = 14; + + private final List> attributeProcessors; + + public ConfiguredFeature(String layerName, TagValueProducer tagValueProducer, FeatureItem feature) { + sources = new HashSet<>(feature.sources()); + + GeometryType geometryType = feature.geometry(); + + //Test to determine whether this type of geometry is included + geometryTest = geometryType.featureTest(); + + //Factory to treat OSM tag values as specific data type values + this.tagValueProducer = tagValueProducer; + + //Test to determine whether this feature is included based on tagging + if (feature.includeWhen() == null) { + tagTest = Expression.TRUE; + } else { + tagTest = matcher(feature.includeWhen(), tagValueProducer); + } + + //Index of zoom ranges for a feature based on what tags are present. + zoomOverride = zoomOverride(feature.zoom()); + + //Test to determine at which zooms to include this feature based on tagging + featureMinZoom = feature.minZoom() == null ? 0 : feature.minZoom(); + featureMaxZoom = feature.maxZoom() == null ? DEFAULT_MAX_ZOOM : feature.maxZoom(); + + //Factory to generate the right feature type from FeatureCollector + geometryFactory = geometryType.geometryFactory(layerName); + + //Configure logic for each attribute in the output tile + attributeProcessors = feature.attributes() + .stream() + .map(this::attributeProcessor) + .toList(); + } + + /** + * Produce an index that matches tags from configuration and returns a minimum zoom level + * + * @param zoom the configured zoom overrides + * @return an index + */ + private Index zoomOverride(Collection zoom) { + if (zoom == null || zoom.isEmpty()) { + return NO_ZOOM_OVERRIDE; + } + + return MultiExpression.of( + zoom.stream() + .map(this::generateOverrideExpression) + .toList()) + .index(); + } + + /** + * Takes the zoom override configuration for a single zoom level and returns an expression that matches tags for that + * level. + * + * @param config zoom override for a single level + * @return matching expression + */ + private Entry generateOverrideExpression(ZoomOverride config) { + return MultiExpression.entry(config.min(), + Expression.or( + config.tag() + .entrySet() + .stream() + .map(this::generateKeyExpression) + .toList())); + } + + /** + * Returns an expression that matches against single key with one or more values + * + * @param keyExpression a map containing a key and one or more values + * @return a matching expression + */ + private Expression generateKeyExpression(Map.Entry keyExpression) { + // Values are either a single value, or a collection + String key = keyExpression.getKey(); + Object rawVal = keyExpression.getValue(); + + if (rawVal instanceof List tagValues) { + return Expression.matchAnyTyped(key, tagValueProducer.valueGetterForKey(key), tagValues); + } + + return Expression.matchAnyTyped(key, tagValueProducer.valueGetterForKey(key), rawVal); + } + + /** + * Produces logic that generates attribute values based on configuration and input data. If both a constantValue + * configuration and a tagValue configuration are set, this is likely a mistake, and the constantValue will take + * precedence. + * + * @param attribute - attribute definition configured from YML + * @return a function that generates an attribute value from a {@link SourceFeature} based on an attribute + * configuration. + */ + private Function attributeValueProducer(AttributeDefinition attribute) { + + Object constVal = attribute.constantValue(); + if (constVal != null) { + return sf -> constVal; + } + + String tagVal = attribute.tagValue(); + if (tagVal != null) { + return tagValueProducer.valueProducerForKey(tagVal); + } + + //Default to producing a tag identical to the input + return tagValueProducer.valueProducerForKey(attribute.key()); + } + + /** + * Generate logic which determines the minimum zoom level for a feature based on a configured pixel size limit. + * + * @param minTilePercent - minimum percentage of a tile that a feature must cover to be shown + * @param minZoom - global minimum zoom for this feature + * @param minZoomByValue - map of tag values to zoom level + * @return minimum zoom function + */ + private static BiFunction attributeZoomThreshold(Double minTilePercent, int minZoom, + Map minZoomByValue) { + + if (minZoom == 0 && minZoomByValue.isEmpty()) { + return null; + } + + ToIntFunction staticZooms = sf -> Math.max(minZoom, minZoomFromTilePercent(sf, minTilePercent)); + + if (minZoomByValue.isEmpty()) { + return (sf, key) -> staticZooms.applyAsInt(sf); + } + + //Attribute value-specific zooms override static zooms + return (sourceFeature, key) -> minZoomByValue.getOrDefault(key, staticZooms.applyAsInt(sourceFeature)); + } + + private static int minZoomFromTilePercent(SourceFeature sf, Double minTilePercent) { + if (minTilePercent == null) { + return 0; + } + try { + return (int) (Math.log(minTilePercent / sf.area()) / LOG4); + } catch (GeometryException e) { + return 14; + } + } + + /** + * Generates a function which produces a fully-configured attribute for a feature. + * + * @param attribute - configuration for this attribute + * @return processing logic + */ + private BiConsumer attributeProcessor(AttributeDefinition attribute) { + var tagKey = attribute.key(); + + var attributeMinZoom = attribute.minZoom(); + attributeMinZoom = attributeMinZoom == null ? 0 : attributeMinZoom; + + var minZoomByValue = attribute.minZoomByValue(); + minZoomByValue = minZoomByValue == null ? Map.of() : minZoomByValue; + + //Workaround because numeric keys are mapped as String + minZoomByValue = tagValueProducer.remapKeysByType(tagKey, minZoomByValue); + + var attributeValueProducer = attributeValueProducer(attribute); + + var attrIncludeWhen = attribute.includeWhen(); + var attrExcludeWhen = attribute.excludeWhen(); + + var attributeTest = + Expression.and( + attrIncludeWhen == null ? Expression.TRUE : matcher(attrIncludeWhen, tagValueProducer), + attrExcludeWhen == null ? Expression.TRUE : not(matcher(attrExcludeWhen, tagValueProducer)) + ).simplify(); + + var minTileCoverage = attrIncludeWhen == null ? null : attribute.minTileCoverSize(); + + BiFunction attributeZoomProducer = + attributeZoomThreshold(minTileCoverage, attributeMinZoom, minZoomByValue); + + if (attributeZoomProducer != null) { + return (sf, f) -> { + if (attributeTest.evaluate(sf)) { + Object value = attributeValueProducer.apply(sf); + f.setAttrWithMinzoom(tagKey, value, attributeZoomProducer.apply(sf, value)); + } + }; + } + + return (sf, f) -> { + if (attributeTest.evaluate(sf)) { + f.setAttr(tagKey, attributeValueProducer.apply(sf)); + } + }; + } + + /** + * Returns an expression that evaluates to true if a source feature should be included in the output. + */ + public Expression matchExpression() { + return Expression.and(geometryTest, tagTest); + } + + /** + * Generates a tile feature based on a source feature. + * + * @param sourceFeature - input source feature + * @param features - output rendered feature collector + */ + public void processFeature(SourceFeature sourceFeature, FeatureCollector features) { + + //Ensure that this feature is from the correct source + if (!sources.contains(sourceFeature.getSource())) { + return; + } + + var minZoom = zoomOverride.getOrElse(sourceFeature, featureMinZoom); + + var f = geometryFactory.apply(features) + .setMinZoom(minZoom) + .setMaxZoom(featureMaxZoom); + + for (var processor : attributeProcessors) { + processor.accept(sourceFeature, f); + } + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java new file mode 100644 index 00000000..d7111136 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java @@ -0,0 +1,92 @@ +package com.onthegomap.planetiler.custommap; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.onthegomap.planetiler.Planetiler; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.custommap.configschema.DataSource; +import com.onthegomap.planetiler.custommap.configschema.DataSourceType; +import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import org.yaml.snakeyaml.Yaml; + +/** + * Main driver to create maps configured by a YAML file. + * + * Parses the config file into a {@link ConfiguredProfile}, loads sources into {@link Planetiler} runner and kicks off + * the map generation process. + */ +public class ConfiguredMapMain { + + private static final Yaml yaml = new Yaml(); + private static final ObjectMapper mapper = new ObjectMapper(); + + /* + * Main entrypoint + */ + public static void main(String... args) throws Exception { + run(Arguments.fromArgsOrConfigFile(args)); + } + + static void run(Arguments args) throws Exception { + var dataDir = Path.of("data"); + var sourcesDir = dataDir.resolve("sources"); + + var schemaFile = args.inputFile( + "schema", + "Location of YML-format schema definition file"); + + var config = loadConfig(schemaFile); + + var planetiler = Planetiler.create(args) + .setProfile(new ConfiguredProfile(config)); + + var sources = config.sources(); + for (var source : sources.entrySet()) { + configureSource(planetiler, sourcesDir, source.getKey(), source.getValue()); + } + + planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")) + .run(); + } + + static SchemaConfig loadConfig(Path schemaFile) throws IOException { + try (var schemaStream = Files.newInputStream(schemaFile)) { + Map parsed = yaml.load(schemaStream); + return mapper.convertValue(parsed, SchemaConfig.class); + } + } + + private static void configureSource(Planetiler planetiler, Path sourcesDir, String sourceName, DataSource source) + throws URISyntaxException { + + DataSourceType sourceType = source.type(); + Path localPath = source.localPath(); + + switch (sourceType) { + case OSM -> { + String url = source.url(); + String[] areaParts = url.split("[:/]"); + String areaFilename = areaParts[areaParts.length - 1]; + String areaName = areaFilename.replaceAll("\\..*$", ""); + if (localPath == null) { + localPath = sourcesDir.resolve(areaName + ".osm.pbf"); + } + planetiler.addOsmSource(sourceName, localPath, url); + } + case SHAPEFILE -> { + String url = source.url(); + if (localPath == null) { + localPath = sourcesDir.resolve(Paths.get(new URI(url).getPath()).getFileName().toString()); + } + planetiler.addShapefileSource(sourceName, localPath, url); + } + default -> throw new IllegalArgumentException("Unhandled source " + sourceType); + } + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java new file mode 100644 index 00000000..8895b825 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java @@ -0,0 +1,69 @@ +package com.onthegomap.planetiler.custommap; + +import static com.onthegomap.planetiler.expression.MultiExpression.Entry; + +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.custommap.configschema.FeatureLayer; +import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; +import com.onthegomap.planetiler.expression.MultiExpression; +import com.onthegomap.planetiler.expression.MultiExpression.Index; +import com.onthegomap.planetiler.reader.SourceFeature; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * A profile configured from a yml file. + */ +public class ConfiguredProfile implements Profile { + + private final SchemaConfig schemaConfig; + + private final Index featureLayerMatcher; + + public ConfiguredProfile(SchemaConfig schemaConfig) { + this.schemaConfig = schemaConfig; + + Collection layers = schemaConfig.layers(); + if (layers == null || layers.isEmpty()) { + throw new IllegalArgumentException("No layers defined"); + } + + TagValueProducer tagValueProducer = new TagValueProducer(schemaConfig.inputMappings()); + + List> configuredFeatureEntries = new ArrayList<>(); + + for (var layer : layers) { + String layerName = layer.name(); + for (var feature : layer.features()) { + var configuredFeature = new ConfiguredFeature(layerName, tagValueProducer, feature); + configuredFeatureEntries.add( + new Entry<>(configuredFeature, configuredFeature.matchExpression())); + } + } + + featureLayerMatcher = MultiExpression.of(configuredFeatureEntries).index(); + } + + @Override + public String name() { + return schemaConfig.schemaName(); + } + + @Override + public String attribution() { + return schemaConfig.attribution(); + } + + @Override + public void processFeature(SourceFeature sourceFeature, FeatureCollector featureCollector) { + featureLayerMatcher.getMatches(sourceFeature) + .forEach(configuredFeature -> configuredFeature.processFeature(sourceFeature, featureCollector)); + } + + @Override + public String description() { + return schemaConfig.schemaDescription(); + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagCriteria.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagCriteria.java new file mode 100644 index 00000000..a2f3888f --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagCriteria.java @@ -0,0 +1,55 @@ +package com.onthegomap.planetiler.custommap; + +import static com.onthegomap.planetiler.expression.Expression.matchAnyTyped; +import static com.onthegomap.planetiler.expression.Expression.matchField; + +import com.onthegomap.planetiler.expression.Expression; +import java.util.Collection; +import java.util.Map; + +/** + * Utility that maps expressions in YAML format to {@link Expression Expressions}. + */ +public class TagCriteria { + + private TagCriteria() { + //Hide implicit public constructor + } + + /** + * Returns a function that determines whether a source feature matches any of the entries in this specification + * + * @param map a map of tag criteria + * @param tagValueProducer a TagValueProducer + * @return a predicate which returns true if this criteria matches + */ + public static Expression matcher(Map map, TagValueProducer tagValueProducer) { + return map.entrySet() + .stream() + .map(entry -> tagCriterionToExpression(tagValueProducer, entry.getKey(), entry.getValue())) + .reduce(Expression::or) + .orElse(Expression.TRUE); + } + + private static Expression tagCriterionToExpression(TagValueProducer tagValueProducer, String key, Object value) { + + //If only a key is provided, with no value, match any object tagged with that key. + if (value == null) { + return matchField(key); + + //If a collection is provided, match any of these values. + } else if (value instanceof Collection values) { + return matchAnyTyped( + key, + tagValueProducer.valueGetterForKey(key), + values.stream().toList()); + + //Otherwise, a key and single value were passed, so match that exact tag + } else { + return matchAnyTyped( + key, + tagValueProducer.valueGetterForKey(key), + value); + } + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagValueProducer.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagValueProducer.java new file mode 100644 index 00000000..6743741d --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TagValueProducer.java @@ -0,0 +1,99 @@ +package com.onthegomap.planetiler.custommap; + +import com.onthegomap.planetiler.reader.WithTags; +import com.onthegomap.planetiler.util.Parse; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.UnaryOperator; + +/** + * Utility that parses attribute values from source features, based on YAML config. + */ +public class TagValueProducer { + + private static final String STRING_DATATYPE = "string"; + private static final String BOOLEAN_DATATYPE = "boolean"; + private static final String DIRECTION_DATATYPE = "direction"; + private static final String LONG_DATATYPE = "long"; + + private static final BiFunction DEFAULT_GETTER = WithTags::getTag; + + private final Map> valueRetriever = new HashMap<>(); + + private final Map keyType = new HashMap<>(); + + private static final Map> inputGetter = + Map.of( + STRING_DATATYPE, WithTags::getString, + BOOLEAN_DATATYPE, WithTags::getBoolean, + DIRECTION_DATATYPE, WithTags::getDirection, + LONG_DATATYPE, WithTags::getLong + ); + + private static final Map> inputParse = + Map.of( + STRING_DATATYPE, s -> s, + BOOLEAN_DATATYPE, Parse::bool, + DIRECTION_DATATYPE, Parse::direction, + LONG_DATATYPE, Parse::parseLong + ); + + public TagValueProducer(Map map) { + if (map == null) { + return; + } + + map.forEach((key, value) -> { + if (value instanceof String stringType) { + valueRetriever.put(key, inputGetter.get(stringType)); + keyType.put(key, stringType); + } else if (value instanceof Map renameMap) { + String output = renameMap.containsKey("output") ? renameMap.get("output").toString() : key; + BiFunction getter = + renameMap.containsKey("type") ? inputGetter.get(renameMap.get("type").toString()) : DEFAULT_GETTER; + //When requesting the output value, actually retrieve the input key with the desired getter + valueRetriever.put(output, + (withTags, requestedKey) -> getter.apply(withTags, key)); + if (renameMap.containsKey("type")) { + keyType.put(output, renameMap.get("type").toString()); + } + } + }); + } + + /** + * Returns a function that extracts the value for {@code key} from a {@link WithTags} instance. + */ + public BiFunction valueGetterForKey(String key) { + return valueRetriever.getOrDefault(key, DEFAULT_GETTER); + } + + /** + * Returns a function that extracts the value for {@code key} from a {@link WithTags} instance. + */ + public Function valueProducerForKey(String key) { + var getter = valueGetterForKey(key); + return withTags -> getter.apply(withTags, key); + } + + /** + * Returns copy of {@code keyedMap} where the keys have been transformed by the parser associated with {code key}. + */ + public Map remapKeysByType(String key, Map keyedMap) { + Map newMap = new LinkedHashMap<>(); + + String dataType = keyType.get(key); + UnaryOperator parser; + + if (dataType == null || (parser = inputParse.get(dataType)) == null) { + newMap.putAll(keyedMap); + } else { + keyedMap.forEach((mapKey, value) -> newMap.put(parser.apply(mapKey), value)); + } + + return newMap; + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java new file mode 100644 index 00000000..4d68117f --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java @@ -0,0 +1,15 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; + +public record AttributeDefinition( + String key, + @JsonProperty("constant_value") Object constantValue, + @JsonProperty("tag_value") String tagValue, + @JsonProperty("include_when") Map includeWhen, + @JsonProperty("exclude_when") Map excludeWhen, + @JsonProperty("min_zoom") Integer minZoom, + @JsonProperty("min_zoom_by_value") Map minZoomByValue, + @JsonProperty("min_tile_cover_size") Double minTileCoverSize +) {} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java new file mode 100644 index 00000000..193e5958 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java @@ -0,0 +1,10 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.nio.file.Path; + +public record DataSource( + DataSourceType type, + String url, + @JsonProperty("local_path") Path localPath +) {} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java new file mode 100644 index 00000000..b128f136 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java @@ -0,0 +1,10 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum DataSourceType { + @JsonProperty("osm") + OSM, + @JsonProperty("shapefile") + SHAPEFILE +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java new file mode 100644 index 00000000..65d72dd8 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java @@ -0,0 +1,17 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.onthegomap.planetiler.geo.GeometryType; +import java.util.Collection; +import java.util.Map; + +public record FeatureItem( + Collection sources, + @JsonProperty("min_zoom") Integer minZoom, + @JsonProperty("max_zoom") Integer maxZoom, + GeometryType geometry, + @JsonProperty("zoom_override") Collection zoom, + @JsonProperty("include_when") Map includeWhen, + @JsonProperty("exclude_when") Map excludeWhen, + Collection attributes +) {} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureLayer.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureLayer.java new file mode 100644 index 00000000..a93e75b8 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureLayer.java @@ -0,0 +1,8 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import java.util.Collection; + +public record FeatureLayer( + String name, + Collection features +) {} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java new file mode 100644 index 00000000..581f3aa9 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java @@ -0,0 +1,27 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collection; +import java.util.Map; + +/** + * An object representation of a vector tile server schema. This object is mapped to a schema YML file using SnakeYAML. + */ +public record SchemaConfig( + @JsonProperty("schema_name") String schemaName, + @JsonProperty("schema_description") String schemaDescription, + String attribution, + Map sources, + @JsonProperty("tag_mappings") Map inputMappings, + Collection layers +) { + + private static final String DEFAULT_ATTRIBUTION = """ + © OpenStreetMap contributors + """.trim(); + + @Override + public String attribution() { + return attribution == null ? DEFAULT_ATTRIBUTION : attribution; + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/ZoomOverride.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/ZoomOverride.java new file mode 100644 index 00000000..10b798da --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/ZoomOverride.java @@ -0,0 +1,12 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import java.util.Map; + +/** + * Configuration item that instructs the renderer to override the default zoom range for features which contain specific + * tag combinations. + */ +public record ZoomOverride( + Integer min, + Integer max, + Map tag) {} diff --git a/planetiler-custommap/src/main/resources/samples/highway_areas.yml b/planetiler-custommap/src/main/resources/samples/highway_areas.yml new file mode 100644 index 00000000..bc28ea73 --- /dev/null +++ b/planetiler-custommap/src/main/resources/samples/highway_areas.yml @@ -0,0 +1,37 @@ +schema_name: Highway areas +schema_description: Features that represent the physical area of roads +attribution: © + OpenStreetMap contributors +sources: + osm: + type: osm + url: geofabrik:poland +tag_mappings: + bridge: boolean + layer: long +layers: +- name: highway_area + features: + - sources: + - osm + geometry: polygon + min_zoom: 14 + include_when: + area:highway: + attributes: + - key: highway + tag_value: area:highway + - key: layer + - key: surface + - key: bridge + - sources: + - osm + geometry: polygon + min_zoom: 14 + include_when: + man_made: bridge + attributes: + - key: man_made + constant_value: bridge + - key: layer + - key: surface \ No newline at end of file diff --git a/planetiler-custommap/src/main/resources/samples/manholes.yml b/planetiler-custommap/src/main/resources/samples/manholes.yml new file mode 100644 index 00000000..08c82841 --- /dev/null +++ b/planetiler-custommap/src/main/resources/samples/manholes.yml @@ -0,0 +1,22 @@ +schema_name: Manhole covers +schema_description: Manhole covers +attribution: © + OpenStreetMap contributors +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- name: manhole + features: + - sources: + - osm + geometry: point + min_zoom: 14 + include_when: + man_made: manhole + attributes: + - key: man_made + - key: manhole + - key: operator + - key: ref \ No newline at end of file diff --git a/planetiler-custommap/src/main/resources/samples/owg_simple.yml b/planetiler-custommap/src/main/resources/samples/owg_simple.yml new file mode 100644 index 00000000..b0f8fb70 --- /dev/null +++ b/planetiler-custommap/src/main/resources/samples/owg_simple.yml @@ -0,0 +1,120 @@ +schema_name: OWG Simple Schema +schema_description: Simple vector tile schema +attribution: © + OpenStreetMap contributors +sources: + water_polygons: + type: shapefile + url: https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip + osm: + type: osm + url: geofabrik:monaco +tag_mappings: + bridge: boolean + intermittent: boolean + layer: long + tunnel: boolean +layers: +- name: water + features: + - sources: + - osm + geometry: polygon + include_when: + natural: water + attributes: + - key: natural + - key: intermittent + include_when: + intermittent: true + - key: name + min_tile_cover_size: 0.01 + include_when: + exclude_when: + tag: + key: water + value: + - river + - canal + - stream + - sources: + - water_polygons + geometry: polygon + include_when: + attributes: + - key: natural + constant_value: water + - sources: + - osm + min_zoom: 7 + geometry: line + include_when: + tag: + key: waterway + value: + - river + - stream + - canal + attributes: + - key: waterway + - key: intermittent + include_when: + intermittent: true + - key: name + min_zoom: 12 +- name: road + features: + - sources: + - osm + geometry: line + include_when: + highway: + - motorway + - trunk + - primary + - secondary + - tertiary + - motorway_link + - trunk_link + - primary_link + - secondary_link + - tertiary_link + - unclassified + - residential + - living_street + - service + - track + min_zoom: 4 + zoom_override: + - min: 5 + tag: + highway: trunk + - min: 7 + tag: + highway: primary + - min: 8 + tag: + highway: secondary + - min: 9 + tag: + highway: + - tertiary + - motorway_link + - trunk_link + - primary_link + - secondary_link + - tertiary_link + - min: 11 + tag: + highway: + - unclassified + - residential + - living_street + - min: 12 + tag: + highway: track + - min: 13 + tag: + highway: service + attributes: + - key: highway diff --git a/planetiler-custommap/src/main/resources/samples/power.yml b/planetiler-custommap/src/main/resources/samples/power.yml new file mode 100644 index 00000000..60828ad0 --- /dev/null +++ b/planetiler-custommap/src/main/resources/samples/power.yml @@ -0,0 +1,35 @@ +schema_name: Power +schema_description: Features that represent electrical power grid +attribution: © + OpenStreetMap contributors +sources: + osm: + type: osm + url: geofabrik:new-jersey +layers: +- name: power + features: + - sources: + - osm + geometry: point + min_zoom: 13 + include_when: + power: + - pole + attributes: + - key: power + - key: ref + - key: height + - key: operator + - sources: + - osm + geometry: line + min_zoom: 12 + include_when: + power: + - line + attributes: + - key: power + - key: voltage + - key: cables + - key: operator \ No newline at end of file diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java new file mode 100644 index 00000000..a90f7b53 --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java @@ -0,0 +1,299 @@ +package com.onthegomap.planetiler.custommap; + +import static com.onthegomap.planetiler.TestUtils.newLineString; +import static com.onthegomap.planetiler.TestUtils.newPolygon; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.*; + +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.FeatureCollector.Feature; +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.custommap.util.TestConfigurableUtils; +import com.onthegomap.planetiler.reader.SimpleFeature; +import com.onthegomap.planetiler.reader.SourceFeature; +import com.onthegomap.planetiler.stats.Stats; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; + +class ConfiguredFeatureTest { + + private static final Function TEST_RESOURCE = TestConfigurableUtils::pathToTestResource; + private static final Function SAMPLE_RESOURCE = TestConfigurableUtils::pathToSample; + private static final Function TEST_INVALID_RESOURCE = TestConfigurableUtils::pathToTestInvalidResource; + + private static final Map waterTags = Map.of( + "natural", "water", + "water", "pond", + "name", "Little Pond", + "test_zoom_tag", "test_zoom_value" + ); + + private static Map motorwayTags = Map.of( + "highway", "motorway", + "layer", "1", + "bridge", "yes", + "tunnel", "yes" + ); + + private static Map trunkTags = Map.of( + "highway", "trunk", + "toll", "yes" + ); + + private static Map primaryTags = Map.of( + "highway", "primary", + "lanes", "2" + + ); + + private static Map highwayAreaTags = Map.of( + "area:highway", "motorway", + "layer", "1", + "bridge", "yes", + "surface", "asphalt" + ); + + private static Map inputMappingTags = Map.of( + "s_type", "string_val", + "l_type", "1", + "b_type", "yes", + "d_type", "yes", + "intermittent", "yes", + "bridge", "yes" + ); + + private static FeatureCollector polygonFeatureCollector() { + var config = PlanetilerConfig.defaults(); + var factory = new FeatureCollector.Factory(config, Stats.inMemory()); + return factory.get(SimpleFeature.create(TestUtils.newPolygon(0, 0, 0.1, 0, 0.1, 0.1, 0, 0), new HashMap<>())); + } + + private static FeatureCollector linestringFeatureCollector() { + var config = PlanetilerConfig.defaults(); + var factory = new FeatureCollector.Factory(config, Stats.inMemory()); + return factory.get(SimpleFeature.create(TestUtils.newLineString(0, 0, 0.1, 0, 0.1, 0.1, 0, 0), new HashMap<>())); + } + + private static Profile loadConfig(Function pathFunction, String filename) throws IOException { + var staticAttributeConfig = pathFunction.apply(filename); + var schema = ConfiguredMapMain.loadConfig(staticAttributeConfig); + return new ConfiguredProfile(schema); + } + + private static void testFeature(Function pathFunction, String schemaFilename, SourceFeature sf, + Supplier fcFactory, + Consumer test, int expectedMatchCount) + throws Exception { + + var profile = loadConfig(pathFunction, schemaFilename); + var fc = fcFactory.get(); + + profile.processFeature(sf, fc); + + var length = new AtomicInteger(0); + + fc.forEach(f -> { + test.accept(f); + length.incrementAndGet(); + }); + + assertEquals(expectedMatchCount, length.get(), "Wrong number of features generated"); + } + + private static void testPolygon(Function pathFunction, String schemaFilename, Map tags, + Consumer test, int expectedMatchCount) + throws Exception { + var sf = + SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList()); + testFeature(pathFunction, schemaFilename, sf, + ConfiguredFeatureTest::polygonFeatureCollector, test, expectedMatchCount); + } + + private static void testLinestring(Function pathFunction, String schemaFilename, + Map tags, Consumer test, int expectedMatchCount) + throws Exception { + var sf = + SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList()); + testFeature(pathFunction, schemaFilename, sf, + ConfiguredFeatureTest::linestringFeatureCollector, test, expectedMatchCount); + } + + @Test + void testStaticAttributeTest() throws Exception { + testPolygon(TEST_RESOURCE, "static_attribute.yml", waterTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals("aTestConstantValue", attr.get("natural")); + }, 1); + } + + @Test + void testTagValueAttributeTest() throws Exception { + testPolygon(TEST_RESOURCE, "tag_attribute.yml", waterTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals("water", attr.get("natural")); + }, 1); + } + + @Test + void testTagIncludeAttributeTest() throws Exception { + testPolygon(TEST_RESOURCE, "tag_include.yml", waterTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals("ok", attr.get("test_include")); + assertFalse(attr.containsKey("test_exclude")); + }, 1); + } + + @Test + void testZoomAttributeTest() throws Exception { + testPolygon(TEST_RESOURCE, "tag_include.yml", waterTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals("test_zoom_value", attr.get("test_zoom_tag")); + + attr = f.getAttrsAtZoom(11); + assertNotEquals("test_zoom_value", attr.get("test_zoom_tag")); + + attr = f.getAttrsAtZoom(9); + assertNotEquals("test_zoom_value", attr.get("test_zoom_tag")); + }, 1); + } + + @Test + void testTagHighwayLinestringTest() throws Exception { + testLinestring(TEST_RESOURCE, "road_motorway.yml", motorwayTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals("motorway", attr.get("highway")); + }, 1); + } + + @Test + void testTagTypeConversionTest() throws Exception { + testLinestring(TEST_RESOURCE, "road_motorway.yml", motorwayTags, f -> { + var attr = f.getAttrsAtZoom(14); + + assertTrue(attr.containsKey("layer"), "Produce attribute layer"); + assertTrue(attr.containsKey("bridge"), "Produce attribute bridge"); + assertTrue(attr.containsKey("tunnel"), "Produce attribute tunnel"); + + assertEquals(1L, attr.get("layer"), "Extract layer as LONG"); + assertEquals(true, attr.get("bridge"), "Extract bridge as tagValue BOOLEAN"); + assertEquals(true, attr.get("tunnel"), "Extract tunnel as constantValue BOOLEAN"); + }, 1); + } + + @Test + void testZoomFilterAttributeTest() throws Exception { + testLinestring(TEST_RESOURCE, "road_motorway.yml", motorwayTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertTrue(attr.containsKey("bridge"), "Produce attribute bridge at z14"); + + attr = f.getAttrsAtZoom(10); + assertFalse(attr.containsKey("bridge"), "Don't produce attribute bridge at z10"); + }, 1); + } + + @Test + void testZoomFilterConditionalTest() throws Exception { + testLinestring(TEST_RESOURCE, "zoom_filter.yml", motorwayTags, f -> { + var attr = f.getAttrsAtZoom(4); + assertEquals("motorway", attr.get("highway"), "Produce attribute highway at z4"); + }, 1); + + testLinestring(TEST_RESOURCE, "zoom_filter.yml", trunkTags, f -> { + assertEquals(5, f.getMinZoom()); + var attr = f.getAttrsAtZoom(5); + assertEquals("trunk", attr.get("highway"), "Produce highway=trunk at z5"); + assertNull(attr.get("toll"), "Skip toll at z5"); + + attr = f.getAttrsAtZoom(6); + assertEquals("trunk", attr.get("highway"), "Produce highway=trunk at z6"); + + attr = f.getAttrsAtZoom(8); + assertEquals("yes", attr.get("toll"), "render toll at z8"); + }, 1); + + testLinestring(TEST_RESOURCE, "zoom_filter.yml", primaryTags, f -> { + var attr = f.getAttrsAtZoom(6); + assertNull(attr.get("highway"), "Skip highway=primary at z6"); + assertNull(attr.get("lanes")); + + attr = f.getAttrsAtZoom(7); + assertEquals("primary", attr.get("highway"), "Produce highway=primary at z7"); + assertNull(attr.get("lanes")); + + attr = f.getAttrsAtZoom(12); + assertEquals("primary", attr.get("highway"), "Produce highway=primary at z12"); + assertEquals(2L, attr.get("lanes")); + }, 1); + } + + @Test + void testAllValuesInKey() throws Exception { + //Show that a key in includeWhen with no values matches all values + testPolygon(SAMPLE_RESOURCE, "highway_areas.yml", highwayAreaTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals(true, attr.get("bridge"), "Produce bridge attribute"); + assertEquals("motorway", attr.get("highway"), "Produce highway area attribute"); + assertEquals("asphalt", attr.get("surface"), "Produce surface attribute"); + assertEquals(1L, attr.get("layer"), "Produce layer attribute"); + }, 1); + } + + @Test + void testInputMapping() throws Exception { + //Show that a key in includeWhen with no values matches all values + testLinestring(TEST_RESOURCE, "data_type_attributes.yml", inputMappingTags, f -> { + var attr = f.getAttrsAtZoom(14); + assertEquals(true, attr.get("b_type"), "Produce boolean"); + assertEquals("string_val", attr.get("s_type"), "Produce string"); + assertEquals(1, attr.get("d_type"), "Produce direction"); + assertEquals(1L, attr.get("l_type"), "Produce long"); + + assertEquals("yes", attr.get("intermittent"), "Produce raw attribute"); + assertEquals(true, attr.get("is_intermittent"), "Produce and rename boolean"); + assertEquals(true, attr.get("bridge"), "Produce boolean from full structure"); + }, 1); + } + + @Test + void testGeometryTypeMismatch() throws Exception { + //Validate that a schema that filters on lines does not match on a polygon feature + var sf = + SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), motorwayTags, "osm", null, 1, + emptyList()); + + testFeature(TEST_RESOURCE, "road_motorway.yml", sf, + ConfiguredFeatureTest::linestringFeatureCollector, f -> { + }, 0); + } + + @Test + void testSourceTypeMismatch() throws Exception { + //Validate that a schema only matches on the specified data source + var sf = + SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1, 0, 0), highwayAreaTags, "not_osm", null, 1, + emptyList()); + + testFeature(SAMPLE_RESOURCE, "highway_areas.yml", sf, + ConfiguredFeatureTest::linestringFeatureCollector, f -> { + }, 0); + } + + @Test + void testInvalidSchemas() throws Exception { + testInvalidSchema("bad_geometry_type.yml", "Profile defined with invalid geometry type"); + testInvalidSchema("no_layers.yml", "Profile defined with no layers"); + } + + private void testInvalidSchema(String filename, String message) { + assertThrows(RuntimeException.class, () -> loadConfig(TEST_INVALID_RESOURCE, filename), message); + } +} diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java new file mode 100644 index 00000000..fe179519 --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java @@ -0,0 +1,103 @@ +package com.onthegomap.planetiler.custommap; + +import static com.onthegomap.planetiler.TestUtils.assertContains; +import static com.onthegomap.planetiler.custommap.util.VerifyMonaco.MONACO_BOUNDS; +import static com.onthegomap.planetiler.util.Gzip.gunzip; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.VectorTile; +import com.onthegomap.planetiler.custommap.util.TestConfigurableUtils; +import com.onthegomap.planetiler.mbtiles.Mbtiles; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Polygon; + +/** + * End-to-end tests for custommap generation. + *

+ * Generates an entire map for the smallest openstreetmap extract available (Monaco) and asserts that expected output + * features exist + */ +class ConfiguredMapTest { + + @TempDir + static Path tmpDir; + private static Mbtiles mbtiles; + + @BeforeAll + public static void runPlanetiler() throws Exception { + Path dbPath = tmpDir.resolve("output.mbtiles"); + ConfiguredMapMain.main( + "generate-custom", + // Use local data extracts instead of downloading + "--schema=" + TestConfigurableUtils.pathToSample("owg_simple.yml"), + "--osm_path=" + TestUtils.pathToResource("monaco-latest.osm.pbf"), + "--water_polygons_path=" + TestUtils.pathToResource("water-polygons-split-3857.zip"), + + // Override temp dir location + "--tmp=" + tmpDir, + + // Override output location + "--mbtiles=" + dbPath + ); + mbtiles = Mbtiles.newReadOnlyDatabase(dbPath); + } + + @AfterAll + public static void close() throws IOException { + mbtiles.close(); + } + + @Test + void testMetadata() { + Map metadata = mbtiles.metadata().getAll(); + assertEquals("OWG Simple Schema", metadata.get("name")); + assertEquals("0", metadata.get("minzoom")); + assertEquals("14", metadata.get("maxzoom")); + assertEquals("baselayer", metadata.get("type")); + assertEquals("pbf", metadata.get("format")); + assertEquals("7.40921,43.72335,7.44864,43.75169", metadata.get("bounds")); + assertEquals("7.42892,43.73752,14", metadata.get("center")); + assertContains("Simple", metadata.get("description")); + assertContains("www.openstreetmap.org/copyright", metadata.get("attribution")); + } + + @Test + void ensureValidGeometries() throws Exception { + Set parsedTiles = TestUtils.getAllTiles(mbtiles); + for (var tileEntry : parsedTiles) { + var decoded = VectorTile.decode(gunzip(tileEntry.bytes())); + for (VectorTile.Feature feature : decoded) { + TestUtils.validateGeometry(feature.geometry().decode()); + } + } + } + + // @Test --TODO FIX after adding water layer + void testContainsOceanPolyons() { + assertMinFeatures("water", Map.of( + "natural", "water" + ), 0, 1, Polygon.class); + } + + @Test + void testRoad() { + assertMinFeatures("road", Map.of( + "highway", "primary" + ), 14, 200, LineString.class); + } + + private static void assertMinFeatures(String layer, Map attrs, int zoom, + int expected, Class clazz) { + TestUtils.assertMinFeatureCount(mbtiles, layer, zoom, attrs, MONACO_BOUNDS, expected, clazz); + } +} diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java new file mode 100644 index 00000000..e0c79190 --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java @@ -0,0 +1,38 @@ +package com.onthegomap.planetiler.custommap; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.jupiter.api.Test; + +class SchemaYAMLLoadTest { + + /** + * Test to ensure that all bundled schemas load to POJOs. + * + * @throws Exception + */ + @Test + void testSchemaLoad() throws IOException { + testSchemasInFolder(Paths.get("src", "main", "resources", "samples")); + testSchemasInFolder(Paths.get("src", "test", "resources", "validSchema")); + } + + private void testSchemasInFolder(Path path) throws IOException { + var schemaFiles = Files.walk(path) + .filter(p -> p.getFileName().toString().endsWith(".yml")) + .toList(); + + assertFalse(schemaFiles.isEmpty(), "No files found"); + + for (Path schemaFile : schemaFiles) { + var schemaConfig = ConfiguredMapMain.loadConfig(schemaFile); + assertNotNull(schemaConfig, () -> "Failed to unmarshall " + schemaFile.toString()); + assertNotNull(new ConfiguredProfile(schemaConfig), () -> "Failed to load profile from " + schemaFile.toString()); + } + } +} diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/TestConfigurableUtils.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/TestConfigurableUtils.java new file mode 100644 index 00000000..1c7f003c --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/TestConfigurableUtils.java @@ -0,0 +1,22 @@ +package com.onthegomap.planetiler.custommap.util; + +import java.nio.file.Path; + +public class TestConfigurableUtils { + public static Path pathToTestResource(String resource) { + return resolve(Path.of("planetiler-custommap", "src", "test", "resources", "validSchema", resource)); + } + + public static Path pathToTestInvalidResource(String resource) { + return resolve(Path.of("planetiler-custommap", "src", "test", "resources", "invalidSchema", resource)); + } + + public static Path pathToSample(String resource) { + return resolve(Path.of("planetiler-custommap", "src", "main", "resources", "samples", resource)); + } + + private static Path resolve(Path pathFromRoot) { + Path cwd = Path.of("").toAbsolutePath(); + return cwd.resolveSibling(pathFromRoot); + } +} diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/VerifyMonaco.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/VerifyMonaco.java new file mode 100644 index 00000000..694dc792 --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/util/VerifyMonaco.java @@ -0,0 +1,44 @@ +package com.onthegomap.planetiler.custommap.util; + +import com.onthegomap.planetiler.mbtiles.Mbtiles; +import com.onthegomap.planetiler.mbtiles.Verify; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; + +/** + * A utility to check the contents of an mbtiles file generated for Monaco. + */ +public class VerifyMonaco { + + public static final Envelope MONACO_BOUNDS = new Envelope(7.40921, 7.44864, 43.72335, 43.75169); + + /** + * Returns a verification result with a basic set of checks against an openmaptiles map built from an extract for + * Monaco. + */ + public static Verify verify(Mbtiles mbtiles) { + Verify verify = Verify.verify(mbtiles); + verify.checkMinFeatureCount(MONACO_BOUNDS, "building", Map.of(), 13, 14, 100, Polygon.class); + verify.checkMinFeatureCount(MONACO_BOUNDS, "transportation", Map.of(), 10, 14, 5, LineString.class); + verify.checkMinFeatureCount(MONACO_BOUNDS, "landcover", Map.of( + "class", "grass", + "subclass", "park" + ), 14, 10, Polygon.class); + verify.checkMinFeatureCount(MONACO_BOUNDS, "water", Map.of("class", "ocean"), 0, 14, 1, Polygon.class); + verify.checkMinFeatureCount(MONACO_BOUNDS, "place", Map.of("class", "country"), 2, 14, 1, Point.class); + return verify; + } + + public static void main(String[] args) throws IOException { + try (var mbtiles = Mbtiles.newReadOnlyDatabase(Path.of(args[0]))) { + var result = verify(mbtiles); + result.print(); + result.failIfErrors(); + } + } +} diff --git a/planetiler-custommap/src/test/resources/invalidSchema/bad_geometry_type.yml b/planetiler-custommap/src/test/resources/invalidSchema/bad_geometry_type.yml new file mode 100644 index 00000000..82f03eb2 --- /dev/null +++ b/planetiler-custommap/src/test/resources/invalidSchema/bad_geometry_type.yml @@ -0,0 +1,18 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- name: testLayer + features: + - sources: + - osm + geometry: smurf + include_when: + natural: water + attributes: + - key: water + - constant_value: wet \ No newline at end of file diff --git a/planetiler-custommap/src/test/resources/invalidSchema/no_layers.yml b/planetiler-custommap/src/test/resources/invalidSchema/no_layers.yml new file mode 100644 index 00000000..03041a9f --- /dev/null +++ b/planetiler-custommap/src/test/resources/invalidSchema/no_layers.yml @@ -0,0 +1,7 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island \ No newline at end of file diff --git a/planetiler-custommap/src/test/resources/validSchema/data_type_attributes.yml b/planetiler-custommap/src/test/resources/validSchema/data_type_attributes.yml new file mode 100644 index 00000000..7121eee8 --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/data_type_attributes.yml @@ -0,0 +1,31 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +tag_mappings: + b_type: boolean + l_type: long + d_type: direction + s_type: string + intermittent: + output: is_intermittent + type: boolean + bridge: + type: boolean +layers: +- name: testLayer + features: + - sources: + - osm + geometry: line + attributes: + - key: b_type + - key: l_type + - key: d_type + - key: s_type + - key: intermittent + - key: is_intermittent + - key: bridge diff --git a/planetiler-custommap/src/test/resources/validSchema/local_path.yml b/planetiler-custommap/src/test/resources/validSchema/local_path.yml new file mode 100644 index 00000000..4178031d --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/local_path.yml @@ -0,0 +1,18 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf +layers: +- name: testLayer + features: + - sources: + - osm + geometry: polygon + include_when: + natural: water + attributes: + - key: natural diff --git a/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml b/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml new file mode 100644 index 00000000..85b21ef5 --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml @@ -0,0 +1,39 @@ +schema_name: OWG Simple Schema +schema_description: Simple vector tile schema +attribution: © + OpenStreetMap contributors +sources: + water_polygons: + type: shapefile + url: https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip + osm: + type: osm + url: geofabrik:rhode-island +tag_mappings: + bridge: boolean # input=bridge, output=bridge, type=boolean + layer: long + tunnel: boolean +layers: +- name: road + features: + - sources: + - osm + min_zoom: 4 + geometry: line + include_when: + highway: motorway + attributes: + - key: highway + - key: bridge + include_when: + bridge: true + min_zoom: 11 + - key: tunnel + constant_value: true + include_when: + tunnel: true + min_zoom: 11 + - key: name + min_zoom: 12 + - key: layer + min_zoom: 13 diff --git a/planetiler-custommap/src/test/resources/validSchema/static_attribute.yml b/planetiler-custommap/src/test/resources/validSchema/static_attribute.yml new file mode 100644 index 00000000..0b20eb0e --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/static_attribute.yml @@ -0,0 +1,18 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- name: testLayer + features: + - sources: + - osm + geometry: polygon + include_when: + natural: water + attributes: + - key: natural + constant_value: aTestConstantValue \ No newline at end of file diff --git a/planetiler-custommap/src/test/resources/validSchema/tag_attribute.yml b/planetiler-custommap/src/test/resources/validSchema/tag_attribute.yml new file mode 100644 index 00000000..6ae3384c --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/tag_attribute.yml @@ -0,0 +1,17 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- name: testLayer + features: + - sources: + - osm + geometry: polygon + include_when: + natural: water + attributes: + - key: natural diff --git a/planetiler-custommap/src/test/resources/validSchema/tag_include.yml b/planetiler-custommap/src/test/resources/validSchema/tag_include.yml new file mode 100644 index 00000000..8adac37d --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/tag_include.yml @@ -0,0 +1,27 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- name: testLayer + features: + - sources: + - osm + min_zoom: 10 + geometry: polygon + include_when: + natural: water + attributes: + - key: test_include + constant_value: ok + include_when: + natural: water + - key: test_exclude + constant_value: bad + include_when: + natural: mud + - key: test_zoom_tag + min_zoom: 12 \ No newline at end of file diff --git a/planetiler-custommap/src/test/resources/validSchema/zoom_filter.yml b/planetiler-custommap/src/test/resources/validSchema/zoom_filter.yml new file mode 100644 index 00000000..ecde0bbc --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/zoom_filter.yml @@ -0,0 +1,37 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +tag_mappings: + lanes: long +layers: +- name: testLayer + features: + - sources: + - osm + geometry: line + min_zoom: 4 + zoom_override: + - min: 5 + tag: + highway: trunk + - min: 7 + tag: + highway: primary + include_when: + highway: + attributes: + - key: highway + min_zoom_by_value: + trunk: 5 + primary: 7 + - key: lanes + min_zoom_by_value: + 4: 9 + 3: 9 + 2: 10 + - key: toll + min_zoom: 8 \ No newline at end of file diff --git a/planetiler-dist/pom.xml b/planetiler-dist/pom.xml index 0f5894ba..fe31089a 100644 --- a/planetiler-dist/pom.xml +++ b/planetiler-dist/pom.xml @@ -32,6 +32,11 @@ planetiler-basemap ${project.parent.version} + + com.onthegomap.planetiler + planetiler-custommap + ${project.parent.version} + com.onthegomap.planetiler planetiler-examples diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index 7a98307a..aa8b3380 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -4,6 +4,7 @@ import com.onthegomap.planetiler.basemap.BasemapMain; import com.onthegomap.planetiler.basemap.util.VerifyMonaco; import com.onthegomap.planetiler.benchmarks.BasemapMapping; import com.onthegomap.planetiler.benchmarks.LongLongMapBench; +import com.onthegomap.planetiler.custommap.ConfiguredMapMain; import com.onthegomap.planetiler.examples.BikeRouteOverlay; import com.onthegomap.planetiler.examples.ToiletsOverlay; import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi; @@ -21,6 +22,7 @@ public class Main { private static final EntryPoint DEFAULT_TASK = BasemapMain::main; private static final Map ENTRY_POINTS = Map.of( "generate-basemap", BasemapMain::main, + "generate-custom", ConfiguredMapMain::main, "basemap", BasemapMain::main, "example-bikeroutes", BikeRouteOverlay::main, "example-toilets", ToiletsOverlay::main, diff --git a/pom.xml b/pom.xml index af51ce00..61f28cde 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ planetiler-core planetiler-basemap + planetiler-custommap planetiler-benchmarks planetiler-examples planetiler-dist @@ -91,6 +92,16 @@ + + org.yaml + snakeyaml + 1.30 + + + org.commonmark + commonmark + 0.18.2 + org.junit.jupiter junit-jupiter-api