kopia lustrzana https://github.com/onthegomap/planetiler
Add whole-tile postprocess hook (#802)
rodzic
6331934d6f
commit
fa7bffb04f
|
@ -1,6 +1,7 @@
|
||||||
package com.onthegomap.planetiler;
|
package com.onthegomap.planetiler;
|
||||||
|
|
||||||
import com.onthegomap.planetiler.geo.GeometryException;
|
import com.onthegomap.planetiler.geo.GeometryException;
|
||||||
|
import com.onthegomap.planetiler.geo.TileCoord;
|
||||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||||
import com.onthegomap.planetiler.reader.SourceFeature;
|
import com.onthegomap.planetiler.reader.SourceFeature;
|
||||||
import com.onthegomap.planetiler.reader.osm.OsmElement;
|
import com.onthegomap.planetiler.reader.osm.OsmElement;
|
||||||
|
@ -103,13 +104,40 @@ public interface Profile {
|
||||||
* @return the new list of output features or {@code null} to not change anything. Set any elements of the list to
|
* @return the new list of output features or {@code null} to not change anything. Set any elements of the list to
|
||||||
* {@code null} if they should be ignored.
|
* {@code null} if they should be ignored.
|
||||||
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
|
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
|
||||||
* the original input features, and continue processing other tiles
|
* the original input features, and continue processing other layers
|
||||||
*/
|
*/
|
||||||
default List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
|
default List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||||
List<VectorTile.Feature> items) throws GeometryException {
|
List<VectorTile.Feature> items) throws GeometryException {
|
||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply any post-processing to layers in an output tile before writing it to the output.
|
||||||
|
* <p>
|
||||||
|
* This is called before {@link #postProcessLayerFeatures(String, int, List)} gets called for each layer. Use this
|
||||||
|
* method if features in one layer should influence features in another layer, to create new layers from existing
|
||||||
|
* ones, or if you need to remove a layer entirely from the output.
|
||||||
|
* <p>
|
||||||
|
* These transformations may add, remove, or change the tags, geometry, or ordering of output features based on other
|
||||||
|
* features present in this tile. See {@link FeatureMerge} class for a set of common transformations that merge
|
||||||
|
* linestrings/polygons.
|
||||||
|
* <p>
|
||||||
|
* Many threads invoke this method concurrently so ensure thread-safe access to any shared data structures.
|
||||||
|
* <p>
|
||||||
|
* The default implementation passes through input features unaltered
|
||||||
|
*
|
||||||
|
* @param tileCoord the tile being post-processed
|
||||||
|
* @param layers all the output features in each layer on this tile
|
||||||
|
* @return the new map from layer to features or {@code null} to not change anything. Set any elements of the lists to
|
||||||
|
* {@code null} if they should be ignored.
|
||||||
|
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
|
||||||
|
* the original input features, and continue processing other tiles
|
||||||
|
*/
|
||||||
|
default Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
|
||||||
|
Map<String, List<VectorTile.Feature>> layers) throws GeometryException {
|
||||||
|
return layers;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the name of the generated tileset to put into {@link Mbtiles} metadata
|
* Returns the name of the generated tileset to put into {@link Mbtiles} metadata
|
||||||
*
|
*
|
||||||
|
@ -158,7 +186,9 @@ public interface Profile {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
default Map<String,String> extraArchiveMetadata() { return Map.of(); }
|
default Map<String, String> extraArchiveMetadata() {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Defines whether {@link Wikidata} should fetch wikidata translations for the input element.
|
* Defines whether {@link Wikidata} should fetch wikidata translations for the input element.
|
||||||
|
|
|
@ -26,6 +26,7 @@ import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.TreeMap;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import javax.annotation.concurrent.NotThreadSafe;
|
import javax.annotation.concurrent.NotThreadSafe;
|
||||||
|
@ -449,28 +450,47 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
|
||||||
if (layerStats != null) {
|
if (layerStats != null) {
|
||||||
tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
|
tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
|
||||||
}
|
}
|
||||||
List<VectorTile.Feature> items = new ArrayList<>(entries.size());
|
List<VectorTile.Feature> items = new ArrayList<>();
|
||||||
String currentLayer = null;
|
String currentLayer = null;
|
||||||
|
Map<String, List<VectorTile.Feature>> layerFeatures = new TreeMap<>();
|
||||||
for (SortableFeature entry : entries) {
|
for (SortableFeature entry : entries) {
|
||||||
var feature = decodeVectorTileFeature(entry);
|
var feature = decodeVectorTileFeature(entry);
|
||||||
String layer = feature.layer();
|
String layer = feature.layer();
|
||||||
|
|
||||||
if (currentLayer == null) {
|
if (currentLayer == null) {
|
||||||
currentLayer = layer;
|
currentLayer = layer;
|
||||||
|
layerFeatures.put(currentLayer, items);
|
||||||
} else if (!currentLayer.equals(layer)) {
|
} else if (!currentLayer.equals(layer)) {
|
||||||
postProcessAndAddLayerFeatures(tile, currentLayer, items);
|
|
||||||
currentLayer = layer;
|
currentLayer = layer;
|
||||||
items.clear();
|
items = new ArrayList<>();
|
||||||
|
layerFeatures.put(layer, items);
|
||||||
}
|
}
|
||||||
|
|
||||||
items.add(feature);
|
items.add(feature);
|
||||||
}
|
}
|
||||||
postProcessAndAddLayerFeatures(tile, currentLayer, items);
|
// first post-process entire tile by invoking postProcessTileFeatures to allow for post-processing that combines
|
||||||
|
// features across different layers, infers new layers, or removes layers
|
||||||
|
try {
|
||||||
|
var initialFeatures = layerFeatures;
|
||||||
|
layerFeatures = profile.postProcessTileFeatures(tileCoord, layerFeatures);
|
||||||
|
if (layerFeatures == null) {
|
||||||
|
layerFeatures = initialFeatures;
|
||||||
|
}
|
||||||
|
} catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
|
||||||
|
handlePostProcessFailure(e, "entire tile");
|
||||||
|
}
|
||||||
|
// then let profiles post-process each layer in isolation with postProcessLayerFeatures
|
||||||
|
for (var entry : layerFeatures.entrySet()) {
|
||||||
|
postProcessAndAddLayerFeatures(tile, entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
return tile;
|
return tile;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
|
private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
|
||||||
List<VectorTile.Feature> features) {
|
List<VectorTile.Feature> features) {
|
||||||
|
if (features == null || features.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
List<VectorTile.Feature> postProcessed = profile
|
List<VectorTile.Feature> postProcessed = profile
|
||||||
.postProcessLayerFeatures(layer, tileCoord.z(), features);
|
.postProcessLayerFeatures(layer, tileCoord.z(), features);
|
||||||
|
@ -482,21 +502,25 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
|
||||||
// also remove points more than --max-point-buffer pixels outside the tile if the
|
// also remove points more than --max-point-buffer pixels outside the tile if the
|
||||||
// user has requested a narrower buffer than the profile provides by default
|
// user has requested a narrower buffer than the profile provides by default
|
||||||
} catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
|
} catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
|
||||||
// failures in tile post-processing happen very late so err on the side of caution and
|
handlePostProcessFailure(e, layer);
|
||||||
// log failures, only throwing when it's a fatal error
|
|
||||||
if (e instanceof GeometryException geoe) {
|
|
||||||
geoe.log(stats, "postprocess_layer",
|
|
||||||
"Caught error postprocessing features for " + layer + " layer on " + tileCoord, config.logJtsExceptions());
|
|
||||||
} else if (e instanceof Error err) {
|
|
||||||
LOGGER.error("Caught fatal error postprocessing features {} {}", layer, tileCoord, e);
|
|
||||||
throw err;
|
|
||||||
} else {
|
|
||||||
LOGGER.error("Caught error postprocessing features {} {}", layer, tileCoord, e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
encoder.addLayerFeatures(layer, features);
|
encoder.addLayerFeatures(layer, features);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void handlePostProcessFailure(Throwable e, String entity) {
|
||||||
|
// failures in tile post-processing happen very late so err on the side of caution and
|
||||||
|
// log failures, only throwing when it's a fatal error
|
||||||
|
if (e instanceof GeometryException geoe) {
|
||||||
|
geoe.log(stats, "postprocess_layer",
|
||||||
|
"Caught error postprocessing features for " + entity + " on " + tileCoord, config.logJtsExceptions());
|
||||||
|
} else if (e instanceof Error err) {
|
||||||
|
LOGGER.error("Caught fatal error postprocessing features {} {}", entity, tileCoord, e);
|
||||||
|
throw err;
|
||||||
|
} else {
|
||||||
|
LOGGER.error("Caught error postprocessing features {} {}", entity, tileCoord, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void add(SortableFeature entry) {
|
void add(SortableFeature entry) {
|
||||||
numFeaturesProcessed.incrementAndGet();
|
numFeaturesProcessed.incrementAndGet();
|
||||||
long key = entry.key();
|
long key = entry.key();
|
||||||
|
|
|
@ -1316,6 +1316,55 @@ class PlanetilerTests {
|
||||||
)), sortListValues(results.tiles));
|
)), sortListValues(results.tiles));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@ValueSource(booleans = {false, true})
|
||||||
|
void postProcessTileFeatures(boolean postProcessLayersToo) throws Exception {
|
||||||
|
double y = 0.5 + Z15_WIDTH / 2;
|
||||||
|
double lat = GeoUtils.getWorldLat(y);
|
||||||
|
|
||||||
|
double x1 = 0.5 + Z15_WIDTH / 4;
|
||||||
|
double lng1 = GeoUtils.getWorldLon(x1);
|
||||||
|
double lng2 = GeoUtils.getWorldLon(x1 + Z15_WIDTH * 10d / 256);
|
||||||
|
|
||||||
|
List<SimpleFeature> features1 = List.of(
|
||||||
|
newReaderFeature(newPoint(lng1, lat), Map.of("from", "a")),
|
||||||
|
newReaderFeature(newPoint(lng2, lat), Map.of("from", "b"))
|
||||||
|
);
|
||||||
|
var testProfile = new Profile.NullProfile() {
|
||||||
|
@Override
|
||||||
|
public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
|
||||||
|
features.point(sourceFeature.getString("from"))
|
||||||
|
.inheritAttrFromSource("from")
|
||||||
|
.setMinZoom(15);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
|
||||||
|
Map<String, List<VectorTile.Feature>> layers) {
|
||||||
|
List<VectorTile.Feature> features = new ArrayList<>();
|
||||||
|
features.addAll(layers.get("a"));
|
||||||
|
features.addAll(layers.get("b"));
|
||||||
|
return Map.of("c", features);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items) {
|
||||||
|
return postProcessLayersToo ? items.reversed() : items;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
var results = run(
|
||||||
|
Map.of("threads", "1", "maxzoom", "15"),
|
||||||
|
(featureGroup, profile, config) -> processReaderFeatures(featureGroup, profile, config, features1),
|
||||||
|
testProfile);
|
||||||
|
|
||||||
|
assertSubmap(sortListValues(Map.of(
|
||||||
|
TileCoord.ofXYZ(Z15_TILES / 2, Z15_TILES / 2, 15), List.of(
|
||||||
|
feature(newPoint(64, 128), "c", Map.of("from", "a")),
|
||||||
|
feature(newPoint(74, 128), "c", Map.of("from", "b"))
|
||||||
|
)
|
||||||
|
)), sortListValues(results.tiles));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testMergeLineStringsIgnoresRoundingIntersections() throws Exception {
|
void testMergeLineStringsIgnoresRoundingIntersections() throws Exception {
|
||||||
double y = 0.5 + Z14_WIDTH / 2;
|
double y = 0.5 + Z14_WIDTH / 2;
|
||||||
|
@ -1681,6 +1730,12 @@ class PlanetilerTests {
|
||||||
throws GeometryException;
|
throws GeometryException;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private interface TilePostprocessFunction {
|
||||||
|
|
||||||
|
Map<String, List<VectorTile.Feature>> process(TileCoord tileCoord, Map<String, List<VectorTile.Feature>> layers)
|
||||||
|
throws GeometryException;
|
||||||
|
}
|
||||||
|
|
||||||
private record PlanetilerResults(
|
private record PlanetilerResults(
|
||||||
Map<TileCoord, List<TestUtils.ComparableFeature>> tiles, Map<String, String> metadata, int tileDataCount
|
Map<TileCoord, List<TestUtils.ComparableFeature>> tiles, Map<String, String> metadata, int tileDataCount
|
||||||
) {}
|
) {}
|
||||||
|
@ -1692,7 +1747,8 @@ class PlanetilerTests {
|
||||||
@Override String version,
|
@Override String version,
|
||||||
BiConsumer<SourceFeature, FeatureCollector> processFeature,
|
BiConsumer<SourceFeature, FeatureCollector> processFeature,
|
||||||
Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
|
Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
|
||||||
LayerPostprocessFunction postprocessLayerFeatures
|
LayerPostprocessFunction postprocessLayerFeatures,
|
||||||
|
TilePostprocessFunction postprocessTileFeatures
|
||||||
) implements Profile {
|
) implements Profile {
|
||||||
|
|
||||||
TestProfile(
|
TestProfile(
|
||||||
|
@ -1701,8 +1757,16 @@ class PlanetilerTests {
|
||||||
LayerPostprocessFunction postprocessLayerFeatures
|
LayerPostprocessFunction postprocessLayerFeatures
|
||||||
) {
|
) {
|
||||||
this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
|
this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
|
||||||
preprocessOsmRelation,
|
preprocessOsmRelation, postprocessLayerFeatures, null);
|
||||||
postprocessLayerFeatures);
|
}
|
||||||
|
|
||||||
|
TestProfile(
|
||||||
|
BiConsumer<SourceFeature, FeatureCollector> processFeature,
|
||||||
|
Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
|
||||||
|
TilePostprocessFunction postprocessTileFeatures
|
||||||
|
) {
|
||||||
|
this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
|
||||||
|
preprocessOsmRelation, null, postprocessTileFeatures);
|
||||||
}
|
}
|
||||||
|
|
||||||
static TestProfile processSourceFeatures(BiConsumer<SourceFeature, FeatureCollector> processFeature) {
|
static TestProfile processSourceFeatures(BiConsumer<SourceFeature, FeatureCollector> processFeature) {
|
||||||
|
@ -1712,7 +1776,7 @@ class PlanetilerTests {
|
||||||
@Override
|
@Override
|
||||||
public List<OsmRelationInfo> preprocessOsmRelation(
|
public List<OsmRelationInfo> preprocessOsmRelation(
|
||||||
OsmElement.Relation relation) {
|
OsmElement.Relation relation) {
|
||||||
return preprocessOsmRelation.apply(relation);
|
return preprocessOsmRelation == null ? null : preprocessOsmRelation.apply(relation);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -1726,7 +1790,13 @@ class PlanetilerTests {
|
||||||
@Override
|
@Override
|
||||||
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
|
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||||
List<VectorTile.Feature> items) throws GeometryException {
|
List<VectorTile.Feature> items) throws GeometryException {
|
||||||
return postprocessLayerFeatures.process(layer, zoom, items);
|
return postprocessLayerFeatures == null ? null : postprocessLayerFeatures.process(layer, zoom, items);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
|
||||||
|
Map<String, List<VectorTile.Feature>> layers) throws GeometryException {
|
||||||
|
return postprocessTileFeatures == null ? null : postprocessTileFeatures.process(tileCoord, layers);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -279,7 +279,7 @@ public class TestUtils {
|
||||||
case UNKNOWN -> throw new IllegalArgumentException("cannot decompress \"UNKNOWN\"");
|
case UNKNOWN -> throw new IllegalArgumentException("cannot decompress \"UNKNOWN\"");
|
||||||
};
|
};
|
||||||
var decoded = VectorTile.decode(bytes).stream()
|
var decoded = VectorTile.decode(bytes).stream()
|
||||||
.map(feature -> feature(decodeSilently(feature.geometry()), feature.attrs())).toList();
|
.map(feature -> feature(decodeSilently(feature.geometry()), feature.layer(), feature.attrs())).toList();
|
||||||
tiles.put(tile.coord(), decoded);
|
tiles.put(tile.coord(), decoded);
|
||||||
}
|
}
|
||||||
return tiles;
|
return tiles;
|
||||||
|
@ -466,11 +466,32 @@ public class TestUtils {
|
||||||
|
|
||||||
public record ComparableFeature(
|
public record ComparableFeature(
|
||||||
GeometryComparision geometry,
|
GeometryComparision geometry,
|
||||||
|
String layer,
|
||||||
Map<String, Object> attrs
|
Map<String, Object> attrs
|
||||||
) {}
|
) {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
return o == this || (o instanceof ComparableFeature other &&
|
||||||
|
geometry.equals(other.geometry) &&
|
||||||
|
attrs.equals(other.attrs) &&
|
||||||
|
(layer == null || other.layer == null || Objects.equals(layer, other.layer)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
int result = geometry.hashCode();
|
||||||
|
result = 31 * result + attrs.hashCode();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ComparableFeature feature(Geometry geom, String layer, Map<String, Object> attrs) {
|
||||||
|
return new ComparableFeature(new NormGeometry(geom), layer, attrs);
|
||||||
|
}
|
||||||
|
|
||||||
public static ComparableFeature feature(Geometry geom, Map<String, Object> attrs) {
|
public static ComparableFeature feature(Geometry geom, Map<String, Object> attrs) {
|
||||||
return new ComparableFeature(new NormGeometry(geom), attrs);
|
return new ComparableFeature(new NormGeometry(geom), null, attrs);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Map<String, Object> toMap(FeatureCollector.Feature feature, int zoom) {
|
public static Map<String, Object> toMap(FeatureCollector.Feature feature, int zoom) {
|
||||||
|
|
Ładowanie…
Reference in New Issue