cleanup forwarding profile and add tests

pull/1/head
Mike Barry 2021-09-18 06:05:50 -04:00
rodzic 6525f03698
commit 7df851ca53
6 zmienionych plików z 424 dodań i 178 usunięć

Wyświetl plik

@ -0,0 +1,196 @@
package com.onthegomap.flatmap;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
/**
* A framework for building complex {@link Profile Profiles} that need to be broken apart into multiple handlers (i.e.
* one per layer).
* <p>
* Individual handlers added with {@link #registerHandler(Handler)} can listen on events by implementing these
* handlers:
* <ul>
* <li>{@link OsmRelationPreprocessor} to process every OSM relation during first pass through OSM file</li>
* <li>{@link FeatureProcessor} to handle features from a particular source (added through {@link #registerSourceHandler(String, FeatureProcessor)})</li>
* <li>{@link FinishHandler} to be notified whenever we finish processing each source</li>
* <li>{@link FeaturePostProcessor} to post-process features in a layer before rendering the output tile</li>
* </ul>
* See {@code OpenMapTilesProfile} for a full implementation using this framework.
*/
public abstract class ForwardingProfile implements Profile {
private final List<Handler> handlers = new ArrayList<>();
/** Handlers that pre-process OSM relations during pass 1 through the data. */
private final List<OsmRelationPreprocessor> osmRelationPreprocessors = new ArrayList<>();
/** Handlers that get a callback when each source is finished reading. */
private final List<FinishHandler> finishHandlers = new ArrayList<>();
/** Map from layer name to its handler if it implements {@link FeaturePostProcessor}. */
private final Map<String, List<FeaturePostProcessor>> postProcessors = new HashMap<>();
/** Map from source ID to its handler if it implements {@link FeatureProcessor}. */
private final Map<String, List<FeatureProcessor>> sourceElementProcessors = new HashMap<>();
/**
* Call {@code processor} for every element in {@code source}.
*
* @param source string ID of the source
* @param processor handler that will process elements in that source
*/
public void registerSourceHandler(String source, FeatureProcessor processor) {
sourceElementProcessors.computeIfAbsent(source, name -> new ArrayList<>())
.add(processor);
}
/**
* Call {@code handler} for different events based on which interfaces {@code handler} implements: {@link
* OsmRelationPreprocessor}, {@link FinishHandler}, or {@link FeaturePostProcessor}.
*/
public void registerHandler(Handler handler) {
this.handlers.add(handler);
if (handler instanceof OsmRelationPreprocessor osmRelationPreprocessor) {
osmRelationPreprocessors.add(osmRelationPreprocessor);
}
if (handler instanceof FinishHandler finishHandler) {
finishHandlers.add(finishHandler);
}
if (handler instanceof FeaturePostProcessor postProcessor) {
postProcessors.computeIfAbsent(postProcessor.name(), name -> new ArrayList<>())
.add(postProcessor);
}
}
@Override
public List<OsmRelationInfo> preprocessOsmRelation(OsmElement.Relation relation) {
// delegate OSM relation pre-processing to each layer, if it implements FeaturePostProcessor
List<OsmRelationInfo> result = null;
for (OsmRelationPreprocessor osmRelationPreprocessor : osmRelationPreprocessors) {
List<OsmRelationInfo> thisResult = osmRelationPreprocessor
.preprocessOsmRelation(relation);
if (thisResult != null) {
if (result == null) {
result = new ArrayList<>(thisResult);
} else {
result.addAll(thisResult);
}
}
}
return result;
}
@Override
public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
// delegate source feature processing to each handler for that source
var handlers = sourceElementProcessors.get(sourceFeature.getSource());
if (handlers != null) {
for (var handler : handlers) {
handler.processFeature(sourceFeature, features);
// TODO extract common handling for expression-based filtering from openmaptiles to this
// common profile when we have another use-case for it.
}
}
}
@Override
public boolean caresAboutSource(String name) {
return sourceElementProcessors.containsKey(name);
}
@Override
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items)
throws GeometryException {
// delegate feature post-processing to each layer, if it implements FeaturePostProcessor
List<FeaturePostProcessor> handlers = postProcessors.get(layer);
List<VectorTile.Feature> result = items;
if (handlers != null) {
for (FeaturePostProcessor handler : handlers) {
var thisResult = handler.postProcess(zoom, result);
if (thisResult != null) {
result = thisResult;
}
}
}
return result;
}
@Override
public void finish(String sourceName, FeatureCollector.Factory featureCollectors,
Consumer<FeatureCollector.Feature> next) {
// delegate finish handling to every layer that implements FinishHandler
for (var handler : finishHandlers) {
handler.finish(sourceName, featureCollectors, next);
}
}
@Override
public void release() {
// release resources used by each handler
handlers.forEach(Handler::release);
}
/** Interface for handlers that this profile forwards to should implement. */
public interface Handler {
/** Free any resources associated with this profile (i.e. shared data structures) */
default void release() {
}
}
public interface HandlerForLayer extends Handler {
/** The layer name this handler is for */
String name();
}
/** Handlers should implement this interface to get notified when a source finishes processing. */
public interface FinishHandler extends Handler {
/**
* Invoked once for each source after all elements for a source have been processed.
*
* @see Profile#finish(String, FeatureCollector.Factory, Consumer)
*/
void finish(String sourceName, FeatureCollector.Factory featureCollectors,
Consumer<FeatureCollector.Feature> emit);
}
/** Handlers should implement this interface to pre-process OSM relations during pass 1 through the data. */
public interface OsmRelationPreprocessor extends Handler {
/**
* Returns information extracted from an OSM relation during pass 1 of the input OSM data to make available when
* processing elements in that relation during pass 2.
*
* @see Profile#preprocessOsmRelation(OsmElement.Relation)
*/
List<OsmRelationInfo> preprocessOsmRelation(OsmElement.Relation relation);
}
/** Handlers should implement this interface to post-process vector tile features before emitting an output tile. */
public interface FeaturePostProcessor extends HandlerForLayer {
/**
* Apply any post-processing to features in this output layer of a tile before writing it to the output file.
*
* @throws GeometryException if the input elements cannot be deserialized, or output elements cannot be serialized
* @see Profile#postProcessLayerFeatures(String, int, List)
*/
List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> items) throws GeometryException;
}
/** Handlers should implement this interface to process input features from a given source ID. */
public interface FeatureProcessor {
/**
* Process an input element from a source feature.
*
* @see Profile#processFeature(SourceFeature, FeatureCollector)
*/
void processFeature(SourceFeature feature, FeatureCollector features);
}
}

Wyświetl plik

@ -6,10 +6,7 @@ import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
import com.onthegomap.flatmap.util.Wikidata;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
/**
@ -33,6 +30,7 @@ import java.util.function.Consumer;
* (i.e. one per layer) and forwarding each element/event to the handlers that care about it.
*/
public interface Profile {
// TODO might want to break this apart into sub-interfaces that ForwardingProfile (and MbtilesMetadata) can use too
/**
* Extracts information from <a href="https://wiki.openstreetmap.org/wiki/Relation">OSM relations</a> that will be
@ -190,174 +188,4 @@ public interface Profile {
}
}
/**
* A profile that delegates handling of input features to individual {@link Handler LayerHandlers} if they implement
* {@link OsmRelationPreprocessor}, {@link FeatureProcessor}, {@link FinishHandler}, or {@link FeaturePostProcessor}.
*/
abstract class ForwardingProfile implements Profile {
private final List<Handler> handlers = new ArrayList<>();
/** Handlers that pre-process OSM relations during pass 1 through the data. */
private final List<OsmRelationPreprocessor> osmRelationPreprocessors = new ArrayList<>();
/** Handlers that get a callback when each source is finished reading. */
private final List<FinishHandler> finishHandlers = new ArrayList<>();
/** Map from layer name to its handler if it implements {@link FeaturePostProcessor}. */
private final Map<String, List<FeaturePostProcessor>> postProcessors = new HashMap<>();
/** Map from source ID to its handler if it implements {@link FeatureProcessor}. */
private final Map<String, List<FeatureProcessor>> sourceElementProcessors = new HashMap<>();
protected void registerSourceHandler(String source, FeatureProcessor processor) {
sourceElementProcessors.computeIfAbsent(source, name -> new ArrayList<>())
.add(processor);
}
protected void registerHandler(Handler handler) {
this.handlers.add(handler);
if (handler instanceof OsmRelationPreprocessor osmRelationPreprocessor) {
osmRelationPreprocessors.add(osmRelationPreprocessor);
}
if (handler instanceof FinishHandler finishHandler) {
finishHandlers.add(finishHandler);
}
if (handler instanceof FeaturePostProcessor postProcessor) {
postProcessors.computeIfAbsent(postProcessor.name(), name -> new ArrayList<>())
.add(postProcessor);
}
}
@Override
public List<OsmRelationInfo> preprocessOsmRelation(OsmElement.Relation relation) {
// delegate OSM relation pre-processing to each layer, if it implements FeaturePostProcessor
List<OsmRelationInfo> result = null;
for (OsmRelationPreprocessor osmRelationPreprocessor : osmRelationPreprocessors) {
List<OsmRelationInfo> thisResult = osmRelationPreprocessor
.preprocessOsmRelation(relation);
if (thisResult != null) {
if (result == null) {
result = new ArrayList<>(thisResult);
} else {
result.addAll(thisResult);
}
}
}
return result;
}
@Override
public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
// delegate source feature processing to each handler for that source
var handlers = sourceElementProcessors.get(sourceFeature.getSource());
if (handlers != null) {
for (var handler : handlers) {
handler.processFeature(sourceFeature, features);
// TODO extract common handling for expression-based filtering from openmaptiles to this
// common profile when we have another use-case for it.
}
}
}
@Override
public boolean caresAboutSource(String name) {
return sourceElementProcessors.containsKey(name);
}
@Override
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items)
throws GeometryException {
// delegate feature post-processing to each layer, if it implements FeaturePostProcessor
List<FeaturePostProcessor> handlers = postProcessors.get(layer);
List<VectorTile.Feature> result = null;
if (handlers != null) {
for (FeaturePostProcessor handler : handlers) {
var thisResult = handler.postProcess(zoom, items);
if (thisResult != null) {
if (result == null) {
result = thisResult;
} else {
result.addAll(thisResult);
}
}
}
}
return result == null ? items : result;
}
@Override
public void finish(String sourceName, FeatureCollector.Factory featureCollectors,
Consumer<FeatureCollector.Feature> next) {
// delegate finish handling to every layer that implements FinishHandler
for (var handler : finishHandlers) {
handler.finish(sourceName, featureCollectors, next);
}
}
@Override
public void release() {
// release resources used by each handler
handlers.forEach(Handler::release);
}
/** Interface for handlers that this profile forwards to should implement. */
public interface Handler {
/** Free any resources associated with this profile (i.e. shared data structures) */
default void release() {
}
}
public interface HandlerForLayer {
/** The layer name this handler is for */
String name();
}
/** Handlers should implement this interface to get notified when a source finishes processing. */
public interface FinishHandler {
/**
* Invoked once for each source after all elements for a source have been processed.
*
* @see Profile#finish(String, FeatureCollector.Factory, Consumer)
*/
void finish(String sourceName, FeatureCollector.Factory featureCollectors,
Consumer<FeatureCollector.Feature> emit);
}
/** Handlers should implement this interface to pre-process OSM relations during pass 1 through the data. */
public interface OsmRelationPreprocessor {
/**
* Returns information extracted from an OSM relation during pass 1 of the input OSM data to make available when
* processing elements in that relation during pass 2.
*
* @see Profile#preprocessOsmRelation(OsmElement.Relation)
*/
List<OsmRelationInfo> preprocessOsmRelation(OsmElement.Relation relation);
}
/** Handlers should implement this interface to post-process vector tile features before emitting an output tile. */
public interface FeaturePostProcessor extends HandlerForLayer {
/**
* Apply any post-processing to features in this output layer of a tile before writing it to the output file.
*
* @throws GeometryException if the input elements cannot be deserialized, or output elements cannot be
* serialized
* @see Profile#postProcessLayerFeatures(String, int, List)
*/
List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> items) throws GeometryException;
}
/** Handlers should implement this interface to process input features from a given source ID. */
public interface FeatureProcessor {
/**
* Process an input element from a source feature.
*
* @see Profile#processFeature(SourceFeature, FeatureCollector)
*/
void processFeature(SourceFeature feature, FeatureCollector features);
}
}
}

Wyświetl plik

@ -243,7 +243,6 @@ public class FlatmapTests {
public void testOverrideMetadata() throws Exception {
var results = runWithReaderFeatures(
Map.of(
"threads", "1",
"mbtiles_name", "mbtiles_name",
"mbtiles_description", "mbtiles_description",
"mbtiles_attribution", "mbtiles_attribution",
@ -1286,6 +1285,59 @@ public class FlatmapTests {
)), sortListValues(results.tiles));
}
@Test
public void testOsmProfileFinishForwardingProfile() throws Exception {
double y = 0.5 + Z14_WIDTH / 2;
double lat = GeoUtils.getWorldLat(y);
double x1 = 0.5 + Z14_WIDTH / 4;
double lng1 = GeoUtils.getWorldLon(x1);
double lng2 = GeoUtils.getWorldLon(x1 + Z14_WIDTH * 10d / 256);
ForwardingProfile profile = new ForwardingProfile() {
@Override
public String name() {
return "test";
}
};
List<SourceFeature> featureList = new CopyOnWriteArrayList<>();
profile.registerSourceHandler("osm", (in, features) -> featureList.add(in));
profile.registerHandler((ForwardingProfile.FinishHandler) (name, featureCollectors, next) -> {
if ("osm".equals(name)) {
for (SourceFeature in : featureList) {
var features = featureCollectors.get(in);
features.point("layer")
.setZoomRange(13, 14)
.inheritAttrFromSource("a");
for (var feature : features) {
next.accept(feature);
}
}
}
});
var results = runWithOsmElements(
Map.of("threads", "1"),
List.of(
with(new ReaderNode(1, lat, lng1), t -> t.setTag("a", 1)),
with(new ReaderNode(2, lat, lng2), t -> t.setTag("a", 3))
),
profile
);
assertSubmap(sortListValues(Map.of(
TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14), List.of(
feature(newPoint(64, 128), Map.of("a", 1L)),
feature(newPoint(74, 128), Map.of("a", 3L))
),
TileCoord.ofXYZ(Z13_TILES / 2, Z13_TILES / 2, 13), List.of(
// merge 32->37 and 37->42 since they have same attrs
feature(newPoint(32, 64), Map.of("a", 1L)),
feature(newPoint(37, 64), Map.of("a", 3L))
)
)), sortListValues(results.tiles));
}
private <K extends Comparable<? super K>, V extends List<?>> Map<K, ?> sortListValues(Map<K, V> input) {
Map<K, List<?>> result = new TreeMap<>();
for (var entry : input.entrySet()) {

Wyświetl plik

@ -0,0 +1,169 @@
package com.onthegomap.flatmap;
import static com.onthegomap.flatmap.TestUtils.assertSubmap;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.reader.SimpleFeature;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import org.junit.jupiter.api.Test;
public class ForwardingProfileTests {
private final ForwardingProfile profile = new ForwardingProfile() {
@Override
public String name() {
return "test";
}
};
@Test
public void testPreprocessOsmRelation() {
record RelA(@Override long id) implements OsmRelationInfo {}
record RelB(@Override long id) implements OsmRelationInfo {}
assertNull(profile.preprocessOsmRelation(new OsmElement.Relation(1)));
profile.registerHandler((ForwardingProfile.OsmRelationPreprocessor) relation -> List.of(new RelA(relation.id())));
assertEquals(List.of(new RelA(1)), profile.preprocessOsmRelation(new OsmElement.Relation(1)));
profile.registerHandler((ForwardingProfile.OsmRelationPreprocessor) relation -> null);
profile.registerHandler((ForwardingProfile.OsmRelationPreprocessor) relation -> List.of(new RelB(relation.id())));
assertEquals(List.of(new RelA(1), new RelB(1)), profile.preprocessOsmRelation(new OsmElement.Relation(1)));
}
private void testFeatures(List<Map<String, Object>> expected, SourceFeature feature) {
List<FeatureCollector.Feature> actualList = TestUtils.processSourceFeature(feature, profile);
assertEquals(expected.size(), actualList.size(), () -> "size: " + actualList);
for (int i = 0; i < expected.size(); i++) {
assertSubmap(expected.get(i), TestUtils.toMap(actualList.get(i), 14));
}
}
@Test
public void testProcessFeature() {
SourceFeature a = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of(), "srca", null, 1);
SourceFeature b = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of(), "srcb", null, 1);
testFeatures(List.of(), a);
testFeatures(List.of(), b);
profile.registerSourceHandler(a.getSource(), (elem, features) -> features.point("a"));
testFeatures(List.of(Map.of(
"_layer", "a"
)), a);
testFeatures(List.of(), b);
profile.registerSourceHandler(b.getSource(), (elem, features) -> features.point("b"));
testFeatures(List.of(Map.of(
"_layer", "a"
)), a);
testFeatures(List.of(Map.of(
"_layer", "b"
)), b);
profile.registerSourceHandler(a.getSource(), (elem, features) -> features.point("a2"));
testFeatures(List.of(Map.of(
"_layer", "a"
), Map.of(
"_layer", "a2"
)), a);
testFeatures(List.of(Map.of(
"_layer", "b"
)), b);
}
@Test
public void testFinishHandler() {
Set<String> finished = new TreeSet<>();
profile.finish("source", null, null);
assertEquals(Set.of(), finished);
profile.registerHandler(
(ForwardingProfile.FinishHandler) (sourceName, featureCollectors, emit) -> finished.add("1-" + sourceName));
profile.finish("source", null, null);
assertEquals(Set.of("1-source"), finished);
finished.clear();
profile.registerHandler(
(ForwardingProfile.FinishHandler) (sourceName, featureCollectors, emit) -> finished.add("2-" + sourceName));
profile.finish("source2", null, null);
assertEquals(Set.of("1-source2", "2-source2"), finished);
}
@Test
public void testFeaturePostProcessor() throws GeometryException {
VectorTile.Feature feature = new VectorTile.Feature(
"layer",
1,
VectorTile.encodeGeometry(GeoUtils.point(0, 0)),
Map.of()
);
assertEquals(List.of(feature), profile.postProcessLayerFeatures("layer", 0, List.of(feature)));
// ignore null response
profile.registerHandler(new ForwardingProfile.FeaturePostProcessor() {
@Override
public List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> items) {
return null;
}
@Override
public String name() {
return "a";
}
});
assertEquals(List.of(feature), profile.postProcessLayerFeatures("a", 0, List.of(feature)));
// empty list removes
profile.registerHandler(new ForwardingProfile.FeaturePostProcessor() {
@Override
public List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> items) {
return List.of();
}
@Override
public String name() {
return "a";
}
});
assertEquals(List.of(), profile.postProcessLayerFeatures("a", 0, List.of(feature)));
// doesn't touch elements in another layer
assertEquals(List.of(feature), profile.postProcessLayerFeatures("b", 0, List.of(feature)));
// 2 handlers for same layer run one after another
var skip1 = new ForwardingProfile.FeaturePostProcessor() {
@Override
public List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> items) {
return items.stream().skip(1).toList();
}
@Override
public String name() {
return "b";
}
};
profile.registerHandler(skip1);
profile.registerHandler(skip1);
profile.registerHandler(new ForwardingProfile.FeaturePostProcessor() {
@Override
public List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> items) {
return null; // ensure that returning null after initial post-processors run keeps the postprocessed result
}
@Override
public String name() {
return "b";
}
});
assertEquals(List.of(feature, feature),
profile.postProcessLayerFeatures("b", 0, List.of(feature, feature, feature, feature)));
assertEquals(List.of(feature, feature, feature, feature),
profile.postProcessLayerFeatures("c", 0, List.of(feature, feature, feature, feature)));
}
}

Wyświetl plik

@ -1,8 +1,8 @@
package com.onthegomap.flatmap.openmaptiles;
import com.onthegomap.flatmap.Profile;
import com.onthegomap.flatmap.ForwardingProfile;
/** Interface for all vector tile layer implementations that {@link OpenMapTilesProfile} delegates to. */
public interface Layer extends
Profile.ForwardingProfile.Handler,
Profile.ForwardingProfile.HandlerForLayer {}
ForwardingProfile.Handler,
ForwardingProfile.HandlerForLayer {}

Wyświetl plik

@ -6,6 +6,7 @@ import static com.onthegomap.flatmap.geo.GeoUtils.EMPTY_POLYGON;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FlatmapRunner;
import com.onthegomap.flatmap.ForwardingProfile;
import com.onthegomap.flatmap.Profile;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
@ -39,7 +40,7 @@ import java.util.List;
* {@link FinishHandler} or post-process features in that layer before rendering the output tile by implementing
* {@link FeaturePostProcessor}.
*/
public class OpenMapTilesProfile extends Profile.ForwardingProfile {
public class OpenMapTilesProfile extends ForwardingProfile {
// IDs used in stats and logs for each input source, as well as argument/config file overrides to source locations
public static final String LAKE_CENTERLINE_SOURCE = "lake_centerlines";