kopia lustrzana https://github.com/onthegomap/planetiler
Use push-down bbox filter for shapefiles
rodzic
d98d5d6ae1
commit
eeeed0aa9e
|
@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory;
|
||||||
public class Bounds {
|
public class Bounds {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(Bounds.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(Bounds.class);
|
||||||
|
public static final Bounds WORLD = new Bounds(null);
|
||||||
|
|
||||||
private Envelope latLon;
|
private Envelope latLon;
|
||||||
private Envelope world;
|
private Envelope world;
|
||||||
|
@ -24,7 +25,7 @@ public class Bounds {
|
||||||
|
|
||||||
private Geometry shape;
|
private Geometry shape;
|
||||||
|
|
||||||
Bounds(Envelope latLon) {
|
public Bounds(Envelope latLon) {
|
||||||
set(latLon);
|
set(latLon);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -36,6 +37,10 @@ public class Bounds {
|
||||||
return world == null ? GeoUtils.WORLD_BOUNDS : world;
|
return world == null ? GeoUtils.WORLD_BOUNDS : world;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean isWorld() {
|
||||||
|
return latLon == null || latLon.equals(GeoUtils.WORLD_LAT_LON_BOUNDS);
|
||||||
|
}
|
||||||
|
|
||||||
public TileExtents tileExtents() {
|
public TileExtents tileExtents() {
|
||||||
if (tileExtents == null) {
|
if (tileExtents == null) {
|
||||||
tileExtents = TileExtents.computeFromWorldBounds(PlanetilerConfig.MAX_MAXZOOM, world(), shape);
|
tileExtents = TileExtents.computeFromWorldBounds(PlanetilerConfig.MAX_MAXZOOM, world(), shape);
|
||||||
|
|
|
@ -2,6 +2,7 @@ package com.onthegomap.planetiler.reader;
|
||||||
|
|
||||||
import com.onthegomap.planetiler.Profile;
|
import com.onthegomap.planetiler.Profile;
|
||||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||||
|
import com.onthegomap.planetiler.config.Bounds;
|
||||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||||
import com.onthegomap.planetiler.stats.Stats;
|
import com.onthegomap.planetiler.stats.Stats;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -19,9 +20,13 @@ import org.geotools.api.referencing.operation.MathTransform;
|
||||||
import org.geotools.api.referencing.operation.OperationNotFoundException;
|
import org.geotools.api.referencing.operation.OperationNotFoundException;
|
||||||
import org.geotools.api.referencing.operation.TransformException;
|
import org.geotools.api.referencing.operation.TransformException;
|
||||||
import org.geotools.data.shapefile.ShapefileDataStore;
|
import org.geotools.data.shapefile.ShapefileDataStore;
|
||||||
|
import org.geotools.factory.CommonFactoryFinder;
|
||||||
import org.geotools.feature.FeatureCollection;
|
import org.geotools.feature.FeatureCollection;
|
||||||
import org.geotools.geometry.jts.JTS;
|
import org.geotools.geometry.jts.JTS;
|
||||||
|
import org.geotools.geometry.jts.ReferencedEnvelope;
|
||||||
import org.geotools.referencing.CRS;
|
import org.geotools.referencing.CRS;
|
||||||
|
import org.geotools.util.factory.GeoTools;
|
||||||
|
import org.locationtech.jts.geom.Envelope;
|
||||||
import org.locationtech.jts.geom.Geometry;
|
import org.locationtech.jts.geom.Geometry;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -45,6 +50,10 @@ public class ShapefileReader extends SimpleReader<SimpleFeature> {
|
||||||
private MathTransform transformToLatLon;
|
private MathTransform transformToLatLon;
|
||||||
|
|
||||||
public ShapefileReader(String sourceProjection, String sourceName, Path input) {
|
public ShapefileReader(String sourceProjection, String sourceName, Path input) {
|
||||||
|
this(sourceProjection, sourceName, input, Bounds.WORLD);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ShapefileReader(String sourceProjection, String sourceName, Path input, Bounds bounds) {
|
||||||
super(sourceName);
|
super(sourceName);
|
||||||
this.layer = input.getFileName().toString().replaceAll("\\.shp$", "");
|
this.layer = input.getFileName().toString().replaceAll("\\.shp$", "");
|
||||||
dataStore = open(input);
|
dataStore = open(input);
|
||||||
|
@ -52,8 +61,6 @@ public class ShapefileReader extends SimpleReader<SimpleFeature> {
|
||||||
String typeName = dataStore.getTypeNames()[0];
|
String typeName = dataStore.getTypeNames()[0];
|
||||||
FeatureSource<SimpleFeatureType, org.geotools.api.feature.simple.SimpleFeature> source = dataStore
|
FeatureSource<SimpleFeatureType, org.geotools.api.feature.simple.SimpleFeature> source = dataStore
|
||||||
.getFeatureSource(typeName);
|
.getFeatureSource(typeName);
|
||||||
|
|
||||||
inputSource = source.getFeatures(Filter.INCLUDE);
|
|
||||||
CoordinateReferenceSystem src =
|
CoordinateReferenceSystem src =
|
||||||
sourceProjection == null ? source.getSchema().getCoordinateReferenceSystem() : CRS.decode(sourceProjection);
|
sourceProjection == null ? source.getSchema().getCoordinateReferenceSystem() : CRS.decode(sourceProjection);
|
||||||
CoordinateReferenceSystem dest = CRS.decode("EPSG:4326", true);
|
CoordinateReferenceSystem dest = CRS.decode("EPSG:4326", true);
|
||||||
|
@ -61,6 +68,26 @@ public class ShapefileReader extends SimpleReader<SimpleFeature> {
|
||||||
if (transformToLatLon.isIdentity()) {
|
if (transformToLatLon.isIdentity()) {
|
||||||
transformToLatLon = null;
|
transformToLatLon = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Filter filter = Filter.INCLUDE;
|
||||||
|
|
||||||
|
Envelope env = bounds.latLon();
|
||||||
|
if (!bounds.isWorld()) {
|
||||||
|
var ff = CommonFactoryFinder.getFilterFactory(GeoTools.getDefaultHints());
|
||||||
|
var schema = source.getSchema();
|
||||||
|
|
||||||
|
String geometryPropertyName = schema.getGeometryDescriptor().getLocalName();
|
||||||
|
|
||||||
|
var bbox = new ReferencedEnvelope(env.getMinX(), env.getMaxX(), env.getMinY(), env.getMaxY(), dest);
|
||||||
|
try {
|
||||||
|
var bbox2 = bbox.transform(schema.getGeometryDescriptor().getCoordinateReferenceSystem(), true);
|
||||||
|
filter = ff.bbox(ff.property(geometryPropertyName), bbox2);
|
||||||
|
} catch (TransformException e) {
|
||||||
|
// just use include filter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inputSource = source.getFeatures(filter);
|
||||||
attributeNames = new String[inputSource.getSchema().getAttributeCount()];
|
attributeNames = new String[inputSource.getSchema().getAttributeCount()];
|
||||||
for (int i = 0; i < attributeNames.length; i++) {
|
for (int i = 0; i < attributeNames.length; i++) {
|
||||||
attributeNames[i] = inputSource.getSchema().getDescriptor(i).getLocalName();
|
attributeNames[i] = inputSource.getSchema().getDescriptor(i).getLocalName();
|
||||||
|
@ -105,7 +132,7 @@ public class ShapefileReader extends SimpleReader<SimpleFeature> {
|
||||||
SourceFeatureProcessor.processFiles(
|
SourceFeatureProcessor.processFiles(
|
||||||
sourceName,
|
sourceName,
|
||||||
sourcePaths,
|
sourcePaths,
|
||||||
path -> new ShapefileReader(sourceProjection, sourceName, path),
|
path -> new ShapefileReader(sourceProjection, sourceName, path, config.bounds()),
|
||||||
writer, config, profile, stats
|
writer, config, profile, stats
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
package com.onthegomap.planetiler.reader;
|
package com.onthegomap.planetiler.reader;
|
||||||
|
|
||||||
|
import static com.onthegomap.planetiler.TestUtils.newPoint;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
import com.onthegomap.planetiler.TestUtils;
|
import com.onthegomap.planetiler.TestUtils;
|
||||||
|
import com.onthegomap.planetiler.config.Bounds;
|
||||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||||
import com.onthegomap.planetiler.stats.Stats;
|
import com.onthegomap.planetiler.stats.Stats;
|
||||||
import com.onthegomap.planetiler.util.FileUtils;
|
import com.onthegomap.planetiler.util.FileUtils;
|
||||||
|
@ -11,9 +13,9 @@ import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.FileSystems;
|
import java.nio.file.FileSystems;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList;
|
||||||
import org.geotools.api.data.SimpleFeatureStore;
|
import org.geotools.api.data.SimpleFeatureStore;
|
||||||
import org.geotools.api.referencing.FactoryException;
|
import org.geotools.api.referencing.FactoryException;
|
||||||
import org.geotools.api.referencing.operation.TransformException;
|
import org.geotools.api.referencing.operation.TransformException;
|
||||||
|
@ -29,12 +31,18 @@ import org.junit.jupiter.api.Timeout;
|
||||||
import org.junit.jupiter.api.condition.DisabledOnOs;
|
import org.junit.jupiter.api.condition.DisabledOnOs;
|
||||||
import org.junit.jupiter.api.condition.OS;
|
import org.junit.jupiter.api.condition.OS;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
import org.locationtech.jts.geom.Envelope;
|
||||||
import org.locationtech.jts.geom.Geometry;
|
import org.locationtech.jts.geom.Geometry;
|
||||||
import org.locationtech.jts.geom.Point;
|
import org.locationtech.jts.geom.Point;
|
||||||
|
|
||||||
class ShapefileReaderTest {
|
class ShapefileReaderTest {
|
||||||
@TempDir
|
@TempDir
|
||||||
private Path tempDir;
|
private Path tempDir;
|
||||||
|
private static final Envelope env = newPoint(-77.12911152370515, 38.79930767201779).getEnvelopeInternal();
|
||||||
|
private static final int numInEnv = 18;
|
||||||
|
static {
|
||||||
|
env.expandBy(0.1);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@Timeout(30)
|
@Timeout(30)
|
||||||
|
@ -55,6 +63,35 @@ class ShapefileReaderTest {
|
||||||
testReadShapefile(dest.resolve("shapefile").resolve("stations.shp"));
|
testReadShapefile(dest.resolve("shapefile").resolve("stations.shp"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Timeout(30)
|
||||||
|
void testReadShapefileWithBoundingBox() {
|
||||||
|
var dest = tempDir.resolve("shapefile.zip");
|
||||||
|
FileUtils.unzipResource("/shapefile.zip", dest);
|
||||||
|
try (
|
||||||
|
var reader = new ShapefileReader(null, "test", dest.resolve("shapefile").resolve("stations.shp"), new Bounds(env))
|
||||||
|
) {
|
||||||
|
for (int i = 1; i <= 2; i++) {
|
||||||
|
assertEquals(numInEnv, reader.getFeatureCount());
|
||||||
|
List<Geometry> points = new CopyOnWriteArrayList<>();
|
||||||
|
WorkerPipeline.start("test", Stats.inMemory())
|
||||||
|
.fromGenerator("source", reader::readFeatures)
|
||||||
|
.addBuffer("reader_queue", 100, 1)
|
||||||
|
.sinkToConsumer("counter", 1, elem -> {
|
||||||
|
assertTrue(elem.getTag("name") instanceof String);
|
||||||
|
assertEquals("test", elem.getSource());
|
||||||
|
assertEquals("stations", elem.getSourceLayer());
|
||||||
|
points.add(elem.latLonGeometry());
|
||||||
|
}).await();
|
||||||
|
assertEquals(numInEnv, points.size());
|
||||||
|
var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
|
||||||
|
var centroid = gc.getCentroid();
|
||||||
|
assertEquals(-77.0934256, centroid.getX(), 1e-5, "iter " + i);
|
||||||
|
assertEquals(38.8509022, centroid.getY(), 1e-5, "iter " + i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testReadShapefileLeniently(@TempDir Path dir) throws IOException, TransformException, FactoryException {
|
void testReadShapefileLeniently(@TempDir Path dir) throws IOException, TransformException, FactoryException {
|
||||||
var shpPath = dir.resolve("test.shp");
|
var shpPath = dir.resolve("test.shp");
|
||||||
|
@ -82,7 +119,7 @@ class ShapefileReaderTest {
|
||||||
featureStore.setTransaction(transaction);
|
featureStore.setTransaction(transaction);
|
||||||
var collection = new DefaultFeatureCollection();
|
var collection = new DefaultFeatureCollection();
|
||||||
var featureBuilder = new SimpleFeatureBuilder(type);
|
var featureBuilder = new SimpleFeatureBuilder(type);
|
||||||
featureBuilder.add(TestUtils.newPoint(1, 2));
|
featureBuilder.add(newPoint(1, 2));
|
||||||
featureBuilder.add(3);
|
featureBuilder.add(3);
|
||||||
var feature = featureBuilder.buildFeature(null);
|
var feature = featureBuilder.buildFeature(null);
|
||||||
collection.add(feature);
|
collection.add(feature);
|
||||||
|
@ -92,7 +129,7 @@ class ShapefileReaderTest {
|
||||||
|
|
||||||
try (var reader = new ShapefileReader(null, "test", shpPath)) {
|
try (var reader = new ShapefileReader(null, "test", shpPath)) {
|
||||||
assertEquals(1, reader.getFeatureCount());
|
assertEquals(1, reader.getFeatureCount());
|
||||||
List<SimpleFeature> features = new ArrayList<>();
|
List<SimpleFeature> features = new CopyOnWriteArrayList<>();
|
||||||
reader.readFeatures(features::add);
|
reader.readFeatures(features::add);
|
||||||
assertEquals(10.5113, features.getFirst().latLonGeometry().getCentroid().getX(), 1e-4);
|
assertEquals(10.5113, features.getFirst().latLonGeometry().getCentroid().getX(), 1e-4);
|
||||||
assertEquals(0, features.getFirst().latLonGeometry().getCentroid().getY(), 1e-4);
|
assertEquals(0, features.getFirst().latLonGeometry().getCentroid().getY(), 1e-4);
|
||||||
|
@ -105,8 +142,8 @@ class ShapefileReaderTest {
|
||||||
|
|
||||||
for (int i = 1; i <= 2; i++) {
|
for (int i = 1; i <= 2; i++) {
|
||||||
assertEquals(86, reader.getFeatureCount());
|
assertEquals(86, reader.getFeatureCount());
|
||||||
List<Geometry> points = new ArrayList<>();
|
List<Geometry> points = new CopyOnWriteArrayList<>();
|
||||||
List<String> names = new ArrayList<>();
|
List<String> names = new CopyOnWriteArrayList<>();
|
||||||
WorkerPipeline.start("test", Stats.inMemory())
|
WorkerPipeline.start("test", Stats.inMemory())
|
||||||
.fromGenerator("source", reader::readFeatures)
|
.fromGenerator("source", reader::readFeatures)
|
||||||
.addBuffer("reader_queue", 100, 1)
|
.addBuffer("reader_queue", 100, 1)
|
||||||
|
@ -117,12 +154,13 @@ class ShapefileReaderTest {
|
||||||
points.add(elem.latLonGeometry());
|
points.add(elem.latLonGeometry());
|
||||||
names.add(elem.getTag("name").toString());
|
names.add(elem.getTag("name").toString());
|
||||||
}).await();
|
}).await();
|
||||||
|
assertEquals(numInEnv, points.stream().filter(point -> env.contains(point.getCoordinate())).count());
|
||||||
assertEquals(86, points.size());
|
assertEquals(86, points.size());
|
||||||
assertTrue(names.contains("Van Dörn Street"));
|
assertTrue(names.contains("Van Dörn Street"));
|
||||||
var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
|
var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
|
||||||
var centroid = gc.getCentroid();
|
var centroid = gc.getCentroid();
|
||||||
assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i);
|
assertEquals(-77.0297995, centroid.getX(), 1e-5, "iter " + i);
|
||||||
assertEquals(38.9119684, centroid.getY(), 5, "iter " + i);
|
assertEquals(38.9119684, centroid.getY(), 1e-5, "iter " + i);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Ładowanie…
Reference in New Issue