kopia lustrzana https://github.com/onthegomap/planetiler
Add basic support for reading GeoPackage files. (#413)
rodzic
aea309e094
commit
ef24e91f0b
|
@ -25,6 +25,7 @@ The `planetiler-core` module includes the following software:
|
|||
- com.github.jnr:jnr-ffi (Apache license)
|
||||
- org.roaringbitmap:RoaringBitmap (Apache license)
|
||||
- org.projectnessie.cel:cel-tools (Apache license)
|
||||
- mil.nga.geopackage:geopackage (MIT license)
|
||||
- Adapted code:
|
||||
- `DouglasPeuckerSimplifier` from [JTS](https://github.com/locationtech/jts) (EDL)
|
||||
- `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
<log4j.version>2.19.0</log4j.version>
|
||||
<prometheus.version>0.16.0</prometheus.version>
|
||||
<protobuf.version>3.21.12</protobuf.version>
|
||||
<geopackage.version>6.6.0</geopackage.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
@ -138,6 +139,11 @@
|
|||
<artifactId>guava</artifactId>
|
||||
<version>31.1-jre</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>mil.nga.geopackage</groupId>
|
||||
<artifactId>geopackage</artifactId>
|
||||
<version>${geopackage.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
|
@ -7,6 +7,7 @@ import com.onthegomap.planetiler.config.Arguments;
|
|||
import com.onthegomap.planetiler.config.MbtilesMetadata;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
|
||||
import com.onthegomap.planetiler.reader.GeoPackageReader;
|
||||
import com.onthegomap.planetiler.reader.NaturalEarthReader;
|
||||
import com.onthegomap.planetiler.reader.ShapefileReader;
|
||||
import com.onthegomap.planetiler.reader.osm.OsmInputFile;
|
||||
|
@ -337,6 +338,31 @@ public class Planetiler {
|
|||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new OGC GeoPackage source that will be processed when {@link #run()} is called.
|
||||
* <p>
|
||||
* If the file does not exist and {@code download=true} argument is set, then the file will first be downloaded from
|
||||
* {@code defaultUrl}.
|
||||
* <p>
|
||||
* To override the location of the {@code geopackage} file, set {@code name_path=newpath.gpkg} in the arguments and to
|
||||
* override the download URL set {@code name_url=http://url/of/file.gpkg}.
|
||||
*
|
||||
* @param name string to use in stats and logs to identify this stage
|
||||
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set
|
||||
* @return this runner instance for chaining
|
||||
* @see GeoPackageReader
|
||||
* @see Downloader
|
||||
*/
|
||||
public Planetiler addGeoPackageSource(String name, Path defaultPath, String defaultUrl) {
|
||||
Path path = getPath(name, "geopackage", defaultPath, defaultUrl);
|
||||
return addStage(name, "Process features in " + path,
|
||||
ifSourceUsed(name,
|
||||
() -> GeoPackageReader.process(name, List.of(path), featureGroup, config, profile, stats)));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Adds a new Natural Earth sqlite file source that will be processed when {@link #run()} is called.
|
||||
* <p>
|
||||
|
|
|
@ -0,0 +1,109 @@
|
|||
package com.onthegomap.planetiler.reader;
|
||||
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import mil.nga.geopackage.GeoPackage;
|
||||
import mil.nga.geopackage.GeoPackageManager;
|
||||
import mil.nga.geopackage.features.user.FeatureColumns;
|
||||
import mil.nga.geopackage.features.user.FeatureDao;
|
||||
import mil.nga.geopackage.geom.GeoPackageGeometryData;
|
||||
import org.geotools.geometry.jts.JTS;
|
||||
import org.geotools.geometry.jts.WKBReader;
|
||||
import org.geotools.referencing.CRS;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.opengis.referencing.crs.CoordinateReferenceSystem;
|
||||
import org.opengis.referencing.operation.MathTransform;
|
||||
|
||||
/**
|
||||
* Utility that reads {@link SourceFeature SourceFeatures} from the vector geometries contained in a GeoPackage file.
|
||||
*/
|
||||
public class GeoPackageReader extends SimpleReader<SimpleFeature> {
|
||||
|
||||
private final GeoPackage geoPackage;
|
||||
|
||||
GeoPackageReader(String sourceName, Path input) {
|
||||
super(sourceName);
|
||||
|
||||
geoPackage = GeoPackageManager.open(false, input.toFile());
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders map features for all elements from an OGC GeoPackage based on the mapping logic defined in {@code
|
||||
* profile}.
|
||||
*
|
||||
* @param sourceName string ID for this reader to use in logs and stats
|
||||
* @param sourcePaths paths to the {@code .gpkg} files on disk
|
||||
* @param writer consumer for rendered features
|
||||
* @param config user-defined parameters controlling number of threads and log interval
|
||||
* @param profile logic that defines what map features to emit for each source feature
|
||||
* @param stats to keep track of counters and timings
|
||||
* @throws IllegalArgumentException if a problem occurs reading the input file
|
||||
*/
|
||||
public static void process(String sourceName, List<Path> sourcePaths, FeatureGroup writer, PlanetilerConfig config,
|
||||
Profile profile, Stats stats) {
|
||||
SourceFeatureProcessor.processFiles(
|
||||
sourceName,
|
||||
sourcePaths,
|
||||
path -> new GeoPackageReader(sourceName, path),
|
||||
writer, config, profile, stats
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getFeatureCount() {
|
||||
long numFeatures = 0;
|
||||
|
||||
for (String name : geoPackage.getFeatureTables()) {
|
||||
FeatureDao features = geoPackage.getFeatureDao(name);
|
||||
numFeatures += features.count();
|
||||
}
|
||||
return numFeatures;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void readFeatures(Consumer<SimpleFeature> next) throws Exception {
|
||||
CoordinateReferenceSystem latLonCRS = CRS.decode("EPSG:4326");
|
||||
long id = 0;
|
||||
|
||||
for (var featureName : geoPackage.getFeatureTables()) {
|
||||
FeatureDao features = geoPackage.getFeatureDao(featureName);
|
||||
|
||||
MathTransform transform = CRS.findMathTransform(
|
||||
CRS.decode("EPSG:" + features.getSrsId()),
|
||||
latLonCRS);
|
||||
|
||||
for (var feature : features.queryForAll()) {
|
||||
GeoPackageGeometryData geometryData = feature.getGeometry();
|
||||
if (geometryData == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Geometry featureGeom = (new WKBReader()).read(geometryData.getWkb());
|
||||
Geometry latLonGeom = (transform.isIdentity()) ? featureGeom : JTS.transform(featureGeom, transform);
|
||||
|
||||
FeatureColumns columns = feature.getColumns();
|
||||
SimpleFeature geom = SimpleFeature.create(latLonGeom, new HashMap<>(columns.columnCount()),
|
||||
sourceName, featureName, ++id);
|
||||
|
||||
for (int i = 0; i < columns.columnCount(); ++i) {
|
||||
if (i != columns.getGeometryIndex()) {
|
||||
geom.setTag(columns.getColumnName(i), feature.getValue(i));
|
||||
}
|
||||
}
|
||||
|
||||
next.accept(geom);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
geoPackage.close();
|
||||
}
|
||||
}
|
|
@ -1667,6 +1667,7 @@ class PlanetilerTests {
|
|||
.addOsmSource("osm", tempOsm)
|
||||
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
|
||||
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
|
||||
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
|
||||
.setOutput("mbtiles", mbtiles)
|
||||
.run();
|
||||
|
||||
|
@ -1743,12 +1744,52 @@ class PlanetilerTests {
|
|||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {
|
||||
"",
|
||||
"--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4",
|
||||
})
|
||||
void testPlanetilerRunnerGeoPackage(String args) throws Exception {
|
||||
Path mbtiles = tempDir.resolve("output.mbtiles");
|
||||
|
||||
Planetiler.create(Arguments.fromArgs((args + " --tmpdir=" + tempDir.resolve("data")).split("\\s+")))
|
||||
.setProfile(new Profile.NullProfile() {
|
||||
@Override
|
||||
public void processFeature(SourceFeature source, FeatureCollector features) {
|
||||
features.point("stations")
|
||||
.setZoomRange(0, 14)
|
||||
.setAttr("name", source.getString("name"));
|
||||
}
|
||||
})
|
||||
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
|
||||
.setOutput("mbtiles", mbtiles)
|
||||
.run();
|
||||
|
||||
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
|
||||
Set<String> uniqueNames = new HashSet<>();
|
||||
long featureCount = 0;
|
||||
var tileMap = TestUtils.getTileMap(db);
|
||||
for (var tile : tileMap.values()) {
|
||||
for (var feature : tile) {
|
||||
feature.geometry().validate();
|
||||
featureCount++;
|
||||
uniqueNames.add((String) feature.attrs().get("name"));
|
||||
}
|
||||
}
|
||||
|
||||
assertTrue(featureCount > 0);
|
||||
assertEquals(86, uniqueNames.size());
|
||||
assertTrue(uniqueNames.contains("Van Dörn Street"));
|
||||
}
|
||||
}
|
||||
|
||||
private void runWithProfile(Path tempDir, Profile profile, boolean force) throws Exception {
|
||||
Planetiler.create(Arguments.of("tmpdir", tempDir, "force", Boolean.toString(force)))
|
||||
.setProfile(profile)
|
||||
.addOsmSource("osm", TestUtils.pathToResource("monaco-latest.osm.pbf"))
|
||||
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
|
||||
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
|
||||
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
|
||||
.setOutput("mbtiles", tempDir.resolve("output.mbtiles"))
|
||||
.run();
|
||||
}
|
||||
|
|
|
@ -0,0 +1,53 @@
|
|||
package com.onthegomap.planetiler.reader;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.onthegomap.planetiler.TestUtils;
|
||||
import com.onthegomap.planetiler.collection.IterableOnce;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Timeout;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
|
||||
class GeoPackageReaderTest {
|
||||
|
||||
@Test
|
||||
@Timeout(30)
|
||||
void testReadGeoPackage() {
|
||||
Path path = TestUtils.pathToResource("geopackage.gpkg");
|
||||
|
||||
try (
|
||||
var reader = new GeoPackageReader("test", path)
|
||||
) {
|
||||
for (int i = 1; i <= 2; i++) {
|
||||
assertEquals(86, reader.getFeatureCount());
|
||||
List<Geometry> points = new ArrayList<>();
|
||||
List<String> names = new ArrayList<>();
|
||||
WorkerPipeline.start("test", Stats.inMemory())
|
||||
.readFromTiny("files", List.of(Path.of("dummy-path")))
|
||||
.addWorker("geopackage", 1, (IterableOnce<Path> p, Consumer<SimpleFeature> next) -> reader.readFeatures(next))
|
||||
.addBuffer("reader_queue", 100, 1)
|
||||
.sinkToConsumer("counter", 1, elem -> {
|
||||
assertTrue(elem.getTag("name") instanceof String);
|
||||
assertEquals("test", elem.getSource());
|
||||
assertEquals("stations", elem.getSourceLayer());
|
||||
points.add(elem.latLonGeometry());
|
||||
names.add(elem.getTag("name").toString());
|
||||
}).await();
|
||||
assertEquals(86, points.size());
|
||||
assertTrue(names.contains("Van Dörn Street"));
|
||||
var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
|
||||
var centroid = gc.getCentroid();
|
||||
assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i);
|
||||
assertEquals(38.9119684, centroid.getY(), 5, "iter " + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
Plik binarny nie jest wyświetlany.
|
@ -57,7 +57,8 @@ examples: [...]
|
|||
A description that tells planetiler how to read geospatial objects with tags from an input file.
|
||||
|
||||
- `type` - Enum representing the file format of the data source, one
|
||||
of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format) or [`shapefile`](https://en.wikipedia.org/wiki/Shapefile)
|
||||
of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format), [`shapefile`](https://en.wikipedia.org/wiki/Shapefile),
|
||||
or [`geopackage`](https://www.geopackage.org/).
|
||||
- `local_path` - Local path to the file to use, inferred from `url` if missing. Can be a string
|
||||
or [expression](#expression) that can reference [argument values](#arguments).
|
||||
- `url` - Location to download the file from if not present at `local_path`.
|
||||
|
|
|
@ -40,7 +40,8 @@
|
|||
"description": "File format of the data source",
|
||||
"enum": [
|
||||
"osm",
|
||||
"shapefile"
|
||||
"shapefile",
|
||||
"geopackage"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
|
|
|
@ -71,6 +71,7 @@ public class ConfiguredMapMain {
|
|||
switch (sourceType) {
|
||||
case OSM -> planetiler.addOsmSource(source.id(), localPath, source.url());
|
||||
case SHAPEFILE -> planetiler.addShapefileSource(source.id(), localPath, source.url());
|
||||
case GEOPACKAGE -> planetiler.addGeoPackageSource(source.id(), localPath, source.url());
|
||||
default -> throw new IllegalArgumentException("Unhandled source type for " + source.id() + ": " + sourceType);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,5 +6,7 @@ public enum DataSourceType {
|
|||
@JsonProperty("osm")
|
||||
OSM,
|
||||
@JsonProperty("shapefile")
|
||||
SHAPEFILE
|
||||
SHAPEFILE,
|
||||
@JsonProperty("geopackage")
|
||||
GEOPACKAGE
|
||||
}
|
||||
|
|
|
@ -9,6 +9,9 @@ sources:
|
|||
osm:
|
||||
type: osm
|
||||
url: geofabrik:rhode-island
|
||||
gpkg:
|
||||
type: geopackage
|
||||
url: https://example.com/geopackage.gpkg
|
||||
tag_mappings:
|
||||
bridge: boolean # input=bridge, output=bridge, type=boolean
|
||||
layer: long
|
||||
|
|
Ładowanie…
Reference in New Issue