Add basic support for reading GeoPackage files. (#413)

shortbread-planet
Erik Price 2023-01-02 09:19:05 -08:00 zatwierdzone przez GitHub
rodzic aea309e094
commit ef24e91f0b
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
12 zmienionych plików z 247 dodań i 3 usunięć

Wyświetl plik

@ -25,6 +25,7 @@ The `planetiler-core` module includes the following software:
- com.github.jnr:jnr-ffi (Apache license)
- org.roaringbitmap:RoaringBitmap (Apache license)
- org.projectnessie.cel:cel-tools (Apache license)
- mil.nga.geopackage:geopackage (MIT license)
- Adapted code:
- `DouglasPeuckerSimplifier` from [JTS](https://github.com/locationtech/jts) (EDL)
- `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)

Wyświetl plik

@ -20,6 +20,7 @@
<log4j.version>2.19.0</log4j.version>
<prometheus.version>0.16.0</prometheus.version>
<protobuf.version>3.21.12</protobuf.version>
<geopackage.version>6.6.0</geopackage.version>
</properties>
<dependencies>
@ -138,6 +139,11 @@
<artifactId>guava</artifactId>
<version>31.1-jre</version>
</dependency>
<dependency>
<groupId>mil.nga.geopackage</groupId>
<artifactId>geopackage</artifactId>
<version>${geopackage.version}</version>
</dependency>
</dependencies>
<build>

Wyświetl plik

@ -7,6 +7,7 @@ import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.MbtilesMetadata;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
import com.onthegomap.planetiler.reader.GeoPackageReader;
import com.onthegomap.planetiler.reader.NaturalEarthReader;
import com.onthegomap.planetiler.reader.ShapefileReader;
import com.onthegomap.planetiler.reader.osm.OsmInputFile;
@ -337,6 +338,31 @@ public class Planetiler {
}));
}
/**
* Adds a new OGC GeoPackage source that will be processed when {@link #run()} is called.
* <p>
* If the file does not exist and {@code download=true} argument is set, then the file will first be downloaded from
* {@code defaultUrl}.
* <p>
* To override the location of the {@code geopackage} file, set {@code name_path=newpath.gpkg} in the arguments and to
* override the download URL set {@code name_url=http://url/of/file.gpkg}.
*
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
* name_url} argument is not set
* @return this runner instance for chaining
* @see GeoPackageReader
* @see Downloader
*/
public Planetiler addGeoPackageSource(String name, Path defaultPath, String defaultUrl) {
Path path = getPath(name, "geopackage", defaultPath, defaultUrl);
return addStage(name, "Process features in " + path,
ifSourceUsed(name,
() -> GeoPackageReader.process(name, List.of(path), featureGroup, config, profile, stats)));
}
/**
* Adds a new Natural Earth sqlite file source that will be processed when {@link #run()} is called.
* <p>

Wyświetl plik

@ -0,0 +1,109 @@
package com.onthegomap.planetiler.reader;
import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.collection.FeatureGroup;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.stats.Stats;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.function.Consumer;
import mil.nga.geopackage.GeoPackage;
import mil.nga.geopackage.GeoPackageManager;
import mil.nga.geopackage.features.user.FeatureColumns;
import mil.nga.geopackage.features.user.FeatureDao;
import mil.nga.geopackage.geom.GeoPackageGeometryData;
import org.geotools.geometry.jts.JTS;
import org.geotools.geometry.jts.WKBReader;
import org.geotools.referencing.CRS;
import org.locationtech.jts.geom.Geometry;
import org.opengis.referencing.crs.CoordinateReferenceSystem;
import org.opengis.referencing.operation.MathTransform;
/**
* Utility that reads {@link SourceFeature SourceFeatures} from the vector geometries contained in a GeoPackage file.
*/
public class GeoPackageReader extends SimpleReader<SimpleFeature> {
private final GeoPackage geoPackage;
GeoPackageReader(String sourceName, Path input) {
super(sourceName);
geoPackage = GeoPackageManager.open(false, input.toFile());
}
/**
* Renders map features for all elements from an OGC GeoPackage based on the mapping logic defined in {@code
* profile}.
*
* @param sourceName string ID for this reader to use in logs and stats
* @param sourcePaths paths to the {@code .gpkg} files on disk
* @param writer consumer for rendered features
* @param config user-defined parameters controlling number of threads and log interval
* @param profile logic that defines what map features to emit for each source feature
* @param stats to keep track of counters and timings
* @throws IllegalArgumentException if a problem occurs reading the input file
*/
public static void process(String sourceName, List<Path> sourcePaths, FeatureGroup writer, PlanetilerConfig config,
Profile profile, Stats stats) {
SourceFeatureProcessor.processFiles(
sourceName,
sourcePaths,
path -> new GeoPackageReader(sourceName, path),
writer, config, profile, stats
);
}
@Override
public long getFeatureCount() {
long numFeatures = 0;
for (String name : geoPackage.getFeatureTables()) {
FeatureDao features = geoPackage.getFeatureDao(name);
numFeatures += features.count();
}
return numFeatures;
}
@Override
public void readFeatures(Consumer<SimpleFeature> next) throws Exception {
CoordinateReferenceSystem latLonCRS = CRS.decode("EPSG:4326");
long id = 0;
for (var featureName : geoPackage.getFeatureTables()) {
FeatureDao features = geoPackage.getFeatureDao(featureName);
MathTransform transform = CRS.findMathTransform(
CRS.decode("EPSG:" + features.getSrsId()),
latLonCRS);
for (var feature : features.queryForAll()) {
GeoPackageGeometryData geometryData = feature.getGeometry();
if (geometryData == null) {
continue;
}
Geometry featureGeom = (new WKBReader()).read(geometryData.getWkb());
Geometry latLonGeom = (transform.isIdentity()) ? featureGeom : JTS.transform(featureGeom, transform);
FeatureColumns columns = feature.getColumns();
SimpleFeature geom = SimpleFeature.create(latLonGeom, new HashMap<>(columns.columnCount()),
sourceName, featureName, ++id);
for (int i = 0; i < columns.columnCount(); ++i) {
if (i != columns.getGeometryIndex()) {
geom.setTag(columns.getColumnName(i), feature.getValue(i));
}
}
next.accept(geom);
}
}
}
@Override
public void close() {
geoPackage.close();
}
}

Wyświetl plik

@ -1667,6 +1667,7 @@ class PlanetilerTests {
.addOsmSource("osm", tempOsm)
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
.setOutput("mbtiles", mbtiles)
.run();
@ -1743,12 +1744,52 @@ class PlanetilerTests {
}
}
@ParameterizedTest
@ValueSource(strings = {
"",
"--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4",
})
void testPlanetilerRunnerGeoPackage(String args) throws Exception {
Path mbtiles = tempDir.resolve("output.mbtiles");
Planetiler.create(Arguments.fromArgs((args + " --tmpdir=" + tempDir.resolve("data")).split("\\s+")))
.setProfile(new Profile.NullProfile() {
@Override
public void processFeature(SourceFeature source, FeatureCollector features) {
features.point("stations")
.setZoomRange(0, 14)
.setAttr("name", source.getString("name"));
}
})
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
.setOutput("mbtiles", mbtiles)
.run();
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
Set<String> uniqueNames = new HashSet<>();
long featureCount = 0;
var tileMap = TestUtils.getTileMap(db);
for (var tile : tileMap.values()) {
for (var feature : tile) {
feature.geometry().validate();
featureCount++;
uniqueNames.add((String) feature.attrs().get("name"));
}
}
assertTrue(featureCount > 0);
assertEquals(86, uniqueNames.size());
assertTrue(uniqueNames.contains("Van Dörn Street"));
}
}
private void runWithProfile(Path tempDir, Profile profile, boolean force) throws Exception {
Planetiler.create(Arguments.of("tmpdir", tempDir, "force", Boolean.toString(force)))
.setProfile(profile)
.addOsmSource("osm", TestUtils.pathToResource("monaco-latest.osm.pbf"))
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
.setOutput("mbtiles", tempDir.resolve("output.mbtiles"))
.run();
}

Wyświetl plik

@ -0,0 +1,53 @@
package com.onthegomap.planetiler.reader;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.onthegomap.planetiler.TestUtils;
import com.onthegomap.planetiler.collection.IterableOnce;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.worker.WorkerPipeline;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.locationtech.jts.geom.Geometry;
class GeoPackageReaderTest {
@Test
@Timeout(30)
void testReadGeoPackage() {
Path path = TestUtils.pathToResource("geopackage.gpkg");
try (
var reader = new GeoPackageReader("test", path)
) {
for (int i = 1; i <= 2; i++) {
assertEquals(86, reader.getFeatureCount());
List<Geometry> points = new ArrayList<>();
List<String> names = new ArrayList<>();
WorkerPipeline.start("test", Stats.inMemory())
.readFromTiny("files", List.of(Path.of("dummy-path")))
.addWorker("geopackage", 1, (IterableOnce<Path> p, Consumer<SimpleFeature> next) -> reader.readFeatures(next))
.addBuffer("reader_queue", 100, 1)
.sinkToConsumer("counter", 1, elem -> {
assertTrue(elem.getTag("name") instanceof String);
assertEquals("test", elem.getSource());
assertEquals("stations", elem.getSourceLayer());
points.add(elem.latLonGeometry());
names.add(elem.getTag("name").toString());
}).await();
assertEquals(86, points.size());
assertTrue(names.contains("Van Dörn Street"));
var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
var centroid = gc.getCentroid();
assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i);
assertEquals(38.9119684, centroid.getY(), 5, "iter " + i);
}
}
}
}

Plik binarny nie jest wyświetlany.

Wyświetl plik

@ -57,7 +57,8 @@ examples: [...]
A description that tells planetiler how to read geospatial objects with tags from an input file.
- `type` - Enum representing the file format of the data source, one
of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format) or [`shapefile`](https://en.wikipedia.org/wiki/Shapefile)
of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format), [`shapefile`](https://en.wikipedia.org/wiki/Shapefile),
or [`geopackage`](https://www.geopackage.org/).
- `local_path` - Local path to the file to use, inferred from `url` if missing. Can be a string
or [expression](#expression) that can reference [argument values](#arguments).
- `url` - Location to download the file from if not present at `local_path`.

Wyświetl plik

@ -40,7 +40,8 @@
"description": "File format of the data source",
"enum": [
"osm",
"shapefile"
"shapefile",
"geopackage"
]
},
"url": {

Wyświetl plik

@ -71,6 +71,7 @@ public class ConfiguredMapMain {
switch (sourceType) {
case OSM -> planetiler.addOsmSource(source.id(), localPath, source.url());
case SHAPEFILE -> planetiler.addShapefileSource(source.id(), localPath, source.url());
case GEOPACKAGE -> planetiler.addGeoPackageSource(source.id(), localPath, source.url());
default -> throw new IllegalArgumentException("Unhandled source type for " + source.id() + ": " + sourceType);
}
}

Wyświetl plik

@ -6,5 +6,7 @@ public enum DataSourceType {
@JsonProperty("osm")
OSM,
@JsonProperty("shapefile")
SHAPEFILE
SHAPEFILE,
@JsonProperty("geopackage")
GEOPACKAGE
}

Wyświetl plik

@ -9,6 +9,9 @@ sources:
osm:
type: osm
url: geofabrik:rhode-island
gpkg:
type: geopackage
url: https://example.com/geopackage.gpkg
tag_mappings:
bridge: boolean # input=bridge, output=bridge, type=boolean
layer: long