From a0f8c67c7828bfce9316535a02d18e64a27ef79e Mon Sep 17 00:00:00 2001
From: Erik Price <188935+erik@users.noreply.github.com>
Date: Wed, 25 Jan 2023 17:56:30 -0800
Subject: [PATCH] Support unzipping GeoPackage sources at runtime (#430)
---
.../com/onthegomap/planetiler/Planetiler.java | 53 +++++++++--
.../planetiler/reader/GeoPackageReader.java | 84 ++++++++++++++----
.../onthegomap/planetiler/util/FileUtils.java | 27 +++++-
.../planetiler/PlanetilerTests.java | 8 +-
.../reader/GeoPackageReaderTest.java | 67 ++++++++------
.../planetiler/util/FileUtilsTest.java | 10 +++
.../src/test/resources/geopackage.gpkg.zip | Bin 0 -> 10501 bytes
.../resources/validSchema/road_motorway.yml | 2 +-
8 files changed, 199 insertions(+), 52 deletions(-)
create mode 100644 planetiler-core/src/test/resources/geopackage.gpkg.zip
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
index 34e6fc0b..292580e7 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
@@ -350,6 +350,48 @@ public class Planetiler {
*
* To override the location of the {@code geopackage} file, set {@code name_path=newpath.gpkg} in the arguments and to
* override the download URL set {@code name_url=http://url/of/file.gpkg}.
+ *
+ * If given a path to a ZIP file containing one or more GeoPackages, each {@code .gpkg} file within will be extracted
+ * to a temporary directory at runtime.
+ *
+ * @param projection the Coordinate Reference System authority code to use, parsed with
+ * {@link org.geotools.referencing.CRS#decode(String)}
+ * @param name string to use in stats and logs to identify this stage
+ * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
+ * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
+ * name_url} argument is not set
+ * @return this runner instance for chaining
+ * @see GeoPackageReader
+ * @see Downloader
+ */
+ public Planetiler addGeoPackageSource(String projection, String name, Path defaultPath, String defaultUrl) {
+ Path path = getPath(name, "geopackage", defaultPath, defaultUrl);
+ return addStage(name, "Process features in " + path,
+ ifSourceUsed(name, () -> {
+ List sourcePaths = List.of(path);
+ if (FileUtils.hasExtension(path, "zip")) {
+ sourcePaths = FileUtils.walkPathWithPattern(path, "*.gpkg");
+ }
+
+ if (sourcePaths.isEmpty()) {
+ throw new IllegalArgumentException("No .gpkg files found in " + path);
+ }
+
+ GeoPackageReader.process(projection, name, sourcePaths, tmpDir, featureGroup, config, profile, stats);
+ }));
+ }
+
+ /**
+ * Adds a new OGC GeoPackage source that will be processed when {@link #run()} is called.
+ *
+ * If the file does not exist and {@code download=true} argument is set, then the file will first be downloaded from
+ * {@code defaultUrl}.
+ *
+ * To override the location of the {@code geopackage} file, set {@code name_path=newpath.gpkg} in the arguments and to
+ * override the download URL set {@code name_url=http://url/of/file.gpkg}.
+ *
+ * If given a path to a ZIP file containing one or more GeoPackages, each {@code .gpkg} file within will be extracted
+ * to a temporary directory at runtime.
*
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
@@ -360,25 +402,23 @@ public class Planetiler {
* @see Downloader
*/
public Planetiler addGeoPackageSource(String name, Path defaultPath, String defaultUrl) {
- Path path = getPath(name, "geopackage", defaultPath, defaultUrl);
- return addStage(name, "Process features in " + path,
- ifSourceUsed(name,
- () -> GeoPackageReader.process(name, List.of(path), featureGroup, config, profile, stats)));
+ return addGeoPackageSource(null, name, defaultPath, defaultUrl);
}
-
/**
* Adds a new Natural Earth sqlite file source that will be processed when {@link #run()} is called.
*
* To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to
* override the download URL set {@code name_url=http://url/of/natural_earth.zip}.
*
+ * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the
* {@code .sqlite} file or a {@code .zip} file containing the sqlite file.
* @return this runner instance for chaining
* @see NaturalEarthReader
*/
+ @Deprecated(forRemoval = true)
public Planetiler addNaturalEarthSource(String name, Path defaultPath) {
return addNaturalEarthSource(name, defaultPath, null);
}
@@ -392,6 +432,8 @@ public class Planetiler {
* To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to
* override the download URL set {@code name_url=http://url/of/natural_earth.zip}.
*
+ * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
+ *
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the
* {@code .sqlite} file or a {@code .zip} file containing the sqlite file.
@@ -401,6 +443,7 @@ public class Planetiler {
* @see NaturalEarthReader
* @see Downloader
*/
+ @Deprecated(forRemoval = true)
public Planetiler addNaturalEarthSource(String name, Path defaultPath, String defaultUrl) {
Path path = getPath(name, "sqlite db", defaultPath, defaultUrl);
return addStage(name, "Process features in " + path, ifSourceUsed(name, () -> NaturalEarthReader
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java
index 15a42b65..b03ed1b6 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java
@@ -4,6 +4,10 @@ import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.collection.FeatureGroup;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.stats.Stats;
+import com.onthegomap.planetiler.util.FileUtils;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
@@ -17,7 +21,7 @@ import org.geotools.geometry.jts.JTS;
import org.geotools.geometry.jts.WKBReader;
import org.geotools.referencing.CRS;
import org.locationtech.jts.geom.Geometry;
-import org.opengis.referencing.crs.CoordinateReferenceSystem;
+import org.opengis.referencing.FactoryException;
import org.opengis.referencing.operation.MathTransform;
/**
@@ -25,32 +29,72 @@ import org.opengis.referencing.operation.MathTransform;
*/
public class GeoPackageReader extends SimpleReader {
+ private Path extractedPath = null;
private final GeoPackage geoPackage;
+ private final MathTransform coordinateTransform;
- GeoPackageReader(String sourceName, Path input) {
+ GeoPackageReader(String sourceProjection, String sourceName, Path input, Path tmpDir) {
super(sourceName);
- geoPackage = GeoPackageManager.open(false, input.toFile());
+ if (sourceProjection != null) {
+ try {
+ var sourceCRS = CRS.decode(sourceProjection);
+ var latLonCRS = CRS.decode("EPSG:4326");
+ coordinateTransform = CRS.findMathTransform(sourceCRS, latLonCRS);
+ } catch (FactoryException e) {
+ throw new FileFormatException("Bad reference system", e);
+ }
+ } else {
+ coordinateTransform = null;
+ }
+
+ try {
+ geoPackage = openGeopackage(input, tmpDir);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
}
+ /**
+ * Create a {@link GeoPackageManager} for the given path. If {@code input} refers to a file within a ZIP archive,
+ * first extract it to a temporary location.
+ */
+ private GeoPackage openGeopackage(Path input, Path tmpDir) throws IOException {
+ var inputUri = input.toUri();
+ if ("jar".equals(inputUri.getScheme())) {
+ extractedPath = Files.createTempFile(tmpDir, "", ".gpkg");
+ try (var inputStream = inputUri.toURL().openStream()) {
+ FileUtils.safeCopy(inputStream, extractedPath);
+ }
+ return GeoPackageManager.open(false, extractedPath.toFile());
+ }
+
+ return GeoPackageManager.open(false, input.toFile());
+ }
+
+
/**
* Renders map features for all elements from an OGC GeoPackage based on the mapping logic defined in {@code
* profile}.
*
- * @param sourceName string ID for this reader to use in logs and stats
- * @param sourcePaths paths to the {@code .gpkg} files on disk
- * @param writer consumer for rendered features
- * @param config user-defined parameters controlling number of threads and log interval
- * @param profile logic that defines what map features to emit for each source feature
- * @param stats to keep track of counters and timings
+ * @param sourceProjection code for the coordinate reference system of the input data, to be parsed by
+ * {@link CRS#decode(String)}
+ * @param sourceName string ID for this reader to use in logs and stats
+ * @param sourcePaths paths to the {@code .gpkg} files on disk
+ * @param tmpDir path to temporary directory for extracting data from zip files
+ * @param writer consumer for rendered features
+ * @param config user-defined parameters controlling number of threads and log interval
+ * @param profile logic that defines what map features to emit for each source feature
+ * @param stats to keep track of counters and timings
* @throws IllegalArgumentException if a problem occurs reading the input file
*/
- public static void process(String sourceName, List sourcePaths, FeatureGroup writer, PlanetilerConfig config,
+ public static void process(String sourceProjection, String sourceName, List sourcePaths, Path tmpDir,
+ FeatureGroup writer, PlanetilerConfig config,
Profile profile, Stats stats) {
SourceFeatureProcessor.processFiles(
sourceName,
sourcePaths,
- path -> new GeoPackageReader(sourceName, path),
+ path -> new GeoPackageReader(sourceProjection, sourceName, path, tmpDir),
writer, config, profile, stats
);
}
@@ -68,15 +112,19 @@ public class GeoPackageReader extends SimpleReader {
@Override
public void readFeatures(Consumer next) throws Exception {
- CoordinateReferenceSystem latLonCRS = CRS.decode("EPSG:4326");
+ var latLonCRS = CRS.decode("EPSG:4326");
long id = 0;
for (var featureName : geoPackage.getFeatureTables()) {
FeatureDao features = geoPackage.getFeatureDao(featureName);
- MathTransform transform = CRS.findMathTransform(
- CRS.decode("EPSG:" + features.getSrsId()),
- latLonCRS);
+ // GeoPackage spec allows this to be 0 (undefined geographic CRS) or
+ // -1 (undefined cartesian CRS). Both cases will throw when trying to
+ // call CRS.decode
+ long srsId = features.getSrsId();
+
+ MathTransform transform = (coordinateTransform != null) ? coordinateTransform :
+ CRS.findMathTransform(CRS.decode("EPSG:" + srsId), latLonCRS);
for (var feature : features.queryForAll()) {
GeoPackageGeometryData geometryData = feature.getGeometry();
@@ -103,7 +151,11 @@ public class GeoPackageReader extends SimpleReader {
}
@Override
- public void close() {
+ public void close() throws IOException {
geoPackage.close();
+
+ if (extractedPath != null) {
+ Files.deleteIfExists(extractedPath);
+ }
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java
index d3c3b4e9..ccca9b45 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java
@@ -253,6 +253,31 @@ public class FileUtils {
}
}
+ /**
+ * Copies bytes from {@code input} to {@code destPath}, ensuring that the size is limited to a reasonable value.
+ *
+ * @throws UncheckedIOException if an IO exception occurs
+ */
+ public static void safeCopy(InputStream inputStream, Path destPath) {
+ try (var outputStream = Files.newOutputStream(destPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
+ int totalSize = 0;
+
+ int nBytes;
+ byte[] buffer = new byte[2048];
+ while ((nBytes = inputStream.read(buffer)) > 0) {
+ outputStream.write(buffer, 0, nBytes);
+ totalSize += nBytes;
+
+ if (totalSize > ZIP_THRESHOLD_SIZE) {
+ throw new IOException("The uncompressed data size " + FORMAT.storage(totalSize) +
+ "B is too much for the application resource capacity");
+ }
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
/**
* Unzips a zip file from an input stream to {@code destDir}.
*
@@ -304,7 +329,7 @@ public class FileUtils {
}
if (totalEntryArchive > ZIP_THRESHOLD_ENTRIES) {
- throw new IOException("Too much entries in this archive " + FORMAT.integer(totalEntryArchive) +
+ throw new IOException("Too many entries in this archive " + FORMAT.integer(totalEntryArchive) +
", can lead to inodes exhaustion of the system");
}
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
index d858721d..f0fdfcfb 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
@@ -1668,7 +1668,7 @@ class PlanetilerTests {
.addOsmSource("osm", tempOsm)
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
- .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
+ .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null)
.setOutput("mbtiles", mbtiles)
.run();
@@ -1749,9 +1749,11 @@ class PlanetilerTests {
@ValueSource(strings = {
"",
"--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4",
+ "--input-file=geopackage.gpkg"
})
void testPlanetilerRunnerGeoPackage(String args) throws Exception {
Path mbtiles = tempDir.resolve("output.mbtiles");
+ String inputFile = Arguments.fromArgs(args).getString("input-file", "", "geopackage.gpkg.zip");
Planetiler.create(Arguments.fromArgs((args + " --tmpdir=" + tempDir.resolve("data")).split("\\s+")))
.setProfile(new Profile.NullProfile() {
@@ -1762,7 +1764,7 @@ class PlanetilerTests {
.setAttr("name", source.getString("name"));
}
})
- .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
+ .addGeoPackageSource("geopackage", TestUtils.pathToResource(inputFile), null)
.setOutput("mbtiles", mbtiles)
.run();
@@ -1790,7 +1792,7 @@ class PlanetilerTests {
.addOsmSource("osm", TestUtils.pathToResource("monaco-latest.osm.pbf"))
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
- .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
+ .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null)
.setOutput("mbtiles", tempDir.resolve("output.mbtiles"))
.run();
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java
index e8594137..3ba212b5 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java
@@ -7,46 +7,61 @@ import com.onthegomap.planetiler.TestUtils;
import com.onthegomap.planetiler.collection.IterableOnce;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.stats.Stats;
+import com.onthegomap.planetiler.util.FileUtils;
import com.onthegomap.planetiler.worker.WorkerPipeline;
+import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
import org.locationtech.jts.geom.Geometry;
class GeoPackageReaderTest {
+ @TempDir
+ static Path tmpDir;
@Test
@Timeout(30)
- void testReadGeoPackage() {
- Path path = TestUtils.pathToResource("geopackage.gpkg");
+ void testReadGeoPackage() throws IOException {
+ Path pathOutsideZip = TestUtils.pathToResource("geopackage.gpkg");
+ Path zipPath = TestUtils.pathToResource("geopackage.gpkg.zip");
+ Path pathInZip = FileUtils.walkPathWithPattern(zipPath, "*.gpkg").get(0);
- try (
- var reader = new GeoPackageReader("test", path)
- ) {
- for (int i = 1; i <= 2; i++) {
- assertEquals(86, reader.getFeatureCount());
- List points = new ArrayList<>();
- List names = new ArrayList<>();
- WorkerPipeline.start("test", Stats.inMemory())
- .readFromTiny("files", List.of(Path.of("dummy-path")))
- .addWorker("geopackage", 1, (IterableOnce p, Consumer next) -> reader.readFeatures(next))
- .addBuffer("reader_queue", 100, 1)
- .sinkToConsumer("counter", 1, elem -> {
- assertTrue(elem.getTag("name") instanceof String);
- assertEquals("test", elem.getSource());
- assertEquals("stations", elem.getSourceLayer());
- points.add(elem.latLonGeometry());
- names.add(elem.getTag("name").toString());
- }).await();
- assertEquals(86, points.size());
- assertTrue(names.contains("Van Dörn Street"));
- var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
- var centroid = gc.getCentroid();
- assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i);
- assertEquals(38.9119684, centroid.getY(), 5, "iter " + i);
+ var projections = new String[]{null, "EPSG:4326"};
+
+ for (var path : List.of(pathOutsideZip, pathInZip)) {
+ for (var proj : projections) {
+ try (
+ var reader = new GeoPackageReader(proj, "test", path, tmpDir)
+ ) {
+ for (int iter = 0; iter < 2; iter++) {
+ String id = "path=" + path + " proj=" + proj + " iter=" + iter;
+ assertEquals(86, reader.getFeatureCount(), id);
+ List points = new ArrayList<>();
+ List names = new ArrayList<>();
+ WorkerPipeline.start("test", Stats.inMemory())
+ .readFromTiny("files", List.of(Path.of("dummy-path")))
+ .addWorker("geopackage", 1,
+ (IterableOnce p, Consumer next) -> reader.readFeatures(next))
+ .addBuffer("reader_queue", 100, 1)
+ .sinkToConsumer("counter", 1, elem -> {
+ assertTrue(elem.getTag("name") instanceof String);
+ assertEquals("test", elem.getSource());
+ assertEquals("stations", elem.getSourceLayer());
+ points.add(elem.latLonGeometry());
+ names.add(elem.getTag("name").toString());
+ }).await();
+ assertEquals(86, points.size(), id);
+ assertTrue(names.contains("Van Dörn Street"), id);
+ var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
+ var centroid = gc.getCentroid();
+ assertEquals(-77.0297995, centroid.getX(), 5, id);
+ assertEquals(38.9119684, centroid.getY(), 5, id);
+ }
+ }
}
}
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java
index 8a94fc13..0dbb5b71 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java
@@ -5,7 +5,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.onthegomap.planetiler.TestUtils;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
@@ -90,6 +92,14 @@ class FileUtilsTest {
);
}
+ @Test
+ void testSafeCopy() throws IOException {
+ var dest = tmpDir.resolve("unzipped");
+ String input = "a1".repeat(1200);
+ FileUtils.safeCopy(new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)), dest);
+ assertEquals(input, Files.readString(dest));
+ }
+
@Test
void testWalkPathWithPatternDirectory() throws IOException {
Path parent = tmpDir.resolve(Path.of("a", "b", "c"));
diff --git a/planetiler-core/src/test/resources/geopackage.gpkg.zip b/planetiler-core/src/test/resources/geopackage.gpkg.zip
new file mode 100644
index 0000000000000000000000000000000000000000..6ee6df871ae631261896f5e1e356356d148d82ce
GIT binary patch
literal 10501
zcmbVy1yEeuwk_@&n&1`)ZowUb26y-19^C0PE(vZy8@J%@4#Bl?2?S{fjl2Gw`gPww
z=iXQE*Z*VHt~qP(T65Lft7?s!V@@?iL?i$l+Mh8h|3(MyU+95{!-b=Uv$1mdWNzVL
zZezu6^U1+RTMHWw6^2)DarWo=%X<1?!oee5!NbA*2?7TP3q%hfgEThpD4P@=DdY>Z
zzI#^}NoeRrKJ5Bh@q9_=<5=wL2)fVddbGawc0=DthP^TnVwY)(u;Vn^I}!mHY=I4z
z>b95-EzA!phNA_^x{OZx_Xer6HBBAdG2fjmv3C6P?v7td+=~QnHn{h5+>eFoj@CEi
zF##C0M#%uRPU~3N3P=NW;f!rGDAn#2p0BU`qQTdpN7Ll`N-sVBf;0V#F2KZ3mf
z-I-t6R$zavhSPziI&cK`7O~AeT4x9MH{WjM)Ls4cJuBC9_i*ompit7LfzPuMsG#3G
z8WR^M6cK?U58)?IEM-(T_T2kCaS3jbLG%NwEXuxP+W4to_AU$}KSan#taqqa+C-7u
zT029BBih!#U5%qDQ1=}lN#q<;sO7Ei%9`XSeY%eyK0be??IqVdtowDmPHBMa1NDtG
zqvd3>mF$BppCkq#?p`mPNFdQV6q`|>cV3PQiv=tSh*lGc3k^zq&sUEOgHlV(G4t12
zC@b83{b6k<8`^&LQzvAyfnvOcE0g4=-jN3{?^->fXW9&J{&q=rJf71@5~y0Cmx7%q
z{Y}+Q?G7h*04&d3`mLT1k|kfnpKmajPIguI?!v}=T$MP1Wjs7<
zX*t>;5HqV%yXeHyts3+CE*U0D|zruuV>pp-Hn4~KT?+J+M7
zk(z1RlCDQR4^Dhkm8hX14M6P8Kfcm$cBjM4AS*(02CNW}
zM00g8*K^J-9>`cjU)<0ev+QX4pkRB5V{I+fozZc2{x^cjNy{QHoipqFrhseq%tE_p
zjtH=qsax{xdKD}PVPeOhaf*D?_D~q7c88gnGiO=Ag0;w>PpxE>-HR|NK%Gz9Yi21w
zAiK6$R=7zRKix4svq=G^fSQ{+sosYFG?Y<;c$=}5cYuyQN+*Bf_#`>T@CX@M7C7<6
zNvW%Pw#zcrIY2*l(w5JGgIJ}3t;qN+W^J5ru-;lQu}|Ska+`($M}7!(D9#Lk!RE(|
zAw}V>SS%&LRxj|wqtD#L-XYouCP+;0L#R0pQe{9@)c~H5aSetJEe$AELwj#DdttP+
zkqcUJw4}D&n8(#HX5?hOWL`g*$d#yDu!ApR^)vi?A-r(qiH#QuJ2?54PV{4&6wfXp
zar-#8pU+CWCsW#;a%+9Hy<#iCyC!V5idmOsqVrd(^ru3*@}aR2EJ6sS_ya!g+gLqz
zC9caLE_{+XNK!pmkd-JjvkYhEJ+m}-1Kk_t@K@f`kAl1L#CxglGYqelzf!&me2H4y
z{b=?#*iKGE`Rrc5u8O=Tv2&0^R!{%fPcFEmdP7q^{QKC8KD8YJQU#3R6
zNFe&40PlzjnYrzuU%+D`BW#5bF{)iaUIpxvVY$tJr?$%;CCR;J8PGeUHe)~0mvwPa
zdrko}X$Yz;HHm)(70Vx9728*ZNqe`?wYN)7elK-ye0VgTwdz@*s?qVV+E@K8)NN|3G`v>J;AC0mTJrfV%v`KK=xl
z8X_k*f|l60Q?zgN1kiAt!oAicS416r>R$Myg~);R4}2jA<8FMrySrG7G|BtGZ)
zVsz2w_SVzb$v7KA-xbjKn|~hSQd4O-XKUE)FxsBo{=71^KE1uBxpA{D+N|nJW)e_v
zb6{wgrF+x?d^p&jxJ$Zy3#{B?!|vexCsf`f@1zn9s^XkJQz8Bxzjmb0pG}{iUouI3
z%>Q=yf7~n{M#x^O!>`pBlgG#Zn6-EDB5wOL4D=72aeOHR#kI?d$mm1Oz!5ssqf`&)>m4KZ~RM9YZ5gmnSa{ds(($|G5p$XH&_5=eU~}
z;(vgF|MUX*&wralb$j>kLF2fdGBa8;*uOq^&hB4|&r>l%QdM4GrLCE8frU$KWuD{C
zBQ6&9ANBo30e80KmlKe0j|+VlP6%=l{Qx3Lax_#VAxY$Xx*FMSmwpd5C@vN@iabbx
z;2yH4^CRtnFSb4e0PcV_qFx#M8!O3?_zT9VA9AMVd3}^e^fME
z@79DgC|#D=PHlUK^dAm^ooUn72jKo2^&u4b%nx&Q^d#`mI)2d3kFzA!k?>~(}-hq
zX#1L&q{qC6i30yv>Zw9XHwf~+cDVzSrljO$#;z??RHI+V=OtNh`FfmA#-IB!Ayi9DQ?Z#{MlFh1MCx1d=S*5rdl?ck521f&NeHRApW(y~y+7iB$W#Jl`_O
z-D_)%9aMP`JuQ4J2J)?XP#d_Yv#_V7-F)E*z?!?PE8~1{N18pkv{c$@E7Lc(eZrxM
zz>fio!jd5|NzqR~_%wzI5uxWsNJu>8nDs7oR+Ow%(~77aL%fSI
zleyj2QN3$YPqTnxH5*NdZ(P~%ZHRh2N&LuiJ(-UrfC_ByJ-NL3TUiBTrb?oj_jr;K
zw$Vx4JJ3oTjQ}b++-UJ$`io!6{FxQBbR06fBH_tLrSJtQNyZr`g>bznMo&yjC3jVDdFl?lc)hR+or!)X{_G
z@&IQhS=a1{YYMt$cd}4qw8y_Y*;UTA%Y}Pg7B*|m`~OFUvVLp&qtRm1$$L8v7~Rzu
zM1bA(EXBPYvcoHm(@W4bx`liFwWML62WNS_1_K<6
z8u{R*l?BHnkC`=%7Vp!$o|{^db>>P{>v4AoxiD5S`VlRy)Vv3I%z7Bn59m)3n{*d3
ze9tknb;t9HVV$hvhnirsynb(P>62^ps~H-Tm7igP=?!M&s;j@EoYSs-nNs*vU6PGs
zELStEtyWcAv~?2B^p{zwR1J@f1S$k51aVx2xc=BX)`5c`47IQivRUlu+F7yW5r5tt
z4IP}-?oQqb&EeEo&B>83joOgso>Pm+e7H{fv@5cUL>0zjF#e3ETNW=c2uo*JT@Vmn
z4QiV^1<>TCd=ND$4oz9JC=O0Rp3QJ(#Xb0PbroiDo)JqRfRSo4S}m=(DodF9amXZX
zc)!`3S+NZxC7AHXiD&C-LNc-gjvWiv8<>!6;Z(uwzWP{i7U9p0^S-lN&cY7lV6D(#
znV``j*L%wC2Or&591aE5d>OkDF1rh{Oo^8KVX@kN
zT8U$QW@BQFbKS`W{r1Uz?lDC+@}Q;DYz^eWPa=)pAC3m?(k?VsvQzcnMn$lUEiDWR
zS@^bIo9U%~<3Wi!an~%=8dxSQ#n29gnhko06c6NXF2dZTCfMKH0SL8!gxKI(_GPKl
z9IDYQmzGzVS0W3jCz62$Skfg@rs8BZ@9;C^Q_V&eBVS%l0$ow`%zR8=6rZg8@{eG3
zK$Z&jCyN8=vJx%RbBPh(w^xH4L{w~I1wKJD1v2|YPK*Jar^S7uVKE*z2RKkiWDWp1
z8ybLh3P}I*>AC4KxMTGk+7Z1b=gu*?o_wJ)St+8%+19iNjQJLWWsuj4jl`Sq+?i3T)JXU^||VgsShC
zMLF?S7lYH!J>G>wrY5M6-bGAy-c0K(EYBiZye6?G(Z-h+jpWSKnN-EEy`6cidZ5@S
zJ0{}jWMdXEG>AaamxYsQ=|b~VV5Cz}__0;`vq=cgLUD12O77S&e)BNftx*_-59Zml
zBzS7BXZ8(I1;bt=%dg(Rk$OCt{V0!DcikNwt-bxs-M@QS&`^YRN=_DH*x0C}!_cU`
z!SON-Zy4i!2qo`rKr-01ZvS8@1GA))#6f0)zk_5nHKJHT3LP#&O8hr{YI}tVeM8r;
zu}&Nh=vp~r0**LCvdmnfH6dfIvR+Gx0R7F!Rc{QJm7m!LjUB6dWC-0inh%WZTo^0y
z*dr@7Sw3}#4dv*yt%ncM86BGZB9CjM99S_J@7}*2u*BakN1;jept&$CV~gYQD_MBF
zU0H(_n9|Zqn+iTNFE*N
zm}s=xs1Wds2HjOZd*u*I^iogspZ+!50E1E#=Qj%akuSxlS7=j?3`9fNpOES#go>VM7NCmHdNf)g!a?aK#cANh>ltIBp
zDUr}*#4KJsP)nWAlb80piur+#>+s5B#CGrUxuA<|<$WQ;4iBX5SH7ru3Ff@-CN<|X
z8g?4*QoQ@qFY9R|&P^$jGUyCCMDsTa*$B_66Pb59W`~*=`j&jyJdfGpa&T3CM^Jy%
zMsE0?+%Kg4`6Bk^J^+BUWKY!w-Hpj&+lCcz@|DXkeUa!(^g#Ef{N^)`uqW_fdsV)O
zD_|I~14s52a@ejb@f3ck=X>BemU96Ps(om)d4V@c2o9xB2&4-0V>BlgS+AyKuG)1Y
zFY$`#?iO(yAQCTBrt-0sF938l>o;uIdMr
zPoFjK>ySM#<%N0Wn%EPD^RxsD+8s6*dP7qQ?hh)wzX}-o^&UDep&C9_s_-&V#O!XQ
z2u%jn?AM-EnDeYAx{m?b!=p-Ltcjy-keR6Ku$m}jmu1WDAGbb#y+il75wuxbJCjQf
z7kP5TyY}mt=h4>-SAoS?VipbcVrQCmTz@1cMcov8OzbN96yQlv!qV5Be?H
zI#q&k|FmY&z6Vl>Ti0nln|V)6qz`|ukX!gQBl#NH{%}pi>-0h#nMqI;r7hzF7&Fu0X4SM?}2K6y}>0DJ&&9tmWZzglsEY`aMv5$^p*m^6k(e
zLk~<0b`iFyF_{`mBO9+PV`erpfX-B)Mc0SZzE`dc@rIrlu&_sb=YQ}*qp<}{Xex;d
zW6@R|vGA&|?v=95+QN8P*iEMdF=tx=@Np)
zFGyf=8j4t~{0Q+l8naqKO5`uG^@jpG~}DTRkvdQgq%!PL>-
zGGIH9G=%J^5g`Yr(bMwQP-4|R=zDsONVZHJUp_oOh(;*(eabBvO|549+-R2e*)I<<
zn)xJ*(E)Hj_z5xOh@jiGhn_7@+j*ueiQ$V@47r=AGoDMr#RZF^7+g$bjNg)J&*;eE
zQ3*-l9rf^%f^1`p
z)l6(F)%wizly+HF*E|}V>N|>|0MzkQv2!1(O>{}gGOp{mO&d%jB#?0-TGs@~-&reK!VkH{dtcWdQW(sMd@sLY9&Zp6#!-{J()mFd
z4e%Zp{?KJ=@`JXy0WV|(qT(J>L!+B}^kn~g;{xcpVlif(;oSJ!q#VRHg>x2nD@D#i
z;abKT(H1~mBxW3a^;O<(@JeET&*4GfY^Sk0cSG(>hriWD%V_Kyt^OohhyBGx5^PU*
z@M?NyEu4uXO;7mNOy)-@?O|puU))MQ^loSaF|Pml=R_UiYuqG(?8%_f^)_QChn8Oz
ztH3ADQZ2vD5YtZiH49n0yj-G1>cjSwSV~`_6nlna@th!TB#q1NfO{;|4+lI1N|BbK
z-&EEf*Qv`y0}~bwSb=Ut4GvSLFO{hqs6iIXW}&1hs;pkMtaYd(jBz`2gb?0BLs-pL
zhRY3(F$PTu!CqwxO4E%K-EhJd4SZ{nJ0-86vAy-nr&Th|Bd_%&a{>w(etP0jEdC`P2AN<
z172LLO+|Ne(kxWP6`I6??=bycPv;cf=CX5|fnm13o6RKUWG_G$;m9=mLA0nvl
z>pwVnj*vZR*#QM%-xgDYy?`X%ru39%0<_zvL8$eVpk~j~<-_&n=Z6^GR~uzl6;*FG
z@QP}rvOWtf(TJ_~eyK37yy>j>9ZbG8f~pb_O|CNE$6`mwC4>+n@kOtA-$JPJ8_Fn5
zOXmAuOecqXR>H>0x_QeR9d1zj!8i&KDRudDn(3yea!SBL%mnT(UlfWpW@ftZh}xA;Mx7+i*G{xAiol~{{Xj(Hog}x
zzM>xy$L8+2$`kda!a;vpG`IBT=SLoiN+z~)<>jUyLFm%wz9lZ*zS-J0Yxr&0o!zJVmiIC`i`M!&_Qac~tq#@uO(7*4
z0pFL!8NIZl%|Aemc!=J}SwqWK`9Uqk9L_m3SYXKbW>xC+SiS(s>8X|E@T@S*b^Rn*o{Bb2bvV
z^`=cfZ8a7b9Ey|VQN#MOzbzOHsk#SGx4wsspPtRp-n8)hlnfmnk>y)_%<2c!MMPVE
zn_SJr?Mt{zAm@wBL+EO+P?@bcV+L+jVK-%HBpTEoiwKKMk2`ZTjZ+
z1x>>d+faDPJQ4GaxYWs*8w#ZKni%GGLWaUU&R9!D@t7@8;Bj^JS)s$Dv>+(CMg>;*
z62@|J=405Q395E!Yw=POF%Cf`}-ZJ{o{|wsQc$hQ!y7$L$q=$
ztD5$b2RP6buEGTkz~5O3NQ&GsGI;m4QtI!bX5u_^ecr~t3DG$Sf_DN5{?Zw~Kp8Zv
zHtIM3y5-yNbQ=B-nG`?|lF9$CFu(!!6u=nFAXB
z+vWX*ATC6#QpLi+IOzHF`3X@9>gE+@r^I}6_;WC5b@@Vl~U=qs<9R-#$~GcIzG
zzf<|C*OGrEpDx3^N_$v+iRiS$$)!~6=3-Nj!ACceKG>1-pF|tcDd)HfZa#k00D|e?
z{zS=uAGs*ck@5XtDC4_q0`TQU{y!w-XKgdcYtk{^3}qQo&>BJB_9kuoB$wm%m~MNJ
zl<^bFY8&s@rpESBEPh+RF-jgXF`IM?ALEzkt+!7P;Y3)fzh9hvlwNAK;QQC_GqJ@N
zfD@mRwnY2!pXS0VuuT`l?ewa}?FDR!AumfkWI6|?+(_8%*v-U`;VM#@&w^4h&nEqY
z;X+UA;r9J*PnF?=_e59xq|w+0Pulk^mub-9E7NJ*-#3S-m&sz_&9a-r_Dh&wdzbzu
zxO&F$vIIJICHrI)7$w$y)L-X4_N;TAR+lzv!sy?0R8}`UMkz8wKlC?wS|IPo?+J*l
z`^97Y-p*a&JWJ7^N~iJt8g0?~tKmOYA&nKLqSZo=isZJ!1}TNrq6TFJbS9@1jgn-c
z!g=;0HjOqJ1%AZ?dZu;8q6v+*_p^EO3X6(%1p}rGfzI(_)hV-r8g0@FP8#KDqu&!|
ztu)#U6e)EZ_3hb<+iRxY$AhYg3l
zeD+(`ZrCqjq*$B?CY5LZsM|9+iAUk+KlRsmKxw0E@to9BAkw+75eycJsy(h&gs{9C
zpeFllk9O!)5be{7(NGZbso;2WftQd*NDrjrEb06)!p+>fIiLAJ1E32sGHLFvC-C%-
zUnKf;Cheg{tUTJR7a#IRRaa-3k?;9Bz!6-^Our71q85q-dq6JND+q3ycxe09g}P>0
zN%n%Fa-_x2zsQQ8TS$i1VM(mC=UW$N+*8G&C=+HZ99w5mdpCHZ*2QWIJwSi1Y3}}n-G88h1O`w+iXKX_nfP*N
z3}1%Tv$pAH=#7fVo5SN8qF}#P2fQAg&d(5Txk)39D?I4^eYfsl+{D?|KG6hXu@k$b
zu1Uk0Id)Lya8ea6hWoVU{iQWc57~BOVe1r}Ul2!b#%IR1cu^buwH_}yhCFAJRx=sG
zeiWu~LNb-eD(HXJi|Y;8-@LB*va1XIlF4T`JX(K9mpV4YAJ`!Fv6iF}zuV82gvFqY
z+j%*k!tk3mZY)J{h(32}Km)x-Yc%tLvNnOf{Fol)EXnWRn8S|C9_r>y%_p9|0YU!z
zSoT`f-HW@6yIglGe|(m|0B_IKuC;UX@L5-){T2PUb)OPd16bVT?saJ>Vx99vRM8s-5?olS2t
zx%%V=faSVx_4#|^)khJmta1c{#Ot|Gc3E}#gGHhPBx%R>e1{SaG|9}b7&wDvll)U^
zTlM|($m8qpG&hm_T@{9ZXrO5;1F=p6kEp`}1`fIo%$tSsijvjW8J_D^>7(GYnI%vW
zAAbD6BRnyLAW#re3Cf}-*s|wV#C*Wosg%xK-#n1g?&t{{2>u@mw;B}KD3Uy-@$>E_
zDIp@h(u$*%US92;VsRAg>i;F_f}5Oix9q)R)+w~#h8mdV+ai{_S--Xs)T>kaOKt!4tOP7JBCdbH(SG~wjl6g|97-Z8ujP%9rc%-NhMi4%ipyMP
zQ82;?k~}*x@iw&KQ)F;ALZ7Xd;MvJ$$Uj8PM$&`o<-1P&vxws#G@z6!M2&(HV0|p^
z(kyYzPg-6e9(&VSS1b
zUTgBmA_zVix;1O8*cF%I{Sy!-lkv)-m))md;9W~01HY>KF7vHNM&C%3^0hsP2WKuz
z9Pufx4>hMBsbSu{To}$Hwl9|`Q^b?pcHF!?2o-7^<`Sb^b%^37tfUJ=6#}2`Be<}r
zMf7XEV1s`Ho)eeo?BQ3oIoBUvO|xKhj)_+S0i?o
zou&=Zj{%V`r6LX(JS7|mpI4R&?XaLm5q90UrMg>WpS|JGJQ{#RP=5^I+tW7^h_P0N
z1|YOBQ~XS$b2cy;`IyRAxOW)cVtlQ(w9_aN2W@C-S-3*%0k}}(VJ<1$X|FKcrmPDJ
zu#2!#bLQt%rbq2s`Ti47j(7S-AOQLhi}K_;T*HQWh6u`K
zQ=dz$vd;TEX2zG(MNlkb!1URXDV!*}m6U{eST_?N(`2vx`+XEcRa7FhBMyf
zq(Cyp-6%Y59-!AXh2uHqe7dKeZs|k)lUTz$d9tp)5KsTY*rd!{yQiBbh_&i0L4;}C
z#xCs?Zz70d@`<5b#`bv+(Pthbw++{hFCfoXo*G&tVM8P;D8@!NQKE;*@G~2XaXqpv
zzRBWRs@*!(ju=Gdtr$ccT#0}K!ZvZ5It-ikE&0{fI4EjI4yzH3e{mYSGEG}yY$9{z
z!27M3w;dsZfy@DsyObR9s6JmMlrg9rYpzwmR3U8LSb&$Wb1JrTnG!AnN<=V(mCDoh
zk4cz0DEGWj1mC1HA1}lyRHXB2PgoMpql*{cNQnVHX2mEBUtX-9809
z&heOW*{MGbA1=J{G3N{r={{+XZt$(B;As47acTBQ&Wc?Ebg}n#lNxzkoe+veE5Cl9
z#vIIk^$IJFgF@?;x*e}gW$TWU)dZU-@6;WH>WQH%ppzCi2S>c;3q6F0
zVu*|LeY?O0)2cOL{-`{4ut-`N1F<7gP{Bn5U13^3duxhiaONzU;Q~#!I3krFf3bnP
zdr$Alz?;kgw5QJqNd3n*Du@P^r~8N}WD;@|^v}W){49?OKP!=g&2LoFS&utpx6%ga
zHw@t$*392*BHXw4nG$FE>??r#apuB!ruPBQ`AY+aNNOw|B|x~q($k8M1l8OUoE>85
zQtL@)c~v}oXQR9lrtax|-f-(hrxQJ7a0tuYPj99PF@Ua)+qmNJZDZ#5hoTZUOaY3H
z+rE^x)%}-6>k?-`whke_v(0XJzaeuW49&Hf8$gi0cvx(v0SJuG3<-We*RhcJyqXa`
z86W3965xuVd35)C_`<8@^)(OCQ$mqeP?~_qQ0*{Nip<*sO6pm2Kj=rWBH;OaX~0-Y
zBDmk92-7yRuR2OO02dx~Yzjn!;)iwPe1thj5a7%a4;(KcV*o`5f|AHz`1_LGEV}z#
zqLm*jNzjT1xO>xVnwTE3r;E+Ofw5L^dDPZUMDwx0o_J#ypZ8cEVbgMm5;j6g5oU*C
zJG!@=M~8Ocntl)B;+FnJ<^a=tag$c8QlqVhJeN*Hq*`v2vCp*^JTGrr3guX^?F^J-
z`$kf1tXxyzDjEh@Gikp`yHSKH^)>t@i{z3-J^;`B5yCCPZ&VxpT-N`
z(T8UuHAPf3nWmK}UdBHign#-D)fC|o0EqvmC-Lv!{_08m7YadtdH;_cifW3;D1S|Z
O|1*sL*u=jUh5KKSmOHcn
literal 0
HcmV?d00001
diff --git a/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml b/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml
index 558e65d0..ec1bd45c 100644
--- a/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml
+++ b/planetiler-custommap/src/test/resources/validSchema/road_motorway.yml
@@ -11,7 +11,7 @@ sources:
url: geofabrik:rhode-island
gpkg:
type: geopackage
- url: https://example.com/geopackage.gpkg
+ url: https://example.com/geopackage.gpkg.zip
tag_mappings:
bridge: boolean # input=bridge, output=bridge, type=boolean
layer: long