diff --git a/flatmap-core/src/main/java/com/onthegomap/flatmap/FlatmapRunner.java b/flatmap-core/src/main/java/com/onthegomap/flatmap/FlatmapRunner.java index e58650a6..491eadce 100644 --- a/flatmap-core/src/main/java/com/onthegomap/flatmap/FlatmapRunner.java +++ b/flatmap-core/src/main/java/com/onthegomap/flatmap/FlatmapRunner.java @@ -4,6 +4,7 @@ import com.onthegomap.flatmap.collection.FeatureGroup; import com.onthegomap.flatmap.collection.LongLongMap; import com.onthegomap.flatmap.config.Arguments; import com.onthegomap.flatmap.config.FlatmapConfig; +import com.onthegomap.flatmap.config.MbtilesMetadata; import com.onthegomap.flatmap.mbiles.MbtilesWriter; import com.onthegomap.flatmap.reader.NaturalEarthReader; import com.onthegomap.flatmap.reader.ShapefileReader; @@ -436,6 +437,7 @@ public class FlatmapRunner { throw new IllegalArgumentException("Can only run once"); } ran = true; + MbtilesMetadata mbtilesMetadata = new MbtilesMetadata(profile, config.arguments()); if (onlyDownloadSources) { // don't check files if not generating map @@ -503,7 +505,7 @@ public class FlatmapRunner { featureGroup.prepare(); - MbtilesWriter.writeOutput(featureGroup, output, profile, config, stats); + MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats); overallTimer.stop(); LOGGER.info("FINISHED!"); diff --git a/flatmap-core/src/main/java/com/onthegomap/flatmap/config/MbtilesMetadata.java b/flatmap-core/src/main/java/com/onthegomap/flatmap/config/MbtilesMetadata.java new file mode 100644 index 00000000..60115f07 --- /dev/null +++ b/flatmap-core/src/main/java/com/onthegomap/flatmap/config/MbtilesMetadata.java @@ -0,0 +1,35 @@ +package com.onthegomap.flatmap.config; + +import com.onthegomap.flatmap.Profile; +import com.onthegomap.flatmap.mbiles.MbtilesWriter; + +/** Controls information that {@link MbtilesWriter} will write to the mbtiles metadata table. */ +public record MbtilesMetadata( + String name, + String description, + String attribution, + String version, + String type +) { + + public MbtilesMetadata(Profile profile) { + this( + profile.name(), + profile.description(), + profile.attribution(), + profile.version(), + profile.isOverlay() ? "overlay" : "baselayer" + ); + } + + public MbtilesMetadata(Profile profile, Arguments args) { + this( + args.getString("mbtiles_name", "'name' attribute for mbtiles metadata", profile.name()), + args.getString("mbtiles_description", "'description' attribute for mbtiles metadata", profile.description()), + args.getString("mbtiles_attribution", "'attribution' attribute for mbtiles metadata", profile.attribution()), + args.getString("mbtiles_version", "'version' attribute for mbtiles metadata", profile.version()), + args.getString("mbtiles_type", "'type' attribute for mbtiles metadata", + profile.isOverlay() ? "overlay" : "baselayer") + ); + } +} diff --git a/flatmap-core/src/main/java/com/onthegomap/flatmap/mbiles/MbtilesWriter.java b/flatmap-core/src/main/java/com/onthegomap/flatmap/mbiles/MbtilesWriter.java index 59eada8e..4e24871d 100644 --- a/flatmap-core/src/main/java/com/onthegomap/flatmap/mbiles/MbtilesWriter.java +++ b/flatmap-core/src/main/java/com/onthegomap/flatmap/mbiles/MbtilesWriter.java @@ -1,9 +1,9 @@ package com.onthegomap.flatmap.mbiles; -import com.onthegomap.flatmap.Profile; import com.onthegomap.flatmap.VectorTile; import com.onthegomap.flatmap.collection.FeatureGroup; import com.onthegomap.flatmap.config.FlatmapConfig; +import com.onthegomap.flatmap.config.MbtilesMetadata; import com.onthegomap.flatmap.geo.TileCoord; import com.onthegomap.flatmap.stats.Counter; import com.onthegomap.flatmap.stats.ProgressLoggers; @@ -43,15 +43,14 @@ import org.slf4j.LoggerFactory; public class MbtilesWriter { private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class); - + private static final long MAX_FEATURES_PER_BATCH = 10_000; + private static final long MAX_TILES_PER_BATCH = 1_000; private final Counter.Readable featuresProcessed; private final Counter memoizedTiles; private final Mbtiles db; private final FlatmapConfig config; - private final Profile profile; private final Stats stats; private final LayerStats layerStats; - private final Counter.Readable[] tilesByZoom; private final Counter.Readable[] totalTileSizesByZoom; private final LongAccumulator[] maxTileSizesByZoom; @@ -59,13 +58,14 @@ public class MbtilesWriter { private final AtomicReference lastTileWritten = new AtomicReference<>(); private final LongAccumulator maxBatchLength = new LongAccumulator(Long::max, 0); private final LongAccumulator minBatchLength = new LongAccumulator(Long::min, Integer.MAX_VALUE); + private final MbtilesMetadata mbtilesMetadata; - MbtilesWriter(FeatureGroup features, Mbtiles db, FlatmapConfig config, Profile profile, Stats stats, - LayerStats layerStats) { + private MbtilesWriter(FeatureGroup features, Mbtiles db, FlatmapConfig config, MbtilesMetadata mbtilesMeatadata, + Stats stats, LayerStats layerStats) { this.features = features; this.db = db; this.config = config; - this.profile = profile; + this.mbtilesMetadata = mbtilesMeatadata; this.stats = stats; this.layerStats = layerStats; tilesByZoom = IntStream.rangeClosed(0, config.maxzoom()) @@ -87,20 +87,20 @@ public class MbtilesWriter { } /** Reads all {@code features}, encodes them in parallel, and writes to {@code outputPath}. */ - public static void writeOutput(FeatureGroup features, Path outputPath, Profile profile, FlatmapConfig config, - Stats stats) { + public static void writeOutput(FeatureGroup features, Path outputPath, MbtilesMetadata mbtilesMetadata, + FlatmapConfig config, Stats stats) { try (Mbtiles output = Mbtiles.newWriteToFileDatabase(outputPath)) { - writeOutput(features, output, () -> FileUtils.fileSize(outputPath), profile, config, stats); + writeOutput(features, output, () -> FileUtils.fileSize(outputPath), mbtilesMetadata, config, stats); } catch (IOException e) { throw new IllegalStateException("Unable to write to " + outputPath, e); } } /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */ - public static void writeOutput(FeatureGroup features, Mbtiles output, DiskBacked fileSize, Profile profile, - FlatmapConfig config, Stats stats) { + public static void writeOutput(FeatureGroup features, Mbtiles output, DiskBacked fileSize, + MbtilesMetadata mbtilesMetadata, FlatmapConfig config, Stats stats) { var timer = stats.startStage("mbtiles"); - MbtilesWriter writer = new MbtilesWriter(features, output, config, profile, stats, + MbtilesWriter writer = new MbtilesWriter(features, output, config, mbtilesMetadata, stats, features.layerStats()); var pipeline = WorkerPipeline.start("mbtiles", stats); @@ -167,6 +167,14 @@ public class MbtilesWriter { timer.stop(); } + private static byte[] gzipCompress(byte[] uncompressedData) throws IOException { + var bos = new ByteArrayOutputStream(uncompressedData.length); + try (var gzipOS = new GZIPOutputStream(bos)) { + gzipOS.write(uncompressedData); + } + return bos.toByteArray(); + } + private String getLastTileLogDetails() { TileCoord lastTile = lastTileWritten.get(); String blurb; @@ -189,37 +197,6 @@ public class MbtilesWriter { return "last tile: " + blurb; } - /** - * Container for a batch of tiles to be processed together in the encoder and writer threads. - *

- * The cost of encoding a tile may vary dramatically by its size (depending on the profile) so batches are sized - * dynamically to put as little as 1 large tile, or as many as 10,000 small tiles in a batch to keep encoding threads - * busy. - * - * @param in the tile data to encode - * @param out the future that encoder thread completes to hand finished tile off to writer thread - */ - private static record TileBatch( - List in, - CompletableFuture> out - ) { - - TileBatch() { - this(new ArrayList<>(), new CompletableFuture<>()); - } - - public int size() { - return in.size(); - } - - public boolean isEmpty() { - return in.isEmpty(); - } - } - - private static final long MAX_FEATURES_PER_BATCH = 10_000; - private static final long MAX_TILES_PER_BATCH = 1_000; - private void readFeaturesAndBatch(Consumer next) { int currentZoom = Integer.MIN_VALUE; TileBatch batch = new TileBatch(); @@ -307,12 +284,12 @@ public class MbtilesWriter { } db.metadata() - .setName(profile.name()) + .setName(mbtilesMetadata.name()) .setFormat("pbf") - .setDescription(profile.description()) - .setAttribution(profile.attribution()) - .setVersion(profile.version()) - .setType(profile.isOverlay() ? "overlay" : "baselayer") + .setDescription(mbtilesMetadata.description()) + .setAttribution(mbtilesMetadata.attribution()) + .setVersion(mbtilesMetadata.version()) + .setType(mbtilesMetadata.type()) .setBoundsAndCenter(config.bounds().latLon()) .setMinzoom(config.minzoom()) .setMaxzoom(config.maxzoom()) @@ -384,11 +361,31 @@ public class MbtilesWriter { return Stream.of(tilesByZoom).mapToLong(c -> c.get()).sum(); } - private static byte[] gzipCompress(byte[] uncompressedData) throws IOException { - var bos = new ByteArrayOutputStream(uncompressedData.length); - try (var gzipOS = new GZIPOutputStream(bos)) { - gzipOS.write(uncompressedData); + /** + * Container for a batch of tiles to be processed together in the encoder and writer threads. + *

+ * The cost of encoding a tile may vary dramatically by its size (depending on the profile) so batches are sized + * dynamically to put as little as 1 large tile, or as many as 10,000 small tiles in a batch to keep encoding threads + * busy. + * + * @param in the tile data to encode + * @param out the future that encoder thread completes to hand finished tile off to writer thread + */ + private static record TileBatch( + List in, + CompletableFuture> out + ) { + + TileBatch() { + this(new ArrayList<>(), new CompletableFuture<>()); + } + + public int size() { + return in.size(); + } + + public boolean isEmpty() { + return in.isEmpty(); } - return bos.toByteArray(); } } diff --git a/flatmap-core/src/test/java/com/onthegomap/flatmap/FlatmapTests.java b/flatmap-core/src/test/java/com/onthegomap/flatmap/FlatmapTests.java index f9eb42f9..54bde648 100644 --- a/flatmap-core/src/test/java/com/onthegomap/flatmap/FlatmapTests.java +++ b/flatmap-core/src/test/java/com/onthegomap/flatmap/FlatmapTests.java @@ -14,6 +14,7 @@ import com.onthegomap.flatmap.collection.FeatureGroup; import com.onthegomap.flatmap.collection.LongLongMap; import com.onthegomap.flatmap.config.Arguments; import com.onthegomap.flatmap.config.FlatmapConfig; +import com.onthegomap.flatmap.config.MbtilesMetadata; import com.onthegomap.flatmap.geo.GeoUtils; import com.onthegomap.flatmap.geo.GeometryException; import com.onthegomap.flatmap.geo.TileCoord; @@ -120,7 +121,8 @@ public class FlatmapTests { runner.run(featureGroup, profile, config); featureGroup.prepare(); try (Mbtiles db = Mbtiles.newInMemoryDatabase()) { - MbtilesWriter.writeOutput(featureGroup, db, () -> 0L, profile, config, stats); + MbtilesWriter.writeOutput(featureGroup, db, () -> 0L, new MbtilesMetadata(profile, config.arguments()), config, + stats); var tileMap = TestUtils.getTileMap(db); tileMap.values().forEach(fs -> fs.forEach(f -> f.geometry().validate())); return new FlatmapResults(tileMap, db.metadata().getAll()); @@ -237,6 +239,31 @@ public class FlatmapTests { ); } + @Test + public void testOverrideMetadata() throws Exception { + var results = runWithReaderFeatures( + Map.of( + "threads", "1", + "mbtiles_name", "mbtiles_name", + "mbtiles_description", "mbtiles_description", + "mbtiles_attribution", "mbtiles_attribution", + "mbtiles_version", "mbtiles_version", + "mbtiles_type", "mbtiles_type" + ), + List.of(), + (sourceFeature, features) -> { + } + ); + assertEquals(Map.of(), results.tiles); + assertSubmap(Map.of( + "name", "mbtiles_name", + "description", "mbtiles_description", + "attribution", "mbtiles_attribution", + "version", "mbtiles_version", + "type", "mbtiles_type" + ), results.metadata); + } + @Test public void testSinglePoint() throws Exception { double x = 0.5 + Z14_WIDTH / 2; diff --git a/flatmap-examples/src/main/java/com/onthegomap/flatmap/examples/ToiletsOverlayLowLevelApi.java b/flatmap-examples/src/main/java/com/onthegomap/flatmap/examples/ToiletsOverlayLowLevelApi.java index 980d347f..3925875c 100644 --- a/flatmap-examples/src/main/java/com/onthegomap/flatmap/examples/ToiletsOverlayLowLevelApi.java +++ b/flatmap-examples/src/main/java/com/onthegomap/flatmap/examples/ToiletsOverlayLowLevelApi.java @@ -6,6 +6,7 @@ import com.onthegomap.flatmap.collection.FeatureGroup; import com.onthegomap.flatmap.collection.LongLongMap; import com.onthegomap.flatmap.config.Arguments; import com.onthegomap.flatmap.config.FlatmapConfig; +import com.onthegomap.flatmap.config.MbtilesMetadata; import com.onthegomap.flatmap.mbiles.MbtilesWriter; import com.onthegomap.flatmap.reader.osm.OsmInputFile; import com.onthegomap.flatmap.reader.osm.OsmReader; @@ -48,9 +49,12 @@ public class ToiletsOverlayLowLevelApi { Stats stats = Stats.inMemory(); Profile profile = new ToiletsOverlay(); - // use default settings, but allow overrides from -Dkey=value jvm arguments + // use default settings, but only allow overrides from -Dkey=value jvm arguments FlatmapConfig config = FlatmapConfig.from(Arguments.fromJvmProperties()); + // extract mbtiles metadata from profile + MbtilesMetadata mbtilesMetadata = new MbtilesMetadata(profile); + // overwrite output each time FileUtils.deleteFile(output); // make sure temp directories exist @@ -102,7 +106,7 @@ public class ToiletsOverlayLowLevelApi { // then process rendered features, grouped by tile, encoding them into binary vector tile format // and writing to the output mbtiles file. - MbtilesWriter.writeOutput(featureGroup, output, profile, config, stats); + MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats); // dump recorded timings at the end stats.printSummary();