diff --git a/NOTICE.md b/NOTICE.md index 6b3e9876..f30d7742 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -46,10 +46,10 @@ Additionally, the `planetiler-basemap` module is based on [OpenMapTiles](https:/ ## Data -|source | license | used as default | included in repo | -|-------|---------|-----------------|------------------| -| OpenStreetMap (OSM) data | [ODBL](https://www.openstreetmap.org/copyright) | yes | yes -| Natural Earth | [public domain](https://www.naturalearthdata.com/about/terms-of-use/) | yes | yes -| OSM Lakelines | [MIT](https://github.com/lukasmartinelli/osm-lakelines), data from OSM [ODBL](https://www.openstreetmap.org/copyright) | yes | no -| OSM Water Polygons | [acknowledgement](https://osmdata.openstreetmap.de/info/license.html), data from OSM [ODBL](https://www.openstreetmap.org/copyright) | yes | yes -| Wikidata name translations | [CCO](https://www.wikidata.org/wiki/Wikidata:Licensing) | no | no +| source | license | used as default | included in repo | +|----------------------------|--------------------------------------------------------------------------------------------------------------------------------------|-----------------|------------------| +| OpenStreetMap (OSM) data | [ODBL](https://www.openstreetmap.org/copyright) | yes | yes | +| Natural Earth | [public domain](https://www.naturalearthdata.com/about/terms-of-use/) | yes | yes | +| OSM Lakelines | [MIT](https://github.com/lukasmartinelli/osm-lakelines), data from OSM [ODBL](https://www.openstreetmap.org/copyright) | yes | no | +| OSM Water Polygons | [acknowledgement](https://osmdata.openstreetmap.de/info/license.html), data from OSM [ODBL](https://www.openstreetmap.org/copyright) | yes | yes | +| Wikidata name translations | [CCO](https://www.wikidata.org/wiki/Wikidata:Licensing) | no | no | diff --git a/README.md b/README.md index 48833945..4f485e79 100644 --- a/README.md +++ b/README.md @@ -118,12 +118,12 @@ See the [planetiler-examples](planetiler-examples) project. Some example runtimes (excluding downloading resources): -| Input | Profile | Machine | Time | mbtiles size | Logs | -| --- | --- | --- | --- | --- | --- | -| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap | DO 16cpu 128GB | 3h9m cpu:42h1m avg:13.3 | 99GB | [logs](planet-logs/v0.1.0-planet-do-16cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-do-16cpu-128gb.nps) | -| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries (67GB) | Basemap | DO 16cpu 128GB | 3h13m cpu:43h40m avg:13.5 | 101GB | [logs](planet-logs/v0.1.0-daylight-do-16cpu-128gb.txt) | -| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap (without z13 building merge) | Linode 50cpu 128GB | 1h9m cpu:24h36m avg:21.2 | 97GB | [logs](planet-logs/v0.1.0-planet-linode-50cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-linode-50cpu-128gb.nps) | -| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB) | 59m cpu:27h6m avg:27.4 | 97GB | [logs](planet-logs/v0.1.0-planet-c5ad-64cpu-128gb.txt) | +| Input | Profile | Machine | Time | mbtiles size | Logs | +|-------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------|-----------------------------|---------------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------| +| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap | DO 16cpu 128GB | 3h9m cpu:42h1m avg:13.3 | 99GB | [logs](planet-logs/v0.1.0-planet-do-16cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-do-16cpu-128gb.nps) | +| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries (67GB) | Basemap | DO 16cpu 128GB | 3h13m cpu:43h40m avg:13.5 | 101GB | [logs](planet-logs/v0.1.0-daylight-do-16cpu-128gb.txt) | +| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap (without z13 building merge) | Linode 50cpu 128GB | 1h9m cpu:24h36m avg:21.2 | 97GB | [logs](planet-logs/v0.1.0-planet-linode-50cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-linode-50cpu-128gb.nps) | +| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB) | 59m cpu:27h6m avg:27.4 | 97GB | [logs](planet-logs/v0.1.0-planet-c5ad-64cpu-128gb.txt) | ## Alternatives diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/BasemapProfile.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/BasemapProfile.java index a9cc055f..f5a8308d 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/BasemapProfile.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/BasemapProfile.java @@ -244,7 +244,7 @@ public class BasemapProfile extends ForwardingProfile { */ public interface IgnoreWikidata {} - private static record RowDispatch( + private record RowDispatch( Tables.Constructor constructor, List> handlers ) {} diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/Generate.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/Generate.java index f9e13971..c019158a 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/Generate.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/Generate.java @@ -387,7 +387,7 @@ public class Generate { } /** The {@code rowClass} of an imposm3 table row and its constructor coerced to a {@link Constructor}. */ - public static record RowClassAndConstructor( + public record RowClassAndConstructor( Class rowClass, Constructor create ) {} @@ -401,7 +401,7 @@ public class Generate { } /** The {@code handlerClass} of a layer handler and it's {@code process} method coerced to a {@link RowHandler}. */ - public static record RowHandlerAndClass( + public record RowHandlerAndClass( Class handlerClass, RowHandler handler ) {} @@ -436,7 +436,7 @@ public class Generate { tablesClass.append(""" /** An OSM element that would appear in the {@code %s} table generated by imposm3. */ - public static record %s(%s) implements Row, %s { + public record %s(%s) implements Row, %s { public %s(SourceFeature source, String mappingKey) { this(%s); } diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java index 9f706213..13ce441f 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java @@ -421,7 +421,7 @@ public class Boundary implements .orElse(null); } - private static record BorderingRegions(Long left, Long right) { + private record BorderingRegions(Long left, Long right) { public static BorderingRegions empty() { return new BorderingRegions(null, null); @@ -432,7 +432,7 @@ public class Boundary implements * Minimal set of information extracted from a boundary relation to be used when processing each way in that * relation. */ - private static record BoundaryRelation( + private record BoundaryRelation( long id, int adminLevel, boolean disputed, @@ -454,7 +454,7 @@ public class Boundary implements } /** Information to hold onto from processing a way in a boundary relation to determine the left/right region ID later. */ - private static record CountryBoundaryComponent( + private record CountryBoundaryComponent( int adminLevel, boolean disputed, boolean maritime, diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Building.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Building.java index 029af367..dd2e61fe 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Building.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Building.java @@ -182,7 +182,7 @@ public class Building implements return (mergeZ13Buildings && zoom == 13) ? FeatureMerge.mergeNearbyPolygons(items, 4, 4, 0.5, 0.5) : items; } - private static record BuildingRelationInfo(long id) implements OsmRelationInfo { + private record BuildingRelationInfo(long id) implements OsmRelationInfo { @Override public long estimateMemoryUsageBytes() { diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java index 4efe834e..f4d3d70a 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java @@ -412,7 +412,7 @@ public class Place implements * Information extracted from a natural earth geographic region that will be inspected when joining with OpenStreetMap * data. */ - private static record NaturalEarthRegion(String name, int rank) { + private record NaturalEarthRegion(String name, int rank) { NaturalEarthRegion(String name, int maxRank, double... ranks) { this(name, (int) Math.ceil(DoubleStream.of(ranks).average().orElse(maxRank))); @@ -423,6 +423,6 @@ public class Place implements * Information extracted from a natural earth place label that will be inspected when joining with OpenStreetMap * data. */ - private static record NaturalEarthPoint(String name, String wikidata, int scaleRank, Set names) {} + private record NaturalEarthPoint(String name, String wikidata, int scaleRank, Set names) {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java index b23024af..d3f5b83c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java @@ -175,7 +175,7 @@ public class FeatureCollector implements Iterable { /** * Creates new feature collector instances for each source feature that we encounter. */ - public static record Factory(PlanetilerConfig config, Stats stats) { + public record Factory(PlanetilerConfig config, Stats stats) { public FeatureCollector get(SourceFeature source) { return new FeatureCollector(source, config, stats); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 96a82641..010bbbf8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -583,14 +583,14 @@ public class Planetiler { } } - private static record Stage(String id, List details, RunnableThatThrows task) { + private record Stage(String id, List details, RunnableThatThrows task) { Stage(String id, String description, RunnableThatThrows task) { this(id, List.of(id + ": " + description), task); } } - private static record ToDownload(String id, String url, Path path) {} + private record ToDownload(String id, String url, Path path) {} - private static record InputPath(String id, Path path) {} + private record InputPath(String id, Path path) {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index b9075b27..621b3415 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -514,7 +514,7 @@ public class VectorTile { * {@code scale == 2} the extent is 4x{@link #EXTENT}. Geometries must be scaled back to 0 using {@link #unscale()} * before outputting to mbtiles. */ - public static record VectorGeometry(int[] commands, GeometryType geomType, int scale) { + public record VectorGeometry(int[] commands, GeometryType geomType, int scale) { public VectorGeometry { if (scale < 0) { @@ -576,7 +576,7 @@ public class VectorTile { * populated when this feature was deserialized from {@link FeatureGroup}, not when parsed from a tile * since vector tile schema does not encode group. */ - public static record Feature( + public record Feature( String layer, long id, VectorGeometry geometry, @@ -770,7 +770,7 @@ public class VectorTile { } } - private static final record EncodedFeature(IntArrayList tags, long id, VectorGeometry geometry) { + private record EncodedFeature(IntArrayList tags, long id, VectorGeometry geometry) { EncodedFeature(Feature in) { this(new IntArrayList(), in.id(), in.geometry()); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java index 336f8409..2b42dfd1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java @@ -308,13 +308,13 @@ public record MultiExpression(List> expressions) { } /** An expression/value pair with unique ID to store whether we evaluated it yet. */ - private static record EntryWithId(T result, Expression expression, int id) {} + private record EntryWithId(T result, Expression expression, int id) {} /** * An {@code expression} to evaluate on input elements and {@code result} value to return when the element matches. */ - public static record Entry(T result, Expression expression) {} + public record Entry(T result, Expression expression) {} /** The result when an expression matches, along with the input element tag {@code keys} that triggered the match. */ - public static record Match(T match, List keys) {} + public record Match(T match, List keys) {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java index 68e1f26b..3ef9c351 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java @@ -480,7 +480,7 @@ public class GeoUtils { } /** Helper class to sort polygons by area of their outer shell. */ - private static record PolyAndArea(Polygon poly, double area) implements Comparable { + private record PolyAndArea(Polygon poly, double area) implements Comparable { PolyAndArea(Polygon poly) { this(poly, Area.ofRing(poly.getExteriorRing().getCoordinateSequence())); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java index 41f76f3d..21c2a51a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java @@ -13,7 +13,7 @@ public enum GeometryType { POLYGON(VectorTileProto.Tile.GeomType.POLYGON, 4); private final VectorTileProto.Tile.GeomType protobufType; - private int minPoints; + private final int minPoints; GeometryType(VectorTileProto.Tile.GeomType protobufType, int minPoints) { this.protobufType = protobufType; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PointIndex.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PointIndex.java index 14e8a548..c1567917 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PointIndex.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PointIndex.java @@ -21,7 +21,7 @@ import org.locationtech.jts.index.strtree.STRtree; @ThreadSafe public class PointIndex { - private static record GeomWithData(Coordinate coord, T data) {} + private record GeomWithData(Coordinate coord, T data) {} private final STRtree index = new STRtree(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java index ec0b7024..bd644d71 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java @@ -19,7 +19,7 @@ import org.locationtech.jts.index.strtree.STRtree; @ThreadSafe public class PolygonIndex { - private static record GeomWithData(Polygon poly, T data) {} + private record GeomWithData(Polygon poly, T data) {} private final STRtree index = new STRtree(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java index e746f16b..0c4bf89d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java @@ -54,7 +54,7 @@ public class TileExtents implements Predicate { * X/Y extents within a given zoom level. {@code minX} and {@code minY} are inclusive and {@code maxX} and {@code * maxY} are exclusive. */ - public static record ForZoom(int minX, int minY, int maxX, int maxY) { + public record ForZoom(int minX, int minY, int maxX, int maxY) { public boolean test(int x, int y) { return testX(x) && testY(y); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index a98a1e1c..38151cb6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -239,7 +239,7 @@ public final class Mbtiles implements Closeable { * @see MBtiles * schema */ - public static record MetadataJson( + public record MetadataJson( @JsonProperty("vector_layers") List vectorLayers ) { @@ -277,7 +277,7 @@ public final class Mbtiles implements Closeable { } } - public static record VectorLayer( + public record VectorLayer( @JsonProperty("id") String id, @JsonProperty("fields") Map fields, @JsonProperty("description") Optional description, @@ -312,7 +312,7 @@ public final class Mbtiles implements Closeable { } /** Contents of a row of the tiles table. */ - public static record TileEntry(TileCoord tile, byte[] bytes) implements Comparable { + public record TileEntry(TileCoord tile, byte[] bytes) implements Comparable { @Override public boolean equals(Object o) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java index 56c06224..57b9445b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java @@ -1,6 +1,7 @@ package com.onthegomap.planetiler.mbtiles; import static com.onthegomap.planetiler.util.Gzip.gzip; +import static com.onthegomap.planetiler.worker.Worker.joinFutures; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.collection.FeatureGroup; @@ -163,10 +164,8 @@ public class MbtilesWriter { .newLine() .add(writer::getLastTileLogDetails); - encodeBranch.awaitAndLog(loggers, config.logInterval()); - if (writeBranch != null) { - writeBranch.awaitAndLog(loggers, config.logInterval()); - } + var doneFuture = writeBranch == null ? encodeBranch.done() : joinFutures(writeBranch.done(), encodeBranch.done()); + loggers.awaitAndLog(doneFuture, config.logInterval()); writer.printTileStats(); timer.stop(); } @@ -362,7 +361,7 @@ public class MbtilesWriter { * @param in the tile data to encode * @param out the future that encoder thread completes to hand finished tile off to writer thread */ - private static record TileBatch( + private record TileBatch( List in, CompletableFuture> out ) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java index 4d312099..50fdcb8d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java @@ -54,11 +54,10 @@ public class Verify { * @param envelope lat/lon bounding box to limit check * @param clazz {@link Geometry} subclass to limit * @return number of features found - * @throws IOException if an error occurs reading from the file * @throws GeometryException if an invalid geometry is encountered */ public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map attrs, Envelope envelope, - Class clazz) throws IOException, GeometryException { + Class clazz) throws GeometryException { int num = 0; for (var tileCoord : db.getAllTileCoords()) { Envelope tileEnv = new Envelope(); @@ -161,7 +160,7 @@ public class Verify { try { int count = getNumFeatures(mbtiles, layer, zoom, tags, bounds, geometryType); return count >= minCount ? Optional.empty() : Optional.of("found " + count); - } catch (IOException | GeometryException e) { + } catch (GeometryException e) { return Optional.of("error: " + e); } }); @@ -232,5 +231,5 @@ public class Verify { return checks.stream().filter(check -> check.error.isPresent()).count(); } - public static record Check(String name, Optional error) {} + public record Check(String name, Optional error) {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmElement.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmElement.java index a6f38e04..a717c494 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmElement.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmElement.java @@ -79,7 +79,7 @@ public interface OsmElement extends WithTags { } /** A node, way, or relation contained in a relation with an optional "role" to clarify the purpose of each member. */ - public static record Member( + public record Member( Type type, long ref, String role diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmInputFile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmInputFile.java index e0dec106..bcb5051b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmInputFile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmInputFile.java @@ -115,7 +115,7 @@ public class OsmInputFile implements Bounds.Provider, OsmSource { return next -> readTo(next, poolName, threads); } - private static record ReaderElementSink(Consumer queue) implements Sink { + private record ReaderElementSink(Consumer queue) implements Sink { @Override public void process(ReaderElement readerElement) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java index d857bcb4..b207a522 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java @@ -446,7 +446,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate { * @param role "role" of the relation member * @param relation user-provided data about the relation from pass1 */ - public static record RelationMember(String role, T relation) {} + public record RelationMember(String role, T relation) {} /** Raw relation membership data that gets encoded/decoded into a long. */ private record RelationMembership(String role, long relationId) {} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/RenderedFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/RenderedFeature.java index 9a55ad6b..f9993017 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/RenderedFeature.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/RenderedFeature.java @@ -32,5 +32,5 @@ public record RenderedFeature( * this is the 4th feature in a group with lowest sort-key then the feature is included if {@code limit * <= 4} */ - public static record Group(long group, int limit) {} + public record Group(long group, int limit) {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java index bf14738b..b54c849f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java @@ -18,12 +18,16 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import javax.management.NotificationEmitter; import javax.management.openmbean.CompositeData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A collection of utilities to gather runtime information about the JVM. */ public class ProcessInfo { + private static final Logger LOGGER = LoggerFactory.getLogger(ProcessInfo.class); + // listen on GC events to track memory pool sizes after each GC private static final AtomicReference> postGcMemoryUsage = new AtomicReference<>(Map.of()); @@ -42,6 +46,13 @@ public class ProcessInfo { }, null, null); } } + + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + if (threadBean.isThreadContentionMonitoringSupported()) { + ManagementFactory.getThreadMXBean().setThreadContentionMonitoringEnabled(true); + } else { + LOGGER.debug("Thread contention monitoring not supported, will not have access to waiting/blocking time stats."); + } } /** @@ -94,9 +105,12 @@ public class ProcessInfo { } /** Processor usage statistics for a thread. */ - public static record ThreadState(String name, Duration cpuTime, Duration userTime, long id) { + public record ThreadState( + String name, Duration cpuTime, Duration userTime, Duration waiting, Duration blocking, long id + ) { - public static final ThreadState DEFAULT = new ThreadState("", Duration.ZERO, Duration.ZERO, -1); + public static final ThreadState DEFAULT = new ThreadState("", Duration.ZERO, Duration.ZERO, Duration.ZERO, + Duration.ZERO, -1); } @@ -134,6 +148,8 @@ public class ProcessInfo { thread.getThreadName(), Duration.ofNanos(threadMXBean.getThreadCpuTime(thread.getThreadId())), Duration.ofNanos(threadMXBean.getThreadUserTime(thread.getThreadId())), + Duration.ofMillis(thread.getWaitedTime()), + Duration.ofMillis(thread.getBlockedTime()), thread.getThreadId() )); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java index 7922c2b9..324c1905 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java @@ -296,7 +296,7 @@ public class Downloader { var inputStream = (ranges || range.start > 0) ? openStreamRange(canonicalUrl, range.start, range.end) : openStream(canonicalUrl); - var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress); + var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress) ) { // ensure this file has been allocated up to the start of this block fileChannel.write(ByteBuffer.allocate(1), range.start); @@ -325,9 +325,9 @@ public class Downloader { .header(USER_AGENT, config.httpUserAgent()); } - static record ResourceMetadata(Optional redirect, String canonicalUrl, long size, boolean acceptRange) {} + record ResourceMetadata(Optional redirect, String canonicalUrl, long size, boolean acceptRange) {} - static record ResourceToDownload( + record ResourceToDownload( String id, String url, Path output, CompletableFuture metadata, AtomicLong progress ) { @@ -347,7 +347,7 @@ public class Downloader { /** * Wrapper for a {@link ReadableByteChannel} that captures progress information. */ - private static record ProgressChannel(ReadableByteChannel inner, AtomicLong progress) implements ReadableByteChannel { + private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress) implements ReadableByteChannel { @Override public int read(ByteBuffer dst) throws IOException { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java index a674032a..b7fcb697 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java @@ -100,9 +100,9 @@ public class Geofabrik { } } - static record PropertiesJson(String id, String parent, String name, Map urls) {} + record PropertiesJson(String id, String parent, String name, Map urls) {} - static record FeatureJson(PropertiesJson properties) {} + record FeatureJson(PropertiesJson properties) {} - static record IndexJson(List features) {} + record IndexJson(List features) {} } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index b9c73501..235db67f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -309,7 +309,7 @@ public class TestUtils { } } - public static record NormGeometry(Geometry geom) implements GeometryComparision { + public record NormGeometry(Geometry geom) implements GeometryComparision { @Override public boolean equals(Object o) { @@ -327,7 +327,7 @@ public class TestUtils { } } - private static record ExactGeometry(Geometry geom) implements GeometryComparision { + private record ExactGeometry(Geometry geom) implements GeometryComparision { @Override public boolean equals(Object o) { @@ -345,7 +345,7 @@ public class TestUtils { } } - public static record TopoGeometry(Geometry geom) implements GeometryComparision { + public record TopoGeometry(Geometry geom) implements GeometryComparision { @Override public boolean equals(Object o) { @@ -363,7 +363,7 @@ public class TestUtils { } } - public static record ComparableFeature( + public record ComparableFeature( GeometryComparision geometry, Map attrs ) {} @@ -490,18 +490,18 @@ public class TestUtils { } @JacksonXmlRootElement(localName = "node") - public static record Node( + public record Node( long id, double lat, double lon ) {} @JacksonXmlRootElement(localName = "nd") - public static record NodeRef( + public record NodeRef( long ref ) {} - public static record Tag(String k, String v) {} + public record Tag(String k, String v) {} - public static record Way( + public record Way( long id, @JacksonXmlProperty(localName = "nd") @JacksonXmlElementWrapper(useWrapping = false) @@ -512,12 +512,12 @@ public class TestUtils { ) {} @JacksonXmlRootElement(localName = "member") - public static record RelationMember( + public record RelationMember( String type, long ref, String role ) {} @JacksonXmlRootElement(localName = "relation") - public static record Relation( + public record Relation( long id, @JacksonXmlProperty(localName = "member") @JacksonXmlElementWrapper(useWrapping = false) @@ -528,7 +528,7 @@ public class TestUtils { ) {} // @JsonIgnoreProperties(ignoreUnknown = true) - public static record OsmXml( + public record OsmXml( String version, String generator, String copyright, @@ -585,7 +585,7 @@ public class TestUtils { int num = Verify.getNumFeatures(db, layer, zoom, attrs, envelope, clazz); assertEquals(expected, num, "z%d features in %s".formatted(zoom, layer)); - } catch (IOException | GeometryException e) { + } catch (GeometryException e) { fail(e); } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java index 94565e50..bbb2bce2 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java @@ -80,7 +80,7 @@ public class FeatureGroupTest { return map; } - private static record Feature(Map attrs, Geometry geom) {} + private record Feature(Map attrs, Geometry geom) {} @Test public void testPutPoints() { diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java index 1326a8cc..4b3b1592 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java @@ -42,7 +42,7 @@ public class BikeRouteOverlay implements Profile { */ // Minimal container for data we extract from OSM bicycle route relations. This is held in RAM so keep it small. - private static record RouteRelationInfo( + private record RouteRelationInfo( // OSM ID of the relation (required): @Override long id, // Values for tags extracted from the OSM relation: