From 8a6997f9dcf618e3710172bf664d284d73f1ecdb Mon Sep 17 00:00:00 2001 From: Michael Barry Date: Tue, 1 Mar 2022 08:43:19 -0500 Subject: [PATCH] Remove graphhopper dependency (#101) --- NOTICE.md | 2 + README.md | 4 +- .../planetiler/basemap/layers/Boundary.java | 4 +- .../basemap/layers/MountainPeak.java | 4 +- .../planetiler/basemap/layers/Park.java | 4 +- .../planetiler/basemap/layers/Place.java | 4 +- .../planetiler/basemap/layers/Poi.java | 4 +- .../basemap/layers/TransportationName.java | 4 +- .../planetiler/basemap/layers/WaterName.java | 4 +- .../planetiler/basemap/layers/Waterway.java | 4 +- planetiler-core/pom.xml | 12 ++-- .../onthegomap/planetiler/FeatureMerge.java | 4 +- .../planetiler/collection/FeatureGroup.java | 3 +- .../planetiler/collection/Hppc.java | 44 ++++++++++++++ .../collection/LongLongMultimap.java | 8 +-- .../reader/osm/OsmMultipolygon.java | 7 +-- .../planetiler/reader/osm/OsmReader.java | 11 ++-- .../planetiler/render/TiledGeometry.java | 8 +-- .../planetiler/stats/ProcessInfo.java | 5 ++ .../planetiler/stats/ProgressLoggers.java | 5 +- .../planetiler/util/Imposm3Parsers.java | 4 +- .../planetiler/util/MemoryEstimator.java | 57 ++----------------- .../onthegomap/planetiler/util/Wikidata.java | 18 +++--- .../planetiler/FeatureMergeTest.java | 10 ++-- 24 files changed, 118 insertions(+), 116 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/collection/Hppc.java diff --git a/NOTICE.md b/NOTICE.md index d0b427b6..cad9af51 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -20,6 +20,8 @@ The `planetiler-core` module includes the following software: - org.xerial:sqlite-jdbc (Apache license) - com.ibm.icu:icu4j ([ICU license](https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE)) - com.google.guava:guava (Apache license) + - org.openstreetmap.osmosis:osmosis-osm-binary (LGPL 3.0) + - com.carrotsearch:hppc (Apache license) - Adapted code: - `DouglasPeuckerSimplifier` from [JTS](https://github.com/locationtech/jts) (EDL) - `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license) diff --git a/README.md b/README.md index 4f485e79..2d8c64d4 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ Planetiler is made possible by these awesome open source projects: and [reference implementation](https://github.com/openmaptiles/openmaptiles) that the [basemap profile](planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers) is based on -- [Graphhopper](https://www.graphhopper.com/) for utilities to process OpenStreetMap data in Java +- [Graphhopper](https://www.graphhopper.com/) for basis of utilities to process OpenStreetMap data in Java - [JTS Topology Suite](https://github.com/locationtech/jts) for working with vector geometries - [Geotools](https://github.com/geotools/geotools) for shapefile processing - [SQLite JDBC Driver](https://github.com/xerial/sqlite-jdbc) for reading Natural Earth data and writing MBTiles files @@ -239,6 +239,8 @@ Planetiler is made possible by these awesome open source projects: - [imposm3](https://github.com/omniscale/imposm3) for the basis of [OSM multipolygon processing](planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java) and [tag parsing utilities](planetiler-core/src/main/java/com/onthegomap/planetiler/util/Imposm3Parsers.java) +- [HPPC](http://labs.carrotsearch.com/) for high-performance primitive Java collections +- [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) for Java utilities to parse OpenStreetMap data See [NOTICE.md](NOTICE.md) for a full list and license details. diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java index 13ce441f..77247ce7 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Boundary.java @@ -42,12 +42,12 @@ import static java.util.stream.Collectors.counting; import static java.util.stream.Collectors.groupingBy; import com.carrotsearch.hppc.LongObjectMap; -import com.graphhopper.coll.GHLongObjectHashMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.FeatureMerge; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; @@ -390,7 +390,7 @@ public class Boundary implements /** Returns a map from region ID to prepared geometry optimized for {@code contains} queries. */ private LongObjectMap prepareRegionPolygons() { LOGGER.info("Creating polygons for " + regionGeometries.size() + " boundaries"); - LongObjectMap countryBoundaries = new GHLongObjectHashMap<>(); + LongObjectMap countryBoundaries = Hppc.newLongObjectHashMap(); for (var entry : regionGeometries.entrySet()) { Long regionId = entry.getKey(); Polygonizer polygonizer = new Polygonizer(); diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/MountainPeak.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/MountainPeak.java index 67869123..d5114a1f 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/MountainPeak.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/MountainPeak.java @@ -38,7 +38,6 @@ package com.onthegomap.planetiler.basemap.layers; import static com.onthegomap.planetiler.basemap.util.Utils.elevationTags; import static com.onthegomap.planetiler.basemap.util.Utils.nullIfEmpty; -import com.carrotsearch.hppc.LongIntHashMap; import com.carrotsearch.hppc.LongIntMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.VectorTile; @@ -46,6 +45,7 @@ import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.reader.SourceFeature; @@ -171,7 +171,7 @@ public class MountainPeak implements @Override public List postProcess(int zoom, List items) { - LongIntMap groupCounts = new LongIntHashMap(); + LongIntMap groupCounts = Hppc.newLongIntHashMap(); for (int i = 0; i < items.size(); i++) { VectorTile.Feature feature = items.get(i); int gridrank = groupCounts.getOrDefault(feature.group(), 1); diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Park.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Park.java index 40ad4667..9db6831f 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Park.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Park.java @@ -39,7 +39,6 @@ import static com.onthegomap.planetiler.basemap.util.Utils.coalesce; import static com.onthegomap.planetiler.basemap.util.Utils.nullIfEmpty; import static com.onthegomap.planetiler.collection.FeatureGroup.SORT_KEY_BITS; -import com.carrotsearch.hppc.LongIntHashMap; import com.carrotsearch.hppc.LongIntMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.FeatureMerge; @@ -48,6 +47,7 @@ import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; @@ -145,7 +145,7 @@ public class Park implements @Override public List postProcess(int zoom, List items) throws GeometryException { // infer the "rank" attribute from point ordering within each label grid square - LongIntMap counts = new LongIntHashMap(); + LongIntMap counts = Hppc.newLongIntHashMap(); for (VectorTile.Feature feature : items) { if (feature.geometry().geomType() == GeometryType.POINT && feature.hasGroup()) { int count = counts.getOrDefault(feature.group(), 0) + 1; diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java index f4d3d70a..be38d5b1 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Place.java @@ -40,7 +40,6 @@ import static com.onthegomap.planetiler.basemap.util.Utils.nullIfEmpty; import static com.onthegomap.planetiler.basemap.util.Utils.nullOrEmpty; import static com.onthegomap.planetiler.collection.FeatureGroup.SORT_KEY_BITS; -import com.carrotsearch.hppc.LongIntHashMap; import com.carrotsearch.hppc.LongIntMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.VectorTile; @@ -48,6 +47,7 @@ import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; @@ -366,7 +366,7 @@ public class Place implements @Override public List postProcess(int zoom, List items) { // infer the rank field from ordering of the place labels with each label grid square - LongIntMap groupCounts = new LongIntHashMap(); + LongIntMap groupCounts = Hppc.newLongIntHashMap(); for (VectorTile.Feature feature : items) { int gridrank = groupCounts.getOrDefault(feature.group(), 1); groupCounts.put(feature.group(), gridrank + 1); diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Poi.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Poi.java index 46f73a20..49b7a4bb 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Poi.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Poi.java @@ -41,7 +41,6 @@ import static com.onthegomap.planetiler.basemap.util.Utils.nullIfLong; import static com.onthegomap.planetiler.basemap.util.Utils.nullOrEmpty; import static java.util.Map.entry; -import com.carrotsearch.hppc.LongIntHashMap; import com.carrotsearch.hppc.LongIntMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.VectorTile; @@ -49,6 +48,7 @@ import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.expression.MultiExpression; import com.onthegomap.planetiler.stats.Stats; @@ -182,7 +182,7 @@ public class Poi implements @Override public List postProcess(int zoom, List items) { // infer the "rank" field from the order of features within each label grid square - LongIntMap groupCounts = new LongIntHashMap(); + LongIntMap groupCounts = Hppc.newLongIntHashMap(); for (VectorTile.Feature feature : items) { int gridrank = groupCounts.getOrDefault(feature.group(), 1); groupCounts.put(feature.group(), gridrank + 1); diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/TransportationName.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/TransportationName.java index 0b25092e..05c2a7c6 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/TransportationName.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/TransportationName.java @@ -41,7 +41,6 @@ import static com.onthegomap.planetiler.basemap.layers.Transportation.isFootwayO import static com.onthegomap.planetiler.basemap.util.Utils.*; import com.carrotsearch.hppc.LongArrayList; -import com.carrotsearch.hppc.LongByteHashMap; import com.carrotsearch.hppc.LongByteMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.FeatureMerge; @@ -51,6 +50,7 @@ import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.reader.osm.OsmElement; import com.onthegomap.planetiler.stats.Stats; @@ -121,7 +121,7 @@ public class TransportationName implements private final boolean limitMerge; private final PlanetilerConfig config; private Transportation transportation; - private final LongByteMap motorwayJunctionHighwayClasses = new LongByteHashMap(); + private final LongByteMap motorwayJunctionHighwayClasses = Hppc.newLongByteHashMap(); public TransportationName(Translations translations, PlanetilerConfig config, Stats stats) { this.config = config; diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/WaterName.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/WaterName.java index fbe29fa8..9998ceb8 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/WaterName.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/WaterName.java @@ -38,12 +38,12 @@ package com.onthegomap.planetiler.basemap.layers; import static com.onthegomap.planetiler.basemap.util.Utils.nullIfEmpty; import com.carrotsearch.hppc.LongObjectMap; -import com.graphhopper.coll.GHLongObjectHashMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.basemap.BasemapProfile; import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; @@ -84,7 +84,7 @@ public class WaterName implements private static final double LOG2 = Math.log(2); private final Translations translations; // need to synchronize updates from multiple threads - private final LongObjectMap lakeCenterlines = new GHLongObjectHashMap<>(); + private final LongObjectMap lakeCenterlines = Hppc.newLongObjectHashMap(); // may be updated concurrently by multiple threads private final ConcurrentSkipListMap importantMarinePoints = new ConcurrentSkipListMap<>(); private final Stats stats; diff --git a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Waterway.java b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Waterway.java index ce1928e1..8b716ccd 100644 --- a/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Waterway.java +++ b/planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers/Waterway.java @@ -39,7 +39,6 @@ import static com.onthegomap.planetiler.basemap.util.Utils.nullIfEmpty; import com.carrotsearch.hppc.LongObjectHashMap; import com.google.common.util.concurrent.AtomicDouble; -import com.graphhopper.coll.GHLongObjectHashMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.FeatureMerge; import com.onthegomap.planetiler.VectorTile; @@ -48,6 +47,7 @@ import com.onthegomap.planetiler.basemap.generated.OpenMapTilesSchema; import com.onthegomap.planetiler.basemap.generated.Tables; import com.onthegomap.planetiler.basemap.util.LanguageUtils; import com.onthegomap.planetiler.basemap.util.Utils; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.reader.SourceFeature; @@ -98,7 +98,7 @@ public class Waterway implements private final Translations translations; private final PlanetilerConfig config; private final Stats stats; - private final LongObjectHashMap riverRelationLengths = new GHLongObjectHashMap<>(); + private final LongObjectHashMap riverRelationLengths = Hppc.newLongObjectHashMap(); public Waterway(Translations translations, PlanetilerConfig config, Stats stats) { this.config = config; diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index 1715da49..f88891f1 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -16,7 +16,6 @@ - 2.4 26.3 2.17.2 0.15.0 @@ -24,9 +23,14 @@ - com.graphhopper - graphhopper-reader-osm - ${graphhopper.version} + com.carrotsearch + hppc + 0.9.1 + + + org.openstreetmap.osmosis + osmosis-osm-binary + 0.48.3 org.locationtech.jts diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 814c101e..9bc96eaa 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -3,7 +3,7 @@ package com.onthegomap.planetiler; import com.carrotsearch.hppc.IntArrayList; import com.carrotsearch.hppc.IntObjectMap; import com.carrotsearch.hppc.IntStack; -import com.graphhopper.coll.GHIntObjectHashMap; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.geo.DouglasPeuckerSimplifier; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; @@ -394,7 +394,7 @@ public class FeatureMerge { env.expandBy(minDist); envelopeIndex.insert(env, i); } - IntObjectMap result = new GHIntObjectHashMap<>(); + IntObjectMap result = Hppc.newIntObjectHashMap(); for (int _i = 0; _i < geometries.size(); _i++) { int i = _i; Geometry a = geometries.get(i); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index 64eeef2e..fe633773 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -1,7 +1,6 @@ package com.onthegomap.planetiler.collection; import com.carrotsearch.hppc.LongLongHashMap; -import com.graphhopper.coll.GHLongLongHashMap; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.config.PlanetilerConfig; @@ -474,7 +473,7 @@ public final class FeatureGroup implements Consumer, IterableHigh Performance Primitive Collections. + */ +public class Hppc { + + public static IntObjectHashMap newIntObjectHashMap() { + return new IntObjectHashMap<>(10, 0.75); + } + + public static ObjectIntHashMap newObjectIntHashMap() { + return new ObjectIntHashMap<>(10, 0.75); + } + + public static LongLongHashMap newLongLongHashMap() { + return new LongLongHashMap(10, 0.75); + } + + public static LongObjectHashMap newLongObjectHashMap() { + return new LongObjectHashMap<>(10, 0.75); + } + + public static LongObjectHashMap newLongObjectHashMap(int size) { + return new LongObjectHashMap<>(size, 0.75); + } + + public static LongIntHashMap newLongIntHashMap() { + return new LongIntHashMap(10, 0.75); + } + + public static LongByteMap newLongByteHashMap() { + return new LongByteHashMap(10, 0.75); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongLongMultimap.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongLongMultimap.java index a36d36bb..1fa15969 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongLongMultimap.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongLongMultimap.java @@ -4,7 +4,7 @@ import static com.onthegomap.planetiler.util.MemoryEstimator.estimateSize; import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.LongIntHashMap; -import com.graphhopper.util.StopWatch; +import com.onthegomap.planetiler.stats.Timer; import com.onthegomap.planetiler.util.MemoryEstimator; import java.util.Arrays; import org.slf4j.Logger; @@ -81,7 +81,7 @@ public interface LongLongMultimap extends MemoryEstimator.HasEstimate { /** Sort the keys and values arrays by key */ private void doPrepare() { - StopWatch watch = new StopWatch().start(); + Timer timer = Timer.start(); LOGGER.debug("Sorting long long multimap..."); long[] sortedKeys = keys.toArray(); @@ -114,7 +114,7 @@ public interface LongLongMultimap extends MemoryEstimator.HasEstimate { } keys.buffer = sortedKeys; values.buffer = sortedValues; - LOGGER.debug("Sorted long long multimap " + watch.stop()); + LOGGER.debug("Sorted long long multimap " + timer.stop()); } @Override @@ -152,7 +152,7 @@ public interface LongLongMultimap extends MemoryEstimator.HasEstimate { class DenseOrderedHppcMultimap implements LongLongMultimap { private static final LongArrayList EMPTY_LIST = new LongArrayList(); - private final LongIntHashMap keyToValuesIndex = new LongIntHashMap(); + private final LongIntHashMap keyToValuesIndex = Hppc.newLongIntHashMap(); // each block starts with a "length" header then contains that number of entries private final LongArrayList values = new LongArrayList(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java index ba93adfc..4e18f594 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java @@ -17,8 +17,7 @@ import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.LongObjectMap; import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.cursors.LongObjectCursor; -import com.graphhopper.coll.GHLongObjectHashMap; -import com.graphhopper.coll.GHObjectIntHashMap; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import java.util.ArrayList; @@ -95,7 +94,7 @@ public class OsmMultipolygon { * @throws GeometryException if building the polygon fails */ public static Geometry build(List rings) throws GeometryException { - ObjectIntMap coordToId = new GHObjectIntHashMap<>(); + ObjectIntMap coordToId = Hppc.newObjectIntHashMap(); List idToCoord = new ArrayList<>(); int id = 0; List idRings = new ArrayList<>(rings.size()); @@ -314,7 +313,7 @@ public class OsmMultipolygon { } static List connectPolygonSegments(List outer) { - LongObjectMap endpointIndex = new GHLongObjectHashMap<>(outer.size() * 2); + LongObjectMap endpointIndex = Hppc.newLongObjectHashMap(outer.size() * 2); List completeRings = new ArrayList<>(outer.size()); for (LongArrayList ids : outer) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java index af9e7376..a7d324dc 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java @@ -1,8 +1,6 @@ package com.onthegomap.planetiler.reader.osm; import static com.onthegomap.planetiler.util.MemoryEstimator.estimateSize; -import static com.onthegomap.planetiler.util.MemoryEstimator.estimateSizeWithoutKeys; -import static com.onthegomap.planetiler.util.MemoryEstimator.estimateSizeWithoutValues; import static com.onthegomap.planetiler.worker.Worker.joinFutures; import com.carrotsearch.hppc.IntObjectHashMap; @@ -13,6 +11,7 @@ import com.carrotsearch.hppc.ObjectIntHashMap; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.collection.FeatureGroup; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.collection.SortableFeature; @@ -81,7 +80,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate { // for routes (750k rels 40m ways) and boundaries (650k rels, 8m ways) // need to store route info to use later when processing ways // <~500mb - private LongObjectHashMap relationInfo = new LongObjectHashMap<>(); + private LongObjectHashMap relationInfo = Hppc.newLongObjectHashMap(); // ~800mb, ~1.6GB when sorting private LongLongMultimap wayToRelations = LongLongMultimap.newSparseUnorderedMultimap(); // for multipolygons need to store way info (20m ways, 800m nodes) to use when processing relations (4.5m) @@ -476,9 +475,9 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate { size += estimateSize(waysInMultipolygon); size += estimateSize(multipolygonWayGeometries); size += estimateSize(wayToRelations); - size += estimateSizeWithoutValues(relationInfo); - size += MemoryEstimator.estimateSizeWithoutValues(roleIdsReverse); - size += estimateSizeWithoutKeys(roleIds); + size += estimateSize(relationInfo); + size += estimateSize(roleIdsReverse); + size += estimateSize(roleIds); size += roleSizes.get(); size += relationInfoSizes.get(); return size; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java index a0ea1201..4ca314d4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java @@ -20,7 +20,7 @@ package com.onthegomap.planetiler.render; import com.carrotsearch.hppc.IntObjectMap; import com.carrotsearch.hppc.cursors.IntCursor; import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.graphhopper.coll.GHIntObjectHashMap; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.collection.IntRangeSet; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.MutableCoordinateSequence; @@ -310,8 +310,8 @@ class TiledGeometry { private IntObjectMap> sliceX(CoordinateSequence segment) { double leftLimit = -buffer; double rightLimit = 1 + buffer; - IntObjectMap> newGeoms = new GHIntObjectHashMap<>(); - IntObjectMap xSlices = new GHIntObjectHashMap<>(); + IntObjectMap> newGeoms = Hppc.newIntObjectHashMap(); + IntObjectMap xSlices = Hppc.newIntObjectHashMap(); int end = segment.size() - 1; for (int i = 0; i < end; i++) { double ax = segment.getX(i); @@ -415,7 +415,7 @@ class TiledGeometry { IntRangeSet rightFilled = null; IntRangeSet leftFilled = null; - IntObjectMap ySlices = new GHIntObjectHashMap<>(); + IntObjectMap ySlices = Hppc.newIntObjectHashMap(); if (x < 0 || x >= maxTilesAtThisZoom) { return null; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java index c971825d..3f42419e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java @@ -105,6 +105,11 @@ public class ProcessInfo { return Runtime.getRuntime().maxMemory(); } + /** Returns the JVM used memory. */ + public static long getUsedMemoryBytes() { + return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + } + /** Processor usage statistics for a thread. */ public record ThreadState( String name, Duration cpuTime, Duration userTime, Duration waiting, Duration blocking, long id diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java index ccd03148..82e1ad80 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java @@ -3,7 +3,6 @@ package com.onthegomap.planetiler.stats; import static com.onthegomap.planetiler.util.Format.padLeft; import static com.onthegomap.planetiler.util.Format.padRight; -import com.graphhopper.util.Helper; import com.onthegomap.planetiler.util.DiskBacked; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.MemoryEstimator; @@ -255,8 +254,8 @@ public class ProgressLoggers { return num > 0.6 ? red(formatted) : num > 0.3 ? yellow(formatted) : formatted; }); loggers.add(new ProgressLogger("mem", - () -> format.storage(Helper.getUsedMB() * Helper.MB, false) + "/" + - format.storage(Helper.getTotalMB() * Helper.MB, false) + + () -> format.storage(ProcessInfo.getUsedMemoryBytes(), false) + "/" + + format.storage(ProcessInfo.getMaxMemoryBytes(), false) + ProcessInfo.getMemoryUsageAfterLastGC().stream() .mapToObj(value -> " postGC: " + blue(format.storage(value, false))) .findFirst() diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Imposm3Parsers.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Imposm3Parsers.java index 070a6646..67e7ce13 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Imposm3Parsers.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Imposm3Parsers.java @@ -14,7 +14,7 @@ package com.onthegomap.planetiler.util; import com.carrotsearch.hppc.ObjectIntMap; -import com.graphhopper.coll.GHObjectIntHashMap; +import com.onthegomap.planetiler.collection.Hppc; import java.util.Map; import java.util.Set; @@ -30,7 +30,7 @@ public class Imposm3Parsers { return object == null ? null : object.toString(); } - private static final ObjectIntMap defaultRank = new GHObjectIntHashMap<>(); + private static final ObjectIntMap defaultRank = Hppc.newObjectIntHashMap(); static { defaultRank.put("minor", 3); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/MemoryEstimator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/MemoryEstimator.java index 1f82c276..1793ffbd 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/MemoryEstimator.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/MemoryEstimator.java @@ -1,14 +1,6 @@ package com.onthegomap.planetiler.util; -import com.carrotsearch.hppc.ByteArrayList; -import com.carrotsearch.hppc.IntArrayList; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.LongArrayList; -import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongIntHashMap; -import com.carrotsearch.hppc.LongLongHashMap; -import com.carrotsearch.hppc.LongObjectHashMap; -import com.carrotsearch.hppc.ObjectIntHashMap; +import com.carrotsearch.hppc.Accountable; /** * Utilities to estimate the size of in-memory objects. @@ -25,30 +17,6 @@ public class MemoryEstimator { return object == null ? 0 : object.estimateMemoryUsageBytes(); } - public static long estimateSize(LongHashSet object) { - return object == null ? 0 : estimateSize(object.keys); - } - - public static long estimateSize(LongLongHashMap object) { - return object == null ? 0 : (estimateSize(object.keys) + estimateSize(object.values)); - } - - public static long estimateSize(LongIntHashMap object) { - return object == null ? 0 : (estimateSize(object.keys) + estimateSize(object.values)); - } - - public static long estimateSize(LongArrayList object) { - return object == null ? 0 : estimateSize(object.buffer); - } - - public static long estimateSize(IntArrayList object) { - return object == null ? 0 : estimateSize(object.buffer); - } - - public static long estimateSize(ByteArrayList object) { - return object == null ? 0 : estimateSize(object.buffer); - } - public static long estimateSize(String string) { return string == null ? 0 : (54 + string.getBytes().length); } @@ -70,27 +38,10 @@ public class MemoryEstimator { } /** - * Estimates the size of the key and value arrays from {@code object} but omits the size of the actual objects that - * values point to. + * Estimates the size of an HPPC {@link Accountable} instance. */ - public static long estimateSizeWithoutValues(IntObjectHashMap object) { - return object == null ? 0 : (estimateSize(object.keys) + estimateSize(object.values)); - } - - /** - * Estimates the size of the key and value arrays from {@code object} but omits the size of the actual objects that - * values point to. - */ - public static long estimateSizeWithoutValues(LongObjectHashMap object) { - return object == null ? 0 : (estimateSize(object.keys) + estimateSize(object.values)); - } - - /** - * Estimates the size of the key and value arrays from {@code object} but omits the size of the actual objects that - * keys point to. - */ - public static long estimateSizeWithoutKeys(ObjectIntHashMap object) { - return object == null ? 0 : (estimateSize(object.keys) + estimateSize(object.values)); + public static long estimateSize(Accountable object) { + return object == null ? 0 : object.ramBytesAllocated(); } public static long estimateArraySize(int length, long entrySize) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index b9449778..33183a9e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -5,7 +5,6 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static com.google.common.net.HttpHeaders.USER_AGENT; import com.carrotsearch.hppc.LongHashSet; -import com.carrotsearch.hppc.LongObjectHashMap; import com.carrotsearch.hppc.LongObjectMap; import com.carrotsearch.hppc.LongSet; import com.carrotsearch.hppc.cursors.LongObjectCursor; @@ -13,9 +12,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.graphhopper.coll.GHLongObjectHashMap; -import com.graphhopper.util.StopWatch; import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.reader.osm.OsmBlockSource; import com.onthegomap.planetiler.reader.osm.OsmElement; @@ -23,6 +21,7 @@ import com.onthegomap.planetiler.reader.osm.OsmInputFile; import com.onthegomap.planetiler.stats.Counter; import com.onthegomap.planetiler.stats.ProgressLoggers; import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.stats.Timer; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.BufferedInputStream; import java.io.BufferedReader; @@ -88,7 +87,7 @@ public class Wikidata { private static LongObjectMap> parseResults(InputStream results) throws IOException { JsonNode node = objectMapper.readTree(results); ArrayNode bindings = (ArrayNode) node.get("results").get("bindings"); - LongObjectMap> resultMap = new LongObjectHashMap<>(); + LongObjectMap> resultMap = Hppc.newLongObjectHashMap(); bindings.elements().forEachRemaining(row -> { long id = extractIdFromWikidataIRI(row.get("id").get("value").asText()); Map map = resultMap.get(id); @@ -163,12 +162,11 @@ public class Wikidata { * Returns translations parsed from {@code path} that was written by a previous run of the downloader. */ public static WikidataTranslations load(Path path) { - StopWatch watch = new StopWatch().start(); + Timer timer = Timer.start(); try (BufferedReader fis = Files.newBufferedReader(path)) { WikidataTranslations result = load(fis); LOGGER.info( - "loaded from " + result.getAll().size() + " mappings from " + path.toAbsolutePath() + " in " + watch - .stop()); + "loaded from " + result.getAll().size() + " mappings from " + path.toAbsolutePath() + " in " + timer.stop()); return result; } catch (IOException e) { LOGGER.info("error loading " + path.toAbsolutePath() + ": " + e); @@ -247,7 +245,7 @@ public class Wikidata { void flush() { try { - StopWatch timer = new StopWatch().start(); + Timer timer = Timer.start(); LongObjectMap> results = queryWikidata(qidsToFetch); batches.inc(); LOGGER.info("Fetched batch " + batches.get() + " (" + qidsToFetch.size() + " qids) " + timer.stop()); @@ -276,7 +274,7 @@ public class Wikidata { private LongObjectMap> queryWikidata(List qidsToFetch) throws IOException, InterruptedException { if (qidsToFetch.isEmpty()) { - return new GHLongObjectHashMap<>(); + return Hppc.newLongObjectHashMap(); } String qidList = qidsToFetch.stream().map(id -> "wd:Q" + id).collect(Collectors.joining(" ")); String query = """ @@ -353,7 +351,7 @@ public class Wikidata { public static class WikidataTranslations implements Translations.TranslationProvider { - private final LongObjectMap> data = new GHLongObjectHashMap<>(); + private final LongObjectMap> data = Hppc.newLongObjectHashMap(); public WikidataTranslations() { } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java index e0283864..668b1e33 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java @@ -6,7 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.carrotsearch.hppc.IntArrayList; import com.carrotsearch.hppc.IntObjectMap; -import com.graphhopper.coll.GHIntObjectHashMap; +import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.mbtiles.Mbtiles; import java.io.IOException; @@ -506,7 +506,7 @@ public class FeatureMergeTest { @SafeVarargs private static IntObjectMap adjacencyListFromGroups(List... groups) { - IntObjectMap result = new GHIntObjectHashMap<>(); + IntObjectMap result = Hppc.newIntObjectHashMap(); for (List group : groups) { for (int i = 0; i < group.size(); i++) { Integer a = group.get(i); @@ -528,7 +528,7 @@ public class FeatureMergeTest { @Test public void testExtractConnectedComponentsEmpty() { assertEquals( - List.of(), FeatureMerge.extractConnectedComponents(new GHIntObjectHashMap<>(), 0) + List.of(), FeatureMerge.extractConnectedComponents(Hppc.newIntObjectHashMap(), 0) ); } @@ -537,7 +537,7 @@ public class FeatureMergeTest { assertEquals( List.of( IntArrayList.from(0) - ), FeatureMerge.extractConnectedComponents(new GHIntObjectHashMap<>(), 1) + ), FeatureMerge.extractConnectedComponents(Hppc.newIntObjectHashMap(), 1) ); } @@ -547,7 +547,7 @@ public class FeatureMergeTest { List.of( IntArrayList.from(0), IntArrayList.from(1) - ), FeatureMerge.extractConnectedComponents(new GHIntObjectHashMap<>(), 2) + ), FeatureMerge.extractConnectedComponents(Hppc.newIntObjectHashMap(), 2) ); }