diff --git a/core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java b/core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java index 921f1a99..fd37a3fc 100644 --- a/core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java +++ b/core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java @@ -47,8 +47,7 @@ public class FeatureCollector implements Iterable { } return geometry(layer, source.worldGeometry()); } catch (GeometryException e) { - stats.dataError("feature_point_" + e.stat()); - LOGGER.warn("Error getting point geometry for " + source + ": " + e.getMessage()); + e.log(stats, "feature_point", "Error getting point geometry for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } @@ -57,8 +56,7 @@ public class FeatureCollector implements Iterable { try { return geometry(layer, source.centroid()); } catch (GeometryException e) { - stats.dataError("feature_centroid_" + e.stat()); - LOGGER.warn("Error getting centroid for " + source + ": " + e.getMessage()); + e.log(stats, "feature_centroid", "Error getting centroid for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } @@ -67,8 +65,7 @@ public class FeatureCollector implements Iterable { try { return geometry(layer, source.line()); } catch (GeometryException e) { - stats.dataError("feature_line_" + e.stat()); - LOGGER.warn("Error constructing line for " + source + ": " + e.getMessage()); + e.log(stats, "feature_line", "Error constructing line for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } @@ -77,8 +74,7 @@ public class FeatureCollector implements Iterable { try { return geometry(layer, source.polygon()); } catch (GeometryException e) { - stats.dataError("feature_polygon_" + e.stat()); - LOGGER.warn("Error constructing polygon for " + source + ": " + e.getMessage()); + e.log(stats, "feature_polygon", "Error constructing polygon for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } @@ -87,8 +83,7 @@ public class FeatureCollector implements Iterable { try { return geometry(layer, source.centroidIfConvex()); } catch (GeometryException e) { - stats.dataError("feature_centroid_if_convex_" + e.stat()); - LOGGER.warn("Error constructing centroid if convex for " + source + ": " + e.getMessage()); + e.log(stats, "feature_centroid_if_convex", "Error constructing centroid if convex for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } @@ -97,8 +92,7 @@ public class FeatureCollector implements Iterable { try { return geometry(layer, source.pointOnSurface()); } catch (GeometryException e) { - stats.dataError("feature_point_on_surface_" + e.stat()); - LOGGER.warn("Error constructing point on surface for " + source + ": " + e.getMessage()); + e.log(stats, "feature_point_on_surface", "Error constructing point on surface for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } diff --git a/core/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java b/core/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java index 01efe266..0d8e7ea4 100644 --- a/core/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java +++ b/core/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java @@ -275,7 +275,7 @@ public class VectorTileEncoder { attrs )); } catch (GeometryException e) { - LOGGER.warn("Error decoding " + tileID + ": " + e); + e.log("Error decoding " + tileID); } } } diff --git a/core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java b/core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java index 69cfcfa4..d4906d16 100644 --- a/core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java +++ b/core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java @@ -388,8 +388,7 @@ public final class FeatureGroup implements Consumer, Iterable try { items = profile.postProcessLayerFeatures(currentLayer, tile.z(), items); } catch (GeometryException e) { - stats.dataError("postprocess_layer_" + e.stat()); - LOGGER.warn("error postprocessing features for " + currentLayer + " layer on " + tile + ": " + e.getMessage()); + e.log(stats, "postprocess_layer", "Error postprocessing features for " + currentLayer + " layer on " + tile); } encoder.addLayerFeatures(currentLayer, items); } diff --git a/core/src/main/java/com/onthegomap/flatmap/geo/GeometryException.java b/core/src/main/java/com/onthegomap/flatmap/geo/GeometryException.java index 515de7e4..ebf1926a 100644 --- a/core/src/main/java/com/onthegomap/flatmap/geo/GeometryException.java +++ b/core/src/main/java/com/onthegomap/flatmap/geo/GeometryException.java @@ -1,7 +1,13 @@ package com.onthegomap.flatmap.geo; +import com.onthegomap.flatmap.monitoring.Stats; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class GeometryException extends Exception { + private static final Logger LOGGER = LoggerFactory.getLogger(GeometryException.class); + private final String stat; public GeometryException(String stat, String message, Throwable cause) { @@ -17,4 +23,33 @@ public class GeometryException extends Exception { public String stat() { return stat; } + + public void log(Stats stats, String statContext, String logContext) { + stats.dataError(statContext + "_" + stat()); + log(logContext); + } + + public void log(String logContext) { + logMessage(logContext + ": " + getMessage()); + } + + void logMessage(String log) { + LOGGER.warn(log); + } + + public static class Verbose extends GeometryException { + + public Verbose(String stat, String message, Throwable cause) { + super(stat, message, cause); + } + + public Verbose(String stat, String message) { + super(stat, message); + } + + @Override + void logMessage(String log) { + LOGGER.trace(log); + } + } } diff --git a/core/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java b/core/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java index 20240953..75dd3a90 100644 --- a/core/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java +++ b/core/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java @@ -135,7 +135,7 @@ public class ProgressLoggers { } public ProgressLoggers addProcessStats() { - add("\n" + " ".repeat(4)); + add("\n" + " ".repeat(3)); addOptionalDeltaLogger("cpus", ProcessInfo::getProcessCpuTime, Format::formatDecimal); addDeltaLogger("gc", ProcessInfo::getGcTime, Format::formatPercent); loggers.add(new ProgressLogger("mem", diff --git a/core/src/main/java/com/onthegomap/flatmap/read/OsmMultipolygon.java b/core/src/main/java/com/onthegomap/flatmap/read/OsmMultipolygon.java index 73c4af64..cf99eb91 100644 --- a/core/src/main/java/com/onthegomap/flatmap/read/OsmMultipolygon.java +++ b/core/src/main/java/com/onthegomap/flatmap/read/OsmMultipolygon.java @@ -106,7 +106,7 @@ public class OsmMultipolygon { ) throws GeometryException { try { if (rings.size() == 0) { - throw new GeometryException("osm_invalid_multipolygon_empty", + throw new GeometryException.Verbose("osm_invalid_multipolygon_empty", "error building multipolygon " + osmId + ": no rings to process"); } List idSegments = connectPolygonSegments(rings); @@ -123,7 +123,7 @@ public class OsmMultipolygon { polygons.sort(BY_AREA_DESCENDING); Set shells = groupParentChildShells(polygons); if (shells.size() == 0) { - throw new GeometryException("osm_invalid_multipolygon_not_closed", + throw new GeometryException.Verbose("osm_invalid_multipolygon_not_closed", "error building multipolygon " + osmId + ": multipolygon not closed"); } else if (shells.size() == 1) { return shells.iterator().next().toPolygon(); diff --git a/core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java b/core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java index 86777d39..54ac5156 100644 --- a/core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java +++ b/core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java @@ -200,8 +200,7 @@ public class FeatureRenderer implements Consumer { emitted++; } } catch (GeometryException e) { - stats.dataError("write_tile_features_" + e.stat()); - LOGGER.warn(e.getMessage() + ": " + tile + " " + feature); + e.log(stats, "write_tile_features", "Error writing tile " + tile + " feature " + feature); } } diff --git a/core/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java b/core/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java index db320246..d3bca041 100644 --- a/core/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java +++ b/core/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java @@ -29,11 +29,11 @@ public class ProgressLoggersTest { // spin waiting for threads to start } - assertEquals("[prefix] | reader( 0%) -> (0/10) -> worker( 0% 0%) -> (0/10) -> writer( 0% 0%)", + assertEquals("[prefix]\n reader( 0%) -> (0/10) -> worker( 0% 0%) -> (0/10) -> writer( 0% 0%)", log.replaceAll("[ 0-9][0-9]%", " 0%")); latch.countDown(); topology.awaitAndLog(loggers, Duration.ofSeconds(10)); - assertEquals("[prefix] | reader( -%) -> (0/10) -> worker( -% -%) -> (0/10) -> writer( -% -%)", + assertEquals("[prefix]\n reader( -%) -> (0/10) -> worker( -% -%) -> (0/10) -> writer( -% -%)", loggers.getLog()); } } diff --git a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Park.java b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Park.java index d4e8bdf4..98994273 100644 --- a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Park.java +++ b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Park.java @@ -31,8 +31,10 @@ public class Park implements private static final Logger LOGGER = LoggerFactory.getLogger(Park.class); private final Translations translations; + private final Stats stats; public Park(Translations translations, Arguments args, Stats stats) { + this.stats = stats; this.translations = translations; } @@ -79,7 +81,7 @@ public class Park implements areaBoost ).setZoomRange(minzoom, 14); } catch (GeometryException e) { - LOGGER.warn("Unable to get park area for " + element.source().id() + ": " + e.getMessage()); + e.log(stats, "omt_park_area", "Unable to get park area for " + element.source().id()); } } } diff --git a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Place.java b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Place.java index 47a8e2e4..175c2397 100644 --- a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Place.java +++ b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Place.java @@ -54,6 +54,7 @@ public class Place implements private static final Logger LOGGER = LoggerFactory.getLogger(Place.class); private final Translations translations; + private final Stats stats; private static record NaturalEarthRegion(String name, int rank) { @@ -77,6 +78,7 @@ public class Place implements public Place(Translations translations, Arguments args, Stats stats) { this.translations = translations; + this.stats = stats; } @Override @@ -111,8 +113,8 @@ public class Place implements )); } } catch (GeometryException e) { - LOGGER - .warn("Error getting geometry for natural earth feature " + table + " " + feature.getTag("ogc_fid") + " " + e); + e.log(stats, "omt_place_ne", + "Error getting geometry for natural earth feature " + table + " " + feature.getTag("ogc_fid")); } } @@ -163,7 +165,8 @@ public class Place implements .setZoomRange(rank - 1, 14) .setZorder(-rank); } catch (GeometryException e) { - LOGGER.warn("Unable to get point for OSM country " + element.source().id()); + e.log(stats, "omt_place_country", + "Unable to get point for OSM country " + element.source().id()); } } @@ -187,7 +190,8 @@ public class Place implements .setZorder(-rank); } } catch (GeometryException e) { - LOGGER.warn("Unable to get point for OSM state " + element.source().id()); + e.log(stats, "omt_place_state", + "Unable to get point for OSM state " + element.source().id()); } } @@ -226,7 +230,8 @@ public class Place implements .setZoomRange(minzoom, 14) .setZorder(zOrder); } catch (GeometryException e) { - LOGGER.warn("Unable to get area for OSM island polygon " + element.source().id() + ": " + e); + e.log(stats, "omt_place_island_poly", + "Unable to get point for OSM island polygon " + element.source().id()); } } @@ -320,7 +325,8 @@ public class Place implements } } } catch (GeometryException e) { - LOGGER.warn("Unable to get area for OSM city " + element.source().id() + ": " + e); + e.log(stats, "omt_place_city", + "Unable to get point for OSM city " + element.source().id()); } } diff --git a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Transportation.java b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Transportation.java index 1926442b..11a3a334 100644 --- a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Transportation.java +++ b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Transportation.java @@ -72,8 +72,10 @@ public class Transportation implements .put(4, 1_000); private static final double PIXEL = 256d / 4096d; private final boolean z13Paths; + private final Stats stats; public Transportation(Translations translations, Arguments args, Stats stats) { + this.stats = stats; this.z13Paths = args.get( "transportation_z13_paths", "transportation(_name) layer: show paths on z13", @@ -149,7 +151,8 @@ public class Transportation implements return; } } catch (GeometryException e) { - LOGGER.warn("Unable to decode pier geometry for " + element.source().id()); + e.log(stats, "omt_transportation_pier", + "Unable to decode pier geometry for " + element.source().id()); return; } minzoom = 13; diff --git a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/TransportationName.java b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/TransportationName.java index 804477d5..b1dfb0e8 100644 --- a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/TransportationName.java +++ b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/TransportationName.java @@ -67,10 +67,12 @@ public class TransportationName implements private final boolean sizeForShield; private final boolean limitMerge; private final boolean z13Paths; + private final Stats stats; private PreparedGeometry greatBritain = null; private AtomicBoolean loggedNoGb = new AtomicBoolean(false); public TransportationName(Translations translations, Arguments args, Stats stats) { + this.stats = stats; this.brunnel = args.get( "transportation_name_brunnel", "transportation_name layer: set to false to omit brunnel and help merge long highways", @@ -230,7 +232,8 @@ public class TransportationName implements relation = new RouteRelation(refMatcher.group(), networkType, 0); } } catch (GeometryException e) { - LOGGER.warn("Unable to test highway against GB route network: " + element.source().id()); + e.log(stats, "omt_transportation_name_gb_test", + "Unable to test highway against GB route network: " + element.source().id()); } } } diff --git a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/WaterName.java b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/WaterName.java index d0a8e7a0..0465736c 100644 --- a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/WaterName.java +++ b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/WaterName.java @@ -33,6 +33,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName, private final Translations translations; private final LongObjectMap lakeCenterlines = new GHLongObjectHashMap<>(); private final TreeMap importantMarinePoints = new TreeMap<>(); + private final Stats stats; @Override public void release() { @@ -42,6 +43,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName, public WaterName(Translations translations, Arguments args, Stats stats) { this.translations = translations; + this.stats = stats; } @Override @@ -69,7 +71,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName, try { lakeCenterlines.put(osmId, feature.worldGeometry()); } catch (GeometryException e) { - LOGGER.warn("Bad lake centerline geometry: " + feature, e); + e.log(stats, "omt_water_name_lakeline", "Bad lake centerline: " + feature); } } } @@ -134,7 +136,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName, .setAttr(Fields.INTERMITTENT, element.isIntermittent() ? 1 : 0) .setZoomRange(minzoom, 14); } catch (GeometryException e) { - LOGGER.warn("Unable to get geometry for water polygon " + element.source().id() + ": " + e); + e.log(stats, "omt_water_polygon", "Unable to get geometry for water polygon " + element.source().id()); } } } diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000..b51de91e --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +mvn -DskipTests=true --projects openmaptiles -am clean package diff --git a/scripts/ubuntu_vm_setup.sh b/scripts/ubuntu_vm_setup.sh new file mode 100644 index 00000000..abb1b438 --- /dev/null +++ b/scripts/ubuntu_vm_setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +set -x + +if test "$#" -ne 1; then + echo "Usage: ubuntu_vm_setup.sh user@ip" + exit 1 +fi + +"$(dirname "$0")"/build.sh + +scp openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-jar-with-dependencies.jar "${1}":flatmap.jar +scp scripts/download-other-sources.sh "${1}":download-other-sources.sh +scp scripts/download-osm.sh "${1}":download-osm.sh +ssh "${1}" " +wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add - && \ +add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \ +apt-get update -y && \ +apt-get install adoptopenjdk-16-hotspot-jre -y && \ +./download-other-sources.sh +"