logging tweaks

core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java

@@ -47,8 +47,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
       }
       return geometry(layer, source.worldGeometry());
     } catch (GeometryException e) {
-      stats.dataError("feature_point_" + e.stat());
+      e.log(stats, "feature_point", "Error getting point geometry for " + source.id());
-      LOGGER.warn("Error getting point geometry for " + source + ": " + e.getMessage());
       return new Feature(layer, EMPTY_GEOM, source.id());
     }
   }
@@ -57,8 +56,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
     try {
       return geometry(layer, source.centroid());
     } catch (GeometryException e) {
-      stats.dataError("feature_centroid_" + e.stat());
+      e.log(stats, "feature_centroid", "Error getting centroid for " + source.id());
-      LOGGER.warn("Error getting centroid for " + source + ": " + e.getMessage());
       return new Feature(layer, EMPTY_GEOM, source.id());
     }
   }
@@ -67,8 +65,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
     try {
       return geometry(layer, source.line());
     } catch (GeometryException e) {
-      stats.dataError("feature_line_" + e.stat());
+      e.log(stats, "feature_line", "Error constructing line for " + source.id());
-      LOGGER.warn("Error constructing line for " + source + ": " + e.getMessage());
       return new Feature(layer, EMPTY_GEOM, source.id());
     }
   }
@@ -77,8 +74,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
     try {
       return geometry(layer, source.polygon());
     } catch (GeometryException e) {
-      stats.dataError("feature_polygon_" + e.stat());
+      e.log(stats, "feature_polygon", "Error constructing polygon for " + source.id());
-      LOGGER.warn("Error constructing polygon for " + source + ": " + e.getMessage());
       return new Feature(layer, EMPTY_GEOM, source.id());
     }
   }
@@ -87,8 +83,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
     try {
       return geometry(layer, source.centroidIfConvex());
     } catch (GeometryException e) {
-      stats.dataError("feature_centroid_if_convex_" + e.stat());
+      e.log(stats, "feature_centroid_if_convex", "Error constructing centroid if convex for " + source.id());
-      LOGGER.warn("Error constructing centroid if convex for " + source + ": " + e.getMessage());
       return new Feature(layer, EMPTY_GEOM, source.id());
     }
   }
@@ -97,8 +92,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
     try {
       return geometry(layer, source.pointOnSurface());
     } catch (GeometryException e) {
-      stats.dataError("feature_point_on_surface_" + e.stat());
+      e.log(stats, "feature_point_on_surface", "Error constructing point on surface for " + source.id());
-      LOGGER.warn("Error constructing point on surface for " + source + ": " + e.getMessage());
       return new Feature(layer, EMPTY_GEOM, source.id());
     }
   }

core/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java

@@ -275,7 +275,7 @@ public class VectorTileEncoder {
               attrs
             ));
           } catch (GeometryException e) {
-            LOGGER.warn("Error decoding " + tileID + ": " + e);
+            e.log("Error decoding " + tileID);
           }
         }
       }

core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java

@@ -388,8 +388,7 @@ public final class FeatureGroup implements Consumer<FeatureSort.Entry>, Iterable
       try {
         items = profile.postProcessLayerFeatures(currentLayer, tile.z(), items);
       } catch (GeometryException e) {
-        stats.dataError("postprocess_layer_" + e.stat());
+        e.log(stats, "postprocess_layer", "Error postprocessing features for " + currentLayer + " layer on " + tile);
-        LOGGER.warn("error postprocessing features for " + currentLayer + " layer on " + tile + ": " + e.getMessage());
       }
       encoder.addLayerFeatures(currentLayer, items);
     }

core/src/main/java/com/onthegomap/flatmap/geo/GeometryException.java

@@ -1,7 +1,13 @@
 package com.onthegomap.flatmap.geo;
 
+import com.onthegomap.flatmap.monitoring.Stats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class GeometryException extends Exception {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(GeometryException.class);
+
   private final String stat;
 
   public GeometryException(String stat, String message, Throwable cause) {
@@ -17,4 +23,33 @@ public class GeometryException extends Exception {
   public String stat() {
     return stat;
   }
+
+  public void log(Stats stats, String statContext, String logContext) {
+    stats.dataError(statContext + "_" + stat());
+    log(logContext);
+  }
+
+  public void log(String logContext) {
+    logMessage(logContext + ": " + getMessage());
+  }
+
+  void logMessage(String log) {
+    LOGGER.warn(log);
+  }
+
+  public static class Verbose extends GeometryException {
+
+    public Verbose(String stat, String message, Throwable cause) {
+      super(stat, message, cause);
+    }
+
+    public Verbose(String stat, String message) {
+      super(stat, message);
+    }
+
+    @Override
+    void logMessage(String log) {
+      LOGGER.trace(log);
+    }
+  }
 }

core/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java

@@ -135,7 +135,7 @@ public class ProgressLoggers {
   }
 
   public ProgressLoggers addProcessStats() {
-    add("\n" + " ".repeat(4));
+    add("\n" + " ".repeat(3));
     addOptionalDeltaLogger("cpus", ProcessInfo::getProcessCpuTime, Format::formatDecimal);
     addDeltaLogger("gc", ProcessInfo::getGcTime, Format::formatPercent);
     loggers.add(new ProgressLogger("mem",

core/src/main/java/com/onthegomap/flatmap/read/OsmMultipolygon.java

@@ -106,7 +106,7 @@ public class OsmMultipolygon {
   ) throws GeometryException {
     try {
       if (rings.size() == 0) {
-        throw new GeometryException("osm_invalid_multipolygon_empty",
+        throw new GeometryException.Verbose("osm_invalid_multipolygon_empty",
           "error building multipolygon " + osmId + ": no rings to process");
       }
       List<LongArrayList> idSegments = connectPolygonSegments(rings);
@@ -123,7 +123,7 @@ public class OsmMultipolygon {
       polygons.sort(BY_AREA_DESCENDING);
       Set<Ring> shells = groupParentChildShells(polygons);
       if (shells.size() == 0) {
-        throw new GeometryException("osm_invalid_multipolygon_not_closed",
+        throw new GeometryException.Verbose("osm_invalid_multipolygon_not_closed",
           "error building multipolygon " + osmId + ": multipolygon not closed");
       } else if (shells.size() == 1) {
         return shells.iterator().next().toPolygon();

core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java

@@ -200,8 +200,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
           emitted++;
         }
       } catch (GeometryException e) {
-        stats.dataError("write_tile_features_" + e.stat());
+        e.log(stats, "write_tile_features", "Error writing tile " + tile + " feature " + feature);
-        LOGGER.warn(e.getMessage() + ": " + tile + " " + feature);
       }
     }
 

core/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java

@@ -29,11 +29,11 @@ public class ProgressLoggersTest {
       // spin waiting for threads to start
     }
 
-    assertEquals("[prefix] | reader( 0%) ->    (0/10) -> worker( 0%  0%) ->    (0/10) -> writer( 0%  0%)",
+    assertEquals("[prefix]\n    reader( 0%) ->    (0/10) -> worker( 0%  0%) ->    (0/10) -> writer( 0%  0%)",
       log.replaceAll("[ 0-9][0-9]%", " 0%"));
     latch.countDown();
     topology.awaitAndLog(loggers, Duration.ofSeconds(10));
-    assertEquals("[prefix] | reader( -%) ->    (0/10) -> worker( -%  -%) ->    (0/10) -> writer( -%  -%)",
+    assertEquals("[prefix]\n    reader( -%) ->    (0/10) -> worker( -%  -%) ->    (0/10) -> writer( -%  -%)",
       loggers.getLog());
   }
 }

openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Park.java

@@ -31,8 +31,10 @@ public class Park implements
 
   private static final Logger LOGGER = LoggerFactory.getLogger(Park.class);
   private final Translations translations;
+  private final Stats stats;
 
   public Park(Translations translations, Arguments args, Stats stats) {
+    this.stats = stats;
     this.translations = translations;
   }
 
@@ -79,7 +81,7 @@ public class Park implements
             areaBoost
           ).setZoomRange(minzoom, 14);
       } catch (GeometryException e) {
-        LOGGER.warn("Unable to get park area for " + element.source().id() + ": " + e.getMessage());
+        e.log(stats, "omt_park_area", "Unable to get park area for " + element.source().id());
       }
     }
   }

openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Place.java

@@ -54,6 +54,7 @@ public class Place implements
   private static final Logger LOGGER = LoggerFactory.getLogger(Place.class);
 
   private final Translations translations;
+  private final Stats stats;
 
   private static record NaturalEarthRegion(String name, int rank) {
 
@@ -77,6 +78,7 @@ public class Place implements
 
   public Place(Translations translations, Arguments args, Stats stats) {
     this.translations = translations;
+    this.stats = stats;
   }
 
   @Override
@@ -111,8 +113,8 @@ public class Place implements
         ));
       }
     } catch (GeometryException e) {
-      LOGGER
+      e.log(stats, "omt_place_ne",
-        .warn("Error getting geometry for natural earth feature " + table + " " + feature.getTag("ogc_fid") + " " + e);
+        "Error getting geometry for natural earth feature " + table + " " + feature.getTag("ogc_fid"));
     }
   }
 
@@ -163,7 +165,8 @@ public class Place implements
         .setZoomRange(rank - 1, 14)
         .setZorder(-rank);
     } catch (GeometryException e) {
-      LOGGER.warn("Unable to get point for OSM country " + element.source().id());
+      e.log(stats, "omt_place_country",
+        "Unable to get point for OSM country " + element.source().id());
     }
   }
 
@@ -187,7 +190,8 @@ public class Place implements
           .setZorder(-rank);
       }
     } catch (GeometryException e) {
-      LOGGER.warn("Unable to get point for OSM state " + element.source().id());
+      e.log(stats, "omt_place_state",
+        "Unable to get point for OSM state " + element.source().id());
     }
   }
 
@@ -226,7 +230,8 @@ public class Place implements
         .setZoomRange(minzoom, 14)
         .setZorder(zOrder);
     } catch (GeometryException e) {
-      LOGGER.warn("Unable to get area for OSM island polygon " + element.source().id() + ": " + e);
+      e.log(stats, "omt_place_island_poly",
+        "Unable to get point for OSM island polygon " + element.source().id());
     }
   }
 
@@ -320,7 +325,8 @@ public class Place implements
           }
         }
       } catch (GeometryException e) {
-        LOGGER.warn("Unable to get area for OSM city " + element.source().id() + ": " + e);
+        e.log(stats, "omt_place_city",
+          "Unable to get point for OSM city " + element.source().id());
       }
     }
 

openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Transportation.java

@@ -72,8 +72,10 @@ public class Transportation implements
     .put(4, 1_000);
   private static final double PIXEL = 256d / 4096d;
   private final boolean z13Paths;
+  private final Stats stats;
 
   public Transportation(Translations translations, Arguments args, Stats stats) {
+    this.stats = stats;
     this.z13Paths = args.get(
       "transportation_z13_paths",
       "transportation(_name) layer: show paths on z13",
@@ -149,7 +151,8 @@ public class Transportation implements
             return;
           }
         } catch (GeometryException e) {
-          LOGGER.warn("Unable to decode pier geometry for " + element.source().id());
+          e.log(stats, "omt_transportation_pier",
+            "Unable to decode pier geometry for " + element.source().id());
           return;
         }
         minzoom = 13;

openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/TransportationName.java

@@ -67,10 +67,12 @@ public class TransportationName implements
   private final boolean sizeForShield;
   private final boolean limitMerge;
   private final boolean z13Paths;
+  private final Stats stats;
   private PreparedGeometry greatBritain = null;
   private AtomicBoolean loggedNoGb = new AtomicBoolean(false);
 
   public TransportationName(Translations translations, Arguments args, Stats stats) {
+    this.stats = stats;
     this.brunnel = args.get(
       "transportation_name_brunnel",
       "transportation_name layer: set to false to omit brunnel and help merge long highways",
@@ -230,7 +232,8 @@ public class TransportationName implements
               relation = new RouteRelation(refMatcher.group(), networkType, 0);
             }
           } catch (GeometryException e) {
-            LOGGER.warn("Unable to test highway against GB route network: " + element.source().id());
+            e.log(stats, "omt_transportation_name_gb_test",
+              "Unable to test highway against GB route network: " + element.source().id());
           }
         }
       }

openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/WaterName.java

@@ -33,6 +33,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
   private final Translations translations;
   private final LongObjectMap<Geometry> lakeCenterlines = new GHLongObjectHashMap<>();
   private final TreeMap<String, Integer> importantMarinePoints = new TreeMap<>();
+  private final Stats stats;
 
   @Override
   public void release() {
@@ -42,6 +43,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
 
   public WaterName(Translations translations, Arguments args, Stats stats) {
     this.translations = translations;
+    this.stats = stats;
   }
 
   @Override
@@ -69,7 +71,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
         try {
           lakeCenterlines.put(osmId, feature.worldGeometry());
         } catch (GeometryException e) {
-          LOGGER.warn("Bad lake centerline geometry: " + feature, e);
+          e.log(stats, "omt_water_name_lakeline", "Bad lake centerline: " + feature);
         }
       }
     }
@@ -134,7 +136,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
           .setAttr(Fields.INTERMITTENT, element.isIntermittent() ? 1 : 0)
           .setZoomRange(minzoom, 14);
       } catch (GeometryException e) {
-        LOGGER.warn("Unable to get geometry for water polygon " + element.source().id() + ": " + e);
+        e.log(stats, "omt_water_polygon", "Unable to get geometry for water polygon " + element.source().id());
       }
     }
   }

scripts/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+mvn -DskipTests=true --projects openmaptiles -am clean package

scripts/ubuntu_vm_setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+if test "$#" -ne 1; then
+  echo "Usage: ubuntu_vm_setup.sh user@ip"
+  exit 1
+fi
+
+"$(dirname "$0")"/build.sh
+
+scp openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-jar-with-dependencies.jar "${1}":flatmap.jar
+scp scripts/download-other-sources.sh "${1}":download-other-sources.sh
+scp scripts/download-osm.sh "${1}":download-osm.sh
+ssh "${1}" "
+wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add - && \
+add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \
+apt-get update -y && \
+apt-get install adoptopenjdk-16-hotspot-jre -y && \
+./download-other-sources.sh
+"