kopia lustrzana https://github.com/onthegomap/planetiler
tile stats
rodzic
db796e1720
commit
5124c4358d
|
@ -227,6 +227,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
private ZoomFunction<Number> pixelTolerance = null;
|
||||
|
||||
private String numPointsAttr = null;
|
||||
private boolean removeHolesBelowMinSize = false;
|
||||
|
||||
private Feature(String layer, Geometry geom, long id) {
|
||||
this.layer = layer;
|
||||
|
@ -731,5 +732,14 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
", attrs=" + attrs +
|
||||
'}';
|
||||
}
|
||||
|
||||
public Feature setRemoveHolesBelowMinSize(boolean b) {
|
||||
this.removeHolesBelowMinSize = b;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean getRemoveHolesBelowMinSize() {
|
||||
return this.removeHolesBelowMinSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -25,6 +25,9 @@ import org.locationtech.jts.geom.Geometry;
|
|||
import org.locationtech.jts.geom.GeometryCollection;
|
||||
import org.locationtech.jts.geom.LineString;
|
||||
import org.locationtech.jts.geom.LinearRing;
|
||||
import org.locationtech.jts.geom.MultiPoint;
|
||||
import org.locationtech.jts.geom.MultiPolygon;
|
||||
import org.locationtech.jts.geom.Point;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
import org.locationtech.jts.geom.Polygonal;
|
||||
import org.locationtech.jts.index.strtree.STRtree;
|
||||
|
@ -87,6 +90,68 @@ public class FeatureMerge {
|
|||
return mergeLineStrings(features, minLength, tolerance, buffer, false);
|
||||
}
|
||||
|
||||
public static List<VectorTile.Feature> mergeMultiPoint(List<VectorTile.Feature> features) {
|
||||
List<VectorTile.Feature> result = new ArrayList<>(features.size());
|
||||
var groupedByAttrs = groupByAttrs(features, result, GeometryType.POINT);
|
||||
for (List<VectorTile.Feature> groupedFeatures : groupedByAttrs) {
|
||||
VectorTile.Feature feature1 = groupedFeatures.get(0);
|
||||
if (groupedFeatures.size() == 1) {
|
||||
result.add(feature1);
|
||||
} else {
|
||||
List<Point> points = new ArrayList<>();
|
||||
for (var feature : groupedFeatures) {
|
||||
try {
|
||||
var geom = feature.geometry().decode();
|
||||
if (geom instanceof Point p) {
|
||||
points.add(p);
|
||||
} else if (geom instanceof MultiPoint mp) {
|
||||
for (int i = 0; i < mp.getNumGeometries(); i++) {
|
||||
points.add((Point) mp.getGeometryN(i));
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn("Unexpected geometry type: {}", geom.getClass());
|
||||
}
|
||||
} catch (GeometryException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
result.add(feature1.copyWithNewGeometry(GeoUtils.combinePoints(points)));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static List<VectorTile.Feature> mergeMultiPolygon(List<VectorTile.Feature> features) {
|
||||
List<VectorTile.Feature> result = new ArrayList<>(features.size());
|
||||
var groupedByAttrs = groupByAttrs(features, result, GeometryType.POLYGON);
|
||||
for (List<VectorTile.Feature> groupedFeatures : groupedByAttrs) {
|
||||
VectorTile.Feature feature1 = groupedFeatures.get(0);
|
||||
if (groupedFeatures.size() == 1) {
|
||||
result.add(feature1);
|
||||
} else {
|
||||
List<Polygon> polys = new ArrayList<>();
|
||||
for (var feature : groupedFeatures) {
|
||||
try {
|
||||
var geom = feature.geometry().decode();
|
||||
if (geom instanceof Polygon p) {
|
||||
polys.add(p);
|
||||
} else if (geom instanceof MultiPolygon mp) {
|
||||
for (int i = 0; i < mp.getNumGeometries(); i++) {
|
||||
polys.add((Polygon) mp.getGeometryN(i));
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn("Unexpected geometry type: {}", geom.getClass());
|
||||
}
|
||||
} catch (GeometryException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
result.add(feature1.copyWithNewGeometry(GeoUtils.combinePolygons(polys)));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges linestrings with the same attributes as {@link #mergeLineStrings(List, Function, double, double, boolean)}
|
||||
* except sets {@code resimplify=false} by default.
|
||||
|
|
|
@ -441,11 +441,9 @@ public class VectorTile {
|
|||
}
|
||||
|
||||
/**
|
||||
* Creates a vector tile protobuf with all features in this tile and serializes it as a byte array.
|
||||
* <p>
|
||||
* Does not compress the result.
|
||||
* Returns a vector tile protobuf object with all features in this tile.
|
||||
*/
|
||||
public byte[] encode() {
|
||||
public VectorTileProto.Tile toProto() {
|
||||
VectorTileProto.Tile.Builder tile = VectorTileProto.Tile.newBuilder();
|
||||
for (Map.Entry<String, Layer> e : layers.entrySet()) {
|
||||
String layerName = e.getKey();
|
||||
|
@ -492,7 +490,16 @@ public class VectorTile {
|
|||
|
||||
tile.addLayers(tileLayer.build());
|
||||
}
|
||||
return tile.build().toByteArray();
|
||||
return tile.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a vector tile protobuf with all features in this tile and serializes it as a byte array.
|
||||
* <p>
|
||||
* Does not compress the result.
|
||||
*/
|
||||
public byte[] encode() {
|
||||
return toProto().toByteArray();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -3,6 +3,7 @@ package com.onthegomap.planetiler.archive;
|
|||
import static com.onthegomap.planetiler.util.Gzip.gzip;
|
||||
import static com.onthegomap.planetiler.worker.Worker.joinFutures;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.onthegomap.planetiler.VectorTile;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
|
@ -18,11 +19,21 @@ import com.onthegomap.planetiler.util.Hashing;
|
|||
import com.onthegomap.planetiler.worker.WorkQueue;
|
||||
import com.onthegomap.planetiler.worker.Worker;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.io.Writer;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.text.NumberFormat;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.Queue;
|
||||
|
@ -34,8 +45,12 @@ import java.util.function.Consumer;
|
|||
import java.util.function.LongSupplier;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.Deflater;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import vector_tile.VectorTileProto;
|
||||
|
||||
/**
|
||||
* Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a
|
||||
|
@ -233,6 +248,7 @@ public class TileArchiveWriter {
|
|||
byte[] lastBytes = null, lastEncoded = null;
|
||||
Long lastTileDataHash = null;
|
||||
boolean lastIsFill = false;
|
||||
List<TileEncodingResult.LayerStats> lastLayerStats = null;
|
||||
boolean skipFilled = config.skipFilledTiles();
|
||||
|
||||
for (TileBatch batch : prev) {
|
||||
|
@ -243,19 +259,24 @@ public class TileArchiveWriter {
|
|||
FeatureGroup.TileFeatures tileFeatures = batch.in.get(i);
|
||||
featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed());
|
||||
byte[] bytes, encoded;
|
||||
List<TileEncodingResult.LayerStats> layerStats;
|
||||
Long tileDataHash;
|
||||
if (tileFeatures.hasSameContents(last)) {
|
||||
bytes = lastBytes;
|
||||
encoded = lastEncoded;
|
||||
tileDataHash = lastTileDataHash;
|
||||
layerStats = lastLayerStats;
|
||||
memoizedTiles.inc();
|
||||
} else {
|
||||
VectorTile en = tileFeatures.getVectorTileEncoder();
|
||||
if (skipFilled && (lastIsFill = en.containsOnlyFills())) {
|
||||
encoded = null;
|
||||
layerStats = null;
|
||||
bytes = null;
|
||||
} else {
|
||||
encoded = en.encode();
|
||||
var proto = en.toProto();
|
||||
layerStats = computeTileStats(proto);
|
||||
encoded = proto.toByteArray();
|
||||
bytes = switch (config.tileCompression()) {
|
||||
case GZIP -> gzip(encoded);
|
||||
case NONE -> encoded;
|
||||
|
@ -267,6 +288,7 @@ public class TileArchiveWriter {
|
|||
encoded.length / 1024);
|
||||
}
|
||||
}
|
||||
lastLayerStats = layerStats;
|
||||
lastEncoded = encoded;
|
||||
lastBytes = bytes;
|
||||
last = tileFeatures;
|
||||
|
@ -285,8 +307,13 @@ public class TileArchiveWriter {
|
|||
totalTileSizesByZoom[zoom].incBy(encodedLength);
|
||||
maxTileSizesByZoom[zoom].accumulate(encodedLength);
|
||||
result.add(
|
||||
new TileEncodingResult(tileFeatures.tileCoord(), bytes,
|
||||
tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash))
|
||||
new TileEncodingResult(
|
||||
tileFeatures.tileCoord(),
|
||||
bytes,
|
||||
encoded == null ? 0 : encoded.length,
|
||||
tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash),
|
||||
layerStats
|
||||
)
|
||||
);
|
||||
}
|
||||
// hand result off to writer
|
||||
|
@ -295,7 +322,46 @@ public class TileArchiveWriter {
|
|||
}
|
||||
}
|
||||
|
||||
public static List<TileEncodingResult.LayerStats> computeTileStats(VectorTileProto.Tile proto) {
|
||||
if (proto == null) {
|
||||
return List.of();
|
||||
}
|
||||
List<TileEncodingResult.LayerStats> result = new ArrayList<>(proto.getLayersCount());
|
||||
for (var layer : proto.getLayersList()) {
|
||||
int attrSize = 0;
|
||||
for (var key : layer.getKeysList().asByteStringList()) {
|
||||
attrSize += key.size();
|
||||
}
|
||||
for (var value : layer.getValuesList()) {
|
||||
attrSize += value.getSerializedSize();
|
||||
}
|
||||
result.add(new TileEncodingResult.LayerStats(
|
||||
layer.getName(),
|
||||
layer.getFeaturesCount(),
|
||||
layer.getSerializedSize(),
|
||||
attrSize,
|
||||
layer.getValuesCount()
|
||||
));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static class FastGzipOutputStream extends GZIPOutputStream {
|
||||
|
||||
public FastGzipOutputStream(OutputStream out) throws IOException {
|
||||
super(out);
|
||||
def.setLevel(Deflater.BEST_SPEED);
|
||||
}
|
||||
}
|
||||
|
||||
private static Writer newWriter(Path path) throws IOException {
|
||||
return new OutputStreamWriter(new FastGzipOutputStream(new BufferedOutputStream(Files.newOutputStream(path,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE))));
|
||||
}
|
||||
|
||||
private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionException, InterruptedException {
|
||||
var f = NumberFormat.getNumberInstance(Locale.getDefault());
|
||||
f.setMaximumFractionDigits(5);
|
||||
|
||||
archive.initialize(tileArchiveMetadata);
|
||||
var order = archive.tileOrder();
|
||||
|
@ -303,7 +369,30 @@ public class TileArchiveWriter {
|
|||
TileCoord lastTile = null;
|
||||
Timer time = null;
|
||||
int currentZ = Integer.MIN_VALUE;
|
||||
try (var tileWriter = archive.newTileWriter()) {
|
||||
LongIntHashMap hashToId = new LongIntHashMap();
|
||||
int tileId = 0;
|
||||
try (
|
||||
var tileWriter = archive.newTileWriter();
|
||||
var tileStats = newWriter(Path.of("tile_stats.tsv.gz"));
|
||||
) {
|
||||
writeTsvLine(tileStats,
|
||||
"z",
|
||||
"x",
|
||||
"y",
|
||||
"hilbert",
|
||||
"tile_bytes",
|
||||
"gzipped_tile_bytes",
|
||||
"deduped_tile_id",
|
||||
"layer",
|
||||
"features",
|
||||
"layer_bytes",
|
||||
"layer_attr_bytes",
|
||||
"layer_attr_values",
|
||||
"min_lon",
|
||||
"max_lon",
|
||||
"min_lat",
|
||||
"max_lat"
|
||||
);
|
||||
for (TileBatch batch : tileBatches) {
|
||||
Queue<TileEncodingResult> encodedTiles = batch.out.get();
|
||||
TileEncodingResult encodedTile;
|
||||
|
@ -323,6 +412,39 @@ public class TileArchiveWriter {
|
|||
time = Timer.start();
|
||||
currentZ = z;
|
||||
}
|
||||
int hilbert = encodedTile.coord().hilbertEncoded();
|
||||
Integer thisTileId;
|
||||
if (encodedTile.tileDataHash().isPresent()) {
|
||||
long hash = encodedTile.tileDataHash().getAsLong();
|
||||
if (hashToId.containsKey(hash)) {
|
||||
thisTileId = hashToId.get(hash);
|
||||
} else {
|
||||
hashToId.put(hash, thisTileId = ++tileId);
|
||||
}
|
||||
} else {
|
||||
thisTileId = null;
|
||||
}
|
||||
Envelope envelope = encodedTile.coord().getEnvelope();
|
||||
for (var layer : encodedTile.layerStats()) {
|
||||
writeTsvLine(tileStats,
|
||||
encodedTile.coord().z(),
|
||||
encodedTile.coord().x(),
|
||||
encodedTile.coord().y(),
|
||||
hilbert,
|
||||
encodedTile.rawTileSize(),
|
||||
encodedTile.tileData().length,
|
||||
thisTileId == null ? "" : thisTileId,
|
||||
layer.name(),
|
||||
layer.features(),
|
||||
layer.totalBytes(),
|
||||
layer.attrBytes(),
|
||||
layer.attrValues(),
|
||||
f.format(envelope.getMinX()),
|
||||
f.format(envelope.getMaxX()),
|
||||
f.format(envelope.getMinY()),
|
||||
f.format(envelope.getMaxY())
|
||||
);
|
||||
}
|
||||
tileWriter.write(encodedTile);
|
||||
|
||||
stats.wroteTile(z, encodedTile.tileData() == null ? 0 : encodedTile.tileData().length);
|
||||
|
@ -331,6 +453,8 @@ public class TileArchiveWriter {
|
|||
lastTileWritten.set(lastTile);
|
||||
}
|
||||
tileWriter.printStats();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
|
||||
if (time != null) {
|
||||
|
@ -341,6 +465,18 @@ public class TileArchiveWriter {
|
|||
archive.finish(tileArchiveMetadata);
|
||||
}
|
||||
|
||||
private static void writeTsvLine(Writer writer, Object... values) throws IOException {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
if (i > 0) {
|
||||
builder.append('\t');
|
||||
}
|
||||
builder.append(values[i]);
|
||||
}
|
||||
builder.append('\n');
|
||||
writer.write(builder.toString());
|
||||
}
|
||||
|
||||
private void printTileStats() {
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
Format format = Format.defaultInstance();
|
||||
|
|
|
@ -2,42 +2,55 @@ package com.onthegomap.planetiler.archive;
|
|||
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalLong;
|
||||
|
||||
public record TileEncodingResult(
|
||||
TileCoord coord,
|
||||
byte[] tileData,
|
||||
int rawTileSize,
|
||||
/** will always be empty in non-compact mode and might also be empty in compact mode */
|
||||
OptionalLong tileDataHash
|
||||
OptionalLong tileDataHash,
|
||||
List<LayerStats> layerStats
|
||||
) {
|
||||
public TileEncodingResult(
|
||||
TileCoord coord,
|
||||
byte[] tileData,
|
||||
OptionalLong tileDataHash
|
||||
) {
|
||||
this(coord, tileData, 0, tileDataHash, List.of());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + Arrays.hashCode(tileData);
|
||||
result = prime * result + Objects.hash(coord, tileDataHash);
|
||||
result = prime * result + Objects.hash(coord, tileDataHash, layerStats);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof TileEncodingResult)) {
|
||||
return false;
|
||||
}
|
||||
TileEncodingResult other = (TileEncodingResult) obj;
|
||||
return Objects.equals(coord, other.coord) && Arrays.equals(tileData, other.tileData) &&
|
||||
Objects.equals(tileDataHash, other.tileDataHash);
|
||||
return this == obj || (obj instanceof TileEncodingResult other &&
|
||||
Objects.equals(coord, other.coord) &&
|
||||
Arrays.equals(tileData, other.tileData) &&
|
||||
Objects.equals(tileDataHash, other.tileDataHash) &&
|
||||
Objects.equals(layerStats, other.layerStats));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TileEncodingResult [coord=" + coord + ", tileData=" + Arrays.toString(tileData) + ", tileDataHash=" +
|
||||
tileDataHash + "]";
|
||||
tileDataHash + ", layerStats=" + layerStats + "]";
|
||||
}
|
||||
|
||||
public record LayerStats(
|
||||
String name,
|
||||
int features,
|
||||
int totalBytes,
|
||||
int attrBytes,
|
||||
int attrValues
|
||||
) {}
|
||||
}
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
*/
|
||||
package com.onthegomap.planetiler.geo;
|
||||
|
||||
import org.locationtech.jts.algorithm.Area;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
import org.locationtech.jts.geom.CoordinateSequence;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
|
@ -45,12 +46,26 @@ public class DouglasPeuckerSimplifier {
|
|||
return (new DPTransformer(distanceTolerance)).transform(geom);
|
||||
}
|
||||
|
||||
public static Geometry simplify(Geometry geom, double distanceTolerance, double minHoleSize) {
|
||||
if (geom.isEmpty()) {
|
||||
return geom.copy();
|
||||
}
|
||||
|
||||
return (new DPTransformer(distanceTolerance, minHoleSize)).transform(geom);
|
||||
}
|
||||
|
||||
private static class DPTransformer extends GeometryTransformer {
|
||||
|
||||
private final double sqTolerance;
|
||||
private final double minHoleSize;
|
||||
|
||||
private DPTransformer(double distanceTolerance) {
|
||||
this(distanceTolerance, 0);
|
||||
}
|
||||
|
||||
private DPTransformer(double distanceTolerance, double minHoleSize) {
|
||||
this.sqTolerance = distanceTolerance * distanceTolerance;
|
||||
this.minHoleSize = minHoleSize;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -142,7 +157,8 @@ public class DouglasPeuckerSimplifier {
|
|||
protected Geometry transformLinearRing(LinearRing geom, Geometry parent) {
|
||||
boolean removeDegenerateRings = parent instanceof Polygon;
|
||||
Geometry simpResult = super.transformLinearRing(geom, parent);
|
||||
if (removeDegenerateRings && !(simpResult instanceof LinearRing)) {
|
||||
if (removeDegenerateRings && (!(simpResult instanceof LinearRing ring) ||
|
||||
(minHoleSize > 0 && Area.ofRing(ring.getCoordinateSequence()) <= minHoleSize))) {
|
||||
return null;
|
||||
}
|
||||
return simpResult;
|
||||
|
|
|
@ -134,6 +134,20 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
|
|||
);
|
||||
}
|
||||
|
||||
public Envelope getEnvelope() {
|
||||
double worldWidthAtZoom = Math.pow(2, z);
|
||||
return new Envelope(
|
||||
GeoUtils.getWorldLon(x / worldWidthAtZoom),
|
||||
GeoUtils.getWorldLon((x + 1) / worldWidthAtZoom),
|
||||
GeoUtils.getWorldLat(y / worldWidthAtZoom),
|
||||
GeoUtils.getWorldLat((y + 1) / worldWidthAtZoom)
|
||||
);
|
||||
}
|
||||
|
||||
public double maxLon() {
|
||||
return GeoUtils.getWorldLon(x / Math.pow(2, z));
|
||||
}
|
||||
|
||||
|
||||
/** Returns a URL that displays the openstreetmap data for this tile. */
|
||||
public String getDebugUrl() {
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
package com.onthegomap.planetiler.pmtiles;
|
||||
|
||||
import com.carrotsearch.hppc.LongObjectHashMap;
|
||||
import com.onthegomap.planetiler.archive.ReadableTileArchive;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileCompression;
|
||||
|
@ -66,6 +67,8 @@ public class ReadablePmtiles implements ReadableTileArchive {
|
|||
return null;
|
||||
}
|
||||
|
||||
private LongObjectHashMap<List<Pmtiles.Entry>> dirCache = new LongObjectHashMap<>();
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("java:S1168")
|
||||
public byte[] getTile(int x, int y, int z) {
|
||||
|
@ -76,12 +79,16 @@ public class ReadablePmtiles implements ReadableTileArchive {
|
|||
int dirLength = (int) header.rootDirLength();
|
||||
|
||||
for (int depth = 0; depth <= 3; depth++) {
|
||||
byte[] dirBytes = getBytes(dirOffset, dirLength);
|
||||
if (header.internalCompression() == Pmtiles.Compression.GZIP) {
|
||||
dirBytes = Gzip.gunzip(dirBytes);
|
||||
}
|
||||
var dir = dirCache.get(dirOffset);
|
||||
if (dir == null) {
|
||||
byte[] dirBytes = getBytes(dirOffset, dirLength);
|
||||
if (header.internalCompression() == Pmtiles.Compression.GZIP) {
|
||||
dirBytes = Gzip.gunzip(dirBytes);
|
||||
}
|
||||
|
||||
var dir = Pmtiles.directoryFromBytes(dirBytes);
|
||||
dir = Pmtiles.directoryFromBytes(dirBytes);
|
||||
dirCache.put(dirOffset, dir);
|
||||
}
|
||||
var entry = findTile(dir, tileId);
|
||||
if (entry != null) {
|
||||
if (entry.runLength() > 0) {
|
||||
|
|
|
@ -0,0 +1,147 @@
|
|||
package com.onthegomap.planetiler.util;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveConfig;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.archive.TileArchives;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import vector_tile.VectorTileProto;
|
||||
|
||||
public class TileStats {
|
||||
|
||||
public static void main(String... args) throws IOException {
|
||||
var arguments = Arguments.fromArgsOrConfigFile(args);
|
||||
var config = PlanetilerConfig.from(arguments);
|
||||
var stats = Stats.inMemory();
|
||||
var inputString = arguments.getString("input", "input file");
|
||||
var output = arguments.file("output", "output file");
|
||||
var input = TileArchiveConfig.from(inputString);
|
||||
var counter = new AtomicLong(0);
|
||||
var timer = stats.startStage("tilestats");
|
||||
record Tile(TileCoord coord, byte[] data) {}
|
||||
try (
|
||||
var reader = TileArchives.newReader(input, config);
|
||||
var result = newWriter(output)
|
||||
) {
|
||||
var pipeline = WorkerPipeline.start("tilestats", stats)
|
||||
.<Tile>fromGenerator("enumerate", next -> {
|
||||
try (var coords = reader.getAllTileCoords()) {
|
||||
while (coords.hasNext()) {
|
||||
var coord = coords.next();
|
||||
next.accept(new Tile(coord, reader.getTile(coord)));
|
||||
}
|
||||
}
|
||||
})
|
||||
.addBuffer("coords", 10_000, 1000)
|
||||
.<TileEncodingResult>addWorker("process", config.featureProcessThreads(), (prev, next) -> {
|
||||
byte[] zipped = null;
|
||||
byte[] unzipped = null;
|
||||
VectorTileProto.Tile decoded = null;
|
||||
long hash = 0;
|
||||
List<TileEncodingResult.LayerStats> tileStats = null;
|
||||
|
||||
for (var coord : prev) {
|
||||
if (!Arrays.equals(zipped, coord.data)) {
|
||||
zipped = coord.data;
|
||||
unzipped = Gzip.gunzip(coord.data);
|
||||
decoded = VectorTileProto.Tile.parseFrom(unzipped);
|
||||
hash = Hashing.fnv1a64(unzipped);
|
||||
tileStats = TileArchiveWriter.computeTileStats(decoded);
|
||||
}
|
||||
next.accept(new TileEncodingResult(coord.coord, coord.data, unzipped.length, OptionalLong.of(hash),
|
||||
tileStats));
|
||||
}
|
||||
})
|
||||
.addBuffer("results", 10_000, 1000)
|
||||
.sinkTo("write", 1, prev -> {
|
||||
writeTsvLine(result,
|
||||
"z",
|
||||
"x",
|
||||
"y",
|
||||
"hilbert",
|
||||
"tile_bytes",
|
||||
"gzipped_tile_bytes",
|
||||
"deduped_tile_id",
|
||||
"layer",
|
||||
"features",
|
||||
"layer_bytes",
|
||||
"layer_attr_bytes",
|
||||
"layer_attr_values"
|
||||
);
|
||||
LongIntHashMap ids = new LongIntHashMap();
|
||||
int num = 0;
|
||||
for (var coord : prev) {
|
||||
int id;
|
||||
if (ids.containsKey(coord.tileDataHash().getAsLong())) {
|
||||
id = ids.get(coord.tileDataHash().getAsLong());
|
||||
} else {
|
||||
ids.put(coord.tileDataHash().getAsLong(), id = ++num);
|
||||
}
|
||||
for (var layer : coord.layerStats()) {
|
||||
writeTsvLine(result,
|
||||
coord.coord().z(),
|
||||
coord.coord().x(),
|
||||
coord.coord().y(),
|
||||
coord.coord().hilbertEncoded(),
|
||||
coord.rawTileSize(),
|
||||
coord.tileData().length,
|
||||
id,
|
||||
layer.name(),
|
||||
layer.features(),
|
||||
layer.totalBytes(),
|
||||
layer.attrBytes(),
|
||||
layer.attrValues()
|
||||
);
|
||||
}
|
||||
counter.incrementAndGet();
|
||||
}
|
||||
});
|
||||
ProgressLoggers loggers = ProgressLoggers.create()
|
||||
.addRateCounter("tiles", counter)
|
||||
.newLine()
|
||||
.addPipelineStats(pipeline)
|
||||
.newLine()
|
||||
.addProcessStats();
|
||||
pipeline.awaitAndLog(loggers, config.logInterval());
|
||||
|
||||
timer.stop();
|
||||
stats.printSummary();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void writeTsvLine(Writer writer, Object... values) throws IOException {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
if (i > 0) {
|
||||
builder.append('\t');
|
||||
}
|
||||
builder.append(values[i]);
|
||||
}
|
||||
builder.append('\n');
|
||||
writer.write(builder.toString());
|
||||
}
|
||||
|
||||
private static Writer newWriter(Path path) throws IOException {
|
||||
return new OutputStreamWriter(
|
||||
new TileArchiveWriter.FastGzipOutputStream(new BufferedOutputStream(Files.newOutputStream(path,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE))));
|
||||
}
|
||||
}
|
|
@ -11,6 +11,7 @@ import com.onthegomap.planetiler.examples.OsmQaTiles;
|
|||
import com.onthegomap.planetiler.examples.ToiletsOverlay;
|
||||
import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi;
|
||||
import com.onthegomap.planetiler.mbtiles.Verify;
|
||||
import com.onthegomap.planetiler.util.TileStats;
|
||||
import java.util.Arrays;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
@ -50,7 +51,8 @@ public class Main {
|
|||
entry("benchmark-longlongmap", LongLongMapBench::main),
|
||||
|
||||
entry("verify-mbtiles", Verify::main),
|
||||
entry("verify-monaco", VerifyMonaco::main)
|
||||
entry("verify-monaco", VerifyMonaco::main),
|
||||
entry("stats", TileStats::main)
|
||||
);
|
||||
|
||||
private static EntryPoint bundledSchema(String path) {
|
||||
|
|
Ładowanie…
Reference in New Issue