kopia lustrzana https://github.com/onthegomap/planetiler
refactor feature data records
rodzic
ca73e5d491
commit
dd505aeb33
6
pom.xml
6
pom.xml
|
@ -130,12 +130,6 @@
|
|||
<version>3.9.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>RELEASE</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.onthegomap.flatmap.collections.MergeSort;
|
||||
import com.onthegomap.flatmap.collections.FeatureSort;
|
||||
import java.util.function.Consumer;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.locationtech.jts.geom.GeometryCollection;
|
||||
|
@ -22,11 +22,11 @@ public class FeatureRenderer {
|
|||
this.config = config;
|
||||
}
|
||||
|
||||
public void renderFeature(RenderableFeature feature, Consumer<MergeSort.Entry> consumer) {
|
||||
public void renderFeature(RenderableFeature feature, Consumer<FeatureSort.Entry> consumer) {
|
||||
renderGeometry(feature.getGeometry(), feature, consumer);
|
||||
}
|
||||
|
||||
public void renderGeometry(Geometry geom, RenderableFeature feature, Consumer<MergeSort.Entry> consumer) {
|
||||
public void renderGeometry(Geometry geom, RenderableFeature feature, Consumer<FeatureSort.Entry> consumer) {
|
||||
// TODO what about converting between area and line?
|
||||
if (geom instanceof Point point) {
|
||||
addPointFeature(feature, point, consumer);
|
||||
|
@ -44,15 +44,15 @@ public class FeatureRenderer {
|
|||
}
|
||||
}
|
||||
|
||||
private void addPointFeature(RenderableFeature feature, Point point, Consumer<MergeSort.Entry> consumer) {
|
||||
private void addPointFeature(RenderableFeature feature, Point point, Consumer<FeatureSort.Entry> consumer) {
|
||||
// TODO render features into tile
|
||||
}
|
||||
|
||||
private void addPointFeature(RenderableFeature feature, MultiPoint points, Consumer<MergeSort.Entry> consumer) {
|
||||
private void addPointFeature(RenderableFeature feature, MultiPoint points, Consumer<FeatureSort.Entry> consumer) {
|
||||
// TODO render features into tile
|
||||
}
|
||||
|
||||
private void addLinearFeature(RenderableFeature feature, Geometry geom, Consumer<MergeSort.Entry> consumer) {
|
||||
private void addLinearFeature(RenderableFeature feature, Geometry geom, Consumer<FeatureSort.Entry> consumer) {
|
||||
// TODO render lines / areas into tile
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,30 +0,0 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap.FeatureMapKey;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap.FeatureMapValue;
|
||||
import java.util.Map;
|
||||
|
||||
public record LayerFeature(
|
||||
boolean hasGroup,
|
||||
long group,
|
||||
int zorder,
|
||||
Map<String, Object> attrs,
|
||||
byte geomType,
|
||||
int[] commands,
|
||||
long id
|
||||
) implements VectorTileFeature {
|
||||
|
||||
public static LayerFeature of(FeatureMapKey key, FeatureMapValue value) {
|
||||
return new LayerFeature(
|
||||
key.hasGroup(),
|
||||
value.group(),
|
||||
key.zOrder(),
|
||||
value.attrs(),
|
||||
value.geomType(),
|
||||
value.commands(),
|
||||
value.featureId()
|
||||
);
|
||||
}
|
||||
|
||||
}
|
|
@ -1,7 +1,6 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures;
|
||||
import com.onthegomap.flatmap.collections.FeatureGroup;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import com.onthegomap.flatmap.monitoring.ProgressLoggers;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
|
@ -33,7 +32,7 @@ public class MbtilesWriter {
|
|||
|
||||
}
|
||||
|
||||
public static void writeOutput(long featureCount, MergeSortFeatureMap features, File output, FlatMapConfig config) {
|
||||
public static void writeOutput(long featureCount, FeatureGroup features, File output, FlatMapConfig config) {
|
||||
Stats stats = config.stats();
|
||||
output.delete();
|
||||
MbtilesWriter writer = new MbtilesWriter(config.stats());
|
||||
|
@ -56,8 +55,8 @@ public class MbtilesWriter {
|
|||
topology.awaitAndLog(loggers, config.logInterval());
|
||||
}
|
||||
|
||||
public void tileEncoder(Supplier<TileFeatures> prev, Consumer<RenderedTile> next) throws Exception {
|
||||
MergeSortFeatureMap.TileFeatures tileFeatures, last = null;
|
||||
public void tileEncoder(Supplier<FeatureGroup.TileFeatures> prev, Consumer<RenderedTile> next) throws Exception {
|
||||
FeatureGroup.TileFeatures tileFeatures, last = null;
|
||||
byte[] lastBytes = null, lastEncoded = null;
|
||||
while ((tileFeatures = prev.get()) != null) {
|
||||
featuresProcessed.addAndGet(tileFeatures.getNumFeatures());
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.onthegomap.flatmap.collections.FeatureGroup;
|
||||
import com.onthegomap.flatmap.collections.FeatureSort;
|
||||
import com.onthegomap.flatmap.collections.LongLongMap;
|
||||
import com.onthegomap.flatmap.collections.MergeSort;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
|
||||
import com.onthegomap.flatmap.profiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.reader.NaturalEarthReader;
|
||||
import com.onthegomap.flatmap.reader.OpenStreetMapReader;
|
||||
|
@ -66,8 +66,8 @@ public class OpenMapTilesMain {
|
|||
FileUtils.forceMkdir(tmpDir.toFile());
|
||||
File nodeDb = tmpDir.resolve("node.db").toFile();
|
||||
LongLongMap nodeLocations = new LongLongMap.MapdbSortedTable(nodeDb);
|
||||
MergeSort featureDb = MergeSort.newExternalMergeSort(tmpDir.resolve("feature.db"), threads, stats);
|
||||
MergeSortFeatureMap featureMap = new MergeSortFeatureMap(featureDb, profile);
|
||||
FeatureSort featureDb = FeatureSort.newExternalMergeSort(tmpDir.resolve("feature.db"), threads, stats);
|
||||
FeatureGroup featureMap = new FeatureGroup(featureDb, profile);
|
||||
FlatMapConfig config = new FlatMapConfig(profile, envelope, threads, stats, logInterval);
|
||||
FeatureRenderer renderer = new FeatureRenderer(config);
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ import com.graphhopper.reader.ReaderElement;
|
|||
import com.graphhopper.reader.osm.pbf.PbfDecoder;
|
||||
import com.graphhopper.reader.osm.pbf.PbfStreamSplitter;
|
||||
import com.graphhopper.reader.osm.pbf.Sink;
|
||||
import com.onthegomap.flatmap.worker.Topology.SourceStep;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.File;
|
||||
|
@ -80,7 +80,7 @@ public class OsmInputFile {
|
|||
}
|
||||
}
|
||||
|
||||
public SourceStep<ReaderElement> read(int threads) {
|
||||
public Topology.SourceStep<ReaderElement> read(int threads) {
|
||||
return next -> readTo(next, threads);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,24 +1,24 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.graphhopper.reader.ReaderRelation;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature;
|
||||
import com.onthegomap.flatmap.reader.OpenStreetMapReader.RelationInfo;
|
||||
import com.onthegomap.flatmap.reader.OpenStreetMapReader;
|
||||
import java.util.List;
|
||||
|
||||
public interface Profile {
|
||||
|
||||
List<RelationInfo> preprocessOsmRelation(ReaderRelation relation);
|
||||
List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation);
|
||||
|
||||
void processFeature(SourceFeature sourceFeature, RenderableFeatures features);
|
||||
|
||||
void release();
|
||||
|
||||
List<VectorTileFeature> postProcessLayerFeatures(String layer, int zoom, List<VectorTileFeature> items);
|
||||
List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTileEncoder.Feature> items);
|
||||
|
||||
class NullProfile implements Profile {
|
||||
|
||||
@Override
|
||||
public List<RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
|
||||
public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -33,8 +33,9 @@ public interface Profile {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<VectorTileFeature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTileFeature> items) {
|
||||
public List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer,
|
||||
int zoom,
|
||||
List<VectorTileEncoder.Feature> items) {
|
||||
return items;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import java.util.Optional;
|
||||
|
||||
public record RenderedFeature(
|
||||
TileCoord tile,
|
||||
VectorTileEncoder.Feature vectorTileFeature,
|
||||
int zOrder,
|
||||
Optional<Group> group
|
||||
) {
|
||||
|
||||
public RenderedFeature {
|
||||
assert vectorTileFeature != null;
|
||||
}
|
||||
|
||||
public static record Group(long group, int limit) {
|
||||
|
||||
}
|
||||
}
|
|
@ -1,7 +1,6 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.graphhopper.reader.ReaderElement;
|
||||
import com.onthegomap.flatmap.Wikidata.WikidataTranslations;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
@ -16,7 +15,7 @@ public class Translations {
|
|||
return null;
|
||||
}
|
||||
|
||||
public void addTranslationProvider(WikidataTranslations load) {
|
||||
public void addTranslationProvider(Wikidata.WikidataTranslations load) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
|
|
|
@ -24,6 +24,7 @@ import com.google.common.primitives.Ints;
|
|||
import com.google.protobuf.InvalidProtocolBufferException;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
@ -66,13 +67,13 @@ public class VectorTileEncoder {
|
|||
private static final double SCALE = ((double) EXTENT) / SIZE;
|
||||
private final Map<String, Layer> layers = new LinkedHashMap<>();
|
||||
|
||||
public static int[] getCommands(Geometry input) {
|
||||
private static int[] getCommands(Geometry input) {
|
||||
var encoder = new CommandEncoder();
|
||||
encoder.accept(input);
|
||||
return encoder.result.toArray();
|
||||
}
|
||||
|
||||
public static VectorTile.Tile.GeomType toGeomType(Geometry geometry) {
|
||||
private static VectorTile.Tile.GeomType toGeomType(Geometry geometry) {
|
||||
if (geometry instanceof Point || geometry instanceof MultiPoint) {
|
||||
return VectorTile.Tile.GeomType.POINT;
|
||||
} else if (geometry instanceof LineString || geometry instanceof MultiLineString) {
|
||||
|
@ -97,7 +98,7 @@ public class VectorTileEncoder {
|
|||
return ((n >> 1) ^ (-(n & 1)));
|
||||
}
|
||||
|
||||
public static Geometry decodeCommands(byte geomTypeByte, int[] commands) {
|
||||
private static Geometry decodeCommands(byte geomTypeByte, int[] commands) {
|
||||
VectorTile.Tile.GeomType geomType = Objects.requireNonNull(VectorTile.Tile.GeomType.forNumber(geomTypeByte));
|
||||
GeometryFactory gf = GeoUtils.gf;
|
||||
int x = 0;
|
||||
|
@ -226,13 +227,13 @@ public class VectorTileEncoder {
|
|||
return geometry;
|
||||
}
|
||||
|
||||
public static List<DecodedFeature> decode(byte[] encoded) {
|
||||
public static List<Feature> decode(byte[] encoded) {
|
||||
try {
|
||||
VectorTile.Tile tile = VectorTile.Tile.parseFrom(encoded);
|
||||
List<DecodedFeature> features = new ArrayList<>();
|
||||
List<Feature> features = new ArrayList<>();
|
||||
for (VectorTile.Tile.Layer layer : tile.getLayersList()) {
|
||||
String layerName = layer.getName();
|
||||
int extent = layer.getExtent();
|
||||
assert layer.getExtent() == 4096;
|
||||
List<String> keys = layer.getKeysList();
|
||||
List<Object> values = new ArrayList<>();
|
||||
|
||||
|
@ -266,12 +267,11 @@ public class VectorTileEncoder {
|
|||
attrs.put(key, value);
|
||||
}
|
||||
Geometry geometry = decodeCommands(feature.getType(), feature.getGeometryList());
|
||||
features.add(new DecodedFeature(
|
||||
features.add(new Feature(
|
||||
layerName,
|
||||
extent,
|
||||
geometry,
|
||||
attrs,
|
||||
feature.getId()
|
||||
feature.getId(),
|
||||
encodeGeometry(geometry),
|
||||
attrs
|
||||
));
|
||||
}
|
||||
}
|
||||
|
@ -285,18 +285,11 @@ public class VectorTileEncoder {
|
|||
return decodeCommands((byte) type.getNumber(), geometryList.stream().mapToInt(i -> i).toArray());
|
||||
}
|
||||
|
||||
public interface VectorTileFeature {
|
||||
|
||||
int[] commands();
|
||||
|
||||
long id();
|
||||
|
||||
byte geomType();
|
||||
|
||||
Map<String, Object> attrs();
|
||||
public static VectorGeometry encodeGeometry(Geometry geometry) {
|
||||
return new VectorGeometry(getCommands(geometry), (byte) toGeomType(geometry).getNumber());
|
||||
}
|
||||
|
||||
public VectorTileEncoder addLayerFeatures(String layerName, List<? extends VectorTileFeature> features) {
|
||||
public VectorTileEncoder addLayerFeatures(String layerName, List<? extends Feature> features) {
|
||||
if (features.isEmpty()) {
|
||||
return this;
|
||||
}
|
||||
|
@ -307,8 +300,8 @@ public class VectorTileEncoder {
|
|||
layers.put(layerName, layer);
|
||||
}
|
||||
|
||||
for (VectorTileFeature inFeature : features) {
|
||||
if (inFeature.commands().length > 0) {
|
||||
for (Feature inFeature : features) {
|
||||
if (inFeature.geometry().commands().length > 0) {
|
||||
EncodedFeature outFeature = new EncodedFeature(inFeature);
|
||||
|
||||
for (Map.Entry<String, ?> e : inFeature.attrs().entrySet()) {
|
||||
|
@ -370,8 +363,8 @@ public class VectorTileEncoder {
|
|||
featureBuilder.setId(feature.id);
|
||||
}
|
||||
|
||||
featureBuilder.setType(VectorTile.Tile.GeomType.forNumber(feature.geometryType));
|
||||
featureBuilder.addAllGeometry(Ints.asList(feature.geometry));
|
||||
featureBuilder.setType(VectorTile.Tile.GeomType.forNumber(feature.geometry().geomType()));
|
||||
featureBuilder.addAllGeometry(Ints.asList(feature.geometry().commands()));
|
||||
tileLayer.addFeatures(featureBuilder.build());
|
||||
}
|
||||
|
||||
|
@ -391,6 +384,63 @@ public class VectorTileEncoder {
|
|||
}
|
||||
}
|
||||
|
||||
public static record VectorGeometry(int[] commands, byte geomType) {
|
||||
|
||||
public Geometry decode() {
|
||||
return decodeCommands(geomType, commands);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
VectorGeometry that = (VectorGeometry) o;
|
||||
|
||||
if (geomType != that.geomType) {
|
||||
return false;
|
||||
}
|
||||
return Arrays.equals(commands, that.commands);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Arrays.hashCode(commands);
|
||||
result = 31 * result + (int) geomType;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "VectorGeometry[" +
|
||||
"commands=int[" + commands.length +
|
||||
"], geomType=" + geomType +
|
||||
" (" + GeomType.forNumber(geomType) +
|
||||
")]";
|
||||
}
|
||||
}
|
||||
|
||||
public static record Feature(
|
||||
String layer,
|
||||
long id,
|
||||
VectorGeometry geometry,
|
||||
Map<String, Object> attrs
|
||||
) {
|
||||
|
||||
public Feature copyWithNewGeometry(Geometry newGeometry) {
|
||||
return new Feature(
|
||||
layer,
|
||||
id,
|
||||
encodeGeometry(newGeometry),
|
||||
attrs
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static class CommandEncoder {
|
||||
|
||||
private final IntArrayList result = new IntArrayList();
|
||||
|
@ -503,23 +553,13 @@ public class VectorTileEncoder {
|
|||
}
|
||||
}
|
||||
|
||||
private static final record EncodedFeature(IntArrayList tags, long id, byte geometryType, int[] geometry) {
|
||||
private static final record EncodedFeature(IntArrayList tags, long id, VectorGeometry geometry) {
|
||||
|
||||
EncodedFeature(VectorTileFeature in) {
|
||||
this(new IntArrayList(), in.id(), in.geomType(), in.commands());
|
||||
EncodedFeature(Feature in) {
|
||||
this(new IntArrayList(), in.id(), in.geometry());
|
||||
}
|
||||
}
|
||||
|
||||
public static final record DecodedFeature(
|
||||
String layerName,
|
||||
int extent,
|
||||
Geometry geometry,
|
||||
Map<String, Object> attributes,
|
||||
long id
|
||||
) {
|
||||
|
||||
}
|
||||
|
||||
private static final class Layer {
|
||||
|
||||
private final List<EncodedFeature> encodedFeatures = new ArrayList<>();
|
||||
|
|
|
@ -27,9 +27,9 @@ import org.jetbrains.annotations.NotNull;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
class ExternalMergeSort implements MergeSort {
|
||||
class ExternalMergeSort implements FeatureSort {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MergeSort.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FeatureSort.class);
|
||||
|
||||
private static final long MAX_CHUNK_SIZE = 1_000_000_000; // 1GB
|
||||
|
||||
|
|
|
@ -0,0 +1,319 @@
|
|||
package com.onthegomap.flatmap.collections;
|
||||
|
||||
import com.carrotsearch.hppc.LongLongHashMap;
|
||||
import com.graphhopper.coll.GHLongLongHashMap;
|
||||
import com.onthegomap.flatmap.Profile;
|
||||
import com.onthegomap.flatmap.RenderedFeature;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
import org.msgpack.core.MessageBufferPacker;
|
||||
import org.msgpack.core.MessagePack;
|
||||
import org.msgpack.core.MessageUnpacker;
|
||||
import org.msgpack.value.Value;
|
||||
import org.msgpack.value.ValueFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public record FeatureGroup(FeatureSort sorter, Profile profile, CommonStringEncoder commonStrings)
|
||||
implements Consumer<FeatureSort.Entry>, Iterable<FeatureGroup.TileFeatures> {
|
||||
|
||||
public static final int Z_ORDER_BITS = 23;
|
||||
public static final int Z_ORDER_MAX = (1 << (Z_ORDER_BITS - 1)) - 1;
|
||||
public static final int Z_ORDER_MIN = -(1 << (Z_ORDER_BITS - 1));
|
||||
private static final int Z_ORDER_MASK = (1 << Z_ORDER_BITS) - 1;
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FeatureGroup.class);
|
||||
private static final ThreadLocal<MessageBufferPacker> messagePackers = ThreadLocal
|
||||
.withInitial(MessagePack::newDefaultBufferPacker);
|
||||
|
||||
public FeatureGroup(FeatureSort sorter, Profile profile) {
|
||||
this(sorter, profile, new CommonStringEncoder());
|
||||
}
|
||||
|
||||
static long encodeSortKey(int tile, byte layer, int zOrder, boolean hasGroup) {
|
||||
zOrder = -zOrder - 1;
|
||||
return ((long) tile << 32L) | ((long) (layer & 0xff) << 24L) | (((zOrder - Z_ORDER_MIN) & Z_ORDER_MASK) << 1L) | (
|
||||
hasGroup ? 1 : 0);
|
||||
}
|
||||
|
||||
static boolean extractHasGroupFromSortKey(long sortKey) {
|
||||
return (sortKey & 1) == 1;
|
||||
}
|
||||
|
||||
static int extractTileFromSortKey(long sortKey) {
|
||||
return (int) (sortKey >> 32L);
|
||||
}
|
||||
|
||||
static byte extractLayerIdFromSortKey(long sortKey) {
|
||||
return (byte) (sortKey >> 24);
|
||||
}
|
||||
|
||||
static int extractZorderFromKey(long sortKey) {
|
||||
return Z_ORDER_MAX - ((int) ((sortKey >> 1) & Z_ORDER_MASK));
|
||||
}
|
||||
|
||||
private static RenderedFeature.Group decodeGroupInfo(byte[] encoded) {
|
||||
try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) {
|
||||
long group = unpacker.unpackLong();
|
||||
int limit = unpacker.unpackInt();
|
||||
return new RenderedFeature.Group(group, limit);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public FeatureSort.Entry encode(RenderedFeature feature) {
|
||||
return new FeatureSort.Entry(
|
||||
encodeSortKey(feature),
|
||||
encodeValue(feature)
|
||||
);
|
||||
}
|
||||
|
||||
private long encodeSortKey(RenderedFeature feature) {
|
||||
var vectorTileFeature = feature.vectorTileFeature();
|
||||
commonStrings.encode(vectorTileFeature.layer());
|
||||
return encodeSortKey(
|
||||
feature.tile().encoded(),
|
||||
commonStrings.encode(vectorTileFeature.layer()),
|
||||
feature.zOrder(),
|
||||
feature.group().isPresent()
|
||||
);
|
||||
}
|
||||
|
||||
private byte[] encodeValue(RenderedFeature feature) {
|
||||
MessageBufferPacker packer = messagePackers.get();
|
||||
packer.clear();
|
||||
try {
|
||||
var groupInfoOption = feature.group();
|
||||
if (groupInfoOption.isPresent()) {
|
||||
var groupInfo = groupInfoOption.get();
|
||||
packer.packLong(groupInfo.group());
|
||||
packer.packInt(groupInfo.limit());
|
||||
}
|
||||
var vectorTileFeature = feature.vectorTileFeature();
|
||||
packer.packLong(vectorTileFeature.id());
|
||||
packer.packByte(vectorTileFeature.geometry().geomType());
|
||||
var attrs = vectorTileFeature.attrs();
|
||||
packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count());
|
||||
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
|
||||
if (entry.getValue() != null) {
|
||||
packer.packByte(commonStrings.encode(entry.getKey()));
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof String) {
|
||||
packer.packValue(ValueFactory.newString((String) value));
|
||||
} else if (value instanceof Integer) {
|
||||
packer.packValue(ValueFactory.newInteger(((Integer) value).longValue()));
|
||||
} else if (value instanceof Long) {
|
||||
packer.packValue(ValueFactory.newInteger((Long) value));
|
||||
} else if (value instanceof Float) {
|
||||
packer.packValue(ValueFactory.newFloat((Float) value));
|
||||
} else if (value instanceof Double) {
|
||||
packer.packValue(ValueFactory.newFloat((Double) value));
|
||||
} else if (value instanceof Boolean) {
|
||||
packer.packValue(ValueFactory.newBoolean((Boolean) value));
|
||||
} else {
|
||||
packer.packValue(ValueFactory.newString(value.toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
int[] commands = vectorTileFeature.geometry().commands();
|
||||
packer.packArrayHeader(commands.length);
|
||||
for (int command : commands) {
|
||||
packer.packInt(command);
|
||||
}
|
||||
packer.close();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
return packer.toByteArray();
|
||||
}
|
||||
|
||||
private VectorTileEncoder.Feature decodeVectorTileFeature(FeatureSort.Entry entry) {
|
||||
try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(entry.value())) {
|
||||
if (extractHasGroupFromSortKey(entry.sortKey())) {
|
||||
unpacker.unpackLong(); // group
|
||||
unpacker.unpackInt(); // groupLimit
|
||||
}
|
||||
long id = unpacker.unpackLong();
|
||||
byte geomType = unpacker.unpackByte();
|
||||
int mapSize = unpacker.unpackMapHeader();
|
||||
Map<String, Object> attrs = new HashMap<>(mapSize);
|
||||
for (int i = 0; i < mapSize; i++) {
|
||||
String key = commonStrings.decode(unpacker.unpackByte());
|
||||
Value v = unpacker.unpackValue();
|
||||
if (v.isStringValue()) {
|
||||
attrs.put(key, v.asStringValue().asString());
|
||||
} else if (v.isIntegerValue()) {
|
||||
attrs.put(key, v.asIntegerValue().toLong());
|
||||
} else if (v.isFloatValue()) {
|
||||
attrs.put(key, v.asFloatValue().toDouble());
|
||||
} else if (v.isBooleanValue()) {
|
||||
attrs.put(key, v.asBooleanValue().getBoolean());
|
||||
}
|
||||
}
|
||||
int commandSize = unpacker.unpackArrayHeader();
|
||||
int[] commands = new int[commandSize];
|
||||
for (int i = 0; i < commandSize; i++) {
|
||||
commands[i] = unpacker.unpackInt();
|
||||
}
|
||||
return new VectorTileEncoder.Feature(
|
||||
commonStrings.decode(extractLayerIdFromSortKey(entry.sortKey())),
|
||||
id,
|
||||
new VectorTileEncoder.VectorGeometry(commands, geomType),
|
||||
attrs
|
||||
);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(FeatureSort.Entry entry) {
|
||||
sorter.add(entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<TileFeatures> iterator() {
|
||||
Iterator<FeatureSort.Entry> entries = sorter.iterator();
|
||||
if (!entries.hasNext()) {
|
||||
return Collections.emptyIterator();
|
||||
}
|
||||
FeatureSort.Entry firstFeature = entries.next();
|
||||
return new Iterator<>() {
|
||||
private FeatureSort.Entry lastFeature = firstFeature;
|
||||
private int lastTileId = extractTileFromSortKey(firstFeature.sortKey());
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return lastFeature != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileFeatures next() {
|
||||
TileFeatures result = new TileFeatures(lastTileId);
|
||||
result.accept(lastFeature);
|
||||
int lastTile = lastTileId;
|
||||
|
||||
while (entries.hasNext()) {
|
||||
FeatureSort.Entry next = entries.next();
|
||||
lastFeature = next;
|
||||
lastTileId = extractTileFromSortKey(lastFeature.sortKey());
|
||||
if (lastTile != lastTileId) {
|
||||
return result;
|
||||
}
|
||||
result.accept(next);
|
||||
}
|
||||
lastFeature = null;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public long getStorageSize() {
|
||||
return sorter.getStorageSize();
|
||||
}
|
||||
|
||||
public class TileFeatures implements Consumer<FeatureSort.Entry> {
|
||||
|
||||
private final TileCoord tile;
|
||||
private final List<FeatureSort.Entry> entries = new ArrayList<>();
|
||||
|
||||
private LongLongHashMap counts = null;
|
||||
private byte layer = Byte.MAX_VALUE;
|
||||
|
||||
public TileFeatures(int tile) {
|
||||
this.tile = TileCoord.decode(tile);
|
||||
}
|
||||
|
||||
public long getNumFeatures() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public TileCoord coord() {
|
||||
return tile;
|
||||
}
|
||||
|
||||
public boolean hasSameContents(TileFeatures other) {
|
||||
if (other == null || other.entries.size() != entries.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < entries.size(); i++) {
|
||||
byte[] a = entries.get(i).value();
|
||||
byte[] b = other.entries.get(i).value();
|
||||
if (!Arrays.equals(a, b)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public VectorTileEncoder getTile() {
|
||||
VectorTileEncoder encoder = new VectorTileEncoder();
|
||||
List<VectorTileEncoder.Feature> items = new ArrayList<>(entries.size());
|
||||
String currentLayer = null;
|
||||
for (int index = entries.size() - 1; index >= 0; index--) {
|
||||
FeatureSort.Entry entry = entries.get(index);
|
||||
|
||||
var feature = decodeVectorTileFeature(entry);
|
||||
String layer = feature.layer();
|
||||
|
||||
if (currentLayer == null) {
|
||||
currentLayer = layer;
|
||||
} else if (!currentLayer.equals(layer)) {
|
||||
encoder.addLayerFeatures(
|
||||
currentLayer,
|
||||
profile.postProcessLayerFeatures(currentLayer, tile.z(), items)
|
||||
);
|
||||
currentLayer = layer;
|
||||
items.clear();
|
||||
}
|
||||
|
||||
items.add(feature);
|
||||
}
|
||||
encoder.addLayerFeatures(
|
||||
currentLayer,
|
||||
profile.postProcessLayerFeatures(currentLayer, tile.z(), items)
|
||||
);
|
||||
return encoder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(FeatureSort.Entry entry) {
|
||||
long sortKey = entry.sortKey();
|
||||
if (extractHasGroupFromSortKey(sortKey)) {
|
||||
byte thisLayer = extractLayerIdFromSortKey(sortKey);
|
||||
if (counts == null) {
|
||||
counts = new GHLongLongHashMap();
|
||||
layer = thisLayer;
|
||||
} else if (thisLayer != layer) {
|
||||
layer = thisLayer;
|
||||
counts.clear();
|
||||
}
|
||||
var groupInfo = decodeGroupInfo(entry.value());
|
||||
long old = counts.getOrDefault(groupInfo.group(), 0);
|
||||
if (groupInfo.limit() > 0 && old >= groupInfo.limit()) {
|
||||
return;
|
||||
}
|
||||
counts.put(groupInfo.group(), old + 1);
|
||||
}
|
||||
entries.add(entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TileFeatures{" +
|
||||
"tile=" + tile +
|
||||
", num entries=" + entries.size() +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
|
@ -7,13 +7,13 @@ import java.util.Arrays;
|
|||
import java.util.List;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
public interface MergeSort extends Iterable<MergeSort.Entry> {
|
||||
public interface FeatureSort extends Iterable<FeatureSort.Entry> {
|
||||
|
||||
static MergeSort newExternalMergeSort(Path tempDir, int threads, Stats stats) {
|
||||
static FeatureSort newExternalMergeSort(Path tempDir, int threads, Stats stats) {
|
||||
return new ExternalMergeSort(tempDir, threads, stats);
|
||||
}
|
||||
|
||||
static MergeSort newExternalMergeSort(Path dir, int workers, int chunkSizeLimit, Stats stats) {
|
||||
static FeatureSort newExternalMergeSort(Path dir, int workers, int chunkSizeLimit, Stats stats) {
|
||||
return new ExternalMergeSort(dir, workers, chunkSizeLimit, stats);
|
||||
}
|
||||
|
|
@ -1,404 +0,0 @@
|
|||
package com.onthegomap.flatmap.collections;
|
||||
|
||||
import com.carrotsearch.hppc.LongLongHashMap;
|
||||
import com.graphhopper.coll.GHLongLongHashMap;
|
||||
import com.onthegomap.flatmap.LayerFeature;
|
||||
import com.onthegomap.flatmap.Profile;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Consumer;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.msgpack.core.MessageBufferPacker;
|
||||
import org.msgpack.core.MessagePack;
|
||||
import org.msgpack.core.MessageUnpacker;
|
||||
import org.msgpack.value.Value;
|
||||
import org.msgpack.value.ValueFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public record MergeSortFeatureMap(MergeSort mergeSort, Profile profile, CommonStringEncoder commonStrings)
|
||||
implements Consumer<MergeSort.Entry>, Iterable<TileFeatures> {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MergeSortFeatureMap.class);
|
||||
|
||||
public MergeSortFeatureMap(MergeSort mergeSort, Profile profile) {
|
||||
this(mergeSort, profile, new CommonStringEncoder());
|
||||
}
|
||||
|
||||
public MergeSort.Entry encode(
|
||||
long featureId,
|
||||
TileCoord tile,
|
||||
String layer,
|
||||
Map<String, Object> attrs,
|
||||
Geometry geom,
|
||||
int zOrder,
|
||||
boolean hasGroup,
|
||||
int groupLimit
|
||||
) {
|
||||
return new MergeSort.Entry(
|
||||
FeatureMapKey.encode(tile.encoded(), commonStrings.encode(layer), zOrder, hasGroup),
|
||||
FeatureMapValue.from(featureId, attrs, geom, hasGroup, groupLimit).encode(commonStrings)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(MergeSort.Entry entry) {
|
||||
mergeSort.add(entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<TileFeatures> iterator() {
|
||||
Iterator<MergeSort.Entry> entries = mergeSort.iterator();
|
||||
if (!entries.hasNext()) {
|
||||
return Collections.emptyIterator();
|
||||
}
|
||||
MergeSort.Entry firstFeature = entries.next();
|
||||
return new Iterator<>() {
|
||||
private MergeSort.Entry lastFeature = firstFeature;
|
||||
private int lastTileId = FeatureMapKey.extractTileFromKey(firstFeature.sortKey());
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return lastFeature != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileFeatures next() {
|
||||
TileFeatures result = new TileFeatures(lastTileId);
|
||||
result.accept(lastFeature);
|
||||
int lastTile = lastTileId;
|
||||
|
||||
while (entries.hasNext()) {
|
||||
MergeSort.Entry next = entries.next();
|
||||
lastFeature = next;
|
||||
lastTileId = FeatureMapKey.extractTileFromKey(lastFeature.sortKey());
|
||||
if (lastTile != lastTileId) {
|
||||
return result;
|
||||
}
|
||||
result.accept(next);
|
||||
}
|
||||
lastFeature = null;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public long getStorageSize() {
|
||||
return mergeSort.getStorageSize();
|
||||
}
|
||||
|
||||
public class TileFeatures implements Consumer<MergeSort.Entry> {
|
||||
|
||||
private final TileCoord tile;
|
||||
private final List<MergeSort.Entry> entries = new ArrayList<>();
|
||||
|
||||
private LongLongHashMap counts = null;
|
||||
private byte layer = Byte.MAX_VALUE;
|
||||
|
||||
public TileFeatures(int tile) {
|
||||
this.tile = TileCoord.decode(tile);
|
||||
}
|
||||
|
||||
public long getNumFeatures() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public TileCoord coord() {
|
||||
return tile;
|
||||
}
|
||||
|
||||
public boolean hasSameContents(TileFeatures other) {
|
||||
if (other == null || other.entries.size() != entries.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < entries.size(); i++) {
|
||||
byte[] a = entries.get(i).value();
|
||||
byte[] b = other.entries.get(i).value();
|
||||
if (!Arrays.equals(a, b)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public VectorTileEncoder getTile() {
|
||||
VectorTileEncoder encoder = new VectorTileEncoder();
|
||||
List<VectorTileFeature> items = new ArrayList<>(entries.size());
|
||||
String currentLayer = null;
|
||||
for (int index = entries.size() - 1; index >= 0; index--) {
|
||||
MergeSort.Entry entry = entries.get(index);
|
||||
|
||||
FeatureMapKey key = FeatureMapKey.decode(entry.sortKey());
|
||||
FeatureMapValue value = FeatureMapValue.decode(entry.value(), key.hasGroup(), commonStrings);
|
||||
String layer = commonStrings.decode(key.layer);
|
||||
|
||||
if (currentLayer == null) {
|
||||
currentLayer = layer;
|
||||
} else if (!currentLayer.equals(layer)) {
|
||||
encoder.addLayerFeatures(
|
||||
currentLayer,
|
||||
profile.postProcessLayerFeatures(currentLayer, tile.z(), items)
|
||||
);
|
||||
currentLayer = layer;
|
||||
items.clear();
|
||||
}
|
||||
|
||||
items.add(LayerFeature.of(key, value));
|
||||
}
|
||||
encoder.addLayerFeatures(
|
||||
currentLayer,
|
||||
profile.postProcessLayerFeatures(currentLayer, tile.z(), items)
|
||||
);
|
||||
return encoder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(MergeSort.Entry entry) {
|
||||
long sortKey = entry.sortKey();
|
||||
if (FeatureMapKey.extractHasGroupFromKey(sortKey)) {
|
||||
byte thisLayer = FeatureMapKey.extractLayerIdFromKey(sortKey);
|
||||
if (counts == null) {
|
||||
counts = new GHLongLongHashMap();
|
||||
layer = thisLayer;
|
||||
} else if (thisLayer != layer) {
|
||||
layer = thisLayer;
|
||||
counts.clear();
|
||||
}
|
||||
var groupInfo = FeatureMapValue.decodeGroupInfo(entry.value());
|
||||
long old = counts.getOrDefault(groupInfo.group, 0);
|
||||
if (groupInfo.limit > 0 && old >= groupInfo.limit) {
|
||||
return;
|
||||
}
|
||||
counts.put(groupInfo.group, old + 1);
|
||||
}
|
||||
entries.add(entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TileFeatures{" +
|
||||
"tile=" + tile +
|
||||
", num entries=" + entries.size() +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
private static final ThreadLocal<MessageBufferPacker> messagePackers = ThreadLocal
|
||||
.withInitial(MessagePack::newDefaultBufferPacker);
|
||||
|
||||
public record RenderedFeature(
|
||||
long featureId,
|
||||
TileCoord tile,
|
||||
String layer,
|
||||
int zOrder,
|
||||
Optional<FeatureMapValue.GroupInfo> groupInfo
|
||||
) {
|
||||
|
||||
}
|
||||
|
||||
public record FeatureMapKey(long encoded, TileCoord tile, byte layer, int zOrder, boolean hasGroup) implements
|
||||
Comparable<FeatureMapKey> {
|
||||
|
||||
private static final int Z_ORDER_MASK = (1 << 23) - 1;
|
||||
public static final int Z_ORDER_MAX = (1 << 22) - 1;
|
||||
public static final int Z_ORDER_MIN = -(1 << 22);
|
||||
public static final int Z_ORDER_BITS = 23;
|
||||
|
||||
public static FeatureMapKey of(int tile, byte layer, int zOrder, boolean hasGroup) {
|
||||
return new FeatureMapKey(encode(tile, layer, zOrder, hasGroup), TileCoord.decode(tile), layer, zOrder, hasGroup);
|
||||
}
|
||||
|
||||
public static FeatureMapKey decode(long encoded) {
|
||||
return of(
|
||||
extractTileFromKey(encoded),
|
||||
extractLayerIdFromKey(encoded),
|
||||
extractZorderFromKey(encoded),
|
||||
extractHasGroupFromKey(encoded)
|
||||
);
|
||||
}
|
||||
|
||||
public static long encode(int tile, byte layer, int zOrder, boolean hasGroup) {
|
||||
zOrder = -zOrder - 1;
|
||||
return ((long) tile << 32L) | ((long) (layer & 0xff) << 24L) | (((zOrder - Z_ORDER_MIN) & Z_ORDER_MASK) << 1L) | (
|
||||
hasGroup ? 1 : 0);
|
||||
}
|
||||
|
||||
public static boolean extractHasGroupFromKey(long sortKey) {
|
||||
return (sortKey & 1) == 1;
|
||||
}
|
||||
|
||||
public static int extractTileFromKey(long sortKey) {
|
||||
return (int) (sortKey >> 32L);
|
||||
}
|
||||
|
||||
public static byte extractLayerIdFromKey(long sortKey) {
|
||||
return (byte) (sortKey >> 24);
|
||||
}
|
||||
|
||||
public static int extractZorderFromKey(long sortKey) {
|
||||
return Z_ORDER_MAX - ((int) ((sortKey >> 1) & Z_ORDER_MASK));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
FeatureMapKey that = (FeatureMapKey) o;
|
||||
|
||||
return encoded == that.encoded;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return (int) (encoded ^ (encoded >>> 32));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(@NotNull FeatureMapKey o) {
|
||||
return Long.compare(encoded, o.encoded);
|
||||
}
|
||||
}
|
||||
|
||||
public static record FeatureMapValue(
|
||||
long featureId,
|
||||
Map<String, Object> attrs,
|
||||
int[] commands,
|
||||
byte geomType,
|
||||
boolean hasGrouping,
|
||||
int groupLimit,
|
||||
long group
|
||||
) {
|
||||
|
||||
public static record GroupInfo(long group, int limit) {
|
||||
|
||||
}
|
||||
|
||||
public static GroupInfo decodeGroupInfo(byte[] encoded) {
|
||||
try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) {
|
||||
long group = unpacker.unpackLong();
|
||||
int limit = unpacker.unpackInt();
|
||||
return new GroupInfo(group, limit);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static FeatureMapValue from(
|
||||
long featureId,
|
||||
Map<String, Object> attrs,
|
||||
Geometry geom,
|
||||
boolean hasGrouping,
|
||||
int groupLimit
|
||||
) {
|
||||
long group = geom.getUserData() instanceof Number number ? number.longValue() : 0;
|
||||
byte geomType = (byte) VectorTileEncoder.toGeomType(geom).getNumber();
|
||||
int[] commands = VectorTileEncoder.getCommands(geom);
|
||||
return new FeatureMapValue(
|
||||
featureId,
|
||||
attrs,
|
||||
commands,
|
||||
geomType,
|
||||
hasGrouping,
|
||||
groupLimit,
|
||||
group
|
||||
);
|
||||
}
|
||||
|
||||
public static FeatureMapValue decode(byte[] encoded, boolean hasGroup, CommonStringEncoder commonStrings) {
|
||||
try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) {
|
||||
long group = 0;
|
||||
int groupLimit = -1;
|
||||
if (hasGroup) {
|
||||
group = unpacker.unpackLong();
|
||||
groupLimit = unpacker.unpackInt();
|
||||
}
|
||||
long id = unpacker.unpackLong();
|
||||
byte geomType = unpacker.unpackByte();
|
||||
int mapSize = unpacker.unpackMapHeader();
|
||||
Map<String, Object> attrs = new HashMap<>(mapSize);
|
||||
for (int i = 0; i < mapSize; i++) {
|
||||
String key = commonStrings.decode(unpacker.unpackByte());
|
||||
Value v = unpacker.unpackValue();
|
||||
if (v.isStringValue()) {
|
||||
attrs.put(key, v.asStringValue().asString());
|
||||
} else if (v.isIntegerValue()) {
|
||||
attrs.put(key, v.asIntegerValue().toLong());
|
||||
} else if (v.isFloatValue()) {
|
||||
attrs.put(key, v.asFloatValue().toDouble());
|
||||
} else if (v.isBooleanValue()) {
|
||||
attrs.put(key, v.asBooleanValue().getBoolean());
|
||||
}
|
||||
}
|
||||
int commandSize = unpacker.unpackArrayHeader();
|
||||
int[] commands = new int[commandSize];
|
||||
for (int i = 0; i < commandSize; i++) {
|
||||
commands[i] = unpacker.unpackInt();
|
||||
}
|
||||
return new FeatureMapValue(id, attrs, commands, geomType, hasGroup, groupLimit, group);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public byte[] encode(CommonStringEncoder commonStrings) {
|
||||
MessageBufferPacker packer = messagePackers.get();
|
||||
packer.clear();
|
||||
try {
|
||||
if (hasGrouping) {
|
||||
packer.packLong(group);
|
||||
packer.packInt(groupLimit);
|
||||
}
|
||||
packer.packLong(featureId);
|
||||
packer.packByte(geomType);
|
||||
packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count());
|
||||
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
|
||||
if (entry.getValue() != null) {
|
||||
packer.packByte(commonStrings.encode(entry.getKey()));
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof String) {
|
||||
packer.packValue(ValueFactory.newString((String) value));
|
||||
} else if (value instanceof Integer) {
|
||||
packer.packValue(ValueFactory.newInteger(((Integer) value).longValue()));
|
||||
} else if (value instanceof Long) {
|
||||
packer.packValue(ValueFactory.newInteger((Long) value));
|
||||
} else if (value instanceof Float) {
|
||||
packer.packValue(ValueFactory.newFloat((Float) value));
|
||||
} else if (value instanceof Double) {
|
||||
packer.packValue(ValueFactory.newFloat((Double) value));
|
||||
} else if (value instanceof Boolean) {
|
||||
packer.packValue(ValueFactory.newBoolean((Boolean) value));
|
||||
} else {
|
||||
packer.packValue(ValueFactory.newString(value.toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
packer.packArrayHeader(commands.length);
|
||||
for (int command : commands) {
|
||||
packer.packInt(command);
|
||||
}
|
||||
packer.close();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
return packer.toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -8,7 +8,6 @@ import static com.onthegomap.flatmap.Format.padRight;
|
|||
|
||||
import com.graphhopper.util.Helper;
|
||||
import com.onthegomap.flatmap.Format;
|
||||
import com.onthegomap.flatmap.monitoring.ProcessInfo.ThreadState;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import com.onthegomap.flatmap.worker.WorkQueue;
|
||||
import com.onthegomap.flatmap.worker.Worker;
|
||||
|
@ -150,7 +149,7 @@ public class ProgressLoggers {
|
|||
public ProgressLoggers addThreadPoolStats(String name, String prefix) {
|
||||
boolean first = loggers.isEmpty() || !(loggers.get(loggers.size() - 1) instanceof TopologyLogger);
|
||||
try {
|
||||
Map<Long, ThreadState> lastThreads = ProcessInfo.getThreadStats();
|
||||
Map<Long, ProcessInfo.ThreadState> lastThreads = ProcessInfo.getThreadStats();
|
||||
AtomicLong lastTime = new AtomicLong(System.nanoTime());
|
||||
loggers.add(new TopologyLogger(() -> {
|
||||
var oldAndNewThreads = new TreeMap<>(lastThreads);
|
||||
|
@ -165,7 +164,7 @@ public class ProgressLoggers {
|
|||
if (!newThreads.containsKey(thread.id())) {
|
||||
return " -%";
|
||||
}
|
||||
long last = lastThreads.getOrDefault(thread.id(), ThreadState.DEFAULT).cpuTimeNanos();
|
||||
long last = lastThreads.getOrDefault(thread.id(), ProcessInfo.ThreadState.DEFAULT).cpuTimeNanos();
|
||||
return padLeft(formatPercent(1d * (thread.cpuTimeNanos() - last) / timeDiff), 3);
|
||||
}).collect(Collectors.joining(" ", "(", ")"));
|
||||
|
||||
|
|
|
@ -1,12 +1,11 @@
|
|||
package com.onthegomap.flatmap.profiles;
|
||||
|
||||
|
||||
import com.graphhopper.reader.ReaderRelation;
|
||||
import com.onthegomap.flatmap.Profile;
|
||||
import com.onthegomap.flatmap.RenderableFeatures;
|
||||
import com.onthegomap.flatmap.SourceFeature;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder;
|
||||
import com.onthegomap.flatmap.reader.OpenStreetMapReader.RelationInfo;
|
||||
import com.onthegomap.flatmap.reader.OpenStreetMapReader;
|
||||
import java.util.List;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
@ -20,13 +19,13 @@ public class OpenMapTilesProfile implements Profile {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<VectorTileEncoder.VectorTileFeature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTileEncoder.VectorTileFeature> items) {
|
||||
public List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTileEncoder.Feature> items) {
|
||||
return items;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
|
||||
public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@ package com.onthegomap.flatmap.reader;
|
|||
import com.onthegomap.flatmap.SourceFeature;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import com.onthegomap.flatmap.worker.Topology.SourceStep;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
|
@ -90,7 +90,7 @@ public class NaturalEarthReader extends Reader {
|
|||
}
|
||||
|
||||
@Override
|
||||
public SourceStep<SourceFeature> read() {
|
||||
public Topology.SourceStep<SourceFeature> read() {
|
||||
return next -> {
|
||||
var tables = tableNames();
|
||||
for (int i = 0; i < tables.size(); i++) {
|
||||
|
|
|
@ -15,10 +15,10 @@ import com.onthegomap.flatmap.Profile;
|
|||
import com.onthegomap.flatmap.RenderableFeature;
|
||||
import com.onthegomap.flatmap.RenderableFeatures;
|
||||
import com.onthegomap.flatmap.SourceFeature;
|
||||
import com.onthegomap.flatmap.collections.FeatureGroup;
|
||||
import com.onthegomap.flatmap.collections.FeatureSort;
|
||||
import com.onthegomap.flatmap.collections.LongLongMap;
|
||||
import com.onthegomap.flatmap.collections.LongLongMultimap;
|
||||
import com.onthegomap.flatmap.collections.MergeSort;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.monitoring.ProgressLoggers;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
|
@ -107,7 +107,7 @@ public class OpenStreetMapReader implements Closeable {
|
|||
topology.awaitAndLog(loggers, config.logInterval());
|
||||
}
|
||||
|
||||
public long pass2(FeatureRenderer renderer, MergeSortFeatureMap writer, int readerThreads, int processThreads,
|
||||
public long pass2(FeatureRenderer renderer, FeatureGroup writer, int readerThreads, int processThreads,
|
||||
FlatMapConfig config) {
|
||||
Profile profile = config.profile();
|
||||
AtomicLong nodesProcessed = new AtomicLong(0);
|
||||
|
@ -119,7 +119,7 @@ public class OpenStreetMapReader implements Closeable {
|
|||
var topology = Topology.start("osm_pass2", stats)
|
||||
.fromGenerator("pbf", osmInputFile.read(readerThreads))
|
||||
.addBuffer("reader_queue", 50_000, 1_000)
|
||||
.<MergeSort.Entry>addWorker("process", processThreads, (prev, next) -> {
|
||||
.<FeatureSort.Entry>addWorker("process", processThreads, (prev, next) -> {
|
||||
RenderableFeatures features = new RenderableFeatures();
|
||||
ReaderElement readerElement;
|
||||
while ((readerElement = prev.get()) != null) {
|
||||
|
|
|
@ -6,12 +6,11 @@ import com.onthegomap.flatmap.Profile;
|
|||
import com.onthegomap.flatmap.RenderableFeature;
|
||||
import com.onthegomap.flatmap.RenderableFeatures;
|
||||
import com.onthegomap.flatmap.SourceFeature;
|
||||
import com.onthegomap.flatmap.collections.MergeSort;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
|
||||
import com.onthegomap.flatmap.collections.FeatureGroup;
|
||||
import com.onthegomap.flatmap.collections.FeatureSort;
|
||||
import com.onthegomap.flatmap.monitoring.ProgressLoggers;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import com.onthegomap.flatmap.worker.Topology.SourceStep;
|
||||
import java.io.Closeable;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
@ -27,7 +26,7 @@ public abstract class Reader implements Closeable {
|
|||
this.stats = stats;
|
||||
}
|
||||
|
||||
public final void process(String name, FeatureRenderer renderer, MergeSortFeatureMap writer, FlatMapConfig config) {
|
||||
public final void process(String name, FeatureRenderer renderer, FeatureGroup writer, FlatMapConfig config) {
|
||||
long featureCount = getCount();
|
||||
int threads = config.threads();
|
||||
Envelope env = config.envelope();
|
||||
|
@ -38,7 +37,7 @@ public abstract class Reader implements Closeable {
|
|||
var topology = Topology.start(name, stats)
|
||||
.fromGenerator("read", read())
|
||||
.addBuffer("read_queue", 1000)
|
||||
.<MergeSort.Entry>addWorker("process", threads, (prev, next) -> {
|
||||
.<FeatureSort.Entry>addWorker("process", threads, (prev, next) -> {
|
||||
RenderableFeatures features = new RenderableFeatures();
|
||||
SourceFeature sourceFeature;
|
||||
while ((sourceFeature = prev.get()) != null) {
|
||||
|
@ -70,7 +69,7 @@ public abstract class Reader implements Closeable {
|
|||
|
||||
public abstract long getCount();
|
||||
|
||||
public abstract SourceStep<SourceFeature> read();
|
||||
public abstract Topology.SourceStep<SourceFeature> read();
|
||||
|
||||
@Override
|
||||
public abstract void close();
|
||||
|
|
|
@ -3,9 +3,9 @@ package com.onthegomap.flatmap.reader;
|
|||
import com.onthegomap.flatmap.FeatureRenderer;
|
||||
import com.onthegomap.flatmap.FlatMapConfig;
|
||||
import com.onthegomap.flatmap.SourceFeature;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
|
||||
import com.onthegomap.flatmap.collections.FeatureGroup;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import com.onthegomap.flatmap.worker.Topology.SourceStep;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.io.Closeable;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
@ -33,14 +33,14 @@ public class ShapefileReader extends Reader implements Closeable {
|
|||
private MathTransform transform;
|
||||
|
||||
public static void process(String sourceProjection, String name, File input, FeatureRenderer renderer,
|
||||
MergeSortFeatureMap writer, FlatMapConfig config) {
|
||||
FeatureGroup writer, FlatMapConfig config) {
|
||||
try (var reader = new ShapefileReader(sourceProjection, input, config.stats())) {
|
||||
reader.process(name, renderer, writer, config);
|
||||
}
|
||||
}
|
||||
|
||||
public static void process(String name, File input, FeatureRenderer renderer,
|
||||
MergeSortFeatureMap writer, FlatMapConfig config) {
|
||||
FeatureGroup writer, FlatMapConfig config) {
|
||||
process(null, name, input, renderer, writer, config);
|
||||
}
|
||||
|
||||
|
@ -105,7 +105,7 @@ public class ShapefileReader extends Reader implements Closeable {
|
|||
}
|
||||
|
||||
@Override
|
||||
public SourceStep<SourceFeature> read() {
|
||||
public Topology.SourceStep<SourceFeature> read() {
|
||||
return next -> {
|
||||
try (var iter = inputSource.features()) {
|
||||
while (iter.hasNext()) {
|
||||
|
|
|
@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
|||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import com.graphhopper.reader.ReaderElement;
|
||||
import com.onthegomap.flatmap.monitoring.Stats.InMemory;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.io.File;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
@ -27,7 +27,7 @@ public class OsmInputFileTest {
|
|||
AtomicInteger nodes = new AtomicInteger(0);
|
||||
AtomicInteger ways = new AtomicInteger(0);
|
||||
AtomicInteger rels = new AtomicInteger(0);
|
||||
Topology.start("test", new InMemory())
|
||||
Topology.start("test", new Stats.InMemory())
|
||||
.fromGenerator("pbf", file.read(2))
|
||||
.addBuffer("reader_queue", 1_000, 100)
|
||||
.sinkToConsumer("counter", 1, elem -> {
|
||||
|
|
|
@ -36,12 +36,6 @@ public class TestUtils {
|
|||
return GeoUtils.gf.createPoint(new CoordinateXY(x, y));
|
||||
}
|
||||
|
||||
public static Point newPointWithUserData(double x, double y, Object userData) {
|
||||
Point point = GeoUtils.gf.createPoint(new CoordinateXY(x, y));
|
||||
point.setUserData(userData);
|
||||
return point;
|
||||
}
|
||||
|
||||
public static MultiPoint newMultiPoint(Point... points) {
|
||||
return GeoUtils.gf.createMultiPoint(points);
|
||||
}
|
||||
|
|
|
@ -30,9 +30,6 @@ import static org.junit.jupiter.api.Assertions.assertNotSame;
|
|||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder.DecodedFeature;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -46,7 +43,6 @@ import org.locationtech.jts.geom.MultiPolygon;
|
|||
import org.locationtech.jts.geom.Point;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
import vector_tile.VectorTile;
|
||||
import vector_tile.VectorTile.Tile.GeomType;
|
||||
|
||||
/**
|
||||
* This class is copied from https://github.com/ElectronicChartCentre/java-vector-tile/blob/master/src/test/java/no/ecc/vectortile/VectorTileEncoderTest.java
|
||||
|
@ -56,13 +52,14 @@ public class VectorTileEncoderTest {
|
|||
// Tests adapted from https://github.com/ElectronicChartCentre/java-vector-tile/blob/master/src/test/java/no/ecc/vectortile/VectorTileEncoderTest.java
|
||||
|
||||
private static List<Integer> getCommands(Geometry geom) {
|
||||
return Ints.asList(VectorTileEncoder.getCommands(TRANSFORM_TO_TILE.transform(geom)));
|
||||
return Ints.asList(VectorTileEncoder.encodeGeometry(TRANSFORM_TO_TILE.transform(geom)).commands());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testToGeomType() {
|
||||
Geometry geometry = gf.createLineString();
|
||||
assertEquals(VectorTile.Tile.GeomType.LINESTRING, VectorTileEncoder.toGeomType(geometry));
|
||||
Geometry geometry = gf.createLineString(new Coordinate[]{new CoordinateXY(1, 2), new CoordinateXY(3, 4)});
|
||||
assertEquals((byte) VectorTile.Tile.GeomType.LINESTRING.getNumber(),
|
||||
VectorTileEncoder.encodeGeometry(geometry).geomType());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -101,22 +98,13 @@ public class VectorTileEncoderTest {
|
|||
)));
|
||||
}
|
||||
|
||||
private static record SimpleVectorTileFeature(
|
||||
int[] commands,
|
||||
long id,
|
||||
byte geomType,
|
||||
Map<String, Object> attrs
|
||||
) implements VectorTileFeature {
|
||||
|
||||
}
|
||||
|
||||
private static VectorTileFeature newVectorTileFeature(Geometry geom, Map<String, Object> attrs) {
|
||||
return new SimpleVectorTileFeature(VectorTileEncoder.getCommands(geom), 1,
|
||||
(byte) VectorTileEncoder.toGeomType(geom).getNumber(), attrs);
|
||||
private static VectorTileEncoder.Feature newVectorTileFeature(String layer, Geometry geom,
|
||||
Map<String, Object> attrs) {
|
||||
return new VectorTileEncoder.Feature(layer, 1, VectorTileEncoder.encodeGeometry(geom), attrs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNullAttributeValue() throws IOException {
|
||||
public void testNullAttributeValue() {
|
||||
VectorTileEncoder vtm = new VectorTileEncoder();
|
||||
Map<String, Object> attrs = new HashMap<>();
|
||||
attrs.put("key1", "value1");
|
||||
|
@ -124,21 +112,23 @@ public class VectorTileEncoderTest {
|
|||
attrs.put("key3", "value3");
|
||||
|
||||
vtm.addLayerFeatures("DEPCNT", List.of(
|
||||
newVectorTileFeature(newPoint(3, 6), attrs)
|
||||
newVectorTileFeature("DEPCNT", newPoint(3, 6), attrs)
|
||||
));
|
||||
|
||||
byte[] encoded = vtm.encode();
|
||||
assertNotSame(0, encoded.length);
|
||||
|
||||
var decoded = VectorTileEncoder.decode(encoded);
|
||||
assertEquals(List.of(new DecodedFeature("DEPCNT", 4096, newPoint(3, 6), Map.of(
|
||||
"key1", "value1",
|
||||
"key3", "value3"
|
||||
), 1)), decoded);
|
||||
assertEquals(List
|
||||
.of(new VectorTileEncoder.Feature("DEPCNT", 1, VectorTileEncoder.encodeGeometry(newPoint(3, 6)), Map.of(
|
||||
"key1", "value1",
|
||||
"key3", "value3"
|
||||
))), decoded);
|
||||
assertSameGeometries(List.of(newPoint(3, 6)), decoded);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAttributeTypes() throws IOException {
|
||||
public void testAttributeTypes() {
|
||||
VectorTileEncoder vtm = new VectorTileEncoder();
|
||||
|
||||
Map<String, Object> attrs = Map.of(
|
||||
|
@ -152,14 +142,14 @@ public class VectorTileEncoderTest {
|
|||
"key8", Boolean.FALSE
|
||||
);
|
||||
|
||||
vtm.addLayerFeatures("DEPCNT", List.of(newVectorTileFeature(newPoint(3, 6), attrs)));
|
||||
vtm.addLayerFeatures("DEPCNT", List.of(newVectorTileFeature("DEPCNT", newPoint(3, 6), attrs)));
|
||||
|
||||
byte[] encoded = vtm.encode();
|
||||
assertNotSame(0, encoded.length);
|
||||
|
||||
List<DecodedFeature> decoded = VectorTileEncoder.decode(encoded);
|
||||
List<VectorTileEncoder.Feature> decoded = VectorTileEncoder.decode(encoded);
|
||||
assertEquals(1, decoded.size());
|
||||
Map<String, Object> decodedAttributes = decoded.get(0).attributes();
|
||||
Map<String, Object> decodedAttributes = decoded.get(0).attrs();
|
||||
assertEquals("value1", decodedAttributes.get("key1"));
|
||||
assertEquals(123L, decodedAttributes.get("key2"));
|
||||
assertEquals(234.1f, decodedAttributes.get("key3"));
|
||||
|
@ -202,7 +192,7 @@ public class VectorTileEncoderTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testMultiPolygon() throws IOException {
|
||||
public void testMultiPolygon() {
|
||||
MultiPolygon mp = newMultiPolygon(
|
||||
(Polygon) newPoint(13, 16).buffer(3),
|
||||
(Polygon) newPoint(24, 25).buffer(5)
|
||||
|
@ -212,19 +202,19 @@ public class VectorTileEncoderTest {
|
|||
Map<String, Object> attrs = Map.of("key1", "value1");
|
||||
|
||||
VectorTileEncoder vtm = new VectorTileEncoder();
|
||||
vtm.addLayerFeatures("mp", List.of(newVectorTileFeature(mp, attrs)));
|
||||
vtm.addLayerFeatures("mp", List.of(newVectorTileFeature("mp", mp, attrs)));
|
||||
|
||||
byte[] encoded = vtm.encode();
|
||||
assertTrue(encoded.length > 0);
|
||||
|
||||
var features = VectorTileEncoder.decode(encoded);
|
||||
assertEquals(1, features.size());
|
||||
MultiPolygon mp2 = (MultiPolygon) features.get(0).geometry();
|
||||
MultiPolygon mp2 = (MultiPolygon) features.get(0).geometry().decode();
|
||||
assertEquals(mp.getNumGeometries(), mp2.getNumGeometries());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGeometryCollectionSilentlyIgnored() throws IOException {
|
||||
public void testGeometryCollectionSilentlyIgnored() {
|
||||
GeometryCollection gc = newGeometryCollection(
|
||||
newPoint(13, 16).buffer(3),
|
||||
newPoint(24, 25)
|
||||
|
@ -232,7 +222,7 @@ public class VectorTileEncoderTest {
|
|||
Map<String, Object> attributes = Map.of("key1", "value1");
|
||||
|
||||
VectorTileEncoder vtm = new VectorTileEncoder();
|
||||
vtm.addLayerFeatures("gc", List.of(newVectorTileFeature(gc, attributes)));
|
||||
vtm.addLayerFeatures("gc", List.of(newVectorTileFeature("gc", gc, attributes)));
|
||||
|
||||
byte[] encoded = vtm.encode();
|
||||
|
||||
|
@ -243,12 +233,12 @@ public class VectorTileEncoderTest {
|
|||
// New tests added:
|
||||
|
||||
@Test
|
||||
public void testRoundTripPoint() throws IOException {
|
||||
public void testRoundTripPoint() {
|
||||
testRoundTripGeometry(gf.createPoint(new CoordinateXY(1, 2)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRoundTripMultipoint() throws IOException {
|
||||
public void testRoundTripMultipoint() {
|
||||
testRoundTripGeometry(gf.createMultiPointFromCoords(new Coordinate[]{
|
||||
new CoordinateXY(1, 2),
|
||||
new CoordinateXY(3, 4)
|
||||
|
@ -256,7 +246,7 @@ public class VectorTileEncoderTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testRoundTripLineString() throws IOException {
|
||||
public void testRoundTripLineString() {
|
||||
testRoundTripGeometry(gf.createLineString(new Coordinate[]{
|
||||
new CoordinateXY(1, 2),
|
||||
new CoordinateXY(3, 4)
|
||||
|
@ -264,7 +254,7 @@ public class VectorTileEncoderTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testRoundTripPolygon() throws IOException {
|
||||
public void testRoundTripPolygon() {
|
||||
testRoundTripGeometry(gf.createPolygon(
|
||||
gf.createLinearRing(new Coordinate[]{
|
||||
new CoordinateXY(0, 0),
|
||||
|
@ -286,7 +276,7 @@ public class VectorTileEncoderTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testRoundTripMultiPolygon() throws IOException {
|
||||
public void testRoundTripMultiPolygon() {
|
||||
testRoundTripGeometry(gf.createMultiPolygon(new Polygon[]{
|
||||
gf.createPolygon(new Coordinate[]{
|
||||
new CoordinateXY(0, 0),
|
||||
|
@ -306,7 +296,7 @@ public class VectorTileEncoderTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testRoundTripAttributes() throws IOException {
|
||||
public void testRoundTripAttributes() {
|
||||
testRoundTripAttrs(Map.of(
|
||||
"string", "string",
|
||||
"long", 1L,
|
||||
|
@ -317,52 +307,53 @@ public class VectorTileEncoderTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testMultipleFeaturesMultipleLayer() throws IOException {
|
||||
public void testMultipleFeaturesMultipleLayer() {
|
||||
Point point = gf.createPoint(new CoordinateXY(0, 0));
|
||||
Map<String, Object> attrs1 = Map.of("a", 1L, "b", 2L);
|
||||
Map<String, Object> attrs2 = Map.of("b", 3L, "c", 2L);
|
||||
byte[] encoded = new VectorTileEncoder().addLayerFeatures("layer1", List.of(
|
||||
new LayerFeature(false, 0, 0, attrs1,
|
||||
(byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 1L),
|
||||
new LayerFeature(false, 0, 0, attrs2,
|
||||
(byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 2L)
|
||||
new VectorTileEncoder.Feature("layer1", 1L, VectorTileEncoder.encodeGeometry(point), attrs1),
|
||||
new VectorTileEncoder.Feature("layer1", 2L, VectorTileEncoder.encodeGeometry(point), attrs2)
|
||||
)).addLayerFeatures("layer2", List.of(
|
||||
new LayerFeature(false, 0, 0, attrs1,
|
||||
(byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 3L)
|
||||
new VectorTileEncoder.Feature("layer2", 3L, VectorTileEncoder.encodeGeometry(point), attrs1)
|
||||
)).encode();
|
||||
|
||||
List<DecodedFeature> decoded = VectorTileEncoder.decode(encoded);
|
||||
assertEquals(attrs1, decoded.get(0).attributes());
|
||||
assertEquals("layer1", decoded.get(0).layerName());
|
||||
List<VectorTileEncoder.Feature> decoded = VectorTileEncoder.decode(encoded);
|
||||
assertEquals(attrs1, decoded.get(0).attrs());
|
||||
assertEquals("layer1", decoded.get(0).layer());
|
||||
|
||||
assertEquals(attrs2, decoded.get(1).attributes());
|
||||
assertEquals("layer1", decoded.get(1).layerName());
|
||||
assertEquals(attrs2, decoded.get(1).attrs());
|
||||
assertEquals("layer1", decoded.get(1).layer());
|
||||
|
||||
assertEquals(attrs1, decoded.get(2).attributes());
|
||||
assertEquals("layer2", decoded.get(2).layerName());
|
||||
assertEquals(attrs1, decoded.get(2).attrs());
|
||||
assertEquals("layer2", decoded.get(2).layer());
|
||||
}
|
||||
|
||||
private void testRoundTripAttrs(Map<String, Object> attrs) throws IOException {
|
||||
private void testRoundTripAttrs(Map<String, Object> attrs) {
|
||||
testRoundTrip(gf.createPoint(new CoordinateXY(0, 0)), "layer", attrs, 1);
|
||||
}
|
||||
|
||||
private void testRoundTripGeometry(Geometry input) throws IOException {
|
||||
private void testRoundTripGeometry(Geometry input) {
|
||||
testRoundTrip(input, "layer", Map.of(), 1);
|
||||
}
|
||||
|
||||
private void testRoundTrip(Geometry input, String layer, Map<String, Object> attrs, long id) throws IOException {
|
||||
int[] commands = VectorTileEncoder.getCommands(input);
|
||||
byte geomType = (byte) VectorTileEncoder.toGeomType(input).ordinal();
|
||||
Geometry output = VectorTileEncoder.decodeCommands(geomType, commands);
|
||||
private void testRoundTrip(Geometry input, String layer, Map<String, Object> attrs, long id) {
|
||||
VectorTileEncoder.VectorGeometry encodedGeom = VectorTileEncoder.encodeGeometry(input);
|
||||
Geometry output = encodedGeom.decode();
|
||||
assertTrue(input.equalsExact(output), "\n" + input + "\n!=\n" + output);
|
||||
|
||||
byte[] encoded = new VectorTileEncoder().addLayerFeatures(layer, List.of(
|
||||
new LayerFeature(false, 0, 0, attrs,
|
||||
(byte) VectorTileEncoder.toGeomType(input).getNumber(), VectorTileEncoder.getCommands(input), id)
|
||||
new VectorTileEncoder.Feature(layer, id, VectorTileEncoder.encodeGeometry(input), attrs)
|
||||
)).encode();
|
||||
|
||||
List<DecodedFeature> decoded = VectorTileEncoder.decode(encoded);
|
||||
DecodedFeature expected = new DecodedFeature(layer, 4096, input, attrs, id);
|
||||
List<VectorTileEncoder.Feature> decoded = VectorTileEncoder.decode(encoded);
|
||||
VectorTileEncoder.Feature expected = new VectorTileEncoder.Feature(layer, id,
|
||||
VectorTileEncoder.encodeGeometry(input), attrs);
|
||||
assertEquals(List.of(expected), decoded);
|
||||
assertSameGeometries(List.of(input), decoded);
|
||||
}
|
||||
|
||||
private void assertSameGeometries(List<Geometry> expected, List<VectorTileEncoder.Feature> actual) {
|
||||
assertEquals(expected, actual.stream().map(d -> d.geometry().decode()).toList());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,8 +6,6 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest;
|
|||
|
||||
import com.graphhopper.reader.ReaderElement;
|
||||
import com.graphhopper.reader.ReaderNode;
|
||||
import com.onthegomap.flatmap.Wikidata.Client;
|
||||
import com.onthegomap.flatmap.Wikidata.WikidataTranslations;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
|
@ -33,7 +31,7 @@ public class WikidataTest {
|
|||
@Test
|
||||
public void testWikidataTranslations() {
|
||||
var expected = Map.of("en", "en value", "es", "es value");
|
||||
WikidataTranslations translations = new WikidataTranslations();
|
||||
Wikidata.WikidataTranslations translations = new Wikidata.WikidataTranslations();
|
||||
assertNull(translations.get(1));
|
||||
translations.put(1, "en", "en value");
|
||||
translations.put(1, "es", "es value");
|
||||
|
@ -49,7 +47,7 @@ public class WikidataTest {
|
|||
@TestFactory
|
||||
public List<DynamicTest> testFetchWikidata() throws IOException, InterruptedException {
|
||||
StringWriter writer = new StringWriter();
|
||||
Client client = Mockito.mock(Client.class, Mockito.RETURNS_SMART_NULLS);
|
||||
Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
|
||||
Wikidata fixture = new Wikidata(writer, client, 2);
|
||||
fixture.fetch(1L);
|
||||
Mockito.verifyNoInteractions(client);
|
||||
|
@ -118,7 +116,7 @@ public class WikidataTest {
|
|||
}),
|
||||
dynamicTest("do not re-request on subsequent loads", () -> {
|
||||
StringWriter writer2 = new StringWriter();
|
||||
Client client2 = Mockito.mock(Client.class, Mockito.RETURNS_SMART_NULLS);
|
||||
Wikidata.Client client2 = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
|
||||
Wikidata fixture2 = new Wikidata(writer2, client2, 2);
|
||||
fixture2.loadExisting(Wikidata.load(new StringReader(writer.toString())));
|
||||
fixture2.fetch(1L);
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
package com.onthegomap.flatmap.collections;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newPoint;
|
||||
import static com.onthegomap.flatmap.TestUtils.newPointWithUserData;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
|
||||
|
||||
import com.onthegomap.flatmap.Profile;
|
||||
import com.onthegomap.flatmap.RenderedFeature;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import java.util.ArrayList;
|
||||
|
@ -16,6 +16,7 @@ import java.util.Comparator;
|
|||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.junit.jupiter.api.DynamicTest;
|
||||
|
@ -25,10 +26,10 @@ import org.junit.jupiter.params.ParameterizedTest;
|
|||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
|
||||
public class MergeSortFeatureMapTest {
|
||||
public class FeatureGroupTest {
|
||||
|
||||
private final List<MergeSort.Entry> list = new ArrayList<>();
|
||||
private final MergeSort sorter = new MergeSort() {
|
||||
private final List<FeatureSort.Entry> list = new ArrayList<>();
|
||||
private final FeatureSort sorter = new FeatureSort() {
|
||||
@Override
|
||||
public void sort() {
|
||||
list.sort(Comparator.naturalOrder());
|
||||
|
@ -50,7 +51,7 @@ public class MergeSortFeatureMapTest {
|
|||
return list.iterator();
|
||||
}
|
||||
};
|
||||
private MergeSortFeatureMap features = new MergeSortFeatureMap(sorter, new Profile.NullProfile());
|
||||
private FeatureGroup features = new FeatureGroup(sorter, new Profile.NullProfile());
|
||||
|
||||
@Test
|
||||
public void testEmpty() {
|
||||
|
@ -61,27 +62,36 @@ public class MergeSortFeatureMapTest {
|
|||
long id = 0;
|
||||
|
||||
private void put(int tile, String layer, Map<String, Object> attrs, Geometry geom) {
|
||||
MergeSort.Entry key = features.encode(id++, TileCoord.decode(tile), layer, attrs, geom, 0, false, 0);
|
||||
features.accept(key);
|
||||
putWithZorder(tile, layer, attrs, geom, 0);
|
||||
}
|
||||
|
||||
private void putWithZorder(int tile, String layer, Map<String, Object> attrs, Geometry geom, int zOrder) {
|
||||
MergeSort.Entry key = features.encode(id++, TileCoord.decode(tile), layer, attrs, geom, zOrder, false, 0);
|
||||
features.accept(key);
|
||||
putWithGroupAndZorder(tile, layer, attrs, geom, zOrder, false, 0, 0);
|
||||
}
|
||||
|
||||
private void putWithGroup(int tile, String layer, Map<String, Object> attrs, Geometry geom, int zOrder, int limit) {
|
||||
MergeSort.Entry key = features.encode(id++, TileCoord.decode(tile), layer, attrs, geom, zOrder, true, limit);
|
||||
features.accept(key);
|
||||
private void putWithGroup(int tile, String layer, Map<String, Object> attrs, Geometry geom, int zOrder, long group,
|
||||
int limit) {
|
||||
putWithGroupAndZorder(tile, layer, attrs, geom, zOrder, true, group, limit);
|
||||
}
|
||||
|
||||
private void putWithGroupAndZorder(int tile, String layer, Map<String, Object> attrs, Geometry geom, int zOrder,
|
||||
boolean hasGroup, long group, int limit) {
|
||||
RenderedFeature feature = new RenderedFeature(
|
||||
TileCoord.decode(tile),
|
||||
new VectorTileEncoder.Feature(layer, id++, VectorTileEncoder.encodeGeometry(geom), attrs),
|
||||
zOrder,
|
||||
hasGroup ? Optional.of(new RenderedFeature.Group(group, limit)) : Optional.empty()
|
||||
);
|
||||
features.accept(features.encode(feature));
|
||||
}
|
||||
|
||||
private Map<Integer, Map<String, List<Feature>>> getFeatures() {
|
||||
Map<Integer, Map<String, List<Feature>>> map = new TreeMap<>();
|
||||
for (MergeSortFeatureMap.TileFeatures tile : features) {
|
||||
for (FeatureGroup.TileFeatures tile : features) {
|
||||
for (var feature : VectorTileEncoder.decode(tile.getTile().encode())) {
|
||||
map.computeIfAbsent(tile.coord().encoded(), (i) -> new TreeMap<>())
|
||||
.computeIfAbsent(feature.layerName(), l -> new ArrayList<>())
|
||||
.add(new Feature(feature.attributes(), feature.geometry()));
|
||||
.computeIfAbsent(feature.layer(), l -> new ArrayList<>())
|
||||
.add(new Feature(feature.attrs(), feature.geometry().decode()));
|
||||
}
|
||||
}
|
||||
return map;
|
||||
|
@ -144,13 +154,13 @@ public class MergeSortFeatureMapTest {
|
|||
public void testLimitPoints() {
|
||||
int x = 5, y = 6;
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 1), 0, 2
|
||||
1, "layer", Map.of("id", 3), newPoint(x, y), 0, 1, 2
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 2
|
||||
1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 2
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 2
|
||||
1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2
|
||||
);
|
||||
sorter.sort();
|
||||
assertEquals(new TreeMap<>(Map.of(
|
||||
|
@ -168,13 +178,13 @@ public class MergeSortFeatureMapTest {
|
|||
public void testLimitPointsInDifferentGroups() {
|
||||
int x = 5, y = 6;
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 2), 0, 2
|
||||
1, "layer", Map.of("id", 3), newPoint(x, y), 0, 2, 2
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 2
|
||||
1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 2
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 2
|
||||
1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2
|
||||
);
|
||||
sorter.sort();
|
||||
assertEquals(new TreeMap<>(Map.of(
|
||||
|
@ -192,13 +202,13 @@ public class MergeSortFeatureMapTest {
|
|||
public void testDontLimitPointsWithGroup() {
|
||||
int x = 5, y = 6;
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 1), 0, 0
|
||||
1, "layer", Map.of("id", 3), newPoint(x, y), 0, 1, 0
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 0
|
||||
1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 0
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 0
|
||||
1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 0
|
||||
);
|
||||
sorter.sort();
|
||||
assertEquals(new TreeMap<>(Map.of(
|
||||
|
@ -214,23 +224,23 @@ public class MergeSortFeatureMapTest {
|
|||
|
||||
@Test
|
||||
public void testProfileChangesGeometry() {
|
||||
features = new MergeSortFeatureMap(sorter, new Profile.NullProfile() {
|
||||
features = new FeatureGroup(sorter, new Profile.NullProfile() {
|
||||
@Override
|
||||
public List<VectorTileEncoder.VectorTileFeature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTileEncoder.VectorTileFeature> items) {
|
||||
public List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTileEncoder.Feature> items) {
|
||||
Collections.reverse(items);
|
||||
return items;
|
||||
}
|
||||
});
|
||||
int x = 5, y = 6;
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 1), 0, 2
|
||||
1, "layer", Map.of("id", 3), newPoint(x, y), 0, 1, 2
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 2
|
||||
1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 2
|
||||
);
|
||||
putWithGroup(
|
||||
1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 2
|
||||
1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2
|
||||
);
|
||||
sorter.sort();
|
||||
assertEquals(new TreeMap<>(Map.of(
|
||||
|
@ -260,14 +270,12 @@ public class MergeSortFeatureMapTest {
|
|||
for (byte layer : layers) {
|
||||
for (int zOrder : zOrders) {
|
||||
for (boolean hasGroup : hasGroups) {
|
||||
MergeSortFeatureMap.FeatureMapKey key = MergeSortFeatureMap.FeatureMapKey
|
||||
.of(tile.encoded(), layer, zOrder, hasGroup);
|
||||
result.add(dynamicTest(key.toString(), () -> {
|
||||
MergeSortFeatureMap.FeatureMapKey decoded = MergeSortFeatureMap.FeatureMapKey.decode(key.encoded());
|
||||
assertEquals(decoded.tile(), tile, "tile");
|
||||
assertEquals(decoded.layer(), layer, "layer");
|
||||
assertEquals(decoded.zOrder(), zOrder, "zOrder");
|
||||
assertEquals(decoded.hasGroup(), hasGroup, "hasGroup");
|
||||
long sortKey = FeatureGroup.encodeSortKey(tile.encoded(), layer, zOrder, hasGroup);
|
||||
result.add(dynamicTest(tile + " " + layer + " " + zOrder + " " + hasGroup, () -> {
|
||||
assertEquals(tile.encoded(), FeatureGroup.extractTileFromSortKey(sortKey), "tile");
|
||||
assertEquals(layer, FeatureGroup.extractLayerIdFromSortKey(sortKey), "layer");
|
||||
assertEquals(zOrder, FeatureGroup.extractZorderFromKey(sortKey), "zOrder");
|
||||
assertEquals(hasGroup, FeatureGroup.extractHasGroupFromSortKey(sortKey), "hasGroup");
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
@ -292,9 +300,9 @@ public class MergeSortFeatureMapTest {
|
|||
int tileB, byte layerB, int zOrderB, boolean hasGroupB
|
||||
) {
|
||||
assertTrue(
|
||||
MergeSortFeatureMap.FeatureMapKey.encode(tileA, layerA, zOrderA, hasGroupA)
|
||||
FeatureGroup.encodeSortKey(tileA, layerA, zOrderA, hasGroupA)
|
||||
<
|
||||
MergeSortFeatureMap.FeatureMapKey.encode(tileB, layerB, zOrderB, hasGroupB)
|
||||
FeatureGroup.encodeSortKey(tileB, layerB, zOrderB, hasGroupB)
|
||||
);
|
||||
}
|
||||
}
|
|
@ -11,29 +11,29 @@ import java.util.Random;
|
|||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
public class MergeSortTest {
|
||||
public class FeatureSortTest {
|
||||
|
||||
@TempDir
|
||||
Path tmpDir;
|
||||
|
||||
private static MergeSort.Entry newEntry(int i) {
|
||||
return new MergeSort.Entry(i, new byte[]{(byte) i});
|
||||
private static FeatureSort.Entry newEntry(int i) {
|
||||
return new FeatureSort.Entry(i, new byte[]{(byte) i});
|
||||
}
|
||||
|
||||
private MergeSort newSorter(int workers, int chunkSizeLimit) {
|
||||
return MergeSort.newExternalMergeSort(tmpDir, workers, chunkSizeLimit, new Stats.InMemory());
|
||||
private FeatureSort newSorter(int workers, int chunkSizeLimit) {
|
||||
return FeatureSort.newExternalMergeSort(tmpDir, workers, chunkSizeLimit, new Stats.InMemory());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmpty() {
|
||||
MergeSort sorter = newSorter(1, 100);
|
||||
FeatureSort sorter = newSorter(1, 100);
|
||||
sorter.sort();
|
||||
assertEquals(List.of(), sorter.toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingle() {
|
||||
MergeSort sorter = newSorter(1, 100);
|
||||
FeatureSort sorter = newSorter(1, 100);
|
||||
sorter.add(newEntry(1));
|
||||
sorter.sort();
|
||||
assertEquals(List.of(newEntry(1)), sorter.toList());
|
||||
|
@ -41,7 +41,7 @@ public class MergeSortTest {
|
|||
|
||||
@Test
|
||||
public void testTwoItemsOneChunk() {
|
||||
MergeSort sorter = newSorter(1, 100);
|
||||
FeatureSort sorter = newSorter(1, 100);
|
||||
sorter.add(newEntry(2));
|
||||
sorter.add(newEntry(1));
|
||||
sorter.sort();
|
||||
|
@ -50,7 +50,7 @@ public class MergeSortTest {
|
|||
|
||||
@Test
|
||||
public void testTwoItemsTwoChunks() {
|
||||
MergeSort sorter = newSorter(1, 0);
|
||||
FeatureSort sorter = newSorter(1, 0);
|
||||
sorter.add(newEntry(2));
|
||||
sorter.add(newEntry(1));
|
||||
sorter.sort();
|
||||
|
@ -59,7 +59,7 @@ public class MergeSortTest {
|
|||
|
||||
@Test
|
||||
public void testTwoWorkers() {
|
||||
MergeSort sorter = newSorter(2, 0);
|
||||
FeatureSort sorter = newSorter(2, 0);
|
||||
sorter.add(newEntry(4));
|
||||
sorter.add(newEntry(3));
|
||||
sorter.add(newEntry(2));
|
||||
|
@ -70,14 +70,14 @@ public class MergeSortTest {
|
|||
|
||||
@Test
|
||||
public void testManyItems() {
|
||||
List<MergeSort.Entry> sorted = new ArrayList<>();
|
||||
List<MergeSort.Entry> shuffled = new ArrayList<>();
|
||||
List<FeatureSort.Entry> sorted = new ArrayList<>();
|
||||
List<FeatureSort.Entry> shuffled = new ArrayList<>();
|
||||
for (int i = 0; i < 10_000; i++) {
|
||||
shuffled.add(newEntry(i));
|
||||
sorted.add(newEntry(i));
|
||||
}
|
||||
Collections.shuffle(shuffled, new Random(0));
|
||||
MergeSort sorter = newSorter(2, 20_000);
|
||||
FeatureSort sorter = newSorter(2, 20_000);
|
||||
shuffled.forEach(sorter::add);
|
||||
sorter.sort();
|
||||
assertEquals(sorted, sorter.toList());
|
|
@ -2,7 +2,6 @@ package com.onthegomap.flatmap.monitoring;
|
|||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import com.onthegomap.flatmap.monitoring.Stats.InMemory;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
|
@ -15,7 +14,7 @@ public class ProgressLoggersTest {
|
|||
@Timeout(10)
|
||||
public void testLogTopology() {
|
||||
var latch = new CountDownLatch(1);
|
||||
var topology = Topology.start("topo", new InMemory())
|
||||
var topology = Topology.start("topo", new Stats.InMemory())
|
||||
.fromGenerator("reader", next -> latch.await())
|
||||
.addBuffer("reader_queue", 10)
|
||||
.addWorker("worker", 2, (a, b) -> latch.await())
|
||||
|
|
|
@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.monitoring.Stats.InMemory;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
|
@ -24,12 +24,12 @@ public class NaturalEarthReaderTest {
|
|||
@Timeout(30)
|
||||
public void testReadNaturalEarth(String filename, @TempDir File tempDir) {
|
||||
var file = new File("src/test/resources/" + filename);
|
||||
try (var reader = new NaturalEarthReader(file, tempDir, new InMemory())) {
|
||||
try (var reader = new NaturalEarthReader(file, tempDir, new Stats.InMemory())) {
|
||||
for (int i = 1; i <= 2; i++) {
|
||||
assertEquals(19, reader.getCount(), "iter " + i);
|
||||
|
||||
List<Geometry> points = new ArrayList<>();
|
||||
Topology.start("test", new InMemory())
|
||||
Topology.start("test", new Stats.InMemory())
|
||||
.fromGenerator("naturalearth", reader.read())
|
||||
.addBuffer("reader_queue", 100, 1)
|
||||
.sinkToConsumer("counter", 1, elem -> {
|
||||
|
|
|
@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.monitoring.Stats.InMemory;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import com.onthegomap.flatmap.worker.Topology;
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
|
@ -16,7 +16,8 @@ import org.locationtech.jts.geom.Geometry;
|
|||
|
||||
public class ShapefileReaderTest {
|
||||
|
||||
private ShapefileReader reader = new ShapefileReader(new File("src/test/resources/shapefile.zip"), new InMemory());
|
||||
private ShapefileReader reader = new ShapefileReader(new File("src/test/resources/shapefile.zip"),
|
||||
new Stats.InMemory());
|
||||
|
||||
@AfterEach
|
||||
public void close() {
|
||||
|
@ -34,7 +35,7 @@ public class ShapefileReaderTest {
|
|||
public void testReadShapefile() {
|
||||
for (int i = 1; i <= 2; i++) {
|
||||
List<Geometry> points = new ArrayList<>();
|
||||
Topology.start("test", new InMemory())
|
||||
Topology.start("test", new Stats.InMemory())
|
||||
.fromGenerator("shapefile", reader.read())
|
||||
.addBuffer("reader_queue", 100, 1)
|
||||
.sinkToConsumer("counter", 1, elem -> {
|
||||
|
|
Ładowanie…
Reference in New Issue