kopia lustrzana https://github.com/onthegomap/planetiler
rodzic
39b7eca2a1
commit
4f059aa8e8
|
@ -82,9 +82,9 @@ tail -f logs.txt
|
|||
```
|
||||
|
||||
It took 3h21m (including 12 minutes downloading source data) to generate a 99GB `output.mbtiles` file. See
|
||||
the [full logs](planet-logs/planet-logs.txt) from this run or this summary that it printed at the end. Notice that it
|
||||
spent almost an hour emitting z13 tiles. That is because the default basemap profile merges nearby building polygons at
|
||||
z13 which is very expensive. You can disable this behavior by setting `--building-merge-z13=false`.
|
||||
the [full logs](planet-logs/v0.1.0-planet-do-16cpu-128gb.txt) from this run or this summary that it printed at the end.
|
||||
Notice that it spent almost an hour emitting z13 tiles. That is because the default basemap profile merges nearby
|
||||
building polygons at z13 which is very expensive. You can disable this behavior by setting `--building-merge-z13=false`.
|
||||
|
||||
```
|
||||
3:21:03 DEB [mbtiles] - Tile stats:
|
||||
|
|
11
README.md
11
README.md
|
@ -91,11 +91,12 @@ See the [flatmap-examples](flatmap-examples) project.
|
|||
|
||||
Some example runtimes (excluding downloading resources):
|
||||
|
||||
| Input | Input Size | Profile | Machine | Time | mbtiles size | Logs |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap | DO 16cpu 128GB RAM | 3h9m (cpu: 42h1m @ 13.3) | 99.6GB | [logs](planet-logs/planet-logs.txt) [VisualVM Profile](planet-logs/planet.nps) |
|
||||
| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries | 68.6GB | Basemap | DO 16cpu 128GB RAM | 3h13m (cpu: 43h40m @ 13.5) | 101.4GB | [logs](planet-logs/logs-daylight.txt) |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB RAM) | 59m26s (cpu: 27h6m @ 27.4) | 97.3GB | [logs](planet-logs/planet-logs-c5ad.txt) |
|
||||
| Input | Profile | Machine | Time | mbtiles size | Logs |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap | DO 16cpu 128GB | 3h9m cpu:42h1m avg:13.3 | 99GB | [logs](planet-logs/v0.1.0-planet-do-16cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-do-16cpu-128gb.nps) |
|
||||
| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries (67GB) | Basemap | DO 16cpu 128GB | 3h13m cpu:43h40m avg:13.5 | 101GB | [logs](planet-logs/v0.1.0-daylight-do-16cpu-128gb.txt) |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap | Linode 50cpu 128GB | 1h9m cpu:24h36m avg:21.2 | 97GB | [logs](planet-logs/v0.1.0-planet-linode-50cpu-128gb.txt), [VisualVM Profile](planet-logs/v0.1.0-planet-linode-50cpu-128gb.nps) |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf (65GB) | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB) | 59m cpu:27h6m avg:27.4 | 97GB | [logs](planet-logs/v0.1.0-planet-c5ad-64cpu-128gb.txt) |
|
||||
|
||||
## Alternatives
|
||||
|
||||
|
|
|
@ -86,10 +86,8 @@ class ExternalMergeSort implements FeatureSort {
|
|||
"Not enough memory to use chunk size " + chunkSizeLimit + " only have " + memory);
|
||||
}
|
||||
this.workers = workers;
|
||||
this.readerLimit = Math.max(1, config.arguments()
|
||||
.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks", 6));
|
||||
this.writerLimit = Math.max(1, config.arguments()
|
||||
.getInteger("sort_max_writers", "maximum number of concurrent write threads to use when sorting chunks", 6));
|
||||
this.readerLimit = Math.max(1, config.sortMaxReaders());
|
||||
this.writerLimit = Math.max(1, config.sortMaxWriters());
|
||||
LOGGER.info("Using merge sort feature map, chunk size=" + (chunkSizeLimit / 1_000_000) + "mb workers=" + workers);
|
||||
try {
|
||||
FileUtils.deleteDirectory(dir);
|
||||
|
|
|
@ -17,6 +17,8 @@ public record FlatmapConfig(
|
|||
boolean emitTilesInOrder,
|
||||
boolean forceOverwrite,
|
||||
boolean gzipTempStorage,
|
||||
int sortMaxReaders,
|
||||
int sortMaxWriters,
|
||||
String nodeMapType,
|
||||
String nodeMapStorage,
|
||||
String httpUserAgent,
|
||||
|
@ -61,6 +63,10 @@ public record FlatmapConfig(
|
|||
arguments.getBoolean("emit_tiles_in_order", "emit tiles in index order", true),
|
||||
arguments.getBoolean("force", "force overwriting output file", false),
|
||||
arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),
|
||||
arguments.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks",
|
||||
6),
|
||||
arguments.getInteger("sort_max_writers", "maximum number of concurrent write threads to use when sorting chunks",
|
||||
6),
|
||||
arguments
|
||||
.getString("nodemap_type", "type of node location map: noop, sortedtable, or sparsearray", "sortedtable"),
|
||||
arguments.getString("nodemap_storage", "storage for location map: mmap or ram", "mmap"),
|
||||
|
|
|
@ -220,6 +220,11 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
String parseThreadPrefix = "pbfpass2";
|
||||
var pipeline = WorkerPipeline.start("osm_pass2", stats)
|
||||
.fromGenerator("pbf", osmInputFile.read(parseThreadPrefix, readerThreads))
|
||||
// TODO should use an adaptive batch size to better utilize lots of cpus:
|
||||
// - make queue size proportional to cores
|
||||
// - much larger batches when processing points
|
||||
// - slightly larger batches when processing ways
|
||||
// - 1_000 is probably fine for relations
|
||||
.addBuffer("reader_queue", 50_000, 1_000)
|
||||
.<SortableFeature>addWorker("process", processThreads, (prev, next) -> {
|
||||
// avoid contention trying to get the thread-local counters by getting them once when thread starts
|
||||
|
|
Plik binarny nie jest wyświetlany.
File diff suppressed because one or more lines are too long
Ładowanie…
Reference in New Issue