kopia lustrzana https://github.com/onthegomap/planetiler
Add `--download-max-bandwidth` option (#467)
rodzic
553fdd708c
commit
78129905e5
|
@ -3,6 +3,7 @@ package com.onthegomap.planetiler.config;
|
|||
import com.onthegomap.planetiler.collection.LongLongMap;
|
||||
import com.onthegomap.planetiler.collection.Storage;
|
||||
import com.onthegomap.planetiler.reader.osm.PolyFileReader;
|
||||
import com.onthegomap.planetiler.util.Parse;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Path;
|
||||
|
@ -40,6 +41,7 @@ public record PlanetilerConfig(
|
|||
int httpRetries,
|
||||
long downloadChunkSizeMB,
|
||||
int downloadThreads,
|
||||
double downloadMaxBandwidth,
|
||||
double minFeatureSizeAtMaxZoom,
|
||||
double minFeatureSizeBelowMaxZoom,
|
||||
double simplifyToleranceAtMaxZoom,
|
||||
|
@ -148,6 +150,8 @@ public record PlanetilerConfig(
|
|||
arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
|
||||
arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
|
||||
arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
|
||||
Parse.bandwidth(arguments.getString("download_max_bandwidth",
|
||||
"Maximum bandwidth to consume when downloading files in units mb/s, mbps, kbps, etc.", "")),
|
||||
arguments.getDouble("min_feature_size_at_max_zoom",
|
||||
"Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming",
|
||||
256d / 4096),
|
||||
|
|
|
@ -3,6 +3,7 @@ package com.onthegomap.planetiler.util;
|
|||
import static com.google.common.net.HttpHeaders.*;
|
||||
import static java.nio.file.StandardOpenOption.WRITE;
|
||||
|
||||
import com.google.common.util.concurrent.RateLimiter;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
|
@ -75,8 +76,10 @@ public class Downloader {
|
|||
private final Stats stats;
|
||||
private final long chunkSizeBytes;
|
||||
private final ResourceUsage diskSpaceCheck = new ResourceUsage("download");
|
||||
private final RateLimiter rateLimiter;
|
||||
|
||||
Downloader(PlanetilerConfig config, Stats stats, long chunkSizeBytes) {
|
||||
this.rateLimiter = config.downloadMaxBandwidth() == 0 ? null : RateLimiter.create(config.downloadMaxBandwidth());
|
||||
this.chunkSizeBytes = chunkSizeBytes;
|
||||
this.config = config;
|
||||
this.stats = stats;
|
||||
|
@ -304,7 +307,7 @@ public class Downloader {
|
|||
try (
|
||||
var inputStream = (ranges || range.start > 0) ? openStreamRange(canonicalUrl, range.start, range.end) :
|
||||
openStream(canonicalUrl);
|
||||
var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress)
|
||||
var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress, rateLimiter)
|
||||
) {
|
||||
// ensure this file has been allocated up to the start of this block
|
||||
fileChannel.write(ByteBuffer.allocate(1), range.start);
|
||||
|
@ -355,12 +358,16 @@ public class Downloader {
|
|||
/**
|
||||
* Wrapper for a {@link ReadableByteChannel} that captures progress information.
|
||||
*/
|
||||
private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress) implements ReadableByteChannel {
|
||||
private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress, RateLimiter rateLimiter)
|
||||
implements ReadableByteChannel {
|
||||
|
||||
@Override
|
||||
public int read(ByteBuffer dst) throws IOException {
|
||||
int n = inner.read(dst);
|
||||
if (n > 0) {
|
||||
if (rateLimiter != null) {
|
||||
rateLimiter.acquire(n);
|
||||
}
|
||||
progress.addAndGet(n);
|
||||
}
|
||||
return n;
|
||||
|
|
|
@ -19,6 +19,10 @@ public class Parse {
|
|||
Pattern.compile(
|
||||
"(?<value>-?[\\d.]+)\\s*((?<mi>mi)|(?<m>m|$)|(?<km>km|kilom)|(?<ft>ft|')|(?<in>in|\")|(?<nmi>nmi|international nautical mile|nautical))",
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
private static final Pattern NUMBER_WITH_UNIT =
|
||||
Pattern.compile(
|
||||
"(?<value>-?[\\d.]+)\\s*(?<unit>[^.\\d]*)",
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
// Ignore warnings about not removing thread local values since planetiler uses dedicated worker threads that release
|
||||
// values when a task is finished and are not re-used.
|
||||
@SuppressWarnings("java:S5164")
|
||||
|
@ -164,7 +168,7 @@ public class Parse {
|
|||
/**
|
||||
* Parses {@code tag} as a measure of distance with unit, converted to a round number of meters or {@code null} if
|
||||
* invalid.
|
||||
*
|
||||
* <p>
|
||||
* See <a href="https://wiki.openstreetmap.org/wiki/Map_features/Units">Map features/Units</a> for the list of
|
||||
* supported units.
|
||||
*/
|
||||
|
@ -205,4 +209,47 @@ public class Parse {
|
|||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a string containing bandwidth, with units of kbps, mb/s, kib/s, etc.
|
||||
* <p>
|
||||
* Returned value is in units of bytes per second, or 0 if no limit specified.
|
||||
*/
|
||||
public static double bandwidth(Object tag) {
|
||||
if (tag != null) {
|
||||
if (tag instanceof Number num) {
|
||||
return num.doubleValue();
|
||||
}
|
||||
var str = tag.toString();
|
||||
var matcher = NUMBER_WITH_UNIT.matcher(str);
|
||||
if (matcher.find()) {
|
||||
try {
|
||||
double value = Double.parseDouble(matcher.group("value"));
|
||||
String unit = matcher.group("unit").toLowerCase(Locale.ROOT).replaceAll("\\s", "");
|
||||
double multiplier = switch (unit) {
|
||||
case "b/s", "" -> 1;
|
||||
case "kb/s" -> 1_000;
|
||||
case "mb/s" -> 1_000_000;
|
||||
case "gb/s" -> 1_000_000_000;
|
||||
case "bps" -> 1d / 8;
|
||||
case "kbps" -> 1_000d / 8;
|
||||
case "mbps" -> 1_000_000d / 8;
|
||||
case "gbps" -> 1_000_000_000d / 8;
|
||||
case "kib/s" -> 1 << 10;
|
||||
case "mib/s" -> 1 << 20;
|
||||
case "gib/s" -> 1 << 30;
|
||||
default -> throw new IllegalArgumentException("Unable to parse bandwidth: " + tag);
|
||||
};
|
||||
double result = value * multiplier;
|
||||
if (result < 0) {
|
||||
throw new IllegalArgumentException("Unable to parse bandwidth: " + tag);
|
||||
}
|
||||
return result;
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Unable to parse bandwidth: " + tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -140,4 +140,28 @@ class ParseTest {
|
|||
void testParseInvalidJvmSize(String input) {
|
||||
assertThrows(IllegalArgumentException.class, () -> Parse.jvmMemoryStringToBytes(input));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@CsvSource(value = {
|
||||
"0, 0",
|
||||
"1, 1",
|
||||
"999999999999, 999999999999",
|
||||
"2b/s, 2",
|
||||
"2kib/s, 2048",
|
||||
"4mib/s, 4194304",
|
||||
"8GiB/s, 8589934592",
|
||||
"2kb/s, 2000",
|
||||
"4mb/s, 4000000",
|
||||
"8Gb/s, 8000000000",
|
||||
"2bps, 0.25",
|
||||
"8bps, 1",
|
||||
"16bps, 2",
|
||||
"8kbps, 1000",
|
||||
"8mbps, 1000000",
|
||||
"8gbps, 1000000000",
|
||||
"garbage, 0"
|
||||
}, nullValues = {"null"})
|
||||
void testParseBandwidth(String input, double expectedOutput) {
|
||||
assertEquals(expectedOutput, Parse.bandwidth(input));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -279,6 +279,9 @@
|
|||
"download_threads": {
|
||||
"description": "Number of parallel threads to use when downloading each file"
|
||||
},
|
||||
"download_max_bandwidth": {
|
||||
"description": "Maximum bandwidth to consume when downloading files in units mb/s, mbps, kbps, etc."
|
||||
},
|
||||
"min_feature_size_at_max_zoom": {
|
||||
"description": "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming"
|
||||
},
|
||||
|
|
|
@ -204,6 +204,7 @@ public class Contexts {
|
|||
argumentValues.put("http_retries", config.httpRetries());
|
||||
argumentValues.put("download_chunk_size_mb", config.downloadChunkSizeMB());
|
||||
argumentValues.put("download_threads", config.downloadThreads());
|
||||
argumentValues.put("download_max_bandwidth", config.downloadMaxBandwidth());
|
||||
argumentValues.put("min_feature_size_at_max_zoom", config.minFeatureSizeAtMaxZoom());
|
||||
argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom());
|
||||
argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());
|
||||
|
|
Ładowanie…
Reference in New Issue