Fix unit tests in other locales (#64)

pull/65/head
Michael Barry 2022-01-27 20:23:24 -05:00 zatwierdzone przez GitHub
rodzic 48ce4cbe23
commit 8d90470d55
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
10 zmienionych plików z 142 dodań i 97 usunięć

Wyświetl plik

@ -10,7 +10,7 @@ on:
jobs:
# TODO: add format/checkstyle
build:
name: Java ${{ matrix.jdk }} / ${{ matrix.os }}
name: Java ${{ matrix.jdk }} / ${{ matrix.os }} ${{ matrix.args }}
strategy:
fail-fast: false
matrix:
@ -19,6 +19,9 @@ jobs:
include:
- os: ubuntu-latest
jdk: 16
- os: ubuntu-latest
jdk: 17
args: "-DargLine='-Duser.language=fr -Duser.country=FR'"
runs-on: ${{ matrix.os }}
timeout-minutes: 15
steps:
@ -31,10 +34,10 @@ jobs:
cache: 'maven'
- name: Build with mvnw (linux/mac)
if: ${{ !contains(matrix.os, 'windows') }}
run: ./mvnw --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
run: ./mvnw ${{matrix.args}} --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
- name: Build with mvnw.cmd (windows)
if: ${{ contains(matrix.os, 'windows') }}
run: mvnw.cmd --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
run: mvnw.cmd ${{matrix.args}} --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
shell: cmd
regenerate:

Wyświetl plik

@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class LongLongMapBench {
public static void main(String[] args) throws InterruptedException {
Format format = Format.defaultInstance();
Path path = Path.of("./llmaptest");
FileUtils.delete(path);
LongLongMap map = LongLongMap.from(args[0], args[1], path);
@ -45,14 +46,14 @@ public class LongLongMapBench {
counter.count = i;
}
long end = System.nanoTime();
String rate = Format.formatNumeric(entries * NANOSECONDS_PER_SECOND / (end - start), false) + "/s";
String rate = format.numeric(entries * NANOSECONDS_PER_SECOND / (end - start), false) + "/s";
System.err.println("Loaded " + entries + " in " + Duration.ofNanos(end - start).toSeconds() + "s (" + rate + ")");
writeRate.set(rate);
}).awaitAndLog(loggers, Duration.ofSeconds(10));
map.get(1);
System.err.println("Storage: " + Format.formatStorage(map.diskUsageBytes(), false));
System.err.println("RAM: " + Format.formatStorage(map.estimateMemoryUsageBytes(), false));
System.err.println("Storage: " + format.storage(map.diskUsageBytes(), false));
System.err.println("RAM: " + format.storage(map.estimateMemoryUsageBytes(), false));
Counter.Readable readCount = Counter.newMultiThreadCounter();
loggers = ProgressLoggers.create()
@ -95,7 +96,7 @@ public class LongLongMapBench {
}
long end = System.nanoTime();
long read = readCount.getAsLong();
String readRate = Format.formatNumeric(read * NANOSECONDS_PER_SECOND / (end - start), false) + "/s";
String readRate = format.numeric(read * NANOSECONDS_PER_SECOND / (end - start), false) + "/s";
System.err.println("Read " + read + " in 30s (" + readRate + ")");
System.err.println(
String.join("\t",
@ -103,9 +104,9 @@ public class LongLongMapBench {
args[1],
args[2],
args[3],
Format.formatStorage(map.estimateMemoryUsageBytes(), false),
Format.formatStorage(map.diskUsageBytes(), false),
Format.formatStorage(FileUtils.size(path), false),
format.storage(map.estimateMemoryUsageBytes(), false),
format.storage(map.diskUsageBytes(), false),
format.storage(FileUtils.size(path), false),
writeRate.get(),
readRate
)

Wyświetl plik

@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
@ -429,7 +430,7 @@ public final class Mbtiles implements Closeable {
/** Data contained in the metadata table. */
public class Metadata {
private static final NumberFormat nf = NumberFormat.getNumberInstance();
private static final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
static {
nf.setMaximumFractionDigits(5);

Wyświetl plik

@ -329,6 +329,7 @@ public class MbtilesWriter {
}
private void printTileStats() {
Format format = Format.defaultInstance();
LOGGER.debug("Tile stats:");
long sumSize = 0;
long sumCount = 0;
@ -340,14 +341,14 @@ public class MbtilesWriter {
sumCount += totalCount;
long maxSize = maxTileSizesByZoom[z].get();
LOGGER.debug("z" + z +
" avg:" + Format.formatStorage(totalSize / Math.max(totalCount, 1), false) +
" max:" + Format.formatStorage(maxSize, false));
" avg:" + format.storage(totalSize / Math.max(totalCount, 1), false) +
" max:" + format.storage(maxSize, false));
}
LOGGER.debug("all" +
" avg:" + Format.formatStorage(sumSize / Math.max(sumCount, 1), false) +
" max:" + Format.formatStorage(maxMax, false));
LOGGER.debug(" # features: " + Format.formatInteger(featuresProcessed.get()));
LOGGER.debug(" # tiles: " + Format.formatInteger(this.tilesEmitted()));
" avg:" + format.storage(sumSize / Math.max(sumCount, 1), false) +
" max:" + format.storage(maxMax, false));
LOGGER.debug(" # features: " + format.integer(featuresProcessed.get()));
LOGGER.debug(" # tiles: " + format.integer(this.tilesEmitted()));
}
private long tilesEmitted() {

Wyświetl plik

@ -2,6 +2,7 @@ package com.onthegomap.planetiler.stats;
import com.onthegomap.planetiler.util.Format;
import java.time.Duration;
import java.util.Locale;
import java.util.Optional;
/**
@ -27,11 +28,16 @@ public record ProcessTime(Duration wall, Optional<Duration> cpu) {
return new ProcessTime(wall.minus(other.wall), cpu.flatMap(thisCpu -> other.cpu.map(thisCpu::minus)));
}
public String toString(Locale locale) {
Format format = Format.forLocale(locale);
Optional<String> deltaCpu = cpu.map(format::seconds);
String avgCpus = cpu.map(cpuTime -> " avg:" + format.decimal(cpuTime.toNanos() * 1d / wall.toNanos()))
.orElse("");
return format.seconds(wall) + " cpu:" + deltaCpu.orElse("-") + avgCpus;
}
@Override
public String toString() {
Optional<String> deltaCpu = cpu.map(Format::formatSeconds);
String avgCpus = cpu.map(cpuTime -> " avg:" + Format.formatDecimal(cpuTime.toNanos() * 1d / wall.toNanos()))
.orElse("");
return Format.formatSeconds(wall) + " cpu:" + deltaCpu.orElse("-") + avgCpus;
return toString(Format.DEFAULT_LOCALE);
}
}

Wyświetl plik

@ -1,6 +1,7 @@
package com.onthegomap.planetiler.stats;
import static com.onthegomap.planetiler.util.Format.*;
import static com.onthegomap.planetiler.util.Format.padLeft;
import static com.onthegomap.planetiler.util.Format.padRight;
import com.graphhopper.util.Helper;
import com.onthegomap.planetiler.util.DiskBacked;
@ -15,6 +16,7 @@ import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
@ -44,6 +46,7 @@ public class ProgressLoggers {
private static final String FG_BLUE = "\u001B[34m";
private static final Logger LOGGER = LoggerFactory.getLogger(ProgressLoggers.class);
private final List<Object> loggers = new ArrayList<>();
private final Format format;
private static String fg(String fg, String string) {
return fg + string + COLOR_RESET;
@ -65,11 +68,16 @@ public class ProgressLoggers {
return fg(FG_BLUE, string);
}
private ProgressLoggers() {
private ProgressLoggers(Locale locale) {
this.format = Format.forLocale(locale);
}
public static ProgressLoggers create() {
return new ProgressLoggers();
return createForLocale(Format.DEFAULT_LOCALE);
}
public static ProgressLoggers createForLocale(Locale locale) {
return new ProgressLoggers(locale);
}
/** Adds "name: [ numCompleted rate/s ]" to the logger. */
@ -98,7 +106,8 @@ public class ProgressLoggers {
}
last.set(valueNow);
lastTime.set(now);
String result = "[ " + formatNumeric(valueNow, true) + " " + formatNumeric(valueDiff / timeDiff, true) + "/s ]";
String result =
"[ " + format.numeric(valueNow, true) + " " + format.numeric(valueDiff / timeDiff, true) + "/s ]";
return color && valueDiff > 0 ? green(result) : result;
}));
return this;
@ -124,7 +133,7 @@ public class ProgressLoggers {
* process.
*/
public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue) {
return addRatePercentCounter(name, total, getValue, n -> Format.formatNumeric(n, true));
return addRatePercentCounter(name, total, getValue, n -> format.numeric(n, true));
}
/**
@ -132,7 +141,7 @@ public class ProgressLoggers {
* process.
*/
public ProgressLoggers addStorageRatePercentCounter(String name, long total, LongSupplier getValue) {
return addRatePercentCounter(name, total, getValue, n -> Format.formatStorage(n, true));
return addRatePercentCounter(name, total, getValue, n -> format.storage(n, true));
}
/**
@ -140,7 +149,7 @@ public class ProgressLoggers {
* process.
*/
public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue,
Function<Number, String> format) {
Function<Number, String> formatter) {
// if there's no total, we can't show progress so fall back to rate logger instead
if (total == 0) {
return addRateCounter(name, getValue, true);
@ -158,8 +167,8 @@ public class ProgressLoggers {
last.set(valueNow);
lastTime.set(now);
String result =
"[ " + format.apply(valueNow) + " " + padLeft(formatPercent(1f * valueNow / total), 4)
+ " " + format.apply(valueDiff / timeDiff) + "/s ]";
"[ " + formatter.apply(valueNow) + " " + padLeft(format.percent(1f * valueNow / total), 4)
+ " " + formatter.apply(valueDiff / timeDiff) + "/s ]";
return valueDiff > 0 ? green(result) : result;
}));
return this;
@ -173,7 +182,7 @@ public class ProgressLoggers {
loggers.add(new ProgressLogger(name, () -> {
long valueNow = getValue.get();
return "[ " + padLeft("" + valueNow, 3) + " / " + padLeft("" + total, 3) + " " + padLeft(
formatPercent(1f * valueNow / total), 4) + " ]";
format.percent(1f * valueNow / total), 4) + " ]";
}));
return this;
}
@ -182,9 +191,9 @@ public class ProgressLoggers {
public ProgressLoggers addQueueStats(WorkQueue<?> queue) {
loggers.add(new WorkerPipelineLogger(() ->
" -> " + padLeft("(" +
formatNumeric(queue.getPending(), false)
format.numeric(queue.getPending(), false)
+ "/" +
formatNumeric(queue.getCapacity(), false)
format.numeric(queue.getCapacity(), false)
+ ")", 9)
));
return this;
@ -209,7 +218,7 @@ public class ProgressLoggers {
return add(() -> {
String bytes;
try {
bytes = formatStorage(Files.size(file), false);
bytes = format.storage(Files.size(file), false);
} catch (IOException e) {
bytes = "-";
}
@ -218,20 +227,20 @@ public class ProgressLoggers {
}
public ProgressLoggers addFileSize(DiskBacked longSupplier) {
return add(() -> " " + padRight(formatStorage(longSupplier.diskUsageBytes(), false), 5));
return add(() -> " " + padRight(format.storage(longSupplier.diskUsageBytes(), false), 5));
}
/** Adds the total of disk and memory usage of {@code thing}. */
public <T extends DiskBacked & MemoryEstimator.HasEstimate> ProgressLoggers addFileSizeAndRam(T thing) {
return add(() -> {
long bytes = thing.diskUsageBytes() + thing.estimateMemoryUsageBytes();
return " " + padRight(formatStorage(bytes, false), 5);
return " " + padRight(format.storage(bytes, false), 5);
});
}
/** Adds the current size of a file on disk. */
public ProgressLoggers addFileSize(String name, DiskBacked file) {
loggers.add(new ProgressLogger(name, () -> formatStorage(file.diskUsageBytes(), true)));
loggers.add(new ProgressLogger(name, () -> format.storage(file.diskUsageBytes(), true)));
return this;
}
@ -240,16 +249,16 @@ public class ProgressLoggers {
* used after last GC to the output.
*/
public ProgressLoggers addProcessStats() {
addOptionalDeltaLogger("cpus", ProcessInfo::getProcessCpuTime, num -> blue(Format.formatDecimal(num)));
addOptionalDeltaLogger("cpus", ProcessInfo::getProcessCpuTime, num -> blue(format.decimal(num)));
addDeltaLogger("gc", ProcessInfo::getGcTime, num -> {
String formatted = Format.formatPercent(num);
String formatted = format.percent(num);
return num > 0.6 ? red(formatted) : num > 0.3 ? yellow(formatted) : formatted;
});
loggers.add(new ProgressLogger("mem",
() -> formatStorage(Helper.getUsedMB() * Helper.MB, false) + "/" +
formatStorage(Helper.getTotalMB() * Helper.MB, false) +
() -> format.storage(Helper.getUsedMB() * Helper.MB, false) + "/" +
format.storage(Helper.getTotalMB() * Helper.MB, false) +
ProcessInfo.getMemoryUsageAfterLastGC().stream()
.mapToObj(value -> " postGC: " + blue(formatStorage(value, false)))
.mapToObj(value -> " postGC: " + blue(format.storage(value, false)))
.findFirst()
.orElse("")
));
@ -298,7 +307,7 @@ public class ProgressLoggers {
return " -%";
}
long last = lastThreads.getOrDefault(thread.id(), ProcessInfo.ThreadState.DEFAULT).cpuTime().toNanos();
return padLeft(formatPercent(1d * (thread.cpuTime().toNanos() - last) / timeDiff), 3);
return padLeft(format.percent(1d * (thread.cpuTime().toNanos() - last) / timeDiff), 3);
}).collect(Collectors.joining(" ", "(", ")"));
lastTime.set(currentTime);
@ -329,7 +338,7 @@ public class ProgressLoggers {
/** Adds the current estimated size of an in-memory object to the output. */
public ProgressLoggers addInMemoryObject(String name, MemoryEstimator.HasEstimate object) {
loggers.add(new ProgressLogger(name, () -> formatStorage(object.estimateMemoryUsageBytes(), true)));
loggers.add(new ProgressLogger(name, () -> format.storage(object.estimateMemoryUsageBytes(), true)));
return this;
}
@ -371,7 +380,7 @@ public class ProgressLoggers {
}
}
private static record ProgressLogger(String name, Supplier<String> fn) {
private record ProgressLogger(String name, Supplier<String> fn) {
@Override
public String toString() {
@ -379,7 +388,7 @@ public class ProgressLoggers {
}
}
private static record WorkerPipelineLogger(Supplier<String> fn) {
private record WorkerPipelineLogger(Supplier<String> fn) {
@Override
public String toString() {

Wyświetl plik

@ -41,6 +41,7 @@ public interface Stats extends AutoCloseable {
* each monitored file.
*/
default void printSummary() {
Format format = Format.defaultInstance();
Logger LOGGER = LoggerFactory.getLogger(getClass());
LOGGER.info("-".repeat(40));
timers().printSummary();
@ -48,7 +49,7 @@ public interface Stats extends AutoCloseable {
for (var entry : monitoredFiles().entrySet()) {
long size = FileUtils.size(entry.getValue());
if (size > 0) {
LOGGER.info("\t" + entry.getKey() + "\t" + Format.formatStorage(size, false) + "B");
LOGGER.info("\t" + entry.getKey() + "\t" + format.storage(size, false) + "B");
}
}
}

Wyświetl plik

@ -2,9 +2,12 @@ package com.onthegomap.planetiler.util;
import java.text.NumberFormat;
import java.time.Duration;
import java.util.Locale;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.text.StringEscapeUtils;
import org.locationtech.jts.geom.Coordinate;
@ -13,7 +16,31 @@ import org.locationtech.jts.geom.Coordinate;
*/
public class Format {
private Format() {
public static final Locale DEFAULT_LOCALE = Locale.getDefault(Locale.Category.FORMAT);
public static final ConcurrentMap<Locale, Format> instances = new ConcurrentHashMap<>();
private final NumberFormat pf;
private final NumberFormat nf;
private final NumberFormat intF;
private Format(Locale locale) {
pf = NumberFormat.getPercentInstance(locale);
pf.setMaximumFractionDigits(0);
nf = NumberFormat.getNumberInstance(locale);
nf.setMaximumFractionDigits(1);
intF = NumberFormat.getNumberInstance(locale);
intF.setMaximumFractionDigits(0);
}
public static Format forLocale(Locale locale) {
Format format = instances.get(locale);
if (format == null) {
format = instances.computeIfAbsent(locale, Format::new);
}
return format;
}
public static Format defaultInstance() {
return forLocale(DEFAULT_LOCALE);
}
private static final NavigableMap<Long, String> STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries(
@ -31,16 +58,6 @@ public class Format {
Map.entry(1_000_000_000_000_000L, "Q")
));
private static final NumberFormat pf = NumberFormat.getPercentInstance();
private static final NumberFormat nf = NumberFormat.getNumberInstance();
private static final NumberFormat intF = NumberFormat.getNumberInstance();
static {
pf.setMaximumFractionDigits(0);
nf.setMaximumFractionDigits(1);
intF.setMaximumFractionDigits(0);
}
public static String padRight(String str, int size) {
StringBuilder strBuilder = new StringBuilder(str);
while (strBuilder.length() < size) {
@ -58,16 +75,16 @@ public class Format {
}
/** Returns a number of bytes formatted like "123" "1.2k" "240M", etc. */
public static String formatStorage(Number num, boolean pad) {
public String storage(Number num, boolean pad) {
return format(num, pad, STORAGE_SUFFIXES);
}
/** Returns a number formatted like "123" "1.2k" "2.5B", etc. */
public static String formatNumeric(Number num, boolean pad) {
public String numeric(Number num, boolean pad) {
return format(num, pad, NUMERIC_SUFFIXES);
}
private static String format(Number num, boolean pad, NavigableMap<Long, String> suffixes) {
private String format(Number num, boolean pad, NavigableMap<Long, String> suffixes) {
long value = num.longValue();
double doubleValue = num.doubleValue();
if (value < 0) {
@ -85,28 +102,28 @@ public class Format {
long truncated = value / (divideBy / 10);
boolean hasDecimal = truncated < 100 && (truncated % 10 != 0);
return padLeft(hasDecimal ? (truncated / 10d) + suffix : (truncated / 10) + suffix, pad ? 4 : 0);
return padLeft(hasDecimal ? decimal(truncated / 10d) + suffix : (truncated / 10) + suffix, pad ? 4 : 0);
}
/** Returns 0.0-1.0 as a "0%" - "100%" with no decimal points. */
public static String formatPercent(double value) {
public String percent(double value) {
return pf.format(value);
}
/** Returns a number formatted with 1 decimal point. */
public static String formatDecimal(double value) {
public String decimal(double value) {
return nf.format(value);
}
/** Returns a number formatted with 0 decimal points. */
public static String formatInteger(Number value) {
public String integer(Number value) {
return intF.format(value);
}
/** Returns a duration formatted as fractional seconds with 1 decimal point. */
public static String formatSeconds(Duration duration) {
public String seconds(Duration duration) {
double seconds = duration.toNanos() * 1d / Duration.ofSeconds(1).toNanos();
return formatDecimal(seconds < 1 ? seconds : Math.round(seconds)) + "s";
return decimal(seconds < 1 ? seconds : Math.round(seconds)) + "s";
}
/** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */

Wyświetl plik

@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.worker.WorkerPipeline;
import java.time.Duration;
import java.util.Locale;
import java.util.concurrent.CountDownLatch;
import java.util.function.Supplier;
import org.junit.jupiter.api.Test;
@ -32,7 +33,7 @@ public class ProgressLoggersTest {
continueLatch.await();
});
var loggers = ProgressLoggers.create()
var loggers = ProgressLoggers.createForLocale(Locale.US)
.newLine()
.addPipelineStats(pipeline);

Wyświetl plik

@ -2,6 +2,7 @@ package com.onthegomap.planetiler.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.Locale;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
@ -9,40 +10,43 @@ public class FormatTest {
@ParameterizedTest
@CsvSource({
"1.5,1",
"999,999",
"1000,1k",
"9999,9.9k",
"10001,10k",
"99999,99k",
"999999,999k",
"9999999,9.9M",
"-9999999,-",
"5.5e12,5.5T",
"1.5,1,en",
"999,999,en",
"1000,1k,en",
"9999,9.9k,en",
"10001,10k,en",
"99999,99k,en",
"999999,999k,en",
"9999999,9.9M,en",
"-9999999,-,en",
"5.5e12,5.5T,en",
"5.5e12,'5,5T',fr",
})
public void testFormatNumeric(Double number, String formatted) {
assertEquals(Format.formatNumeric(number, false), formatted);
public void testFormatNumeric(Double number, String expected, Locale locale) {
assertEquals(expected, Format.forLocale(locale).numeric(number, false));
}
@ParameterizedTest
@CsvSource({
"999,999",
"1000,1k",
"9999,9.9k",
"5.5e9,5.5G",
"999,999,en",
"1000,1k,en",
"9999,9.9k,en",
"5.5e9,5.5G,en",
"5.5e9,'5,5G',fr",
})
public void testFormatStorage(Double number, String formatted) {
assertEquals(formatted, Format.formatStorage(number, false));
public void testFormatStorage(Double number, String expected, Locale locale) {
assertEquals(expected, Format.forLocale(locale).storage(number, false));
}
@ParameterizedTest
@CsvSource({
"0,0%",
"1,100%",
"0.11111,11%",
"0,0%,en",
"1,100%,en",
"0.11111,11%,en",
"0.11111,11 %,fr",
})
public void testFormatPercent(Double number, String formatted) {
assertEquals(formatted, Format.formatPercent(number));
public void testFormatPercent(Double number, String formatted, Locale locale) {
assertEquals(formatted, Format.forLocale(locale).percent(number));
}
@ParameterizedTest
@ -60,12 +64,13 @@ public class FormatTest {
@ParameterizedTest
@CsvSource({
"0,0",
"0.1,0.1",
"0.11,0.1",
"1111.11,'1,111.1'",
"0,0,en",
"0.1,0.1,en",
"0.11,0.1,en",
"1111.11,'1,111.1',en",
"1111.11,'1.111,1',it",
})
public void testFormatDecimal(Double in, String out) {
assertEquals(out, Format.formatDecimal(in));
public void testFormatDecimal(Double in, String out, Locale locale) {
assertEquals(out, Format.forLocale(locale).decimal(in));
}
}