mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Expose arguments via config (#363)

pull/364/head
Michael Barry 2022-10-04 19:57:59 -04:00 zatwierdzone przez GitHub
rodzic 83148052b0
commit 0eb148ee3c
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
47 zmienionych plików z 1945 dodań i 261 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

13
planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
Wyświetl plik

@ -95,8 +95,7 @@ public class Planetiler {
private Planetiler(Arguments arguments) {
this.arguments = arguments;
stats = arguments.getStats();
overallTimer = stats.startStage("overall");
LogUtil.clearStage();
overallTimer = stats.startStageQuietly("overall");
config = PlanetilerConfig.from(arguments);
tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp"));
onlyDownloadSources = arguments.getBoolean("only_download", "download source data then exit", false);
@ -115,16 +114,6 @@ public class Planetiler {
return new Planetiler(arguments);
}
/**
* Returns a new empty runner that will get configuration from {@code arguments} to the main method, JVM properties,
* environmental variables, or a config file specified in {@code config} argument.
*
* @param arguments array of string arguments provided to {@code public static void main(String[] args)} entrypoint
*/
public static Planetiler create(String... arguments) {
return new Planetiler(Arguments.fromArgsOrConfigFile(arguments));
}
/**
* Adds a new {@code .osm.pbf} source that will be processed when {@link #run()} is called.
* <p>

114
planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java
Wyświetl plik

@ -1,5 +1,7 @@
package com.onthegomap.planetiler.config;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.stats.Stats;
import java.io.IOException;
@ -8,13 +10,15 @@ import java.nio.file.Path;
import java.time.Duration;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import java.util.stream.Stream;
import org.locationtech.jts.geom.Envelope;
@ -29,11 +33,22 @@ public class Arguments {
private static final Logger LOGGER = LoggerFactory.getLogger(Arguments.class);
private final Function<String, String> provider;
private final UnaryOperator<String> provider;
private final Supplier<? extends Collection<String>> keys;
private boolean silent = false;
private Arguments(UnaryOperator<String> provider) {
private Arguments(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys) {
this.provider = provider;
this.keys = keys;
}
private static Arguments from(UnaryOperator<String> provider, Supplier<? extends Collection<String>> rawKeys,
UnaryOperator<String> forward, UnaryOperator<String> reverse) {
Supplier<List<String>> keys = () -> rawKeys.get().stream().flatMap(key -> {
String reversed = reverse.apply(key);
return key.equalsIgnoreCase(reversed) ? Stream.empty() : Stream.of(reversed);
}).toList();
return new Arguments(key -> provider.apply(forward.apply(key)), keys);
}
/**
@ -42,7 +57,17 @@ public class Arguments {
* For example to set {@code key=value}: {@code java -Dplanetiler.key=value -jar ...}
*/
public static Arguments fromJvmProperties() {
return new Arguments(key -> System.getProperty("planetiler." + key));
return fromJvmProperties(
System::getProperty,
() -> System.getProperties().stringPropertyNames()
);
}
static Arguments fromJvmProperties(UnaryOperator<String> getter, Supplier<? extends Collection<String>> keys) {
return from(getter, keys,
key -> "planetiler." + key.toLowerCase(Locale.ROOT),
key -> key.replaceFirst("^planetiler\\.", "").toLowerCase(Locale.ROOT)
);
}
/**
@ -51,7 +76,27 @@ public class Arguments {
* For example to set {@code key=value}: {@code PLANETILER_KEY=value java -jar ...}
*/
public static Arguments fromEnvironment() {
return new Arguments(key -> System.getenv("PLANETILER_" + key.toUpperCase(Locale.ROOT)));
return fromEnvironment(
System::getenv,
() -> System.getenv().keySet()
);
}
static Arguments fromEnvironment(UnaryOperator<String> getter, Supplier<Set<String>> keys) {
return from(getter, keys,
key -> "PLANETILER_" + key.toUpperCase(Locale.ROOT),
key -> key.replaceFirst("^PLANETILER_", "").toLowerCase(Locale.ROOT)
);
}
/**
* Returns arguments parsed from a {@link Properties} object.
*/
public static Arguments from(Properties properties) {
return new Arguments(
properties::getProperty,
properties::stringPropertyNames
);
}
/**
@ -97,7 +142,7 @@ public class Arguments {
Properties properties = new Properties();
try (var reader = Files.newBufferedReader(path)) {
properties.load(reader);
return new Arguments(properties::getProperty);
return from(properties);
} catch (IOException e) {
throw new IllegalArgumentException("Unable to load config file: " + path, e);
}
@ -147,7 +192,7 @@ public class Arguments {
}
public static Arguments of(Map<String, String> map) {
return new Arguments(map::get);
return new Arguments(map::get, map::keySet);
}
/** Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs. */
@ -177,10 +222,20 @@ public class Arguments {
* @return arguments instance that checks {@code this} first and if a match is not found then {@code other}
*/
public Arguments orElse(Arguments other) {
return new Arguments(key -> {
String ourResult = get(key);
return ourResult != null ? ourResult : other.get(key);
});
var result = new Arguments(
key -> {
String ourResult = get(key);
return ourResult != null ? ourResult : other.get(key);
},
() -> Stream.concat(
other.keys.get().stream(),
keys.get().stream()
).distinct().toList()
);
if (silent) {
result.silence();
}
return result;
}
String getArg(String key) {
@ -218,7 +273,7 @@ public class Arguments {
return result;
}
private void logArgValue(String key, String description, Object result) {
protected void logArgValue(String key, String description, Object result) {
if (!silent) {
LOGGER.debug("argument: {}={} ({})", key, result, description);
}
@ -332,13 +387,13 @@ public class Arguments {
public Stats getStats() {
String prometheus = getArg("pushgateway");
if (prometheus != null && !prometheus.isBlank()) {
LOGGER.info("Using prometheus push gateway stats");
LOGGER.info("argument: stats=use prometheus push gateway stats");
String job = getString("pushgateway.job", "prometheus pushgateway job ID", "planetiler");
Duration interval = getDuration("pushgateway.interval", "how often to send stats to prometheus push gateway",
"15s");
return Stats.prometheusPushGateway(prometheus, job, interval);
} else {
LOGGER.info("Using in-memory stats");
LOGGER.info("argument: stats=use in-memory stats");
return Stats.inMemory();
}
}
@ -390,4 +445,35 @@ public class Arguments {
logArgValue(key, description, parsed);
return parsed;
}
/**
* Returns a map from all the arguments provided to their values.
*/
public Map<String, String> toMap() {
Map<String, String> result = new HashMap<>();
for (var key : keys.get()) {
result.put(key, get(key));
}
return result;
}
/** Returns a copy of this {@code Arguments} instance that logs each extracted argument value exactly once. */
public Arguments withExactlyOnceLogging() {
Multiset<String> logged = HashMultiset.create();
return new Arguments(this.provider, this.keys) {
@Override
protected void logArgValue(String key, String description, Object result) {
int count = logged.add(key, 1);
if (count == 0) {
super.logArgValue(key, description, result);
} else if (count == 3000) {
LOGGER.warn("Too many requests for argument '{}', result should be cached", key);
}
}
};
}
public boolean silenced() {
return silent;
}
}

2
planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
Wyświetl plik

@ -124,7 +124,7 @@ public record PlanetilerConfig(
maxzoom,
renderMaxzoom,
arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false),
arguments.getBoolean("optimize_db", "optimize mbtiles after writing", false),
arguments.getBoolean("optimize_db", "Vacuum analyze mbtiles after writing", false),
arguments.getBoolean("emit_tiles_in_order", "emit tiles in index order", true),
arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),

31
planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java
Wyświetl plik

@ -32,13 +32,21 @@ public enum DataType implements BiFunction<WithTags, String, Object> {
this(id, (d, k) -> parser.apply(d.getTag(k)), parser);
}
@Override
public Object apply(WithTags withTags, String string) {
return this.getter.apply(withTags, string);
}
public Object convertFrom(Object value) {
return this.parser.apply(value);
/** Returns the data type associated with {@code value}, or {@link #GET_TAG} as a fallback. */
public static DataType typeOf(Object value) {
if (value instanceof String) {
return GET_STRING;
} else if (value instanceof Integer) {
return GET_INT;
} else if (value instanceof Long) {
return GET_LONG;
} else if (value instanceof Double) {
return GET_DOUBLE;
} else if (value instanceof Boolean) {
return GET_BOOLEAN;
} else {
return GET_TAG;
}
}
/** Returns the data type associated with {@code id}, or {@link #GET_TAG} as a fallback. */
@ -51,6 +59,15 @@ public enum DataType implements BiFunction<WithTags, String, Object> {
return GET_TAG;
}
@Override
public Object apply(WithTags withTags, String string) {
return this.getter.apply(withTags, string);
}
public Object convertFrom(Object value) {
return this.parser.apply(value);
}
public String id() {
return id;
}

26
planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
Wyświetl plik

@ -62,11 +62,31 @@ public interface Stats extends AutoCloseable {
* Also sets the "stage" prefix that shows up in the logs to {@code name}.
*/
default Timers.Finishable startStage(String name) {
LogUtil.setStage(name);
var timer = timers().startTimer(name);
return startStage(name, true);
}
/**
* Same as {@link #startStage(String)} except does not log that it started, or set the logging prefix.
*/
default Timers.Finishable startStageQuietly(String name) {
return startStage(name, false);
}
/**
* Records that a long-running task with {@code name} has started and returns a handle to call when finished.
* <p>
* Also sets the "stage" prefix that shows up in the logs to {@code name} if {@code log} is true.
*/
default Timers.Finishable startStage(String name, boolean log) {
if (log) {
LogUtil.setStage(name);
}
var timer = timers().startTimer(name, log);
return () -> {
timer.stop();
LogUtil.clearStage();
if (log) {
LogUtil.clearStage();
}
};
}

14
planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Timers.java
Wyświetl plik

@ -91,16 +91,22 @@ public class Timers {
}
public Finishable startTimer(String name) {
return startTimer(name, true);
}
public Finishable startTimer(String name, boolean logStart) {
Timer timer = Timer.start();
Stage stage = new Stage(timer);
timers.put(name, stage);
Stage last = currentStage.getAndSet(stage);
LOGGER.info("");
LOGGER.info("Starting...");
if (logStart) {
LOGGER.info("");
LOGGER.info("Starting...");
}
return () -> {
LOGGER.info("Finished in " + timers.get(name).timer.stop());
LOGGER.info("Finished in {}", timers.get(name).timer.stop());
for (var details : getStageDetails(name, true)) {
LOGGER.info(" " + details);
LOGGER.info(" {}", details);
}
currentStage.set(last);
};

17
planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileWatcher.java
Wyświetl plik

@ -1,7 +1,5 @@
package com.onthegomap.planetiler.util;
import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -104,19 +102,4 @@ public class FileWatcher {
}
}
}
@FunctionalInterface
public interface FunctionThatThrows<I, O> {
@SuppressWarnings("java:S112")
O apply(I value) throws Exception;
default O runAndWrapException(I value) {
try {
return apply(value);
} catch (Exception e) {
return throwFatalException(e);
}
}
}
}

18
planetiler-core/src/main/java/com/onthegomap/planetiler/util/FunctionThatThrows.java 100644
Wyświetl plik

@ -0,0 +1,18 @@
package com.onthegomap.planetiler.util;
import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
@FunctionalInterface
public interface FunctionThatThrows<I, O> {
@SuppressWarnings("java:S112")
O apply(I value) throws Exception;
default O runAndWrapException(I value) {
try {
return apply(value);
} catch (Exception e) {
return throwFatalException(e);
}
}
}

46
planetiler-core/src/main/java/com/onthegomap/planetiler/util/Memoized.java 100644
Wyświetl plik

@ -0,0 +1,46 @@
package com.onthegomap.planetiler.util;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
/**
* Caches the value of a function, so it only gets called once for each unique input, including when it throws an
* exception.
*/
public class Memoized<I, O> implements Function<I, O> {
private final ConcurrentHashMap<I, Try<O>> cache = new ConcurrentHashMap<>();
private final Function<I, Try<O>> supplier;
private Memoized(FunctionThatThrows<I, O> supplier) {
this.supplier = i -> Try.apply(() -> supplier.apply(i));
}
/** Returns a memoized version of {@code supplier} that gets called only once for each input. */
public static <I, O> Memoized<I, O> memoize(FunctionThatThrows<I, O> supplier) {
return new Memoized<>(supplier);
}
@Override
public O apply(I i) {
Try<O> result = cache.get(i);
if (result == null) {
result = cache.computeIfAbsent(i, supplier);
}
return result.get();
}
/** Returns a success or failure wrapper for the function call. */
public Try<O> tryApply(I i) {
Try<O> result = cache.get(i);
if (result == null) {
result = cache.computeIfAbsent(i, supplier);
}
return result;
}
/** Returns a success or failure wrapper for the function call, and casting the result to {@code clazz}. */
public <T> Try<T> tryApply(I i, Class<T> clazz) {
return tryApply(i).cast(clazz);
}
}

29
planetiler-core/src/main/java/com/onthegomap/planetiler/util/Try.java
Wyświetl plik

@ -1,5 +1,7 @@
package com.onthegomap.planetiler.util;
import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
/**
* A container for the result of an operation that may succeed or fail.
*
@ -45,12 +47,35 @@ public interface Try<T> {
return null;
}
record Success<T> (T get) implements Try<T> {}
/** If success, then tries to cast the result to {@code clazz}, turning into a failure if not possible. */
default <O> Try<O> cast(Class<O> clazz) {
return map(clazz::cast);
}
/**
* If this is a success, then maps the value through {@code fn}, returning the new value in a {@link Success} if
* successful, or {@link Failure} if the mapping function threw an exception.
*/
<O> Try<O> map(FunctionThatThrows<T, O> fn);
record Success<T> (T get) implements Try<T> {
@Override
public <O> Try<O> map(FunctionThatThrows<T, O> fn) {
return Try.apply(() -> fn.apply(get));
}
}
record Failure<T> (@Override Exception exception) implements Try<T> {
@Override
public T get() {
throw new IllegalStateException(exception);
return throwFatalException(exception);
}
@Override
@SuppressWarnings("unchecked")
public <O> Try<O> map(FunctionThatThrows<T, O> fn) {
return (Try<O>) this;
}
}

8
planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedError.java 100644
Wyświetl plik

@ -0,0 +1,8 @@
package com.onthegomap.planetiler;
/** A fatal error intentionally thrown by a test. */
public class ExpectedError extends Error {
public ExpectedError() {
super("expected error", null, true, false);
}
}

2
planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedException.java
Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.planetiler;
/** An exception intentionally thrown by a test. */
public class ExpectedException extends Error {
public class ExpectedException extends RuntimeException {
public ExpectedException() {
super("expected exception", null, true, false);
}

2
planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
Wyświetl plik

@ -871,7 +871,7 @@ class PlanetilerTests {
with(new OsmElement.Node(1, 0, 0), t -> t.setTag("attr", "value"))
),
(in, features) -> {
throw new ExpectedException();
throw new ExpectedError();
}
));
}

98
planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java
Wyświetl plik

@ -2,11 +2,13 @@ package com.onthegomap.planetiler.config;
import static org.junit.jupiter.api.Assertions.*;
import com.onthegomap.planetiler.ExpectedException;
import com.onthegomap.planetiler.TestUtils;
import com.onthegomap.planetiler.reader.osm.OsmInputFile;
import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Envelope;
@ -181,4 +183,100 @@ class ArgumentsTest {
assertTrue(args.getBoolean("force1", "force1", false));
assertTrue(args.getBoolean("force2", "force2", false));
}
@Test
void testListArgumentValuesFromCommandLine() {
assertEquals(Map.of(), Arguments.fromArgs().toMap());
assertEquals(Map.of(
"key", "value",
"key2", "value2",
"force1", "true",
"force2", "true"
), Arguments.fromArgs(
"--key value --key2 value2 --force1 --force2".split("\\s+")
).toMap());
}
@Test
void testListArgumentValuesFromMap() {
assertEquals(Map.of(), Arguments.of(Map.of()).toMap());
assertEquals(Map.of("a", "1", "b", "2"), Arguments.of(Map.of("a", "1", "b", "2")).toMap());
}
@Test
void testListArgumentValuesFromConfigFile() {
Arguments args = Arguments.fromConfigFile(TestUtils.pathToResource("test.properties"));
assertEquals(Map.of("key1", "value1fromfile", "key2", "value2fromfile"), args.toMap());
}
@Test
void testListArgumentsFromEnvironment() {
Map<String, String> env = Map.of(
"OTHER", "value",
"PLANETILEROTHER", "VALUE",
"PLANETILER_KEY1", "value1",
"PLANETILER_KEY2", "value2"
);
Arguments args = Arguments.fromEnvironment(env::get, env::keySet);
assertEquals(Map.of(
"key1", "value1",
"key2", "value2"
), args.toMap());
}
@Test
void testListArgumentsFromJvmProperties() {
Map<String, String> jvm = Map.of(
"OTHER", "value",
"PLANETILEROTHER", "VALUE",
"PLANETILER_KEY1", "value1",
"PLANETILER_KEY2", "value2",
"planetiler.key3", "value4"
);
Arguments args = Arguments.fromJvmProperties(jvm::get, jvm::keySet);
assertEquals(Map.of(
"key3", "value4"
), args.toMap());
}
@Test
void testListArgumentsFromMerged() {
Map<String, String> env = Map.of(
"OTHER", "value",
"PLANETILEROTHER", "VALUE",
"PLANETILER_KEY1", "value1",
"PLANETILER_KEY2", "value2",
"planetiler.key3", "value3"
);
Map<String, String> jvm = Map.of(
"other", "value",
"PLANETILEROTHER", "VALUE",
"PLANETILER_KEY1", "value1",
"PLANETILER_KEY2", "value2",
"planetiler.key3", "value4"
);
Arguments args = Arguments.fromJvmProperties(jvm::get, jvm::keySet)
.orElse(Arguments.fromEnvironment(env::get, env::keySet));
assertEquals(Map.of(
"key3", "value4",
"key1", "value1",
"key2", "value2"
), args.toMap());
}
@Test
void testDontAccessArgListUntilUsed() {
Map<String, String> env = Map.of(
"OTHER", "value",
"PLANETILEROTHER", "VALUE",
"PLANETILER_KEY1", "value1",
"PLANETILER_KEY2", "value2",
"planetiler.key3", "value3"
);
Arguments args = Arguments.fromEnvironment(env::get, () -> {
throw new ExpectedException();
});
assertEquals("value1", args.getString("key1", ""));
assertThrows(ExpectedException.class, args::toMap);
}
}

59
planetiler-core/src/test/java/com/onthegomap/planetiler/util/MemoizedTest.java 100644
Wyświetl plik

@ -0,0 +1,59 @@
package com.onthegomap.planetiler.util;
import static com.onthegomap.planetiler.util.Memoized.memoize;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.onthegomap.planetiler.ExpectedException;
import org.junit.jupiter.api.Test;
class MemoizedTest {
int calls = 0;
@Test
void testMemoize() {
Memoized<Integer, Integer> memoized = memoize(i -> {
calls++;
return i + 1;
});
assertEquals(0, calls);
assertEquals(1, memoized.apply(0));
assertEquals(1, calls);
assertEquals(1, memoized.apply(0));
assertEquals(1, memoized.tryApply(0).get());
assertEquals(1, calls);
assertEquals(2, memoized.apply(1));
assertEquals(2, memoized.apply(1));
assertEquals(2, calls);
}
@Test
void testThrowException() {
Memoized<Integer, Integer> memoized = memoize(i -> {
calls++;
throw new ExpectedException();
});
assertEquals(0, calls);
assertThrows(ExpectedException.class, () -> memoized.apply(0));
assertThrows(ExpectedException.class, () -> memoized.apply(0));
assertTrue(memoized.tryApply(0).isFailure());
assertEquals(1, calls);
assertThrows(ExpectedException.class, () -> memoized.apply(1));
assertThrows(ExpectedException.class, () -> memoized.apply(1));
assertTrue(memoized.tryApply(1).isFailure());
assertEquals(2, calls);
}
@Test
void testTryCast() {
Memoized<Integer, Integer> memoized = memoize(i -> {
calls++;
return i + 1;
});
assertEquals(1, memoized.tryApply(0, Number.class).get());
var failed = memoized.tryApply(0, String.class);
assertTrue(failed.isFailure());
assertThrows(ClassCastException.class, failed::get);
}
}

10
planetiler-core/src/test/java/com/onthegomap/planetiler/util/TryTest.java
Wyświetl plik

@ -1,7 +1,9 @@
package com.onthegomap.planetiler.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
@ -22,4 +24,12 @@ class TryTest {
assertEquals(Try.failure(exception), result);
assertThrows(IllegalStateException.class, result::get);
}
@Test
void cast() {
var result = Try.apply(() -> 1);
assertEquals(Try.success(1), result.cast(Number.class));
assertTrue(result.cast(String.class).isFailure());
assertInstanceOf(ClassCastException.class, result.cast(String.class).exception());
}
}

124
planetiler-custommap/README.md
Wyświetl plik

@ -33,6 +33,8 @@ The root of the schema has the following attributes:
- `examples` - A list of [Test Case](#test-case) input features and the vector tile features they should map to, or a
relative path to a file with those examples in it. Run planetiler with `verify schema_file.yml` to see
if they work as expected.
- `args` - Set default values for arguments that can be referenced later in the config and overridden from the
command-line or environmental variables. See [Arguments](#arguments).
- `definitions` - An unparsed spot where you can
define [anchor labels](#anchors-and-aliases) to be used in other parts of the
schema
@ -46,6 +48,7 @@ attribution: <a href="https://www.openstreetmap.org/copyright" target="_blank">&
sources: { ... }
tag_mappings: { ... }
layers: [...]
args: { ... }
examples: [...]
```
@ -55,10 +58,12 @@ A description that tells planetiler how to read geospatial objects with tags fro
- `type` - Enum representing the file format of the data source, one
of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format) or [`shapefile`](https://en.wikipedia.org/wiki/Shapefile)
- `local_path` - Local path to the file to use, inferred from `url` if missing
- `local_path` - Local path to the file to use, inferred from `url` if missing. Can be a string
or [expression](#expression) that can reference [argument values](#arguments).
- `url` - Location to download the file from if not present at `local_path`.
For [geofabrik](https://download.geofabrik.de/) named areas, use `geofabrik:`
prefixes, for example `geofabrik:rhode-island`.
For [geofabrik](https://download.geofabrik.de/) named areas, use `geofabrik:` prefixes, for
example `geofabrik:rhode-island`. Can be a string or [expression](#expression) that can
reference [argument values](#arguments).
For example:
@ -89,6 +94,101 @@ tag_mappings:
type: integer
```
## Arguments
A map from argument name to its definition. Arguments can be referenced later in the config and
overridden from the command-line or environmental variables. Argument definitions can either be an object with these
properties, or just the default value:
- `default` - Default value for the argument. Can be an [expression](#expression) that references other arguments.
- `description` - Description of the argument to print when parsing it.
- `type` - [Data type](#data-type) to use when parsing the value. If missing, then inferred from the default value.
For example:
```yaml
# Define an "area" argument with default value "switzerland"
# and a "path" argument that defaults to the area with .osm.pbf extension
args:
area:
description: Geofabrik area to download
default: switzerland
osm_local_path: '${ args.area + ".osm.pbf" }'
# Use the value of the "area" and "path" arguments to construct the source definition
sources:
osm:
type: osm
url: '${ "geofabrik:" + args.area }'
local_path: '${ args.osm_local_path }'
```
You can pass in `--area=france` from the command line to set download URL to `geofabrik:france` and local path
to `france.osm.pbf`. Planetiler searches for argument values in this order:
1. Command-line arguments `--area=france`
2. JVM Properties with "planetiler." prefix: `java -Dplanetiler.area=france ...`
3. Environmental variables with "PLANETILER_" prefix: `PLANETILER_AREA=france java ...`
4. Default value from the config
Argument values are available from the [`args` variable](#root-context) in
an [inline script expression](#inline-script-expression) or the [`arg_value` expression](#argument-value-expression).
### Built-in arguments
`args` can also be used to set the default value for built-in arguments that control planetiler's behavior:
<!--
to regenerate:
cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.properties | to_entries[] | "- `" + .key + "` - " + .value.description' | pbcopy
-->
- `threads` - Default number of threads to use.
- `write_threads` - Default number of threads to use when writing temp features
- `process_threads` - Default number of threads to use when processing input features
- `feature_read_threads` - Default number of threads to use when reading features at tile write time
- `minzoom` - Minimum tile zoom level to emit
- `maxzoom` - Maximum tile zoom level to emit
- `render_maxzoom` - Maximum rendering zoom level up to
- `skip_mbtiles_index_creation` - Skip adding index to mbtiles file
- `optimize_db` - Vacuum analyze mbtiles file after writing
- `emit_tiles_in_order` - Emit vector tiles in index order
- `force` - Overwriting output file and ignore warnings
- `gzip_temp` - Gzip temporary feature storage (uses more CPU, but less disk space)
- `mmap_temp` - Use memory-mapped IO for temp feature files
- `sort_max_readers` - Maximum number of concurrent read threads to use when sorting chunks
- `sort_max_writers` - Maximum number of concurrent write threads to use when sorting chunks
- `nodemap_type` - Type of node location map
- `nodemap_storage` - Storage for node location map
- `nodemap_madvise` - Use linux madvise(random) for node locations
- `multipolygon_geometry_storage` - Storage for multipolygon geometries
- `multipolygon_geometry_madvise` - Use linux madvise(random) for multiplygon geometries
- `http_user_agent` - User-Agent header to set when downloading files over HTTP
- `http_retries` - Retries to use when downloading files over HTTP
- `download_chunk_size_mb` - Size of file chunks to download in parallel in megabytes
- `download_threads` - Number of parallel threads to use when downloading each file
- `min_feature_size_at_max_zoom` - Default value for the minimum size in tile pixels of features to emit at the maximum
zoom level to allow for overzooming
- `min_feature_size` - Default value for the minimum size in tile pixels of features to emit below the maximum zoom
level
- `simplify_tolerance_at_max_zoom` - Default value for the tile pixel tolerance to use when simplifying features at the
maximum zoom level to allow for overzooming
- `simplify_tolerance` - Default value for the tile pixel tolerance to use when simplifying features below the maximum
zoom level
- `compact_db` - Reduce the DB size by separating and deduping the tile data
- `skip_filled_tiles` - Skip writing tiles containing only polygon fills to the output
- `tile_warning_size_mb` - Maximum size in megabytes of a tile to emit a warning about
For example:
```yaml
# Tell planetiler to download sources using 10 threads
args:
download_threads: 10
```
Built-in arguments can also be accessed from the config file if desired: `${ args.download_threads }`.
## Layer
A layer contains a thematically-related set of features from one or more input sources.
@ -167,6 +267,8 @@ To define the value, use one of:
- `coalesce` - A [Coalesce Expression](#coalesce-expression) that sets this attribute to the first non-null match from a
list of expressions.
- `tag_value` - A [Tag Value Expression](#tag-value-expression) that sets this attribute to the value for a tag.
- `arg_value` - An [Argument Value Expression](#argument-value-expression) that sets this attribute to the value for a
tag.
For example:
@ -217,6 +319,17 @@ value:
tag_value: natural
```
### Argument Value Expression
Use `arg_value:` to return the value of an argument set in the [Arguments](#arguments) section, or overridden from the
command-line or environment.
```yaml
# return value for "attr_value" argument
value:
arg_value: attr_value
```
### Coalesce Expression
Use `coalesce: [expression, expression, ...]` to make the expression evaluate to the first non-null result of a list of
@ -321,8 +434,11 @@ nested, so each child context can also access the variables from its parent.
> ##### root context
>
> defines no variables
> Available variables:
> - `args` - a map from [argument](#arguments) name to value, see also [built-in arguments](#built-in-arguments) that
>
>> are always available.
>>
>> ##### process feature context
>>
>> Context available when processing an input feature, for example testing whether to include it from `include_when`.

260
planetiler-custommap/planetiler.schema.json
Wyświetl plik

@ -45,10 +45,11 @@
},
"url": {
"description": "Location to download the file from. For geofabrik named areas, use `geofabrik:` prefixes, for example `geofabrik:rhode-island`.",
"type": "string"
"$ref": "#/$defs/expression"
},
"local_path": {
"description": "Local path to the file to use, inferred from `url` if missing"
"description": "Local path to the file to use, inferred from `url` if missing",
"$ref": "#/$defs/expression"
}
},
"anyOf": [
@ -111,6 +112,246 @@
}
}
},
"args": {
"description": "Set default values for built-in command-line arguments or expose new command-line arguments.",
"type": "object",
"properties": {
"threads": {
"description": "Default number of threads to use."
},
"write_threads": {
"description": "Default number of threads to use when writing temp features"
},
"process_threads": {
"description": "Default number of threads to use when processing input features"
},
"feature_read_threads": {
"description": "Default number of threads to use when reading features at tile write time"
},
"minzoom": {
"description": "Minimum tile zoom level to emit"
},
"maxzoom": {
"description": "Maximum tile zoom level to emit"
},
"render_maxzoom": {
"description": "Maximum rendering zoom level up to"
},
"skip_mbtiles_index_creation": {
"description": "Skip adding index to mbtiles file",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"optimize_db": {
"description": "Vacuum analyze mbtiles file after writing",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"emit_tiles_in_order": {
"description": "Emit vector tiles in index order",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"force": {
"description": "Overwriting output file and ignore warnings",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"gzip_temp": {
"description": "Gzip temporary feature storage (uses more CPU, but less disk space)",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"mmap_temp": {
"description": "Use memory-mapped IO for temp feature files",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"sort_max_readers": {
"description": "Maximum number of concurrent read threads to use when sorting chunks"
},
"sort_max_writers": {
"description": "Maximum number of concurrent write threads to use when sorting chunks"
},
"nodemap_type": {
"description": "Type of node location map",
"anyOf": [
{
"type": "string"
},
{
"enum": [
"array",
"sparsearray",
"sortedtable",
"noop"
]
}
]
},
"nodemap_storage": {
"description": "Storage for node location map",
"anyOf": [
{
"type": "string"
},
{
"enum": [
"ram",
"mmap",
"direct"
]
}
]
},
"nodemap_madvise": {
"description": "Use linux madvise(random) for node locations",
"anyOf": [
{
"type": "boolean"
},
{
"type": "string"
}
]
},
"multipolygon_geometry_storage": {
"description": "Storage for multipolygon geometries",
"anyOf": [
{
"type": "string"
},
{
"enum": [
"ram",
"mmap",
"direct"
]
}
]
},
"multipolygon_geometry_madvise": {
"description": "Use linux madvise(random) for multiplygon geometries",
"anyOf": [
{
"type": "boolean"
},
{
"type": "string"
}
]
},
"http_user_agent": {
"description": "User-Agent header to set when downloading files over HTTP"
},
"http_retries": {
"description": "Retries to use when downloading files over HTTP"
},
"download_chunk_size_mb": {
"description": "Size of file chunks to download in parallel in megabytes"
},
"download_threads": {
"description": "Number of parallel threads to use when downloading each file"
},
"min_feature_size_at_max_zoom": {
"description": "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming"
},
"min_feature_size": {
"description": "Default value for the minimum size in tile pixels of features to emit below the maximum zoom level"
},
"simplify_tolerance_at_max_zoom": {
"description": "Default value for the tile pixel tolerance to use when simplifying features at the maximum zoom level to allow for overzooming"
},
"simplify_tolerance": {
"description": "Default value for the tile pixel tolerance to use when simplifying features below the maximum zoom level"
},
"compact_db": {
"description": "Reduce the DB size by separating and deduping the tile data",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"skip_filled_tiles": {
"description": "Skip writing tiles containing only polygon fills to the output",
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"tile_warning_size_mb": {
"description": "Maximum size in megabytes of a tile to emit a warning about"
}
},
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"type": "object",
"properties": {
"default": {
"description": "Default value for this argument (can be an expression).",
"$ref": "#/$defs/expression"
},
"type": {
"description": "Data type of this argument.",
"$ref": "#/$defs/datatype"
},
"description": {
"description": "Description for this argument.",
"type": "string"
}
}
}
]
}
},
"examples": {
"description": "Example input features and the vector tile features they map to, or a relative path to a file with those examples in it.",
"oneOf": [
@ -208,6 +449,9 @@
{
"$ref": "#/$defs/expression_tag_value"
},
{
"$ref": "#/$defs/expression_arg_value"
},
{
"$ref": "#/$defs/expression_value"
},
@ -316,6 +560,9 @@
{
"$ref": "#/$defs/expression_tag_value"
},
{
"$ref": "#/$defs/expression_arg_value"
},
{
"$ref": "#/$defs/expression_value"
},
@ -365,6 +612,15 @@
}
}
},
"expression_arg_value": {
"type": "object",
"properties": {
"arg_value": {
"description": "Returns the value associated with an argument",
"$ref": "#/$defs/expression"
}
}
},
"expression_value": {
"type": "object",
"properties": {

20
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java
Wyświetl plik

@ -10,6 +10,8 @@ import com.onthegomap.planetiler.custommap.expression.ScriptContext;
import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
import com.onthegomap.planetiler.expression.DataType;
import com.onthegomap.planetiler.expression.MultiExpression;
import com.onthegomap.planetiler.util.Memoized;
import com.onthegomap.planetiler.util.Try;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -28,6 +30,8 @@ import java.util.stream.Stream;
*/
public class ConfigExpressionParser<I extends ScriptContext> {
private static final Memoized<EvaluateInput, ?> MEMOIZED = Memoized.memoize(arg -> ConfigExpressionParser
.parse(arg.expression, TagValueProducer.EMPTY, arg.root.description(), arg.clazz).apply(arg.root));
private final TagValueProducer tagValueProducer;
private final ScriptEnvironment<I> input;
@ -50,6 +54,17 @@ public class ConfigExpressionParser<I extends ScriptContext> {
return new ConfigExpressionParser<>(tagValueProducer, context).parse(object, outputClass).simplify();
}
/**
* Attempts to evaluate {@code expression} from a yaml config, using only globally-available environmental variables
* from the {@code root} context.
*/
public static <T> Try<T> tryStaticEvaluate(Contexts.Root root, Object expression, Class<T> resultType) {
if (expression == null) {
return Try.success(null);
}
return MEMOIZED.tryApply(new EvaluateInput(root, expression, resultType), resultType);
}
private <O> ConfigExpression<I, O> parse(Object object, Class<O> output) {
if (object == null) {
return ConfigExpression.constOf(null);
@ -82,6 +97,9 @@ public class ConfigExpressionParser<I extends ScriptContext> {
} else if (keys.equals(Set.of("tag_value"))) {
var tagProducer = parse(map.get("tag_value"), String.class);
return getTag(signature(output), tagProducer);
} else if (keys.equals(Set.of("arg_value"))) {
var keyProducer = parse(map.get("arg_value"), String.class);
return getArg(signature(output), keyProducer);
} else if (keys.equals(Set.of("value"))) {
return parse(map.get("value"), output);
}
@ -134,4 +152,6 @@ public class ConfigExpressionParser<I extends ScriptContext> {
private <O> Signature<I, O> signature(Class<O> outputClass) {
return new Signature<>(input, outputClass);
}
private record EvaluateInput(Contexts.Root root, Object expression, Class<?> clazz) {}
}

30
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java
Wyświetl plik

@ -8,6 +8,7 @@ import com.onthegomap.planetiler.FeatureCollector.Feature;
import com.onthegomap.planetiler.custommap.configschema.AttributeDefinition;
import com.onthegomap.planetiler.custommap.configschema.FeatureGeometry;
import com.onthegomap.planetiler.custommap.configschema.FeatureItem;
import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
import com.onthegomap.planetiler.expression.Expression;
import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.reader.SourceFeature;
@ -34,9 +35,13 @@ public class ConfiguredFeature {
private final TagValueProducer tagValueProducer;
private final List<BiConsumer<Contexts.FeaturePostMatch, Feature>> featureProcessors;
private final Set<String> sources;
private final ScriptEnvironment<Contexts.ProcessFeature> processFeatureContext;
private final ScriptEnvironment<Contexts.FeatureAttribute> featureAttributeContext;
private ScriptEnvironment<Contexts.FeaturePostMatch> featurePostMatchContext;
public ConfiguredFeature(String layer, TagValueProducer tagValueProducer, FeatureItem feature) {
public ConfiguredFeature(String layer, TagValueProducer tagValueProducer, FeatureItem feature,
Contexts.Root rootContext) {
sources = Set.copyOf(feature.source());
FeatureGeometry geometryType = feature.geometry();
@ -46,6 +51,9 @@ public class ConfiguredFeature {
//Factory to treat OSM tag values as specific data type values
this.tagValueProducer = tagValueProducer;
processFeatureContext = Contexts.ProcessFeature.description(rootContext);
featurePostMatchContext = Contexts.FeaturePostMatch.description(rootContext);
featureAttributeContext = Contexts.FeatureAttribute.description(rootContext);
//Test to determine whether this feature is included based on tagging
Expression filter;
@ -53,13 +61,15 @@ public class ConfiguredFeature {
filter = Expression.TRUE;
} else {
filter =
BooleanExpressionParser.parse(feature.includeWhen(), tagValueProducer, Contexts.ProcessFeature.DESCRIPTION);
BooleanExpressionParser.parse(feature.includeWhen(), tagValueProducer,
processFeatureContext);
}
if (feature.excludeWhen() != null) {
filter = Expression.and(
filter,
Expression.not(
BooleanExpressionParser.parse(feature.excludeWhen(), tagValueProducer, Contexts.ProcessFeature.DESCRIPTION))
BooleanExpressionParser.parse(feature.excludeWhen(), tagValueProducer,
processFeatureContext))
);
}
tagTest = filter;
@ -86,7 +96,7 @@ public class ConfiguredFeature {
var expression = ConfigExpressionParser.parse(
input,
tagValueProducer,
Contexts.FeaturePostMatch.DESCRIPTION,
featurePostMatchContext,
clazz
);
if (expression.equals(constOf(null))) {
@ -140,12 +150,14 @@ public class ConfiguredFeature {
value.put("value", attribute.value());
} else if (attribute.tagValue() != null) {
value.put("tag_value", attribute.tagValue());
} else if (attribute.argValue() != null) {
value.put("arg_value", attribute.argValue());
} else {
value.put("tag_value", attribute.key());
}
}
return ConfigExpressionParser.parse(value, tagValueProducer, Contexts.FeaturePostMatch.DESCRIPTION, Object.class);
return ConfigExpressionParser.parse(value, tagValueProducer, featurePostMatchContext, Object.class);
}
/**
@ -160,7 +172,7 @@ public class ConfiguredFeature {
Double minTilePercent, Object rawMinZoom, Map<Object, Integer> minZoomByValue) {
var result = ConfigExpressionParser.parse(rawMinZoom, tagValueProducer,
Contexts.FeatureAttribute.DESCRIPTION, Integer.class);
featureAttributeContext, Integer.class);
if ((result.equals(constOf(0)) ||
result.equals(constOf(null))) && minZoomByValue.isEmpty()) {
@ -206,9 +218,11 @@ public class ConfiguredFeature {
var attributeTest =
Expression.and(
attrIncludeWhen == null ? Expression.TRUE :
BooleanExpressionParser.parse(attrIncludeWhen, tagValueProducer, Contexts.FeaturePostMatch.DESCRIPTION),
BooleanExpressionParser.parse(attrIncludeWhen, tagValueProducer,
featurePostMatchContext),
attrExcludeWhen == null ? Expression.TRUE :
not(BooleanExpressionParser.parse(attrExcludeWhen, tagValueProducer, Contexts.FeaturePostMatch.DESCRIPTION))
not(BooleanExpressionParser.parse(attrExcludeWhen, tagValueProducer,
featurePostMatchContext))
).simplify();
var minTileCoverage = attrIncludeWhen == null ? null : attribute.minTileCoverSize();

64
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java
Wyświetl plik

@ -2,14 +2,11 @@ package com.onthegomap.planetiler.custommap;
import com.onthegomap.planetiler.Planetiler;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.DataSource;
import com.onthegomap.planetiler.custommap.configschema.DataSourceType;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import java.net.URI;
import java.net.URISyntaxException;
import com.onthegomap.planetiler.custommap.expression.ParseException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Main driver to create maps configured by a YAML file.
@ -23,69 +20,58 @@ public class ConfiguredMapMain {
* Main entrypoint
*/
public static void main(String... args) throws Exception {
run(Arguments.fromArgsOrConfigFile(args));
}
static void run(Arguments args) throws Exception {
Arguments arguments = Arguments.fromEnvOrArgs(args);
var dataDir = Path.of("data");
var sourcesDir = dataDir.resolve("sources");
var schemaFile = args.getString(
var schemaFile = arguments.getString(
"schema",
"Location of YML-format schema definition file"
);
var path = Path.of(schemaFile);
SchemaConfig config;
SchemaConfig schema;
if (Files.exists(path)) {
config = SchemaConfig.load(path);
schema = SchemaConfig.load(path);
} else {
// if the file doesn't exist, check if it's bundled in the jar
schemaFile = schemaFile.startsWith("/samples/") ? schemaFile : "/samples/" + schemaFile;
if (ConfiguredMapMain.class.getResource(schemaFile) != null) {
config = YAML.loadResource(schemaFile, SchemaConfig.class);
schema = YAML.loadResource(schemaFile, SchemaConfig.class);
} else {
throw new IllegalArgumentException("Schema file not found: " + schemaFile);
}
}
var planetiler = Planetiler.create(args)
.setProfile(new ConfiguredProfile(config));
// use default argument values from config file as fallback if not set from command-line or env vars
Contexts.Root rootContext = Contexts.buildRootContext(arguments, schema.args());
var sources = config.sources();
for (var source : sources.entrySet()) {
configureSource(planetiler, sourcesDir, source.getKey(), source.getValue());
var planetiler = Planetiler.create(rootContext.arguments());
var profile = new ConfiguredProfile(schema, rootContext);
planetiler.setProfile(profile);
for (var source : profile.sources()) {
configureSource(planetiler, sourcesDir, source);
}
planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles"))
.run();
planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")).run();
}
private static void configureSource(Planetiler planetiler, Path sourcesDir, String sourceName, DataSource source)
throws URISyntaxException {
private static void configureSource(Planetiler planetiler, Path sourcesDir, Source source) {
DataSourceType sourceType = source.type();
Path localPath = source.localPath();
if (localPath == null) {
if (source.url() == null) {
throw new ParseException("Must provide either a url or path for " + source.id());
}
localPath = sourcesDir.resolve(source.defaultFileUrl());
}
switch (sourceType) {
case OSM -> {
String url = source.url();
String[] areaParts = url.split("[:/]");
String areaFilename = areaParts[areaParts.length - 1];
String areaName = areaFilename.replaceAll("\\..*$", "");
if (localPath == null) {
localPath = sourcesDir.resolve(areaName + ".osm.pbf");
}
planetiler.addOsmSource(sourceName, localPath, url);
}
case SHAPEFILE -> {
String url = source.url();
if (localPath == null) {
localPath = sourcesDir.resolve(Paths.get(new URI(url).getPath()).getFileName().toString());
}
planetiler.addShapefileSource(sourceName, localPath, url);
}
default -> throw new IllegalArgumentException("Unhandled source " + sourceType);
case OSM -> planetiler.addOsmSource(source.id(), localPath, source.url());
case SHAPEFILE -> planetiler.addShapefileSource(source.id(), localPath, source.url());
default -> throw new IllegalArgumentException("Unhandled source type for " + source.id() + ": " + sourceType);
}
}
}

33
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java
Wyświetl plik

@ -10,6 +10,7 @@ import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.expression.MultiExpression;
import com.onthegomap.planetiler.expression.MultiExpression.Index;
import com.onthegomap.planetiler.reader.SourceFeature;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -22,27 +23,29 @@ import java.util.stream.Collectors;
*/
public class ConfiguredProfile implements Profile {
private final SchemaConfig schemaConfig;
private final SchemaConfig schema;
private final Map<String, Index<ConfiguredFeature>> featureLayerMatcher;
private final TagValueProducer tagValueProducer;
private final Contexts.Root rootContext;
public ConfiguredProfile(SchemaConfig schemaConfig) {
this.schemaConfig = schemaConfig;
public ConfiguredProfile(SchemaConfig schema, Contexts.Root rootContext) {
this.schema = schema;
this.rootContext = rootContext;
Collection<FeatureLayer> layers = schemaConfig.layers();
Collection<FeatureLayer> layers = schema.layers();
if (layers == null || layers.isEmpty()) {
throw new IllegalArgumentException("No layers defined");
}
tagValueProducer = new TagValueProducer(schemaConfig.inputMappings());
tagValueProducer = new TagValueProducer(schema.inputMappings());
Map<String, List<MultiExpression.Entry<ConfiguredFeature>>> configuredFeatureEntries = new HashMap<>();
for (var layer : layers) {
String layerId = layer.id();
for (var feature : layer.features()) {
var configuredFeature = new ConfiguredFeature(layerId, tagValueProducer, feature);
var configuredFeature = new ConfiguredFeature(layerId, tagValueProducer, feature, rootContext);
var entry = new Entry<>(configuredFeature, configuredFeature.matchExpression());
for (var source : feature.source()) {
var list = configuredFeatureEntries.computeIfAbsent(source, s -> new ArrayList<>());
@ -58,17 +61,17 @@ public class ConfiguredProfile implements Profile {
@Override
public String name() {
return schemaConfig.schemaName();
return schema.schemaName();
}
@Override
public String attribution() {
return schemaConfig.attribution();
return schema.attribution();
}
@Override
public void processFeature(SourceFeature sourceFeature, FeatureCollector featureCollector) {
var context = new Contexts.ProcessFeature(sourceFeature, tagValueProducer);
var context = rootContext.createProcessFeatureContext(sourceFeature, tagValueProducer);
var index = featureLayerMatcher.get(sourceFeature.getSource());
if (index != null) {
var matches = index.getMatchesWithTriggers(context);
@ -83,6 +86,16 @@ public class ConfiguredProfile implements Profile {
@Override
public String description() {
return schemaConfig.schemaDescription();
return schema.schemaDescription();
}
public List<Source> sources() {
List<Source> sources = new ArrayList<>();
schema.sources().forEach((key, value) -> {
var url = ConfigExpressionParser.tryStaticEvaluate(rootContext, value.url(), String.class).get();
var path = ConfigExpressionParser.tryStaticEvaluate(rootContext, value.localPath(), String.class).get();
sources.add(new Source(key, value.type(), url, path == null ? null : Path.of(path)));
});
return sources;
}
}

309
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
Wyświetl plik

@ -1,14 +1,34 @@
package com.onthegomap.planetiler.custommap;
import com.google.api.expr.v1alpha1.Constant;
import com.google.api.expr.v1alpha1.Decl;
import com.google.api.expr.v1alpha1.Type;
import com.google.protobuf.NullValue;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.custommap.expression.ParseException;
import com.onthegomap.planetiler.custommap.expression.ScriptContext;
import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
import com.onthegomap.planetiler.expression.DataType;
import com.onthegomap.planetiler.reader.SourceFeature;
import com.onthegomap.planetiler.reader.WithGeometryType;
import com.onthegomap.planetiler.reader.WithTags;
import com.onthegomap.planetiler.util.Try;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.projectnessie.cel.checker.Decls;
import org.projectnessie.cel.common.types.NullT;
import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry;
import org.projectnessie.cel.common.types.ref.TypeAdapter;
import org.projectnessie.cel.common.types.ref.Val;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Wrapper objects that provide all available inputs to different parts of planetiler schema configs at runtime.
@ -17,35 +37,256 @@ import org.projectnessie.cel.common.types.NullT;
* that all global variables from a parent context are also available to its child context.
*/
public class Contexts {
private static final Logger LOGGER = LoggerFactory.getLogger(Contexts.class);
private static Object wrapNullable(Object nullable) {
return nullable == null ? NullT.NullValue : nullable;
}
public static Root root() {
return new Root();
public static Root emptyRoot() {
return new Root(Arguments.of().silence(), Map.of());
}
/**
* Returns a {@link Root} context built from {@code schemaArgs} argument definitions and {@code origArguments}
* arguments provided from the command-line/environment.
* <p>
* Arguments may depend on the value of other arguments so this iteratively evaluates the arguments until their values
* settle.
*
* @throws ParseException if the argument definitions are malformed, or if there's an infinite loop
*/
public static Contexts.Root buildRootContext(Arguments origArguments, Map<String, Object> schemaArgs) {
boolean loggingEnabled = !origArguments.silenced();
origArguments.silence();
Map<String, String> argDescriptions = new LinkedHashMap<>();
Map<String, Object> unparsedSchemaArgs = new HashMap<>(schemaArgs);
Map<String, Object> parsedSchemaArgs = new HashMap<>(origArguments.toMap());
Contexts.Root result = new Root(origArguments, parsedSchemaArgs);
Arguments arguments = origArguments;
int iters = 0;
// arguments may reference the value of other arguments, so continue parsing until they all succeed...
while (!unparsedSchemaArgs.isEmpty()) {
final var root = result;
final var args = arguments;
Map<String, Exception> failures = new HashMap<>();
Map.copyOf(unparsedSchemaArgs).forEach((key, value) -> {
boolean builtin = root.builtInArgs.contains(key);
String description;
Object defaultValueObject;
DataType type = null;
if (value instanceof Map<?, ?> map) {
if (builtin) {
throw new ParseException("Cannot override built-in argument: " + key);
}
var typeObject = map.get("type");
if (typeObject != null) {
type = DataType.from(Objects.toString(typeObject));
}
var descriptionObject = map.get("description");
description = descriptionObject == null ? "no description provided" : descriptionObject.toString();
defaultValueObject = map.get("default");
if (type != null) {
var fromArgs = args.getString(key, description, null);
if (fromArgs != null) {
parsedSchemaArgs.put(key, type.convertFrom(fromArgs));
}
}
} else {
defaultValueObject = value;
description = "no description provided";
}
argDescriptions.put(key, description);
Try<Object> defaultValue = ConfigExpressionParser.tryStaticEvaluate(root, defaultValueObject, Object.class);
if (defaultValue.isSuccess()) {
Object raw = defaultValue.get();
String asString = Objects.toString(raw);
if (type == null) {
type = DataType.typeOf(raw);
}
var stringResult = args.getString(key, description, asString);
Object castedResult = type.convertFrom(stringResult);
if (stringResult == null) {
throw new ParseException("Missing required parameter: " + key + "(" + description + ")");
} else if (castedResult == null) {
throw new ParseException("Cannot convert value for " + key + " to " + type.id() + ": " + stringResult);
}
parsedSchemaArgs.put(key, castedResult);
unparsedSchemaArgs.remove(key);
} else {
failures.put(key, defaultValue.exception());
}
});
arguments = origArguments.orElse(Arguments.of(parsedSchemaArgs.entrySet().stream().collect(Collectors.toMap(
Map.Entry::getKey,
e -> Objects.toString(e.getValue()))
)));
result = new Root(arguments, parsedSchemaArgs);
if (iters++ > 100) {
failures
.forEach(
(key, failure) -> LOGGER.error("Error computing {}:\n{}", key,
ExceptionUtils.getRootCause(failure).toString().indent(4)));
throw new ParseException("Infinite loop while processing arguments: " + unparsedSchemaArgs.keySet());
}
}
var finalArguments = loggingEnabled ? arguments.withExactlyOnceLogging() : arguments.silence();
if (loggingEnabled) {
argDescriptions.forEach((key, description) -> finalArguments.getString(key, description, null));
}
return new Root(finalArguments, parsedSchemaArgs);
}
/**
* Root context available everywhere in a planetiler schema config.
* <p>
* Holds argument values parsed from the schema config and command-line args.
*/
public record Root() implements ScriptContext {
public static final class Root implements ScriptContext {
private static final TypeAdapter TYPE_ADAPTER = ProtoTypeRegistry.newRegistry();
private final Arguments arguments;
private final PlanetilerConfig config;
private final ScriptEnvironment<Root> description;
private final Map<String, Val> bindings = new HashMap<>();
private final Map<String, Object> argumentValues = new HashMap<>();
public final Set<String> builtInArgs;
// TODO add argument parsing
public static final ScriptEnvironment<Root> DESCRIPTION =
ScriptEnvironment.root().forInput(Root.class);
public Arguments arguments() {
return arguments;
}
public PlanetilerConfig config() {
return config;
}
@Override
public boolean equals(Object o) {
return this == o || (o instanceof Root root && argumentValues.equals(root.argumentValues));
}
@Override
public int hashCode() {
return Objects.hash(argumentValues);
}
@Override
public Object argument(String key) {
return argumentValues.get(key);
}
private Root(Arguments arguments, Map<String, Object> schemaArgs) {
this.arguments = arguments;
this.config = PlanetilerConfig.from(arguments);
argumentValues.put("threads", config.threads());
argumentValues.put("write_threads", config.featureWriteThreads());
argumentValues.put("process_threads", config.featureProcessThreads());
argumentValues.put("feature_read_threads", config.featureReadThreads());
// args.put("loginterval", config.logInterval());
argumentValues.put("minzoom", config.minzoom());
argumentValues.put("maxzoom", config.maxzoom());
argumentValues.put("render_maxzoom", config.maxzoomForRendering());
argumentValues.put("skip_mbtiles_index_creation", config.skipIndexCreation());
argumentValues.put("optimize_db", config.optimizeDb());
argumentValues.put("emit_tiles_in_order", config.emitTilesInOrder());
argumentValues.put("force", config.force());
argumentValues.put("gzip_temp", config.gzipTempStorage());
argumentValues.put("mmap_temp", config.mmapTempStorage());
argumentValues.put("sort_max_readers", config.sortMaxReaders());
argumentValues.put("sort_max_writers", config.sortMaxWriters());
argumentValues.put("nodemap_type", config.nodeMapType());
argumentValues.put("nodemap_storage", config.nodeMapStorage());
argumentValues.put("nodemap_madvise", config.nodeMapMadvise());
argumentValues.put("multipolygon_geometry_storage", config.multipolygonGeometryStorage());
argumentValues.put("multipolygon_geometry_madvise", config.multipolygonGeometryMadvise());
argumentValues.put("http_user_agent", config.httpUserAgent());
// args.put("http_timeout", config.httpTimeout());
argumentValues.put("http_retries", config.httpRetries());
argumentValues.put("download_chunk_size_mb", config.downloadChunkSizeMB());
argumentValues.put("download_threads", config.downloadThreads());
argumentValues.put("min_feature_size_at_max_zoom", config.minFeatureSizeAtMaxZoom());
argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom());
argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());
argumentValues.put("simplify_tolerance", config.simplifyToleranceBelowMaxZoom());
argumentValues.put("compact_db", config.compactDb());
argumentValues.put("skip_filled_tiles", config.skipFilledTiles());
argumentValues.put("tile_warning_size_mb", config.tileWarningSizeBytes());
builtInArgs = Set.copyOf(argumentValues.keySet());
schemaArgs.forEach(argumentValues::putIfAbsent);
config.arguments().toMap().forEach(argumentValues::putIfAbsent);
argumentValues.forEach((k, v) -> bindings.put("args." + k, TYPE_ADAPTER.nativeToValue(v)));
bindings.put("args", TYPE_ADAPTER.nativeToValue(this.argumentValues));
description = ScriptEnvironment.root(this).forInput(Root.class)
.withDeclarations(
argumentValues.entrySet().stream()
.map(entry -> decl(entry.getKey(), entry.getValue()))
.toList()
).withDeclarations(
Decls.newVar("args", Decls.newMapType(Decls.String, Decls.Any))
);
}
private Decl decl(String name, Object value) {
Type type;
var builder = Constant.newBuilder();
if (value instanceof String s) {
builder.setStringValue(s);
type = Decls.String;
} else if (value instanceof Boolean b) {
builder.setBoolValue(b);
type = Decls.Bool;
} else if (value instanceof Long || value instanceof Integer) {
builder.setInt64Value(((Number) value).longValue());
type = Decls.Int;
} else if (value instanceof Double || value instanceof Float) {
builder.setDoubleValue(((Number) value).doubleValue());
type = Decls.Double;
} else if (value == null) {
builder.setNullValue(NullValue.NULL_VALUE);
type = Decls.Null;
} else {
throw new IllegalArgumentException(
"Unrecognized constant type: " + value + " (" + value.getClass().getName() + ")");
}
return Decls.newConst("args." + name, type, builder.build());
}
public ScriptEnvironment<Root> description() {
return description;
}
@Override
public Object apply(String input) {
return bindings.get(input);
}
public ProcessFeature createProcessFeatureContext(SourceFeature sourceFeature, TagValueProducer tagValueProducer) {
return new ProcessFeature(this, sourceFeature, tagValueProducer);
}
}
private interface NestedContext extends ScriptContext {
default Root root() {
return null;
}
@Override
default Object argument(String key) {
return root().argument(key);
}
}
/**
* Makes nested contexts adhere to {@link WithTags} and {@link WithGeometryType} by recursively fetching source
* feature from the root context.
*/
private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType {
private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType, NestedContext {
default FeatureContext parent() {
return null;
}
@ -54,6 +295,11 @@ public class Contexts {
return parent().feature();
}
@Override
default Root root() {
return parent().root();
}
@Override
default Map<String, Object> tags() {
return feature().tags();
@ -87,7 +333,10 @@ public class Contexts {
* @param feature The input feature being processed
* @param tagValueProducer Common parsing for input feature tags
*/
public record ProcessFeature(@Override SourceFeature feature, @Override TagValueProducer tagValueProducer)
public record ProcessFeature(
@Override Root root, @Override SourceFeature feature,
@Override TagValueProducer tagValueProducer
)
implements FeatureContext {
private static final String FEATURE_TAGS = "feature.tags";
@ -95,14 +344,16 @@ public class Contexts {
private static final String FEATURE_SOURCE = "feature.source";
private static final String FEATURE_SOURCE_LAYER = "feature.source_layer";
public static final ScriptEnvironment<ProcessFeature> DESCRIPTION = ScriptEnvironment.root()
.forInput(ProcessFeature.class)
.withDeclarations(
Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)),
Decls.newVar(FEATURE_ID, Decls.Int),
Decls.newVar(FEATURE_SOURCE, Decls.String),
Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String)
);
public static ScriptEnvironment<ProcessFeature> description(Root root) {
return root.description()
.forInput(ProcessFeature.class)
.withDeclarations(
Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)),
Decls.newVar(FEATURE_ID, Decls.Int),
Decls.newVar(FEATURE_SOURCE, Decls.String),
Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String)
);
}
@Override
public Object apply(String key) {
@ -127,7 +378,7 @@ public class Contexts {
/**
* Context available after a feature has been matched.
*
* <p>
* Adds {@code match_key} and {@code match_value} variables that capture which tag key/value caused the feature to be
* included.
*
@ -139,12 +390,14 @@ public class Contexts {
private static final String MATCH_KEY = "match_key";
private static final String MATCH_VALUE = "match_value";
public static final ScriptEnvironment<FeaturePostMatch> DESCRIPTION = ProcessFeature.DESCRIPTION
.forInput(FeaturePostMatch.class)
.withDeclarations(
Decls.newVar(MATCH_KEY, Decls.String),
Decls.newVar(MATCH_VALUE, Decls.Any)
);
public static ScriptEnvironment<FeaturePostMatch> description(Root root) {
return ProcessFeature.description(root)
.forInput(FeaturePostMatch.class)
.withDeclarations(
Decls.newVar(MATCH_KEY, Decls.String),
Decls.newVar(MATCH_VALUE, Decls.Any)
);
}
@Override
public Object apply(String key) {
@ -182,10 +435,14 @@ public class Contexts {
* @param value Value of the attribute
*/
public record FeatureAttribute(@Override FeaturePostMatch parent, Object value) implements FeatureContext {
private static final String VALUE = "value";
public static final ScriptEnvironment<FeatureAttribute> DESCRIPTION = FeaturePostMatch.DESCRIPTION
.forInput(FeatureAttribute.class)
.withDeclarations(Decls.newVar(VALUE, Decls.Any));
public static ScriptEnvironment<FeatureAttribute> description(Root root) {
return FeaturePostMatch.description(root)
.forInput(FeatureAttribute.class)
.withDeclarations(Decls.newVar(VALUE, Decls.Any));
}
@Override
public Object apply(String key) {

23
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java 100644
Wyświetl plik

@ -0,0 +1,23 @@
package com.onthegomap.planetiler.custommap;
import com.onthegomap.planetiler.custommap.configschema.DataSourceType;
import java.nio.file.Path;
/** A parsed source definition from a config file. */
public record Source(
String id,
DataSourceType type,
String url,
Path localPath
) {
public String defaultFileUrl() {
String result = url
.replaceFirst("^https?://", "")
.replaceAll("[\\W&&[^.]]+", "_");
if (type == DataSourceType.OSM && !result.endsWith(".pbf")) {
result = result + ".osm.pbf";
}
return result;
}
}

1
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java
Wyświetl plik

@ -14,6 +14,7 @@ public record AttributeDefinition(
// pass-through to value expression
@JsonProperty("value") Object value,
@JsonProperty("tag_value") String tagValue,
@JsonProperty("arg_value") String argValue,
Object type,
Object coalesce
) {}

5
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java
Wyświetl plik

@ -1,10 +1,9 @@
package com.onthegomap.planetiler.custommap.configschema;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.nio.file.Path;
public record DataSource(
DataSourceType type,
String url,
@JsonProperty("local_path") Path localPath
Object url,
@JsonProperty("local_path") Object localPath
) {}

8
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java
Wyświetl plik

@ -17,7 +17,8 @@ public record SchemaConfig(
Object definitions,
@JsonProperty("tag_mappings") Map<String, Object> inputMappings,
Collection<FeatureLayer> layers,
Object examples
Object examples,
Map<String, Object> args
) {
private static final String DEFAULT_ATTRIBUTION = """
@ -29,6 +30,11 @@ public record SchemaConfig(
return attribution == null ? DEFAULT_ATTRIBUTION : attribution;
}
@Override
public Map<String, Object> args() {
return args == null ? Map.of() : args;
}
public static SchemaConfig load(Path path) {
return YAML.load(path, SchemaConfig.class);
}

37
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
Wyświetl plik

@ -45,6 +45,11 @@ public interface ConfigExpression<I extends ScriptContext, O>
return new GetTag<>(signature, tag);
}
static <I extends ScriptContext, O> ConfigExpression<I, O> getArg(Signature<I, O> signature,
ConfigExpression<I, String> tag) {
return new GetArg<>(signature, tag);
}
static <I extends ScriptContext, O> ConfigExpression<I, O> cast(Signature<I, O> signature,
ConfigExpression<I, ?> input, DataType dataType) {
return new Cast<>(signature, input, dataType);
@ -163,13 +168,16 @@ public interface ConfigExpression<I extends ScriptContext, O>
default -> {
var result = children.stream()
.flatMap(
child -> child instanceof Coalesce<I, O> childCoalesce ? childCoalesce.children.stream() :
child -> child instanceof Coalesce<?, ?> childCoalesce ? childCoalesce.children.stream() :
Stream.of(child))
.filter(child -> !child.equals(constOf(null)))
.distinct()
.toList();
var indexOfFirstConst = result.stream().takeWhile(d -> !(d instanceof ConfigExpression.Const<I, O>)).count();
yield coalesce(result.stream().limit(indexOfFirstConst + 1).toList());
var indexOfFirstConst = result.stream().takeWhile(d -> !(d instanceof ConfigExpression.Const<?, ?>)).count();
yield coalesce(result.stream().map(d -> {
@SuppressWarnings("unchecked") ConfigExpression<I, O> casted = (ConfigExpression<I, O>) d;
return casted;
}).limit(indexOfFirstConst + 1).toList());
}
};
}
@ -210,6 +218,29 @@ public interface ConfigExpression<I extends ScriptContext, O>
}
}
/** An expression that returns the value associated a given argument at runtime. */
record GetArg<I extends ScriptContext, O> (
Signature<I, O> signature,
ConfigExpression<I, String> arg
) implements ConfigExpression<I, O> {
@Override
public O apply(I i) {
return TypeConversion.convert(i.argument(arg.apply(i)), signature.out);
}
@Override
public ConfigExpression<I, O> simplifyOnce() {
var key = arg.simplifyOnce();
if (key instanceof ConfigExpression.Const<I, String> constKey) {
var rawResult = signature.in.root().argument(constKey.value);
return constOf(TypeConversion.convert(rawResult, signature.out));
} else {
return new GetArg<>(signature, key);
}
}
}
/** An expression that converts the input to a desired output {@link DataType} at runtime. */
record Cast<I extends ScriptContext, O> (
Signature<I, O> signature,

31
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java
Wyświetl plik

@ -1,12 +1,10 @@
package com.onthegomap.planetiler.custommap.expression;
import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.TypeConversion;
import com.onthegomap.planetiler.custommap.expression.stdlib.PlanetilerStdLib;
import com.onthegomap.planetiler.util.Memoized;
import com.onthegomap.planetiler.util.Try;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import org.projectnessie.cel.extension.StringsLib;
import org.projectnessie.cel.tools.Script;
@ -23,6 +21,8 @@ import org.projectnessie.cel.tools.ScriptHost;
public class ConfigExpressionScript<I extends ScriptContext, O> implements ConfigExpression<I, O> {
private static final Pattern EXPRESSION_PATTERN = Pattern.compile("^\\s*\\$\\{(.*)}\\s*$");
private static final Pattern ESCAPED_EXPRESSION_PATTERN = Pattern.compile("^\\s*\\\\+\\$\\{(.*)}\\s*$");
private static final Memoized<ConfigExpressionScript<?, ?>, ?> staticEvaluationCache =
Memoized.memoize(ConfigExpressionScript::doStaticEvaluate);
private final Script script;
private final Class<O> returnType;
private final String scriptText;
@ -107,9 +107,6 @@ public class ConfigExpressionScript<I extends ScriptContext, O> implements Confi
if (!description.declarations().isEmpty()) {
scriptBuilder.withDeclarations(description.declarations());
}
if (!description.types().isEmpty()) {
scriptBuilder.withTypes(description.types());
}
var script = scriptBuilder.build();
return new ConfigExpressionScript<>(string, script, description, expected);
@ -132,17 +129,16 @@ public class ConfigExpressionScript<I extends ScriptContext, O> implements Confi
// ignore the parsed script object
return this == o || (o instanceof ConfigExpressionScript<?, ?> config &&
returnType.equals(config.returnType) &&
scriptText.equals(config.scriptText));
scriptText.equals(config.scriptText) &&
descriptor.equals(config.descriptor));
}
@Override
public int hashCode() {
// ignore the parsed script object
return Objects.hash(returnType, scriptText);
return Objects.hash(returnType, scriptText, descriptor);
}
private static final Map<ConfigExpressionScript<?, ?>, Boolean> staticEvaluationCache = new ConcurrentHashMap<>();
/**
* Attempts to parse and evaluate this script in an environment with no variables.
* <p>
@ -152,19 +148,12 @@ public class ConfigExpressionScript<I extends ScriptContext, O> implements Confi
public Try<O> tryStaticEvaluate() {
// type checking can be expensive when run hundreds of times simplifying expressions iteratively and it never
// changes for a given script and input environment, so cache results between calls.
boolean canStaticEvaluate =
staticEvaluationCache.computeIfAbsent(this, config -> config.doTryStaticEvaluate().isSuccess());
if (canStaticEvaluate) {
return doTryStaticEvaluate();
} else {
return Try.failure(new IllegalStateException());
}
return staticEvaluationCache.tryApply(this, returnType);
}
private Try<O> doTryStaticEvaluate() {
return Try
.apply(
() -> ConfigExpressionScript.parse(scriptText, Contexts.Root.DESCRIPTION, returnType).apply(Contexts.root()));
private O doStaticEvaluate() {
return ConfigExpressionScript.parse(scriptText, descriptor.root().description(), returnType)
.apply(descriptor.root());
}
@Override

4
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java
Wyświetl plik

@ -22,4 +22,8 @@ public interface ScriptContext extends Function<String, Object>, WithTags {
default TagValueProducer tagValueProducer() {
return TagValueProducer.EMPTY;
}
default Object argument(String key) {
return null;
}
}

23
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
Wyświetl plik

@ -1,35 +1,44 @@
package com.onthegomap.planetiler.custommap.expression;
import com.google.api.expr.v1alpha1.Decl;
import com.onthegomap.planetiler.custommap.Contexts;
import java.util.List;
import java.util.stream.Stream;
/**
* Type definitions for the environment that a script expression runs in.
*
* @param types Additional types available.
* @param declarations Global variable types
* @param clazz Class of the input context type
* @param <T> The runtime expression context type
*/
public record ScriptEnvironment<T extends ScriptContext> (List<Object> types, List<Decl> declarations, Class<T> clazz) {
public record ScriptEnvironment<T extends ScriptContext> (List<Decl> declarations, Class<T> clazz, Contexts.Root root) {
private static <T> List<T> concat(List<T> a, List<T> b) {
return Stream.concat(a.stream(), b.stream()).toList();
}
private static <T> List<T> concat(List<T> a, T[] b) {
return Stream.concat(a.stream(), Stream.of(b)).toList();
}
/** Returns a copy of this environment with a new input type {@code U}. */
public <U extends ScriptContext> ScriptEnvironment<U> forInput(Class<U> newClazz) {
return new ScriptEnvironment<>(types, declarations, newClazz);
return new ScriptEnvironment<>(declarations, newClazz, root);
}
/** Returns a copy of this environment with a list of variable declarations appended to the global environment. */
public ScriptEnvironment<T> withDeclarations(Decl... others) {
return new ScriptEnvironment<>(types, concat(declarations, others), clazz);
return new ScriptEnvironment<>(concat(declarations, others), clazz, root);
}
/** Returns an empty environment with no variables defined. */
public static ScriptEnvironment<ScriptContext> root() {
return new ScriptEnvironment<>(List.of(), List.of(), ScriptContext.class);
/** Returns a copy of this environment with a list of variable declarations appended to the global environment. */
public ScriptEnvironment<T> withDeclarations(List<Decl> others) {
return new ScriptEnvironment<>(concat(declarations, others), clazz, root);
}
/** Returns an empty environment with only static global variables (like command-line args) defined. */
public static ScriptEnvironment<ScriptContext> root(Contexts.Root root) {
return new ScriptEnvironment<>(List.of(), ScriptContext.class, root);
}
/** Returns true if this contains a variable declaration for {@code variable}. */

28
planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
Wyświetl plik

@ -6,6 +6,7 @@ import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.custommap.ConfiguredProfile;
import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.YAML;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.geo.GeometryType;
@ -55,13 +56,13 @@ public class SchemaValidator {
PrintStream output = System.out;
output.println("OK");
var paths = validateFromCli(schema, arguments, output);
var paths = validateFromCli(schema, output);
if (watch) {
output.println();
output.println("Watching filesystem for changes...");
var watcher = FileWatcher.newWatcher(paths.toArray(Path[]::new));
watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, arguments, output));
watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, output));
}
}
@ -69,32 +70,32 @@ public class SchemaValidator {
return t != null && (cause.isInstance(t) || hasCause(t.getCause(), cause));
}
static Set<Path> validateFromCli(Path schema, Arguments args, PrintStream output) {
static Set<Path> validateFromCli(Path schemaPath, PrintStream output) {
Set<Path> pathsToWatch = new HashSet<>();
pathsToWatch.add(schema);
pathsToWatch.add(schemaPath);
output.println();
output.println("Validating...");
output.println();
SchemaValidator.Result result;
try {
var parsedSchema = SchemaConfig.load(schema);
var examples = parsedSchema.examples();
var schema = SchemaConfig.load(schemaPath);
var examples = schema.examples();
// examples can either be embedded in the yaml file, or referenced
SchemaSpecification spec;
if (examples instanceof String s) {
var path = Path.of(s);
if (!path.isAbsolute()) {
path = schema.resolveSibling(path);
path = schemaPath.resolveSibling(path);
}
// if referenced, make sure we watch that file for changes
pathsToWatch.add(path);
spec = SchemaSpecification.load(path);
} else if (examples != null) {
spec = YAML.convertValue(parsedSchema, SchemaSpecification.class);
spec = YAML.convertValue(schema, SchemaSpecification.class);
} else {
spec = new SchemaSpecification(List.of());
}
result = validate(parsedSchema, spec, args);
result = validate(schema, spec);
} catch (Exception exception) {
Throwable rootCause = ExceptionUtils.getRootCause(exception);
if (hasCause(exception, com.onthegomap.planetiler.custommap.expression.ParseException.class)) {
@ -175,13 +176,14 @@ public class SchemaValidator {
* Returns the result of validating the profile defined by {@code schema} against the examples in
* {@code specification}.
*/
public static Result validate(SchemaConfig schema, SchemaSpecification specification, Arguments args) {
return validate(new ConfiguredProfile(schema), specification, args);
public static Result validate(SchemaConfig schema, SchemaSpecification specification) {
var context = Contexts.buildRootContext(Arguments.of().silence(), schema.args());
return validate(new ConfiguredProfile(schema, context), specification, context.config());
}
/** Returns the result of validating {@code profile} against the examples in {@code specification}. */
public static Result validate(Profile profile, SchemaSpecification specification, Arguments args) {
var featureCollectorFactory = new FeatureCollector.Factory(PlanetilerConfig.from(args.silence()), Stats.inMemory());
public static Result validate(Profile profile, SchemaSpecification specification, PlanetilerConfig config) {
var featureCollectorFactory = new FeatureCollector.Factory(config, Stats.inMemory());
return new Result(specification.examples().stream().map(example -> new ExampleResult(example, Try.apply(() -> {
List<String> issues = new ArrayList<>();
var input = example.input();

9
planetiler-custommap/src/main/resources/samples/shortbread.yml
Wyświetl plik

@ -2,6 +2,13 @@ schema_name: Shortbread
schema_description: A basic, lean, general-purpose vector tile schema for OpenStreetMap data. See https://shortbread.geofabrik.de/
attribution: <a href="https://www.openstreetmap.org/copyright" target="_blank">&copy; OpenStreetMap contributors</a>
examples: shortbread.spec.yml
args:
area:
description: Geofabrik area to download
default: massachusetts
osm_url:
description: OSM URL to download
default: '${ "geofabrik:" + args.area }'
sources:
ocean:
type: shapefile
@ -11,7 +18,7 @@ sources:
url: https://shortbread.geofabrik.de/shapefiles/admin-points-4326.zip
osm:
type: osm
url: geofabrik:massachusetts
url: '${ args.osm_url }'
definitions:
# TODO let attribute definitions set multiple keys so you can just use `- *names`
attributes:

3
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java
Wyświetl plik

@ -3,7 +3,6 @@ package com.onthegomap.planetiler.custommap;
import static com.onthegomap.planetiler.expression.Expression.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
import com.onthegomap.planetiler.expression.Expression;
import java.util.Map;
import org.junit.jupiter.api.Test;
@ -13,7 +12,7 @@ class BooleanExpressionParserTest {
private static void assertParse(String yaml, Expression parsed) {
Object expression = YAML.load(yaml, Object.class);
var actual = BooleanExpressionParser.parse(expression, TVP, ScriptEnvironment.root());
var actual = BooleanExpressionParser.parse(expression, TVP, TestContexts.ROOT_CONTEXT);
assertEquals(
parsed.simplify().generateJavaCode(),
actual.simplify().generateJavaCode()

3
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java
Wyświetl plik

@ -1,5 +1,6 @@
package com.onthegomap.planetiler.custommap;
import static com.onthegomap.planetiler.custommap.TestContexts.PROCESS_FEATURE;
import static com.onthegomap.planetiler.custommap.expression.ConfigExpression.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -16,7 +17,7 @@ import org.junit.jupiter.params.provider.ValueSource;
class ConfigExpressionParserTest {
private static final TagValueProducer TVP = new TagValueProducer(Map.of());
private static final ConfigExpression.Signature<Contexts.ProcessFeature, Object> FEATURE_SIGNATURE =
signature(Contexts.ProcessFeature.DESCRIPTION, Object.class);
signature(PROCESS_FEATURE, Object.class);
private static <O> void assertParse(String yaml, ConfigExpression<?, ?> parsed, Class<O> clazz) {
Object expression = YAML.load(yaml, Object.class);

266
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
Wyświetl plik

@ -9,13 +9,16 @@ import static org.junit.jupiter.api.Assertions.*;
import com.onthegomap.planetiler.FeatureCollector;
import com.onthegomap.planetiler.FeatureCollector.Feature;
import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.custommap.configschema.DataSourceType;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.custommap.util.TestConfigurableUtils;
import com.onthegomap.planetiler.reader.SimpleFeature;
import com.onthegomap.planetiler.reader.SourceFeature;
import com.onthegomap.planetiler.stats.Stats;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
@ -26,6 +29,7 @@ import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
class ConfiguredFeatureTest {
private PlanetilerConfig planetilerConfig = PlanetilerConfig.defaults();
private static final Function<String, Path> TEST_RESOURCE = TestConfigurableUtils::pathToTestResource;
private static final Function<String, Path> SAMPLE_RESOURCE = TestConfigurableUtils::pathToSample;
@ -74,32 +78,35 @@ class ConfiguredFeatureTest {
"bridge", "yes"
);
private static Profile loadConfig(Function<String, Path> pathFunction, String filename) {
private Profile loadConfig(Function<String, Path> pathFunction, String filename) {
var staticAttributeConfig = pathFunction.apply(filename);
var schema = SchemaConfig.load(staticAttributeConfig);
return new ConfiguredProfile(schema);
var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args());
planetilerConfig = root.config();
return new ConfiguredProfile(schema, root);
}
private static Profile loadConfig(String config) {
private ConfiguredProfile loadConfig(String config) {
var schema = SchemaConfig.load(config);
return new ConfiguredProfile(schema);
var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args());
planetilerConfig = root.config();
return new ConfiguredProfile(schema, root);
}
private static void testFeature(Function<String, Path> pathFunction, String schemaFilename, SourceFeature sf,
private void testFeature(Function<String, Path> pathFunction, String schemaFilename, SourceFeature sf,
Consumer<Feature> test, int expectedMatchCount) {
var profile = loadConfig(pathFunction, schemaFilename);
testFeature(sf, test, expectedMatchCount, profile);
}
private static void testFeature(String config, SourceFeature sf, Consumer<Feature> test, int expectedMatchCount) {
private void testFeature(String config, SourceFeature sf, Consumer<Feature> test, int expectedMatchCount) {
var profile = loadConfig(config);
testFeature(sf, test, expectedMatchCount, profile);
}
private static void testFeature(SourceFeature sf, Consumer<Feature> test, int expectedMatchCount, Profile profile) {
var config = PlanetilerConfig.defaults();
var factory = new FeatureCollector.Factory(config, Stats.inMemory());
private void testFeature(SourceFeature sf, Consumer<Feature> test, int expectedMatchCount, Profile profile) {
var factory = new FeatureCollector.Factory(planetilerConfig, Stats.inMemory());
var fc = factory.get(sf);
profile.processFeature(sf, fc);
@ -114,14 +121,14 @@ class ConfiguredFeatureTest {
assertEquals(expectedMatchCount, length.get(), "Wrong number of features generated");
}
private static void testPolygon(String config, Map<String, Object> tags,
private void testPolygon(String config, Map<String, Object> tags,
Consumer<Feature> test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList());
testFeature(config, sf, test, expectedMatchCount);
}
private static void testPoint(String config, Map<String, Object> tags,
private void testPoint(String config, Map<String, Object> tags,
Consumer<Feature> test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newPoint(0, 0), tags, "osm", null, 1, emptyList());
@ -129,21 +136,21 @@ class ConfiguredFeatureTest {
}
private static void testLinestring(String config,
private void testLinestring(String config,
Map<String, Object> tags, Consumer<Feature> test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList());
testFeature(config, sf, test, expectedMatchCount);
}
private static void testPolygon(Function<String, Path> pathFunction, String schemaFilename, Map<String, Object> tags,
private void testPolygon(Function<String, Path> pathFunction, String schemaFilename, Map<String, Object> tags,
Consumer<Feature> test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList());
testFeature(pathFunction, schemaFilename, sf, test, expectedMatchCount);
}
private static void testLinestring(Function<String, Path> pathFunction, String schemaFilename,
private void testLinestring(Function<String, Path> pathFunction, String schemaFilename,
Map<String, Object> tags, Consumer<Feature> test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList());
@ -784,4 +791,235 @@ class ConfiguredFeatureTest {
private void testInvalidSchema(String filename, String message) {
assertThrows(RuntimeException.class, () -> loadConfig(TEST_INVALID_RESOURCE, filename), message);
}
@ParameterizedTest
@ValueSource(strings = {
"arg_value: argument",
"value: '${ args.argument }'",
"value: '${ args[\"argument\"] }'",
})
void testUseArgumentNotDefined(String string) {
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
"argument", "value"
)));
var config = """
sources:
osm:
type: osm
url: geofabrik:rhode-island
local_path: data/rhode-island.osm.pbf
layers:
- id: testLayer
features:
- source: osm
geometry: point
include_when:
natural: water
attributes:
- key: key
%s
""".formatted(string);
testPoint(config, Map.of(
"natural", "water"
), feature -> {
assertEquals("value", feature.getAttrsAtZoom(14).get("key"));
}, 1);
}
@ParameterizedTest
@ValueSource(strings = {
"arg_value: threads",
"value: '${ args.threads }'",
"value: '${ args[\"threads\"] }'",
})
void testOverrideArgument(String string) {
var config = """
sources:
osm:
type: osm
url: geofabrik:rhode-island
local_path: data/rhode-island.osm.pbf
args:
threads: 2
layers:
- id: testLayer
features:
- source: osm
geometry: point
include_when:
natural: water
attributes:
- key: key
type: integer
%s
""".formatted(string);
testPoint(config, Map.of(
"natural", "water"
), feature -> {
assertEquals(2, feature.getAttrsAtZoom(14).get("key"));
}, 1);
}
@Test
void testDefineArgument() {
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
"custom_overridden_arg", "test2",
"custom_simple_overridden_int_arg", "3"
)));
var config = """
sources:
osm:
type: osm
url: geofabrik:rhode-island
local_path: data/rhode-island.osm.pbf
args:
custom_int_arg:
type: integer
description: test arg out
default: 12
custom_boolean_arg:
type: boolean
description: test boolean arg out
default: true
custom_overridden_arg:
default: test
custom_simple_string_arg: value
custom_simple_int_arg: 1
custom_simple_double_arg: 1.5
custom_simple_bool_arg: true
custom_simple_overridden_int_arg: 2
layers:
- id: testLayer
features:
- source: osm
geometry: point
include_when:
natural: water
attributes:
- key: int
value: '${ args.custom_int_arg }'
- key: bool
value: '${ args["custom_boolean_arg"] }'
- key: overridden
arg_value: custom_overridden_arg
- key: custom_simple_string_arg
arg_value: custom_simple_string_arg
- key: custom_simple_int_arg
arg_value: custom_simple_int_arg
- key: custom_simple_bool_arg
arg_value: custom_simple_bool_arg
- key: custom_simple_overridden_int_arg
arg_value: custom_simple_overridden_int_arg
- key: custom_simple_double_arg
arg_value: custom_simple_double_arg
- key: custom_simple_int_arg_as_string
arg_value: custom_simple_int_arg
type: string
""";
testPoint(config, Map.of(
"natural", "water"
), feature -> {
assertEquals(12L, feature.getAttrsAtZoom(14).get("int"));
assertEquals(true, feature.getAttrsAtZoom(14).get("bool"));
assertEquals("test2", feature.getAttrsAtZoom(14).get("overridden"));
assertEquals("value", feature.getAttrsAtZoom(14).get("custom_simple_string_arg"));
assertEquals(1, feature.getAttrsAtZoom(14).get("custom_simple_int_arg"));
assertEquals("1", feature.getAttrsAtZoom(14).get("custom_simple_int_arg_as_string"));
assertEquals(1.5, feature.getAttrsAtZoom(14).get("custom_simple_double_arg"));
assertEquals(true, feature.getAttrsAtZoom(14).get("custom_simple_bool_arg"));
assertEquals(3, feature.getAttrsAtZoom(14).get("custom_simple_overridden_int_arg"));
}, 1);
}
@Test
void testDefineArgumentsUsingExpressions() {
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
"custom_overridden_arg", "test2",
"custom_simple_overridden_int_arg", "3"
)));
var config = """
sources:
osm:
type: osm
url: geofabrik:rhode-island
local_path: data/rhode-island.osm.pbf
args:
arg1:
type: long
description: test arg out
default: '${ 2 - 1 }'
arg2: '${ 2 - 1 }'
arg3:
default: '${ 2 - 1 }'
arg4: ${ args.arg3 + 1 }
layers:
- id: testLayer
features:
- source: osm
geometry: point
attributes:
- key: arg1
arg_value: arg1
- key: arg2
arg_value: arg2
- key: arg3
arg_value: arg3
- key: arg4
arg_value: arg4
""";
testPoint(config, Map.of(), feature -> {
assertEquals(1L, feature.getAttrsAtZoom(14).get("arg1"));
assertEquals(1L, feature.getAttrsAtZoom(14).get("arg2"));
assertEquals(1L, feature.getAttrsAtZoom(14).get("arg3"));
assertEquals(2L, feature.getAttrsAtZoom(14).get("arg4"));
}, 1);
}
@Test
void testUseArgumentInSourceUrlPath() {
var config = """
args:
area: rhode-island
url: '${ "geofabrik:" + args.area }'
sources:
osm:
type: osm
url: '${ args.url }'
layers:
- id: testLayer
features:
- source: osm
geometry: point
""";
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
"area", "boston"
)));
assertEquals(List.of(new Source(
"osm",
DataSourceType.OSM,
"geofabrik:boston",
null
)), loadConfig(config).sources());
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
assertEquals(List.of(new Source(
"osm",
DataSourceType.OSM,
"geofabrik:rhode-island",
null
)), loadConfig(config).sources());
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
assertEquals("geofabrik_rhode_island.osm.pbf", loadConfig(config).sources().get(0).defaultFileUrl());
this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
"url", "https://example.com/file.osm.pbf"
)));
assertEquals("example.com_file.osm.pbf", loadConfig(config).sources().get(0).defaultFileUrl());
}
}

228
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java 100644
Wyświetl plik

@ -0,0 +1,228 @@
package com.onthegomap.planetiler.custommap;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.expression.ParseException;
import java.util.Map;
import org.junit.jupiter.api.Test;
class ContextsTest {
@Test
void testParseEmptyArgs() {
Map<String, String> cliArgs = Map.of();
Map<String, Object> schemaArgs = Map.of();
Contexts.buildRootContext(Arguments.of(cliArgs), schemaArgs);
}
@Test
void setPlanetilerConfigThroughCli() {
Map<String, String> cliArgs = Map.of("threads", "9999");
Map<String, Object> schemaArgs = Map.of("threads", "8888");
var result = Contexts.buildRootContext(Arguments.of(cliArgs), schemaArgs);
assertEquals(9999, result.config().threads());
}
@Test
void setPlanetilerConfigThroughSchemaArgs() {
assertEquals(8888, Contexts.buildRootContext(
Arguments.of(Map.of()),
Map.of("threads", "8888")
).config().threads());
assertEquals(8888, Contexts.buildRootContext(
Arguments.of(Map.of()),
Map.of("threads", 8888)
).config().threads());
}
@Test
void configDefinesDefaultArgumentValue() {
assertEquals(8888, Contexts.buildRootContext(
Arguments.of(Map.of()),
Map.of(
"arg", 8888
)
).argument("arg"));
}
@Test
void overrideDefaultArgFromConfig() {
assertEquals(9999, Contexts.buildRootContext(
Arguments.of(Map.of("arg", "9999")),
Map.of(
"arg", 8888
)
).argument("arg"));
}
@Test
void defineArgTypeInConfig() {
assertEquals(8888, Contexts.buildRootContext(
Arguments.of(),
Map.of(
"arg", Map.of(
"default", "8888",
"type", "integer"
)
)
).argument("arg"));
}
@Test
void implyTypeFromDefaultValue() {
assertEquals("8888", Contexts.buildRootContext(
Arguments.of(),
Map.of(
"arg", "8888"
)
).argument("arg"));
assertEquals("9999", Contexts.buildRootContext(
Arguments.of("arg", "9999"),
Map.of(
"arg", "8888"
)
).argument("arg"));
assertEquals(8888, Contexts.buildRootContext(
Arguments.of(),
Map.of(
"arg", 8888
)
).argument("arg"));
assertEquals(9999, Contexts.buildRootContext(
Arguments.of("arg", "9999"),
Map.of(
"arg", 8888
)
).argument("arg"));
}
@Test
void computeDefaultValueUsingExpression() {
assertEquals(2, Contexts.buildRootContext(
Arguments.of(),
Map.of(
"arg", Map.of(
"default", "${ 1+1 }",
"type", "integer"
)
)
).argument("arg"));
}
@Test
void implyTypeFromExpressionValue() {
assertEquals(2L, Contexts.buildRootContext(
Arguments.of(),
Map.of(
"arg", "${ 1+1 }"
)
).argument("arg"));
}
@Test
void referenceOtherArgInExpression() {
Map<String, Object> configArgs = Map.of(
"arg1", 1,
"arg2", "${ args.arg1 + 1}"
);
assertEquals(2L, Contexts.buildRootContext(
Arguments.of(),
configArgs
).argument("arg2"));
assertEquals(3L, Contexts.buildRootContext(
Arguments.of(Map.of("arg1", "2")),
configArgs
).argument("arg2"));
assertEquals(10L, Contexts.buildRootContext(
Arguments.of(Map.of("arg2", "10")),
configArgs
).argument("arg2"));
}
@Test
void referenceOtherArgInExpressionTwice() {
Map<String, Object> configArgs = Map.of(
"arg1", 1,
"arg2", "${ args.arg1 + 1}",
"arg3", "${ args.arg2 + 1}"
);
assertEquals(3L, Contexts.buildRootContext(
Arguments.of(),
configArgs
).argument("arg3"));
}
@Test
void failOnInfiniteLoop() {
Map<String, Object> configArgs = Map.of(
"arg1", Map.of(
"default", "${ args.arg3 + 1 }",
"type", "long"
),
"arg2", "${ args.arg1 + 1}",
"arg3", "${ args.arg2 + 1}"
);
var empty = Arguments.of();
assertThrows(ParseException.class, () -> Contexts.buildRootContext(
empty,
configArgs
));
// but if you break the chain it's OK?
assertEquals(3L, Contexts.buildRootContext(
Arguments.of(Map.of("arg1", "1")),
configArgs
).argument("arg3"));
}
@Test
void setPlanetilerConfigFromOtherArg() {
assertEquals(8888, Contexts.buildRootContext(
Arguments.of(Map.of()),
Map.of(
"other", "8888",
"threads", "${ args.other }"
)
).config().threads());
}
@Test
void testCantRedefineBuiltin() {
var fromCli = Arguments.of(Map.of());
Map<String, Object> fromConfig = Map.of(
"threads", Map.of(
"default", 4,
"type", "string"
)
);
assertThrows(ParseException.class, () -> Contexts.buildRootContext(fromCli, fromConfig));
}
@Test
void testDefineRequiredArg() {
var argsWithoutValue = Arguments.of(Map.of());
var argsWithValue = Arguments.of(Map.of("arg", "3"));
Map<String, Object> fromConfig = Map.of(
"arg", Map.of(
"type", "integer",
"description", "desc"
)
);
assertThrows(ParseException.class, () -> Contexts.buildRootContext(argsWithoutValue, fromConfig));
assertEquals(3, Contexts.buildRootContext(argsWithValue, fromConfig).argument("arg"));
}
@Test
void setPlanetilerConfigFromOtherPlanetilerConfig() {
var root = Contexts.buildRootContext(
Arguments.of(Map.of()),
Map.of(
"mmap", "${ args.threads < 1 }"
)
);
assertTrue(root.config().mmapTempStorage());
}
}

4
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java
Wyświetl plik

@ -2,7 +2,6 @@ package com.onthegomap.planetiler.custommap;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.custommap.validator.SchemaSpecification;
import com.onthegomap.planetiler.custommap.validator.SchemaValidator;
@ -21,8 +20,7 @@ class SchemaTests {
var base = Path.of("src", "main", "resources", "samples");
var result = SchemaValidator.validate(
SchemaConfig.load(base.resolve(schema)),
SchemaSpecification.load(base.resolve(spec)),
Arguments.of()
SchemaSpecification.load(base.resolve(spec))
);
return result.results().stream()
.map(test -> dynamicTest(test.example().name(), () -> {

4
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java
Wyświetl plik

@ -3,6 +3,7 @@ package com.onthegomap.planetiler.custommap;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import java.io.IOException;
import java.nio.file.Files;
@ -33,8 +34,9 @@ class SchemaYAMLLoadTest {
for (Path schemaFile : schemaFiles) {
var schemaConfig = SchemaConfig.load(schemaFile);
var root = Contexts.buildRootContext(Arguments.of(), schemaConfig.args());
assertNotNull(schemaConfig, () -> "Failed to unmarshall " + schemaFile.toString());
assertNotNull(new ConfiguredProfile(schemaConfig), () -> "Failed to load profile from " + schemaFile.toString());
new ConfiguredProfile(schemaConfig, root);
}
}
}

3
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java
Wyświetl plik

@ -1,5 +1,6 @@
package com.onthegomap.planetiler.custommap;
import static com.onthegomap.planetiler.custommap.TestContexts.ROOT;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.geo.GeoUtils;
@ -16,7 +17,7 @@ class TagValueProducerTest {
assertEquals(expected, tvp.valueForKey(wrapped, key));
assertEquals(expected, tvp.valueGetterForKey(key).apply(wrapped, key));
assertEquals(expected, tvp.valueProducerForKey(key)
.apply(new Contexts.ProcessFeature(SimpleFeature.create(GeoUtils.EMPTY_GEOMETRY, tags), tvp)
.apply(ROOT.createProcessFeatureContext(SimpleFeature.create(GeoUtils.EMPTY_GEOMETRY, tags), tvp)
.createPostMatchContext(List.of())));
}

14
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java 100644
Wyświetl plik

@ -0,0 +1,14 @@
package com.onthegomap.planetiler.custommap;
import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
public class TestContexts {
public static final Contexts.Root ROOT = Contexts.emptyRoot();
public static final ScriptEnvironment<Contexts.Root> ROOT_CONTEXT = ROOT.description();
public static final ScriptEnvironment<Contexts.ProcessFeature> PROCESS_FEATURE =
Contexts.ProcessFeature.description(ROOT);
public static final ScriptEnvironment<Contexts.FeaturePostMatch> FEATURE_POST_MATCH =
Contexts.FeaturePostMatch.description(ROOT);
public static final ScriptEnvironment<Contexts.FeatureAttribute> FEATURE_ATTRIBUTE =
Contexts.FeatureAttribute.description(ROOT);
}

17
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java
Wyświetl plik

@ -4,7 +4,7 @@ import static com.onthegomap.planetiler.expression.Expression.and;
import static com.onthegomap.planetiler.expression.Expression.or;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.TestContexts;
import com.onthegomap.planetiler.expression.Expression;
import org.junit.jupiter.api.Test;
@ -12,11 +12,20 @@ class BooleanExpressionScriptTest {
@Test
void testSimplify() {
assertEquals(Expression.TRUE,
and(or(BooleanExpressionScript.script("1+1<3", Contexts.Root.DESCRIPTION))).simplify());
and(or(BooleanExpressionScript.script("1+1<3", TestContexts.ROOT_CONTEXT))).simplify());
assertEquals(Expression.FALSE,
and(or(BooleanExpressionScript.script("1+1>3", Contexts.Root.DESCRIPTION))).simplify());
and(or(BooleanExpressionScript.script("1+1>3", TestContexts.ROOT_CONTEXT))).simplify());
var other = BooleanExpressionScript.script("feature.tags.natural", Contexts.ProcessFeature.DESCRIPTION);
var other =
BooleanExpressionScript.script("feature.tags.natural", TestContexts.PROCESS_FEATURE);
assertEquals(other, and(or(other)).simplify());
}
@Test
void testSimplifyInlinesArguments() {
assertEquals(Expression.TRUE,
and(or(BooleanExpressionScript.script("args.threads > 0", TestContexts.ROOT_CONTEXT))).simplify());
assertEquals(Expression.FALSE,
and(or(BooleanExpressionScript.script("args.threads < 0", TestContexts.ROOT_CONTEXT))).simplify());
}
}

70
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java
Wyświetl plik

@ -1,6 +1,9 @@
package com.onthegomap.planetiler.custommap.expression;
import static com.onthegomap.planetiler.TestUtils.newPoint;
import static com.onthegomap.planetiler.custommap.TestContexts.FEATURE_POST_MATCH;
import static com.onthegomap.planetiler.custommap.TestContexts.PROCESS_FEATURE;
import static com.onthegomap.planetiler.custommap.TestContexts.ROOT_CONTEXT;
import static com.onthegomap.planetiler.custommap.expression.ConfigExpression.*;
import static com.onthegomap.planetiler.expression.Expression.matchAny;
import static com.onthegomap.planetiler.expression.Expression.or;
@ -8,8 +11,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.TagValueProducer;
import com.onthegomap.planetiler.custommap.TestContexts;
import com.onthegomap.planetiler.expression.DataType;
import com.onthegomap.planetiler.expression.Expression;
import com.onthegomap.planetiler.expression.MultiExpression;
@ -19,10 +24,9 @@ import java.util.Map;
import org.junit.jupiter.api.Test;
class ConfigExpressionTest {
private static final ConfigExpression.Signature<Contexts.Root, Integer> ROOT =
signature(Contexts.Root.DESCRIPTION, Integer.class);
private static final ConfigExpression.Signature<Contexts.Root, Integer> ROOT = signature(ROOT_CONTEXT, Integer.class);
private static final ConfigExpression.Signature<Contexts.ProcessFeature, Integer> FEATURE_SIGNATURE =
signature(Contexts.ProcessFeature.DESCRIPTION, Integer.class);
signature(PROCESS_FEATURE, Integer.class);
@Test
void testConst() {
@ -32,7 +36,7 @@ class ConfigExpressionTest {
@Test
void testVariable() {
var feature = SimpleFeature.create(newPoint(0, 0), Map.of("a", "b", "c", 1), "source", "source_layer", 99);
var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()));
var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()));
// simple match
assertEquals("source", variable(FEATURE_SIGNATURE.withOutput(String.class), "feature.source").apply(context));
assertEquals("source_layer",
@ -45,32 +49,32 @@ class ConfigExpressionTest {
@Test
void testCoalesce() {
assertNull(coalesce(List.of()).apply(Contexts.root()));
assertNull(coalesce(List.of()).apply(TestContexts.ROOT));
assertNull(coalesce(
List.of(
constOf(null)
)).apply(Contexts.root()));
)).apply(TestContexts.ROOT));
assertEquals(2, coalesce(
List.of(
constOf(null),
constOf(2)
)).apply(Contexts.root()));
)).apply(TestContexts.ROOT));
assertEquals(1, coalesce(
List.of(
constOf(1),
constOf(2)
)).apply(Contexts.root()));
)).apply(TestContexts.ROOT));
}
@Test
void testDynamic() {
assertEquals(1, script(ROOT, "5 - 4").apply(Contexts.root()));
assertEquals(1, script(ROOT, "5 - 4").apply(TestContexts.ROOT));
}
@Test
void testMatch() {
var feature = SimpleFeature.create(newPoint(0, 0), Map.of("a", "b", "c", 1));
var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()));
var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()));
// simple match
assertEquals(2, match(FEATURE_SIGNATURE, MultiExpression.of(List.of(
MultiExpression.entry(constOf(1),
@ -280,33 +284,30 @@ class ConfigExpressionTest {
assertEquals(
"123",
getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")).apply(
new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of())))
TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of())))
);
assertEquals(
123,
getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc"))
.apply(new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of("abc", "integer"))))
.apply(TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of("abc", "integer"))))
);
assertEquals(
123,
getTag(signature(Contexts.FeaturePostMatch.DESCRIPTION, Object.class), constOf("abc"))
.apply(new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of("abc", "integer")))
getTag(signature(FEATURE_POST_MATCH, Object.class), constOf("abc"))
.apply(TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of("abc", "integer")))
.createPostMatchContext(List.of()))
);
assertEquals(
null,
getTag(signature(Contexts.Root.DESCRIPTION, Object.class), constOf("abc"))
.apply(Contexts.root())
);
assertNull(getTag(signature(ROOT_CONTEXT, Object.class), constOf("abc"))
.apply(TestContexts.ROOT));
}
@Test
void testCastGetTag() {
var feature = SimpleFeature.create(newPoint(0, 0), Map.of("abc", "123"), "source", "source_layer", 99);
var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()));
var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()));
var expression = cast(
FEATURE_SIGNATURE.withOutput(Integer.class),
getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")),
@ -328,7 +329,7 @@ class ConfigExpressionTest {
constOf("123"),
DataType.GET_INT
);
assertEquals(123, expression.apply(Contexts.root()));
assertEquals(123, expression.apply(TestContexts.ROOT));
}
@Test
@ -341,4 +342,31 @@ class ConfigExpressionTest {
).simplify()
);
}
@Test
void testSimplifyGetArg() {
var args = Arguments.of(Map.of("arg", "12"));
var root = Contexts.buildRootContext(args, Map.of());
var context = signature(root.description(), Integer.class);
assertEquals(constOf(12),
getArg(
context,
constOf("arg")
).simplify()
);
assertEquals(constOf(12),
script(
context,
"args.arg"
).simplify()
);
assertEquals(constOf(12),
script(
context,
"args['arg']"
).simplify()
);
}
}

9
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java
Wyświetl plik

@ -63,4 +63,13 @@ class DataTypeTest {
assertEquals(0, DataType.from("direction").convertFrom("no"));
assertEquals(0, DataType.from("direction").convertFrom("false"));
}
@Test
void testTypeOf() {
assertEquals(DataType.GET_DOUBLE, DataType.typeOf(1.5));
assertEquals(DataType.GET_LONG, DataType.typeOf(1L));
assertEquals(DataType.GET_INT, DataType.typeOf(1));
assertEquals(DataType.GET_BOOLEAN, DataType.typeOf(true));
assertEquals(DataType.GET_STRING, DataType.typeOf("string"));
}
}

22
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java
Wyświetl plik

@ -3,6 +3,9 @@ package com.onthegomap.planetiler.custommap.expression;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.Contexts;
import java.util.Map;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
@ -52,10 +55,25 @@ class ExpressionTests {
"min([1.1, 2.2, 3.3])|1.1|double",
"max([1])|1|long",
"min([1])|1|long",
"args.arg_from_config|1|long",
"args.arg_from_config + 1|2|long",
"args[\"arg_from_config\"]|1|long",
"args[\"arg_from_config\"] + 1|2|long",
"args.overridden_arg_from_config|4|long",
"args[\"overridden_arg_from_config\"] + 1|5|long",
"args.arg_from_cli|2|string",
"args[\"arg_from_cli\"]|2|string",
}, delimiter = '|')
void testExpression(String in, String expected, String type) {
var expression = ConfigExpressionScript.parse(in, ScriptEnvironment.root());
var result = expression.apply(ScriptContext.empty());
Map<String, Object> configFromSchema = Map.of("arg_from_config", 1, "overridden_arg_from_config", 3);
Map<String, String> configFromCli = Map.of("arg_from_cli", "2", "overridden_arg_from_config", "4");
var context = Contexts.buildRootContext(
Arguments.of(configFromCli),
configFromSchema
);
var expression = ConfigExpressionScript.parse(in, context.description());
var result = expression.apply(context);
switch (type) {
case "long" -> assertEquals(Long.valueOf(expected), result);
case "double" -> assertEquals(Double.valueOf(expected), result);

57
planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java
Wyświetl plik

@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -25,12 +24,10 @@ class SchemaValidatorTest {
record Result(SchemaValidator.Result output, String cliOutput) {}
Result validate(String schema, String spec) throws IOException {
var args = Arguments.of();
private Result validate(String schema, String spec) throws IOException {
var result = SchemaValidator.validate(
SchemaConfig.load(schema),
SchemaSpecification.load(spec),
args
SchemaSpecification.load(spec)
);
for (var example : result.results()) {
if (example.issues().isFailure()) {
@ -39,30 +36,29 @@ class SchemaValidatorTest {
}
// also exercise the cli writer and return what it would have printed to stdout
var cliOutput = validateCli(Files.writeString(tmpDir.resolve("schema"),
schema + "\nexamples: " + Files.writeString(tmpDir.resolve("spec.yml"), spec)), args);
schema + "\nexamples: " + Files.writeString(tmpDir.resolve("spec.yml"), spec)));
// also test the case where the examples are embedded in the schema itself
assertEquals(
cliOutput,
validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\n" + spec), args)
validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\n" + spec))
);
// also test where examples points to a relative path (written in previous step)
assertEquals(
cliOutput,
validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\nexamples: spec.yml"), args)
validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\nexamples: spec.yml"))
);
return new Result(result, cliOutput);
}
private String validateCli(Path path, Arguments args) {
private String validateCli(Path path) {
try (
var baos = new ByteArrayOutputStream();
var printStream = new PrintStream(baos, true, StandardCharsets.UTF_8)
) {
SchemaValidator.validateFromCli(
path,
args,
printStream
);
return baos.toString(StandardCharsets.UTF_8);
@ -180,4 +176,45 @@ class SchemaValidatorTest {
);
assertFalse(results.output.ok(), results.toString());
}
@Test
void testValidationWiresInArguments() throws IOException {
var results = validate(
"""
sources:
osm:
type: osm
url: geofabrik:rhode-island
args:
key: default_value
layers:
- id: water
features:
- source: osm
geometry: polygon
include_when:
natural: water
attributes:
- key: from_arg
arg_value: key
- key: threads
value: '${ args.threads + 1 }'
""",
"""
examples:
- name: test output
input:
source: osm
geometry: polygon
tags:
natural: water
output:
layer: water
tags:
from_arg: default_value
threads: %s
""".formatted(1 + Math.max(Runtime.getRuntime().availableProcessors(), 2))
);
assertTrue(results.output.ok(), results.toString());
}
}

8
quickstart.sh
Wyświetl plik

@ -16,6 +16,7 @@ DRY_RUN=""
VERSION="latest"
DOCKER_DIR="$(pwd)/data"
TASK="openmaptiles"
BUILD="true"
# Parse args into env vars
while [[ $# -gt 0 ]]; do
@ -25,6 +26,7 @@ while [[ $# -gt 0 ]]; do
--dockerdir) DOCKER_DIR="$2"; shift ;;
--jar) METHOD="jar" ;;
--build|--source) METHOD="build" ;;
--skipbuild) METHOD="build"; BUILD="false" ;;
--version=*) VERSION="${1#*=}" ;;
--version) VERSION="$2"; shift ;;
@ -134,8 +136,10 @@ case $METHOD in
run "$JAVA" "${JVM_ARGS}" -jar planetiler.jar "${PLANETILER_ARGS[@]}"
;;
build)
echo "Building planetiler..."
run ./mvnw -q -DskipTests --projects planetiler-dist -am clean package
if [ "$BUILD" == "true" ]; then
echo "Building planetiler..."
run ./mvnw -q -DskipTests --projects planetiler-dist -am clean package
fi
run "$JAVA" "${JVM_ARGS}" -jar planetiler-dist/target/*with-deps.jar "${PLANETILER_ARGS[@]}"
;;
esac