diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 1df70343..6b283e34 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -95,8 +95,7 @@ public class Planetiler { private Planetiler(Arguments arguments) { this.arguments = arguments; stats = arguments.getStats(); - overallTimer = stats.startStage("overall"); - LogUtil.clearStage(); + overallTimer = stats.startStageQuietly("overall"); config = PlanetilerConfig.from(arguments); tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp")); onlyDownloadSources = arguments.getBoolean("only_download", "download source data then exit", false); @@ -115,16 +114,6 @@ public class Planetiler { return new Planetiler(arguments); } - /** - * Returns a new empty runner that will get configuration from {@code arguments} to the main method, JVM properties, - * environmental variables, or a config file specified in {@code config} argument. - * - * @param arguments array of string arguments provided to {@code public static void main(String[] args)} entrypoint - */ - public static Planetiler create(String... arguments) { - return new Planetiler(Arguments.fromArgsOrConfigFile(arguments)); - } - /** * Adds a new {@code .osm.pbf} source that will be processed when {@link #run()} is called. *

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java index bb266cf3..015543bf 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java @@ -1,5 +1,7 @@ package com.onthegomap.planetiler.config; +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.stats.Stats; import java.io.IOException; @@ -8,13 +10,15 @@ import java.nio.file.Path; import java.time.Duration; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TreeMap; -import java.util.function.Function; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Stream; import org.locationtech.jts.geom.Envelope; @@ -29,11 +33,22 @@ public class Arguments { private static final Logger LOGGER = LoggerFactory.getLogger(Arguments.class); - private final Function provider; + private final UnaryOperator provider; + private final Supplier> keys; private boolean silent = false; - private Arguments(UnaryOperator provider) { + private Arguments(UnaryOperator provider, Supplier> keys) { this.provider = provider; + this.keys = keys; + } + + private static Arguments from(UnaryOperator provider, Supplier> rawKeys, + UnaryOperator forward, UnaryOperator reverse) { + Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> { + String reversed = reverse.apply(key); + return key.equalsIgnoreCase(reversed) ? Stream.empty() : Stream.of(reversed); + }).toList(); + return new Arguments(key -> provider.apply(forward.apply(key)), keys); } /** @@ -42,7 +57,17 @@ public class Arguments { * For example to set {@code key=value}: {@code java -Dplanetiler.key=value -jar ...} */ public static Arguments fromJvmProperties() { - return new Arguments(key -> System.getProperty("planetiler." + key)); + return fromJvmProperties( + System::getProperty, + () -> System.getProperties().stringPropertyNames() + ); + } + + static Arguments fromJvmProperties(UnaryOperator getter, Supplier> keys) { + return from(getter, keys, + key -> "planetiler." + key.toLowerCase(Locale.ROOT), + key -> key.replaceFirst("^planetiler\\.", "").toLowerCase(Locale.ROOT) + ); } /** @@ -51,7 +76,27 @@ public class Arguments { * For example to set {@code key=value}: {@code PLANETILER_KEY=value java -jar ...} */ public static Arguments fromEnvironment() { - return new Arguments(key -> System.getenv("PLANETILER_" + key.toUpperCase(Locale.ROOT))); + return fromEnvironment( + System::getenv, + () -> System.getenv().keySet() + ); + } + + static Arguments fromEnvironment(UnaryOperator getter, Supplier> keys) { + return from(getter, keys, + key -> "PLANETILER_" + key.toUpperCase(Locale.ROOT), + key -> key.replaceFirst("^PLANETILER_", "").toLowerCase(Locale.ROOT) + ); + } + + /** + * Returns arguments parsed from a {@link Properties} object. + */ + public static Arguments from(Properties properties) { + return new Arguments( + properties::getProperty, + properties::stringPropertyNames + ); } /** @@ -97,7 +142,7 @@ public class Arguments { Properties properties = new Properties(); try (var reader = Files.newBufferedReader(path)) { properties.load(reader); - return new Arguments(properties::getProperty); + return from(properties); } catch (IOException e) { throw new IllegalArgumentException("Unable to load config file: " + path, e); } @@ -147,7 +192,7 @@ public class Arguments { } public static Arguments of(Map map) { - return new Arguments(map::get); + return new Arguments(map::get, map::keySet); } /** Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs. */ @@ -177,10 +222,20 @@ public class Arguments { * @return arguments instance that checks {@code this} first and if a match is not found then {@code other} */ public Arguments orElse(Arguments other) { - return new Arguments(key -> { - String ourResult = get(key); - return ourResult != null ? ourResult : other.get(key); - }); + var result = new Arguments( + key -> { + String ourResult = get(key); + return ourResult != null ? ourResult : other.get(key); + }, + () -> Stream.concat( + other.keys.get().stream(), + keys.get().stream() + ).distinct().toList() + ); + if (silent) { + result.silence(); + } + return result; } String getArg(String key) { @@ -218,7 +273,7 @@ public class Arguments { return result; } - private void logArgValue(String key, String description, Object result) { + protected void logArgValue(String key, String description, Object result) { if (!silent) { LOGGER.debug("argument: {}={} ({})", key, result, description); } @@ -332,13 +387,13 @@ public class Arguments { public Stats getStats() { String prometheus = getArg("pushgateway"); if (prometheus != null && !prometheus.isBlank()) { - LOGGER.info("Using prometheus push gateway stats"); + LOGGER.info("argument: stats=use prometheus push gateway stats"); String job = getString("pushgateway.job", "prometheus pushgateway job ID", "planetiler"); Duration interval = getDuration("pushgateway.interval", "how often to send stats to prometheus push gateway", "15s"); return Stats.prometheusPushGateway(prometheus, job, interval); } else { - LOGGER.info("Using in-memory stats"); + LOGGER.info("argument: stats=use in-memory stats"); return Stats.inMemory(); } } @@ -390,4 +445,35 @@ public class Arguments { logArgValue(key, description, parsed); return parsed; } + + /** + * Returns a map from all the arguments provided to their values. + */ + public Map toMap() { + Map result = new HashMap<>(); + for (var key : keys.get()) { + result.put(key, get(key)); + } + return result; + } + + /** Returns a copy of this {@code Arguments} instance that logs each extracted argument value exactly once. */ + public Arguments withExactlyOnceLogging() { + Multiset logged = HashMultiset.create(); + return new Arguments(this.provider, this.keys) { + @Override + protected void logArgValue(String key, String description, Object result) { + int count = logged.add(key, 1); + if (count == 0) { + super.logArgValue(key, description, result); + } else if (count == 3000) { + LOGGER.warn("Too many requests for argument '{}', result should be cached", key); + } + } + }; + } + + public boolean silenced() { + return silent; + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index cb8db4c5..4e4f397a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -124,7 +124,7 @@ public record PlanetilerConfig( maxzoom, renderMaxzoom, arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false), - arguments.getBoolean("optimize_db", "optimize mbtiles after writing", false), + arguments.getBoolean("optimize_db", "Vacuum analyze mbtiles after writing", false), arguments.getBoolean("emit_tiles_in_order", "emit tiles in index order", true), arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false), arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false), diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java index 896287e1..ab93fdef 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java @@ -32,13 +32,21 @@ public enum DataType implements BiFunction { this(id, (d, k) -> parser.apply(d.getTag(k)), parser); } - @Override - public Object apply(WithTags withTags, String string) { - return this.getter.apply(withTags, string); - } - - public Object convertFrom(Object value) { - return this.parser.apply(value); + /** Returns the data type associated with {@code value}, or {@link #GET_TAG} as a fallback. */ + public static DataType typeOf(Object value) { + if (value instanceof String) { + return GET_STRING; + } else if (value instanceof Integer) { + return GET_INT; + } else if (value instanceof Long) { + return GET_LONG; + } else if (value instanceof Double) { + return GET_DOUBLE; + } else if (value instanceof Boolean) { + return GET_BOOLEAN; + } else { + return GET_TAG; + } } /** Returns the data type associated with {@code id}, or {@link #GET_TAG} as a fallback. */ @@ -51,6 +59,15 @@ public enum DataType implements BiFunction { return GET_TAG; } + @Override + public Object apply(WithTags withTags, String string) { + return this.getter.apply(withTags, string); + } + + public Object convertFrom(Object value) { + return this.parser.apply(value); + } + public String id() { return id; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java index 15025d4f..3ac15efc 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java @@ -62,11 +62,31 @@ public interface Stats extends AutoCloseable { * Also sets the "stage" prefix that shows up in the logs to {@code name}. */ default Timers.Finishable startStage(String name) { - LogUtil.setStage(name); - var timer = timers().startTimer(name); + return startStage(name, true); + } + + /** + * Same as {@link #startStage(String)} except does not log that it started, or set the logging prefix. + */ + default Timers.Finishable startStageQuietly(String name) { + return startStage(name, false); + } + + /** + * Records that a long-running task with {@code name} has started and returns a handle to call when finished. + *

+ * Also sets the "stage" prefix that shows up in the logs to {@code name} if {@code log} is true. + */ + default Timers.Finishable startStage(String name, boolean log) { + if (log) { + LogUtil.setStage(name); + } + var timer = timers().startTimer(name, log); return () -> { timer.stop(); - LogUtil.clearStage(); + if (log) { + LogUtil.clearStage(); + } }; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Timers.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Timers.java index 4722dc93..3ce21270 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Timers.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Timers.java @@ -91,16 +91,22 @@ public class Timers { } public Finishable startTimer(String name) { + return startTimer(name, true); + } + + public Finishable startTimer(String name, boolean logStart) { Timer timer = Timer.start(); Stage stage = new Stage(timer); timers.put(name, stage); Stage last = currentStage.getAndSet(stage); - LOGGER.info(""); - LOGGER.info("Starting..."); + if (logStart) { + LOGGER.info(""); + LOGGER.info("Starting..."); + } return () -> { - LOGGER.info("Finished in " + timers.get(name).timer.stop()); + LOGGER.info("Finished in {}", timers.get(name).timer.stop()); for (var details : getStageDetails(name, true)) { - LOGGER.info(" " + details); + LOGGER.info(" {}", details); } currentStage.set(last); }; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileWatcher.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileWatcher.java index ab08f945..0a0d904e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileWatcher.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileWatcher.java @@ -1,7 +1,5 @@ package com.onthegomap.planetiler.util; -import static com.onthegomap.planetiler.util.Exceptions.throwFatalException; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -104,19 +102,4 @@ public class FileWatcher { } } } - - @FunctionalInterface - public interface FunctionThatThrows { - - @SuppressWarnings("java:S112") - O apply(I value) throws Exception; - - default O runAndWrapException(I value) { - try { - return apply(value); - } catch (Exception e) { - return throwFatalException(e); - } - } - } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FunctionThatThrows.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FunctionThatThrows.java new file mode 100644 index 00000000..736c8e14 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FunctionThatThrows.java @@ -0,0 +1,18 @@ +package com.onthegomap.planetiler.util; + +import static com.onthegomap.planetiler.util.Exceptions.throwFatalException; + +@FunctionalInterface +public interface FunctionThatThrows { + + @SuppressWarnings("java:S112") + O apply(I value) throws Exception; + + default O runAndWrapException(I value) { + try { + return apply(value); + } catch (Exception e) { + return throwFatalException(e); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Memoized.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Memoized.java new file mode 100644 index 00000000..6e85b342 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Memoized.java @@ -0,0 +1,46 @@ +package com.onthegomap.planetiler.util; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +/** + * Caches the value of a function, so it only gets called once for each unique input, including when it throws an + * exception. + */ +public class Memoized implements Function { + private final ConcurrentHashMap> cache = new ConcurrentHashMap<>(); + private final Function> supplier; + + private Memoized(FunctionThatThrows supplier) { + this.supplier = i -> Try.apply(() -> supplier.apply(i)); + } + + /** Returns a memoized version of {@code supplier} that gets called only once for each input. */ + public static Memoized memoize(FunctionThatThrows supplier) { + return new Memoized<>(supplier); + } + + + @Override + public O apply(I i) { + Try result = cache.get(i); + if (result == null) { + result = cache.computeIfAbsent(i, supplier); + } + return result.get(); + } + + /** Returns a success or failure wrapper for the function call. */ + public Try tryApply(I i) { + Try result = cache.get(i); + if (result == null) { + result = cache.computeIfAbsent(i, supplier); + } + return result; + } + + /** Returns a success or failure wrapper for the function call, and casting the result to {@code clazz}. */ + public Try tryApply(I i, Class clazz) { + return tryApply(i).cast(clazz); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Try.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Try.java index 69e5061e..4cef4263 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Try.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Try.java @@ -1,5 +1,7 @@ package com.onthegomap.planetiler.util; +import static com.onthegomap.planetiler.util.Exceptions.throwFatalException; + /** * A container for the result of an operation that may succeed or fail. * @@ -45,12 +47,35 @@ public interface Try { return null; } - record Success (T get) implements Try {} + /** If success, then tries to cast the result to {@code clazz}, turning into a failure if not possible. */ + default Try cast(Class clazz) { + return map(clazz::cast); + } + + /** + * If this is a success, then maps the value through {@code fn}, returning the new value in a {@link Success} if + * successful, or {@link Failure} if the mapping function threw an exception. + */ + Try map(FunctionThatThrows fn); + + record Success (T get) implements Try { + + @Override + public Try map(FunctionThatThrows fn) { + return Try.apply(() -> fn.apply(get)); + } + } record Failure (@Override Exception exception) implements Try { @Override public T get() { - throw new IllegalStateException(exception); + return throwFatalException(exception); + } + + @Override + @SuppressWarnings("unchecked") + public Try map(FunctionThatThrows fn) { + return (Try) this; } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedError.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedError.java new file mode 100644 index 00000000..a81ab979 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedError.java @@ -0,0 +1,8 @@ +package com.onthegomap.planetiler; + +/** A fatal error intentionally thrown by a test. */ +public class ExpectedError extends Error { + public ExpectedError() { + super("expected error", null, true, false); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedException.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedException.java index dcddacb2..eab1dd11 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedException.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/ExpectedException.java @@ -1,7 +1,7 @@ package com.onthegomap.planetiler; /** An exception intentionally thrown by a test. */ -public class ExpectedException extends Error { +public class ExpectedException extends RuntimeException { public ExpectedException() { super("expected exception", null, true, false); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 3dcf473d..c8a1f7ee 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -871,7 +871,7 @@ class PlanetilerTests { with(new OsmElement.Node(1, 0, 0), t -> t.setTag("attr", "value")) ), (in, features) -> { - throw new ExpectedException(); + throw new ExpectedError(); } )); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java index 3898ecec..c38f9384 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java @@ -2,11 +2,13 @@ package com.onthegomap.planetiler.config; import static org.junit.jupiter.api.Assertions.*; +import com.onthegomap.planetiler.ExpectedException; import com.onthegomap.planetiler.TestUtils; import com.onthegomap.planetiler.reader.osm.OsmInputFile; import java.nio.file.Path; import java.time.Duration; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.locationtech.jts.geom.Envelope; @@ -181,4 +183,100 @@ class ArgumentsTest { assertTrue(args.getBoolean("force1", "force1", false)); assertTrue(args.getBoolean("force2", "force2", false)); } + + @Test + void testListArgumentValuesFromCommandLine() { + assertEquals(Map.of(), Arguments.fromArgs().toMap()); + assertEquals(Map.of( + "key", "value", + "key2", "value2", + "force1", "true", + "force2", "true" + ), Arguments.fromArgs( + "--key value --key2 value2 --force1 --force2".split("\\s+") + ).toMap()); + } + + @Test + void testListArgumentValuesFromMap() { + assertEquals(Map.of(), Arguments.of(Map.of()).toMap()); + assertEquals(Map.of("a", "1", "b", "2"), Arguments.of(Map.of("a", "1", "b", "2")).toMap()); + } + + @Test + void testListArgumentValuesFromConfigFile() { + Arguments args = Arguments.fromConfigFile(TestUtils.pathToResource("test.properties")); + assertEquals(Map.of("key1", "value1fromfile", "key2", "value2fromfile"), args.toMap()); + } + + @Test + void testListArgumentsFromEnvironment() { + Map env = Map.of( + "OTHER", "value", + "PLANETILEROTHER", "VALUE", + "PLANETILER_KEY1", "value1", + "PLANETILER_KEY2", "value2" + ); + Arguments args = Arguments.fromEnvironment(env::get, env::keySet); + assertEquals(Map.of( + "key1", "value1", + "key2", "value2" + ), args.toMap()); + } + + @Test + void testListArgumentsFromJvmProperties() { + Map jvm = Map.of( + "OTHER", "value", + "PLANETILEROTHER", "VALUE", + "PLANETILER_KEY1", "value1", + "PLANETILER_KEY2", "value2", + "planetiler.key3", "value4" + ); + Arguments args = Arguments.fromJvmProperties(jvm::get, jvm::keySet); + assertEquals(Map.of( + "key3", "value4" + ), args.toMap()); + } + + @Test + void testListArgumentsFromMerged() { + Map env = Map.of( + "OTHER", "value", + "PLANETILEROTHER", "VALUE", + "PLANETILER_KEY1", "value1", + "PLANETILER_KEY2", "value2", + "planetiler.key3", "value3" + ); + Map jvm = Map.of( + "other", "value", + "PLANETILEROTHER", "VALUE", + "PLANETILER_KEY1", "value1", + "PLANETILER_KEY2", "value2", + "planetiler.key3", "value4" + ); + Arguments args = Arguments.fromJvmProperties(jvm::get, jvm::keySet) + .orElse(Arguments.fromEnvironment(env::get, env::keySet)); + assertEquals(Map.of( + "key3", "value4", + "key1", "value1", + "key2", "value2" + ), args.toMap()); + } + + @Test + void testDontAccessArgListUntilUsed() { + Map env = Map.of( + "OTHER", "value", + "PLANETILEROTHER", "VALUE", + "PLANETILER_KEY1", "value1", + "PLANETILER_KEY2", "value2", + "planetiler.key3", "value3" + ); + Arguments args = Arguments.fromEnvironment(env::get, () -> { + throw new ExpectedException(); + }); + assertEquals("value1", args.getString("key1", "")); + assertThrows(ExpectedException.class, args::toMap); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/MemoizedTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/MemoizedTest.java new file mode 100644 index 00000000..27a1203e --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/MemoizedTest.java @@ -0,0 +1,59 @@ +package com.onthegomap.planetiler.util; + +import static com.onthegomap.planetiler.util.Memoized.memoize; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.onthegomap.planetiler.ExpectedException; +import org.junit.jupiter.api.Test; + +class MemoizedTest { + int calls = 0; + + @Test + void testMemoize() { + Memoized memoized = memoize(i -> { + calls++; + return i + 1; + }); + assertEquals(0, calls); + assertEquals(1, memoized.apply(0)); + assertEquals(1, calls); + assertEquals(1, memoized.apply(0)); + assertEquals(1, memoized.tryApply(0).get()); + assertEquals(1, calls); + assertEquals(2, memoized.apply(1)); + assertEquals(2, memoized.apply(1)); + assertEquals(2, calls); + } + + @Test + void testThrowException() { + Memoized memoized = memoize(i -> { + calls++; + throw new ExpectedException(); + }); + assertEquals(0, calls); + assertThrows(ExpectedException.class, () -> memoized.apply(0)); + assertThrows(ExpectedException.class, () -> memoized.apply(0)); + assertTrue(memoized.tryApply(0).isFailure()); + assertEquals(1, calls); + assertThrows(ExpectedException.class, () -> memoized.apply(1)); + assertThrows(ExpectedException.class, () -> memoized.apply(1)); + assertTrue(memoized.tryApply(1).isFailure()); + assertEquals(2, calls); + } + + @Test + void testTryCast() { + Memoized memoized = memoize(i -> { + calls++; + return i + 1; + }); + assertEquals(1, memoized.tryApply(0, Number.class).get()); + var failed = memoized.tryApply(0, String.class); + assertTrue(failed.isFailure()); + assertThrows(ClassCastException.class, failed::get); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TryTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TryTest.java index 40e2bf00..4a4ee1ee 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TryTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TryTest.java @@ -1,7 +1,9 @@ package com.onthegomap.planetiler.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -22,4 +24,12 @@ class TryTest { assertEquals(Try.failure(exception), result); assertThrows(IllegalStateException.class, result::get); } + + @Test + void cast() { + var result = Try.apply(() -> 1); + assertEquals(Try.success(1), result.cast(Number.class)); + assertTrue(result.cast(String.class).isFailure()); + assertInstanceOf(ClassCastException.class, result.cast(String.class).exception()); + } } diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md index 525104fe..e3bb2cfe 100644 --- a/planetiler-custommap/README.md +++ b/planetiler-custommap/README.md @@ -33,6 +33,8 @@ The root of the schema has the following attributes: - `examples` - A list of [Test Case](#test-case) input features and the vector tile features they should map to, or a relative path to a file with those examples in it. Run planetiler with `verify schema_file.yml` to see if they work as expected. +- `args` - Set default values for arguments that can be referenced later in the config and overridden from the + command-line or environmental variables. See [Arguments](#arguments). - `definitions` - An unparsed spot where you can define [anchor labels](#anchors-and-aliases) to be used in other parts of the schema @@ -46,6 +48,7 @@ attribution: & sources: { ... } tag_mappings: { ... } layers: [...] +args: { ... } examples: [...] ``` @@ -55,10 +58,12 @@ A description that tells planetiler how to read geospatial objects with tags fro - `type` - Enum representing the file format of the data source, one of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format) or [`shapefile`](https://en.wikipedia.org/wiki/Shapefile) -- `local_path` - Local path to the file to use, inferred from `url` if missing +- `local_path` - Local path to the file to use, inferred from `url` if missing. Can be a string + or [expression](#expression) that can reference [argument values](#arguments). - `url` - Location to download the file from if not present at `local_path`. - For [geofabrik](https://download.geofabrik.de/) named areas, use `geofabrik:` - prefixes, for example `geofabrik:rhode-island`. + For [geofabrik](https://download.geofabrik.de/) named areas, use `geofabrik:` prefixes, for + example `geofabrik:rhode-island`. Can be a string or [expression](#expression) that can + reference [argument values](#arguments). For example: @@ -89,6 +94,101 @@ tag_mappings: type: integer ``` +## Arguments + +A map from argument name to its definition. Arguments can be referenced later in the config and +overridden from the command-line or environmental variables. Argument definitions can either be an object with these +properties, or just the default value: + +- `default` - Default value for the argument. Can be an [expression](#expression) that references other arguments. +- `description` - Description of the argument to print when parsing it. +- `type` - [Data type](#data-type) to use when parsing the value. If missing, then inferred from the default value. + +For example: + +```yaml +# Define an "area" argument with default value "switzerland" +# and a "path" argument that defaults to the area with .osm.pbf extension +args: + area: + description: Geofabrik area to download + default: switzerland + osm_local_path: '${ args.area + ".osm.pbf" }' + +# Use the value of the "area" and "path" arguments to construct the source definition +sources: + osm: + type: osm + url: '${ "geofabrik:" + args.area }' + local_path: '${ args.osm_local_path }' +``` + +You can pass in `--area=france` from the command line to set download URL to `geofabrik:france` and local path +to `france.osm.pbf`. Planetiler searches for argument values in this order: + +1. Command-line arguments `--area=france` +2. JVM Properties with "planetiler." prefix: `java -Dplanetiler.area=france ...` +3. Environmental variables with "PLANETILER_" prefix: `PLANETILER_AREA=france java ...` +4. Default value from the config + +Argument values are available from the [`args` variable](#root-context) in +an [inline script expression](#inline-script-expression) or the [`arg_value` expression](#argument-value-expression). + +### Built-in arguments + +`args` can also be used to set the default value for built-in arguments that control planetiler's behavior: + + +- `threads` - Default number of threads to use. +- `write_threads` - Default number of threads to use when writing temp features +- `process_threads` - Default number of threads to use when processing input features +- `feature_read_threads` - Default number of threads to use when reading features at tile write time +- `minzoom` - Minimum tile zoom level to emit +- `maxzoom` - Maximum tile zoom level to emit +- `render_maxzoom` - Maximum rendering zoom level up to +- `skip_mbtiles_index_creation` - Skip adding index to mbtiles file +- `optimize_db` - Vacuum analyze mbtiles file after writing +- `emit_tiles_in_order` - Emit vector tiles in index order +- `force` - Overwriting output file and ignore warnings +- `gzip_temp` - Gzip temporary feature storage (uses more CPU, but less disk space) +- `mmap_temp` - Use memory-mapped IO for temp feature files +- `sort_max_readers` - Maximum number of concurrent read threads to use when sorting chunks +- `sort_max_writers` - Maximum number of concurrent write threads to use when sorting chunks +- `nodemap_type` - Type of node location map +- `nodemap_storage` - Storage for node location map +- `nodemap_madvise` - Use linux madvise(random) for node locations +- `multipolygon_geometry_storage` - Storage for multipolygon geometries +- `multipolygon_geometry_madvise` - Use linux madvise(random) for multiplygon geometries +- `http_user_agent` - User-Agent header to set when downloading files over HTTP +- `http_retries` - Retries to use when downloading files over HTTP +- `download_chunk_size_mb` - Size of file chunks to download in parallel in megabytes +- `download_threads` - Number of parallel threads to use when downloading each file +- `min_feature_size_at_max_zoom` - Default value for the minimum size in tile pixels of features to emit at the maximum + zoom level to allow for overzooming +- `min_feature_size` - Default value for the minimum size in tile pixels of features to emit below the maximum zoom + level +- `simplify_tolerance_at_max_zoom` - Default value for the tile pixel tolerance to use when simplifying features at the + maximum zoom level to allow for overzooming +- `simplify_tolerance` - Default value for the tile pixel tolerance to use when simplifying features below the maximum + zoom level +- `compact_db` - Reduce the DB size by separating and deduping the tile data +- `skip_filled_tiles` - Skip writing tiles containing only polygon fills to the output +- `tile_warning_size_mb` - Maximum size in megabytes of a tile to emit a warning about + +For example: + +```yaml +# Tell planetiler to download sources using 10 threads +args: + download_threads: 10 +``` + +Built-in arguments can also be accessed from the config file if desired: `${ args.download_threads }`. + ## Layer A layer contains a thematically-related set of features from one or more input sources. @@ -167,6 +267,8 @@ To define the value, use one of: - `coalesce` - A [Coalesce Expression](#coalesce-expression) that sets this attribute to the first non-null match from a list of expressions. - `tag_value` - A [Tag Value Expression](#tag-value-expression) that sets this attribute to the value for a tag. +- `arg_value` - An [Argument Value Expression](#argument-value-expression) that sets this attribute to the value for a + tag. For example: @@ -217,6 +319,17 @@ value: tag_value: natural ``` +### Argument Value Expression + +Use `arg_value:` to return the value of an argument set in the [Arguments](#arguments) section, or overridden from the +command-line or environment. + +```yaml +# return value for "attr_value" argument +value: + arg_value: attr_value +``` + ### Coalesce Expression Use `coalesce: [expression, expression, ...]` to make the expression evaluate to the first non-null result of a list of @@ -321,8 +434,11 @@ nested, so each child context can also access the variables from its parent. > ##### root context > -> defines no variables +> Available variables: +> - `args` - a map from [argument](#arguments) name to value, see also [built-in arguments](#built-in-arguments) that > +>> are always available. +>> >> ##### process feature context >> >> Context available when processing an input feature, for example testing whether to include it from `include_when`. diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json index d8dbeaec..79b6b1b4 100644 --- a/planetiler-custommap/planetiler.schema.json +++ b/planetiler-custommap/planetiler.schema.json @@ -45,10 +45,11 @@ }, "url": { "description": "Location to download the file from. For geofabrik named areas, use `geofabrik:` prefixes, for example `geofabrik:rhode-island`.", - "type": "string" + "$ref": "#/$defs/expression" }, "local_path": { - "description": "Local path to the file to use, inferred from `url` if missing" + "description": "Local path to the file to use, inferred from `url` if missing", + "$ref": "#/$defs/expression" } }, "anyOf": [ @@ -111,6 +112,246 @@ } } }, + "args": { + "description": "Set default values for built-in command-line arguments or expose new command-line arguments.", + "type": "object", + "properties": { + "threads": { + "description": "Default number of threads to use." + }, + "write_threads": { + "description": "Default number of threads to use when writing temp features" + }, + "process_threads": { + "description": "Default number of threads to use when processing input features" + }, + "feature_read_threads": { + "description": "Default number of threads to use when reading features at tile write time" + }, + "minzoom": { + "description": "Minimum tile zoom level to emit" + }, + "maxzoom": { + "description": "Maximum tile zoom level to emit" + }, + "render_maxzoom": { + "description": "Maximum rendering zoom level up to" + }, + "skip_mbtiles_index_creation": { + "description": "Skip adding index to mbtiles file", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "optimize_db": { + "description": "Vacuum analyze mbtiles file after writing", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "emit_tiles_in_order": { + "description": "Emit vector tiles in index order", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "force": { + "description": "Overwriting output file and ignore warnings", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "gzip_temp": { + "description": "Gzip temporary feature storage (uses more CPU, but less disk space)", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "mmap_temp": { + "description": "Use memory-mapped IO for temp feature files", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "sort_max_readers": { + "description": "Maximum number of concurrent read threads to use when sorting chunks" + }, + "sort_max_writers": { + "description": "Maximum number of concurrent write threads to use when sorting chunks" + }, + "nodemap_type": { + "description": "Type of node location map", + "anyOf": [ + { + "type": "string" + }, + { + "enum": [ + "array", + "sparsearray", + "sortedtable", + "noop" + ] + } + ] + }, + "nodemap_storage": { + "description": "Storage for node location map", + "anyOf": [ + { + "type": "string" + }, + { + "enum": [ + "ram", + "mmap", + "direct" + ] + } + ] + }, + "nodemap_madvise": { + "description": "Use linux madvise(random) for node locations", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] + }, + "multipolygon_geometry_storage": { + "description": "Storage for multipolygon geometries", + "anyOf": [ + { + "type": "string" + }, + { + "enum": [ + "ram", + "mmap", + "direct" + ] + } + ] + }, + "multipolygon_geometry_madvise": { + "description": "Use linux madvise(random) for multiplygon geometries", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] + }, + "http_user_agent": { + "description": "User-Agent header to set when downloading files over HTTP" + }, + "http_retries": { + "description": "Retries to use when downloading files over HTTP" + }, + "download_chunk_size_mb": { + "description": "Size of file chunks to download in parallel in megabytes" + }, + "download_threads": { + "description": "Number of parallel threads to use when downloading each file" + }, + "min_feature_size_at_max_zoom": { + "description": "Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming" + }, + "min_feature_size": { + "description": "Default value for the minimum size in tile pixels of features to emit below the maximum zoom level" + }, + "simplify_tolerance_at_max_zoom": { + "description": "Default value for the tile pixel tolerance to use when simplifying features at the maximum zoom level to allow for overzooming" + }, + "simplify_tolerance": { + "description": "Default value for the tile pixel tolerance to use when simplifying features below the maximum zoom level" + }, + "compact_db": { + "description": "Reduce the DB size by separating and deduping the tile data", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "skip_filled_tiles": { + "description": "Skip writing tiles containing only polygon fills to the output", + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "tile_warning_size_mb": { + "description": "Maximum size in megabytes of a tile to emit a warning about" + } + }, + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "default": { + "description": "Default value for this argument (can be an expression).", + "$ref": "#/$defs/expression" + }, + "type": { + "description": "Data type of this argument.", + "$ref": "#/$defs/datatype" + }, + "description": { + "description": "Description for this argument.", + "type": "string" + } + } + } + ] + } + }, "examples": { "description": "Example input features and the vector tile features they map to, or a relative path to a file with those examples in it.", "oneOf": [ @@ -208,6 +449,9 @@ { "$ref": "#/$defs/expression_tag_value" }, + { + "$ref": "#/$defs/expression_arg_value" + }, { "$ref": "#/$defs/expression_value" }, @@ -316,6 +560,9 @@ { "$ref": "#/$defs/expression_tag_value" }, + { + "$ref": "#/$defs/expression_arg_value" + }, { "$ref": "#/$defs/expression_value" }, @@ -365,6 +612,15 @@ } } }, + "expression_arg_value": { + "type": "object", + "properties": { + "arg_value": { + "description": "Returns the value associated with an argument", + "$ref": "#/$defs/expression" + } + } + }, "expression_value": { "type": "object", "properties": { diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java index 72ee0538..beaf5bca 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java @@ -10,6 +10,8 @@ import com.onthegomap.planetiler.custommap.expression.ScriptContext; import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment; import com.onthegomap.planetiler.expression.DataType; import com.onthegomap.planetiler.expression.MultiExpression; +import com.onthegomap.planetiler.util.Memoized; +import com.onthegomap.planetiler.util.Try; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -28,6 +30,8 @@ import java.util.stream.Stream; */ public class ConfigExpressionParser { + private static final Memoized MEMOIZED = Memoized.memoize(arg -> ConfigExpressionParser + .parse(arg.expression, TagValueProducer.EMPTY, arg.root.description(), arg.clazz).apply(arg.root)); private final TagValueProducer tagValueProducer; private final ScriptEnvironment input; @@ -50,6 +54,17 @@ public class ConfigExpressionParser { return new ConfigExpressionParser<>(tagValueProducer, context).parse(object, outputClass).simplify(); } + /** + * Attempts to evaluate {@code expression} from a yaml config, using only globally-available environmental variables + * from the {@code root} context. + */ + public static Try tryStaticEvaluate(Contexts.Root root, Object expression, Class resultType) { + if (expression == null) { + return Try.success(null); + } + return MEMOIZED.tryApply(new EvaluateInput(root, expression, resultType), resultType); + } + private ConfigExpression parse(Object object, Class output) { if (object == null) { return ConfigExpression.constOf(null); @@ -82,6 +97,9 @@ public class ConfigExpressionParser { } else if (keys.equals(Set.of("tag_value"))) { var tagProducer = parse(map.get("tag_value"), String.class); return getTag(signature(output), tagProducer); + } else if (keys.equals(Set.of("arg_value"))) { + var keyProducer = parse(map.get("arg_value"), String.class); + return getArg(signature(output), keyProducer); } else if (keys.equals(Set.of("value"))) { return parse(map.get("value"), output); } @@ -134,4 +152,6 @@ public class ConfigExpressionParser { private Signature signature(Class outputClass) { return new Signature<>(input, outputClass); } + + private record EvaluateInput(Contexts.Root root, Object expression, Class clazz) {} } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java index 223fa1ba..02023ee3 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java @@ -8,6 +8,7 @@ import com.onthegomap.planetiler.FeatureCollector.Feature; import com.onthegomap.planetiler.custommap.configschema.AttributeDefinition; import com.onthegomap.planetiler.custommap.configschema.FeatureGeometry; import com.onthegomap.planetiler.custommap.configschema.FeatureItem; +import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment; import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.reader.SourceFeature; @@ -34,9 +35,13 @@ public class ConfiguredFeature { private final TagValueProducer tagValueProducer; private final List> featureProcessors; private final Set sources; + private final ScriptEnvironment processFeatureContext; + private final ScriptEnvironment featureAttributeContext; + private ScriptEnvironment featurePostMatchContext; - public ConfiguredFeature(String layer, TagValueProducer tagValueProducer, FeatureItem feature) { + public ConfiguredFeature(String layer, TagValueProducer tagValueProducer, FeatureItem feature, + Contexts.Root rootContext) { sources = Set.copyOf(feature.source()); FeatureGeometry geometryType = feature.geometry(); @@ -46,6 +51,9 @@ public class ConfiguredFeature { //Factory to treat OSM tag values as specific data type values this.tagValueProducer = tagValueProducer; + processFeatureContext = Contexts.ProcessFeature.description(rootContext); + featurePostMatchContext = Contexts.FeaturePostMatch.description(rootContext); + featureAttributeContext = Contexts.FeatureAttribute.description(rootContext); //Test to determine whether this feature is included based on tagging Expression filter; @@ -53,13 +61,15 @@ public class ConfiguredFeature { filter = Expression.TRUE; } else { filter = - BooleanExpressionParser.parse(feature.includeWhen(), tagValueProducer, Contexts.ProcessFeature.DESCRIPTION); + BooleanExpressionParser.parse(feature.includeWhen(), tagValueProducer, + processFeatureContext); } if (feature.excludeWhen() != null) { filter = Expression.and( filter, Expression.not( - BooleanExpressionParser.parse(feature.excludeWhen(), tagValueProducer, Contexts.ProcessFeature.DESCRIPTION)) + BooleanExpressionParser.parse(feature.excludeWhen(), tagValueProducer, + processFeatureContext)) ); } tagTest = filter; @@ -86,7 +96,7 @@ public class ConfiguredFeature { var expression = ConfigExpressionParser.parse( input, tagValueProducer, - Contexts.FeaturePostMatch.DESCRIPTION, + featurePostMatchContext, clazz ); if (expression.equals(constOf(null))) { @@ -140,12 +150,14 @@ public class ConfiguredFeature { value.put("value", attribute.value()); } else if (attribute.tagValue() != null) { value.put("tag_value", attribute.tagValue()); + } else if (attribute.argValue() != null) { + value.put("arg_value", attribute.argValue()); } else { value.put("tag_value", attribute.key()); } } - return ConfigExpressionParser.parse(value, tagValueProducer, Contexts.FeaturePostMatch.DESCRIPTION, Object.class); + return ConfigExpressionParser.parse(value, tagValueProducer, featurePostMatchContext, Object.class); } /** @@ -160,7 +172,7 @@ public class ConfiguredFeature { Double minTilePercent, Object rawMinZoom, Map minZoomByValue) { var result = ConfigExpressionParser.parse(rawMinZoom, tagValueProducer, - Contexts.FeatureAttribute.DESCRIPTION, Integer.class); + featureAttributeContext, Integer.class); if ((result.equals(constOf(0)) || result.equals(constOf(null))) && minZoomByValue.isEmpty()) { @@ -206,9 +218,11 @@ public class ConfiguredFeature { var attributeTest = Expression.and( attrIncludeWhen == null ? Expression.TRUE : - BooleanExpressionParser.parse(attrIncludeWhen, tagValueProducer, Contexts.FeaturePostMatch.DESCRIPTION), + BooleanExpressionParser.parse(attrIncludeWhen, tagValueProducer, + featurePostMatchContext), attrExcludeWhen == null ? Expression.TRUE : - not(BooleanExpressionParser.parse(attrExcludeWhen, tagValueProducer, Contexts.FeaturePostMatch.DESCRIPTION)) + not(BooleanExpressionParser.parse(attrExcludeWhen, tagValueProducer, + featurePostMatchContext)) ).simplify(); var minTileCoverage = attrIncludeWhen == null ? null : attribute.minTileCoverSize(); diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java index 9a787d0c..5377bac9 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java @@ -2,14 +2,11 @@ package com.onthegomap.planetiler.custommap; import com.onthegomap.planetiler.Planetiler; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.custommap.configschema.DataSource; import com.onthegomap.planetiler.custommap.configschema.DataSourceType; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; -import java.net.URI; -import java.net.URISyntaxException; +import com.onthegomap.planetiler.custommap.expression.ParseException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; /** * Main driver to create maps configured by a YAML file. @@ -23,69 +20,58 @@ public class ConfiguredMapMain { * Main entrypoint */ public static void main(String... args) throws Exception { - run(Arguments.fromArgsOrConfigFile(args)); - } - - static void run(Arguments args) throws Exception { + Arguments arguments = Arguments.fromEnvOrArgs(args); var dataDir = Path.of("data"); var sourcesDir = dataDir.resolve("sources"); - var schemaFile = args.getString( + var schemaFile = arguments.getString( "schema", "Location of YML-format schema definition file" ); var path = Path.of(schemaFile); - SchemaConfig config; + SchemaConfig schema; if (Files.exists(path)) { - config = SchemaConfig.load(path); + schema = SchemaConfig.load(path); } else { // if the file doesn't exist, check if it's bundled in the jar schemaFile = schemaFile.startsWith("/samples/") ? schemaFile : "/samples/" + schemaFile; if (ConfiguredMapMain.class.getResource(schemaFile) != null) { - config = YAML.loadResource(schemaFile, SchemaConfig.class); + schema = YAML.loadResource(schemaFile, SchemaConfig.class); } else { throw new IllegalArgumentException("Schema file not found: " + schemaFile); } } - var planetiler = Planetiler.create(args) - .setProfile(new ConfiguredProfile(config)); + // use default argument values from config file as fallback if not set from command-line or env vars + Contexts.Root rootContext = Contexts.buildRootContext(arguments, schema.args()); - var sources = config.sources(); - for (var source : sources.entrySet()) { - configureSource(planetiler, sourcesDir, source.getKey(), source.getValue()); + var planetiler = Planetiler.create(rootContext.arguments()); + var profile = new ConfiguredProfile(schema, rootContext); + planetiler.setProfile(profile); + + for (var source : profile.sources()) { + configureSource(planetiler, sourcesDir, source); } - planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")) - .run(); + planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")).run(); } - private static void configureSource(Planetiler planetiler, Path sourcesDir, String sourceName, DataSource source) - throws URISyntaxException { + private static void configureSource(Planetiler planetiler, Path sourcesDir, Source source) { DataSourceType sourceType = source.type(); Path localPath = source.localPath(); + if (localPath == null) { + if (source.url() == null) { + throw new ParseException("Must provide either a url or path for " + source.id()); + } + localPath = sourcesDir.resolve(source.defaultFileUrl()); + } switch (sourceType) { - case OSM -> { - String url = source.url(); - String[] areaParts = url.split("[:/]"); - String areaFilename = areaParts[areaParts.length - 1]; - String areaName = areaFilename.replaceAll("\\..*$", ""); - if (localPath == null) { - localPath = sourcesDir.resolve(areaName + ".osm.pbf"); - } - planetiler.addOsmSource(sourceName, localPath, url); - } - case SHAPEFILE -> { - String url = source.url(); - if (localPath == null) { - localPath = sourcesDir.resolve(Paths.get(new URI(url).getPath()).getFileName().toString()); - } - planetiler.addShapefileSource(sourceName, localPath, url); - } - default -> throw new IllegalArgumentException("Unhandled source " + sourceType); + case OSM -> planetiler.addOsmSource(source.id(), localPath, source.url()); + case SHAPEFILE -> planetiler.addShapefileSource(source.id(), localPath, source.url()); + default -> throw new IllegalArgumentException("Unhandled source type for " + source.id() + ": " + sourceType); } } } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java index 5fa8677a..4650cdf2 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java @@ -10,6 +10,7 @@ import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import com.onthegomap.planetiler.expression.MultiExpression; import com.onthegomap.planetiler.expression.MultiExpression.Index; import com.onthegomap.planetiler.reader.SourceFeature; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -22,27 +23,29 @@ import java.util.stream.Collectors; */ public class ConfiguredProfile implements Profile { - private final SchemaConfig schemaConfig; + private final SchemaConfig schema; private final Map> featureLayerMatcher; private final TagValueProducer tagValueProducer; + private final Contexts.Root rootContext; - public ConfiguredProfile(SchemaConfig schemaConfig) { - this.schemaConfig = schemaConfig; + public ConfiguredProfile(SchemaConfig schema, Contexts.Root rootContext) { + this.schema = schema; + this.rootContext = rootContext; - Collection layers = schemaConfig.layers(); + Collection layers = schema.layers(); if (layers == null || layers.isEmpty()) { throw new IllegalArgumentException("No layers defined"); } - tagValueProducer = new TagValueProducer(schemaConfig.inputMappings()); + tagValueProducer = new TagValueProducer(schema.inputMappings()); Map>> configuredFeatureEntries = new HashMap<>(); for (var layer : layers) { String layerId = layer.id(); for (var feature : layer.features()) { - var configuredFeature = new ConfiguredFeature(layerId, tagValueProducer, feature); + var configuredFeature = new ConfiguredFeature(layerId, tagValueProducer, feature, rootContext); var entry = new Entry<>(configuredFeature, configuredFeature.matchExpression()); for (var source : feature.source()) { var list = configuredFeatureEntries.computeIfAbsent(source, s -> new ArrayList<>()); @@ -58,17 +61,17 @@ public class ConfiguredProfile implements Profile { @Override public String name() { - return schemaConfig.schemaName(); + return schema.schemaName(); } @Override public String attribution() { - return schemaConfig.attribution(); + return schema.attribution(); } @Override public void processFeature(SourceFeature sourceFeature, FeatureCollector featureCollector) { - var context = new Contexts.ProcessFeature(sourceFeature, tagValueProducer); + var context = rootContext.createProcessFeatureContext(sourceFeature, tagValueProducer); var index = featureLayerMatcher.get(sourceFeature.getSource()); if (index != null) { var matches = index.getMatchesWithTriggers(context); @@ -83,6 +86,16 @@ public class ConfiguredProfile implements Profile { @Override public String description() { - return schemaConfig.schemaDescription(); + return schema.schemaDescription(); + } + + public List sources() { + List sources = new ArrayList<>(); + schema.sources().forEach((key, value) -> { + var url = ConfigExpressionParser.tryStaticEvaluate(rootContext, value.url(), String.class).get(); + var path = ConfigExpressionParser.tryStaticEvaluate(rootContext, value.localPath(), String.class).get(); + sources.add(new Source(key, value.type(), url, path == null ? null : Path.of(path))); + }); + return sources; } } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java index 04ed7384..73a9e74d 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java @@ -1,14 +1,34 @@ package com.onthegomap.planetiler.custommap; +import com.google.api.expr.v1alpha1.Constant; +import com.google.api.expr.v1alpha1.Decl; +import com.google.api.expr.v1alpha1.Type; +import com.google.protobuf.NullValue; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.custommap.expression.ParseException; import com.onthegomap.planetiler.custommap.expression.ScriptContext; import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment; +import com.onthegomap.planetiler.expression.DataType; import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.WithGeometryType; import com.onthegomap.planetiler.reader.WithTags; +import com.onthegomap.planetiler.util.Try; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.projectnessie.cel.checker.Decls; import org.projectnessie.cel.common.types.NullT; +import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry; +import org.projectnessie.cel.common.types.ref.TypeAdapter; +import org.projectnessie.cel.common.types.ref.Val; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Wrapper objects that provide all available inputs to different parts of planetiler schema configs at runtime. @@ -17,35 +37,256 @@ import org.projectnessie.cel.common.types.NullT; * that all global variables from a parent context are also available to its child context. */ public class Contexts { + private static final Logger LOGGER = LoggerFactory.getLogger(Contexts.class); private static Object wrapNullable(Object nullable) { return nullable == null ? NullT.NullValue : nullable; } - public static Root root() { - return new Root(); + public static Root emptyRoot() { + return new Root(Arguments.of().silence(), Map.of()); + } + + /** + * Returns a {@link Root} context built from {@code schemaArgs} argument definitions and {@code origArguments} + * arguments provided from the command-line/environment. + *

+ * Arguments may depend on the value of other arguments so this iteratively evaluates the arguments until their values + * settle. + * + * @throws ParseException if the argument definitions are malformed, or if there's an infinite loop + */ + public static Contexts.Root buildRootContext(Arguments origArguments, Map schemaArgs) { + boolean loggingEnabled = !origArguments.silenced(); + origArguments.silence(); + Map argDescriptions = new LinkedHashMap<>(); + Map unparsedSchemaArgs = new HashMap<>(schemaArgs); + Map parsedSchemaArgs = new HashMap<>(origArguments.toMap()); + Contexts.Root result = new Root(origArguments, parsedSchemaArgs); + Arguments arguments = origArguments; + int iters = 0; + // arguments may reference the value of other arguments, so continue parsing until they all succeed... + while (!unparsedSchemaArgs.isEmpty()) { + final var root = result; + final var args = arguments; + Map failures = new HashMap<>(); + + Map.copyOf(unparsedSchemaArgs).forEach((key, value) -> { + boolean builtin = root.builtInArgs.contains(key); + String description; + Object defaultValueObject; + DataType type = null; + if (value instanceof Map map) { + if (builtin) { + throw new ParseException("Cannot override built-in argument: " + key); + } + var typeObject = map.get("type"); + if (typeObject != null) { + type = DataType.from(Objects.toString(typeObject)); + } + var descriptionObject = map.get("description"); + description = descriptionObject == null ? "no description provided" : descriptionObject.toString(); + defaultValueObject = map.get("default"); + if (type != null) { + var fromArgs = args.getString(key, description, null); + if (fromArgs != null) { + parsedSchemaArgs.put(key, type.convertFrom(fromArgs)); + } + } + } else { + defaultValueObject = value; + description = "no description provided"; + } + argDescriptions.put(key, description); + Try defaultValue = ConfigExpressionParser.tryStaticEvaluate(root, defaultValueObject, Object.class); + if (defaultValue.isSuccess()) { + Object raw = defaultValue.get(); + String asString = Objects.toString(raw); + if (type == null) { + type = DataType.typeOf(raw); + } + var stringResult = args.getString(key, description, asString); + Object castedResult = type.convertFrom(stringResult); + if (stringResult == null) { + throw new ParseException("Missing required parameter: " + key + "(" + description + ")"); + } else if (castedResult == null) { + throw new ParseException("Cannot convert value for " + key + " to " + type.id() + ": " + stringResult); + } + parsedSchemaArgs.put(key, castedResult); + unparsedSchemaArgs.remove(key); + } else { + failures.put(key, defaultValue.exception()); + } + }); + + arguments = origArguments.orElse(Arguments.of(parsedSchemaArgs.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + e -> Objects.toString(e.getValue())) + ))); + result = new Root(arguments, parsedSchemaArgs); + if (iters++ > 100) { + failures + .forEach( + (key, failure) -> LOGGER.error("Error computing {}:\n{}", key, + ExceptionUtils.getRootCause(failure).toString().indent(4))); + throw new ParseException("Infinite loop while processing arguments: " + unparsedSchemaArgs.keySet()); + } + } + var finalArguments = loggingEnabled ? arguments.withExactlyOnceLogging() : arguments.silence(); + if (loggingEnabled) { + argDescriptions.forEach((key, description) -> finalArguments.getString(key, description, null)); + } + return new Root(finalArguments, parsedSchemaArgs); } /** * Root context available everywhere in a planetiler schema config. + *

+ * Holds argument values parsed from the schema config and command-line args. */ - public record Root() implements ScriptContext { + public static final class Root implements ScriptContext { + private static final TypeAdapter TYPE_ADAPTER = ProtoTypeRegistry.newRegistry(); + private final Arguments arguments; + private final PlanetilerConfig config; + private final ScriptEnvironment description; + private final Map bindings = new HashMap<>(); + private final Map argumentValues = new HashMap<>(); + public final Set builtInArgs; - // TODO add argument parsing - public static final ScriptEnvironment DESCRIPTION = - ScriptEnvironment.root().forInput(Root.class); + public Arguments arguments() { + return arguments; + } + + public PlanetilerConfig config() { + return config; + } + + @Override + public boolean equals(Object o) { + return this == o || (o instanceof Root root && argumentValues.equals(root.argumentValues)); + } + + @Override + public int hashCode() { + return Objects.hash(argumentValues); + } + + @Override + public Object argument(String key) { + return argumentValues.get(key); + } + + private Root(Arguments arguments, Map schemaArgs) { + this.arguments = arguments; + this.config = PlanetilerConfig.from(arguments); + argumentValues.put("threads", config.threads()); + argumentValues.put("write_threads", config.featureWriteThreads()); + argumentValues.put("process_threads", config.featureProcessThreads()); + argumentValues.put("feature_read_threads", config.featureReadThreads()); + // args.put("loginterval", config.logInterval()); + argumentValues.put("minzoom", config.minzoom()); + argumentValues.put("maxzoom", config.maxzoom()); + argumentValues.put("render_maxzoom", config.maxzoomForRendering()); + argumentValues.put("skip_mbtiles_index_creation", config.skipIndexCreation()); + argumentValues.put("optimize_db", config.optimizeDb()); + argumentValues.put("emit_tiles_in_order", config.emitTilesInOrder()); + argumentValues.put("force", config.force()); + argumentValues.put("gzip_temp", config.gzipTempStorage()); + argumentValues.put("mmap_temp", config.mmapTempStorage()); + argumentValues.put("sort_max_readers", config.sortMaxReaders()); + argumentValues.put("sort_max_writers", config.sortMaxWriters()); + argumentValues.put("nodemap_type", config.nodeMapType()); + argumentValues.put("nodemap_storage", config.nodeMapStorage()); + argumentValues.put("nodemap_madvise", config.nodeMapMadvise()); + argumentValues.put("multipolygon_geometry_storage", config.multipolygonGeometryStorage()); + argumentValues.put("multipolygon_geometry_madvise", config.multipolygonGeometryMadvise()); + argumentValues.put("http_user_agent", config.httpUserAgent()); + // args.put("http_timeout", config.httpTimeout()); + argumentValues.put("http_retries", config.httpRetries()); + argumentValues.put("download_chunk_size_mb", config.downloadChunkSizeMB()); + argumentValues.put("download_threads", config.downloadThreads()); + argumentValues.put("min_feature_size_at_max_zoom", config.minFeatureSizeAtMaxZoom()); + argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom()); + argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom()); + argumentValues.put("simplify_tolerance", config.simplifyToleranceBelowMaxZoom()); + argumentValues.put("compact_db", config.compactDb()); + argumentValues.put("skip_filled_tiles", config.skipFilledTiles()); + argumentValues.put("tile_warning_size_mb", config.tileWarningSizeBytes()); + builtInArgs = Set.copyOf(argumentValues.keySet()); + + schemaArgs.forEach(argumentValues::putIfAbsent); + config.arguments().toMap().forEach(argumentValues::putIfAbsent); + + argumentValues.forEach((k, v) -> bindings.put("args." + k, TYPE_ADAPTER.nativeToValue(v))); + bindings.put("args", TYPE_ADAPTER.nativeToValue(this.argumentValues)); + + description = ScriptEnvironment.root(this).forInput(Root.class) + .withDeclarations( + argumentValues.entrySet().stream() + .map(entry -> decl(entry.getKey(), entry.getValue())) + .toList() + ).withDeclarations( + Decls.newVar("args", Decls.newMapType(Decls.String, Decls.Any)) + ); + } + + private Decl decl(String name, Object value) { + Type type; + var builder = Constant.newBuilder(); + if (value instanceof String s) { + builder.setStringValue(s); + type = Decls.String; + } else if (value instanceof Boolean b) { + builder.setBoolValue(b); + type = Decls.Bool; + } else if (value instanceof Long || value instanceof Integer) { + builder.setInt64Value(((Number) value).longValue()); + type = Decls.Int; + } else if (value instanceof Double || value instanceof Float) { + builder.setDoubleValue(((Number) value).doubleValue()); + type = Decls.Double; + } else if (value == null) { + builder.setNullValue(NullValue.NULL_VALUE); + type = Decls.Null; + } else { + throw new IllegalArgumentException( + "Unrecognized constant type: " + value + " (" + value.getClass().getName() + ")"); + } + return Decls.newConst("args." + name, type, builder.build()); + } + + public ScriptEnvironment description() { + return description; + } @Override public Object apply(String input) { + return bindings.get(input); + } + + public ProcessFeature createProcessFeatureContext(SourceFeature sourceFeature, TagValueProducer tagValueProducer) { + return new ProcessFeature(this, sourceFeature, tagValueProducer); + } + } + + private interface NestedContext extends ScriptContext { + + default Root root() { return null; } + + @Override + default Object argument(String key) { + return root().argument(key); + } } /** * Makes nested contexts adhere to {@link WithTags} and {@link WithGeometryType} by recursively fetching source * feature from the root context. */ - private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType { + private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType, NestedContext { + default FeatureContext parent() { return null; } @@ -54,6 +295,11 @@ public class Contexts { return parent().feature(); } + @Override + default Root root() { + return parent().root(); + } + @Override default Map tags() { return feature().tags(); @@ -87,7 +333,10 @@ public class Contexts { * @param feature The input feature being processed * @param tagValueProducer Common parsing for input feature tags */ - public record ProcessFeature(@Override SourceFeature feature, @Override TagValueProducer tagValueProducer) + public record ProcessFeature( + @Override Root root, @Override SourceFeature feature, + @Override TagValueProducer tagValueProducer + ) implements FeatureContext { private static final String FEATURE_TAGS = "feature.tags"; @@ -95,14 +344,16 @@ public class Contexts { private static final String FEATURE_SOURCE = "feature.source"; private static final String FEATURE_SOURCE_LAYER = "feature.source_layer"; - public static final ScriptEnvironment DESCRIPTION = ScriptEnvironment.root() - .forInput(ProcessFeature.class) - .withDeclarations( - Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)), - Decls.newVar(FEATURE_ID, Decls.Int), - Decls.newVar(FEATURE_SOURCE, Decls.String), - Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String) - ); + public static ScriptEnvironment description(Root root) { + return root.description() + .forInput(ProcessFeature.class) + .withDeclarations( + Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)), + Decls.newVar(FEATURE_ID, Decls.Int), + Decls.newVar(FEATURE_SOURCE, Decls.String), + Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String) + ); + } @Override public Object apply(String key) { @@ -127,7 +378,7 @@ public class Contexts { /** * Context available after a feature has been matched. - * + *

* Adds {@code match_key} and {@code match_value} variables that capture which tag key/value caused the feature to be * included. * @@ -139,12 +390,14 @@ public class Contexts { private static final String MATCH_KEY = "match_key"; private static final String MATCH_VALUE = "match_value"; - public static final ScriptEnvironment DESCRIPTION = ProcessFeature.DESCRIPTION - .forInput(FeaturePostMatch.class) - .withDeclarations( - Decls.newVar(MATCH_KEY, Decls.String), - Decls.newVar(MATCH_VALUE, Decls.Any) - ); + public static ScriptEnvironment description(Root root) { + return ProcessFeature.description(root) + .forInput(FeaturePostMatch.class) + .withDeclarations( + Decls.newVar(MATCH_KEY, Decls.String), + Decls.newVar(MATCH_VALUE, Decls.Any) + ); + } @Override public Object apply(String key) { @@ -182,10 +435,14 @@ public class Contexts { * @param value Value of the attribute */ public record FeatureAttribute(@Override FeaturePostMatch parent, Object value) implements FeatureContext { + private static final String VALUE = "value"; - public static final ScriptEnvironment DESCRIPTION = FeaturePostMatch.DESCRIPTION - .forInput(FeatureAttribute.class) - .withDeclarations(Decls.newVar(VALUE, Decls.Any)); + + public static ScriptEnvironment description(Root root) { + return FeaturePostMatch.description(root) + .forInput(FeatureAttribute.class) + .withDeclarations(Decls.newVar(VALUE, Decls.Any)); + } @Override public Object apply(String key) { diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java new file mode 100644 index 00000000..102e1209 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java @@ -0,0 +1,23 @@ +package com.onthegomap.planetiler.custommap; + +import com.onthegomap.planetiler.custommap.configschema.DataSourceType; +import java.nio.file.Path; + +/** A parsed source definition from a config file. */ +public record Source( + String id, + DataSourceType type, + String url, + Path localPath +) { + + public String defaultFileUrl() { + String result = url + .replaceFirst("^https?://", "") + .replaceAll("[\\W&&[^.]]+", "_"); + if (type == DataSourceType.OSM && !result.endsWith(".pbf")) { + result = result + ".osm.pbf"; + } + return result; + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java index 3f6d1cbd..6e29b15e 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java @@ -14,6 +14,7 @@ public record AttributeDefinition( // pass-through to value expression @JsonProperty("value") Object value, @JsonProperty("tag_value") String tagValue, + @JsonProperty("arg_value") String argValue, Object type, Object coalesce ) {} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java index 193e5958..6296a64d 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java @@ -1,10 +1,9 @@ package com.onthegomap.planetiler.custommap.configschema; import com.fasterxml.jackson.annotation.JsonProperty; -import java.nio.file.Path; public record DataSource( DataSourceType type, - String url, - @JsonProperty("local_path") Path localPath + Object url, + @JsonProperty("local_path") Object localPath ) {} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java index 9b1f1e44..c4b70912 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java @@ -17,7 +17,8 @@ public record SchemaConfig( Object definitions, @JsonProperty("tag_mappings") Map inputMappings, Collection layers, - Object examples + Object examples, + Map args ) { private static final String DEFAULT_ATTRIBUTION = """ @@ -29,6 +30,11 @@ public record SchemaConfig( return attribution == null ? DEFAULT_ATTRIBUTION : attribution; } + @Override + public Map args() { + return args == null ? Map.of() : args; + } + public static SchemaConfig load(Path path) { return YAML.load(path, SchemaConfig.class); } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java index f8882411..1ccf4994 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java @@ -45,6 +45,11 @@ public interface ConfigExpression return new GetTag<>(signature, tag); } + static ConfigExpression getArg(Signature signature, + ConfigExpression tag) { + return new GetArg<>(signature, tag); + } + static ConfigExpression cast(Signature signature, ConfigExpression input, DataType dataType) { return new Cast<>(signature, input, dataType); @@ -163,13 +168,16 @@ public interface ConfigExpression default -> { var result = children.stream() .flatMap( - child -> child instanceof Coalesce childCoalesce ? childCoalesce.children.stream() : + child -> child instanceof Coalesce childCoalesce ? childCoalesce.children.stream() : Stream.of(child)) .filter(child -> !child.equals(constOf(null))) .distinct() .toList(); - var indexOfFirstConst = result.stream().takeWhile(d -> !(d instanceof ConfigExpression.Const)).count(); - yield coalesce(result.stream().limit(indexOfFirstConst + 1).toList()); + var indexOfFirstConst = result.stream().takeWhile(d -> !(d instanceof ConfigExpression.Const)).count(); + yield coalesce(result.stream().map(d -> { + @SuppressWarnings("unchecked") ConfigExpression casted = (ConfigExpression) d; + return casted; + }).limit(indexOfFirstConst + 1).toList()); } }; } @@ -210,6 +218,29 @@ public interface ConfigExpression } } + /** An expression that returns the value associated a given argument at runtime. */ + record GetArg ( + Signature signature, + ConfigExpression arg + ) implements ConfigExpression { + + @Override + public O apply(I i) { + return TypeConversion.convert(i.argument(arg.apply(i)), signature.out); + } + + @Override + public ConfigExpression simplifyOnce() { + var key = arg.simplifyOnce(); + if (key instanceof ConfigExpression.Const constKey) { + var rawResult = signature.in.root().argument(constKey.value); + return constOf(TypeConversion.convert(rawResult, signature.out)); + } else { + return new GetArg<>(signature, key); + } + } + } + /** An expression that converts the input to a desired output {@link DataType} at runtime. */ record Cast ( Signature signature, diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java index 52c3818c..1aa77b01 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java @@ -1,12 +1,10 @@ package com.onthegomap.planetiler.custommap.expression; -import com.onthegomap.planetiler.custommap.Contexts; import com.onthegomap.planetiler.custommap.TypeConversion; import com.onthegomap.planetiler.custommap.expression.stdlib.PlanetilerStdLib; +import com.onthegomap.planetiler.util.Memoized; import com.onthegomap.planetiler.util.Try; -import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import org.projectnessie.cel.extension.StringsLib; import org.projectnessie.cel.tools.Script; @@ -23,6 +21,8 @@ import org.projectnessie.cel.tools.ScriptHost; public class ConfigExpressionScript implements ConfigExpression { private static final Pattern EXPRESSION_PATTERN = Pattern.compile("^\\s*\\$\\{(.*)}\\s*$"); private static final Pattern ESCAPED_EXPRESSION_PATTERN = Pattern.compile("^\\s*\\\\+\\$\\{(.*)}\\s*$"); + private static final Memoized, ?> staticEvaluationCache = + Memoized.memoize(ConfigExpressionScript::doStaticEvaluate); private final Script script; private final Class returnType; private final String scriptText; @@ -107,9 +107,6 @@ public class ConfigExpressionScript implements Confi if (!description.declarations().isEmpty()) { scriptBuilder.withDeclarations(description.declarations()); } - if (!description.types().isEmpty()) { - scriptBuilder.withTypes(description.types()); - } var script = scriptBuilder.build(); return new ConfigExpressionScript<>(string, script, description, expected); @@ -132,17 +129,16 @@ public class ConfigExpressionScript implements Confi // ignore the parsed script object return this == o || (o instanceof ConfigExpressionScript config && returnType.equals(config.returnType) && - scriptText.equals(config.scriptText)); + scriptText.equals(config.scriptText) && + descriptor.equals(config.descriptor)); } @Override public int hashCode() { // ignore the parsed script object - return Objects.hash(returnType, scriptText); + return Objects.hash(returnType, scriptText, descriptor); } - private static final Map, Boolean> staticEvaluationCache = new ConcurrentHashMap<>(); - /** * Attempts to parse and evaluate this script in an environment with no variables. *

@@ -152,19 +148,12 @@ public class ConfigExpressionScript implements Confi public Try tryStaticEvaluate() { // type checking can be expensive when run hundreds of times simplifying expressions iteratively and it never // changes for a given script and input environment, so cache results between calls. - boolean canStaticEvaluate = - staticEvaluationCache.computeIfAbsent(this, config -> config.doTryStaticEvaluate().isSuccess()); - if (canStaticEvaluate) { - return doTryStaticEvaluate(); - } else { - return Try.failure(new IllegalStateException()); - } + return staticEvaluationCache.tryApply(this, returnType); } - private Try doTryStaticEvaluate() { - return Try - .apply( - () -> ConfigExpressionScript.parse(scriptText, Contexts.Root.DESCRIPTION, returnType).apply(Contexts.root())); + private O doStaticEvaluate() { + return ConfigExpressionScript.parse(scriptText, descriptor.root().description(), returnType) + .apply(descriptor.root()); } @Override diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java index 9dd8f22a..84bec706 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java @@ -22,4 +22,8 @@ public interface ScriptContext extends Function, WithTags { default TagValueProducer tagValueProducer() { return TagValueProducer.EMPTY; } + + default Object argument(String key) { + return null; + } } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java index 089bfdc0..244a4122 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java @@ -1,35 +1,44 @@ package com.onthegomap.planetiler.custommap.expression; import com.google.api.expr.v1alpha1.Decl; +import com.onthegomap.planetiler.custommap.Contexts; import java.util.List; import java.util.stream.Stream; /** * Type definitions for the environment that a script expression runs in. * - * @param types Additional types available. * @param declarations Global variable types * @param clazz Class of the input context type * @param The runtime expression context type */ -public record ScriptEnvironment (List types, List declarations, Class clazz) { +public record ScriptEnvironment (List declarations, Class clazz, Contexts.Root root) { + private static List concat(List a, List b) { + return Stream.concat(a.stream(), b.stream()).toList(); + } + private static List concat(List a, T[] b) { return Stream.concat(a.stream(), Stream.of(b)).toList(); } /** Returns a copy of this environment with a new input type {@code U}. */ public ScriptEnvironment forInput(Class newClazz) { - return new ScriptEnvironment<>(types, declarations, newClazz); + return new ScriptEnvironment<>(declarations, newClazz, root); } /** Returns a copy of this environment with a list of variable declarations appended to the global environment. */ public ScriptEnvironment withDeclarations(Decl... others) { - return new ScriptEnvironment<>(types, concat(declarations, others), clazz); + return new ScriptEnvironment<>(concat(declarations, others), clazz, root); } - /** Returns an empty environment with no variables defined. */ - public static ScriptEnvironment root() { - return new ScriptEnvironment<>(List.of(), List.of(), ScriptContext.class); + /** Returns a copy of this environment with a list of variable declarations appended to the global environment. */ + public ScriptEnvironment withDeclarations(List others) { + return new ScriptEnvironment<>(concat(declarations, others), clazz, root); + } + + /** Returns an empty environment with only static global variables (like command-line args) defined. */ + public static ScriptEnvironment root(Contexts.Root root) { + return new ScriptEnvironment<>(List.of(), ScriptContext.class, root); } /** Returns true if this contains a variable declaration for {@code variable}. */ diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java index a1f4416c..1964b963 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java @@ -6,6 +6,7 @@ import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.custommap.ConfiguredProfile; +import com.onthegomap.planetiler.custommap.Contexts; import com.onthegomap.planetiler.custommap.YAML; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import com.onthegomap.planetiler.geo.GeometryType; @@ -55,13 +56,13 @@ public class SchemaValidator { PrintStream output = System.out; output.println("OK"); - var paths = validateFromCli(schema, arguments, output); + var paths = validateFromCli(schema, output); if (watch) { output.println(); output.println("Watching filesystem for changes..."); var watcher = FileWatcher.newWatcher(paths.toArray(Path[]::new)); - watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, arguments, output)); + watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, output)); } } @@ -69,32 +70,32 @@ public class SchemaValidator { return t != null && (cause.isInstance(t) || hasCause(t.getCause(), cause)); } - static Set validateFromCli(Path schema, Arguments args, PrintStream output) { + static Set validateFromCli(Path schemaPath, PrintStream output) { Set pathsToWatch = new HashSet<>(); - pathsToWatch.add(schema); + pathsToWatch.add(schemaPath); output.println(); output.println("Validating..."); output.println(); SchemaValidator.Result result; try { - var parsedSchema = SchemaConfig.load(schema); - var examples = parsedSchema.examples(); + var schema = SchemaConfig.load(schemaPath); + var examples = schema.examples(); // examples can either be embedded in the yaml file, or referenced SchemaSpecification spec; if (examples instanceof String s) { var path = Path.of(s); if (!path.isAbsolute()) { - path = schema.resolveSibling(path); + path = schemaPath.resolveSibling(path); } // if referenced, make sure we watch that file for changes pathsToWatch.add(path); spec = SchemaSpecification.load(path); } else if (examples != null) { - spec = YAML.convertValue(parsedSchema, SchemaSpecification.class); + spec = YAML.convertValue(schema, SchemaSpecification.class); } else { spec = new SchemaSpecification(List.of()); } - result = validate(parsedSchema, spec, args); + result = validate(schema, spec); } catch (Exception exception) { Throwable rootCause = ExceptionUtils.getRootCause(exception); if (hasCause(exception, com.onthegomap.planetiler.custommap.expression.ParseException.class)) { @@ -175,13 +176,14 @@ public class SchemaValidator { * Returns the result of validating the profile defined by {@code schema} against the examples in * {@code specification}. */ - public static Result validate(SchemaConfig schema, SchemaSpecification specification, Arguments args) { - return validate(new ConfiguredProfile(schema), specification, args); + public static Result validate(SchemaConfig schema, SchemaSpecification specification) { + var context = Contexts.buildRootContext(Arguments.of().silence(), schema.args()); + return validate(new ConfiguredProfile(schema, context), specification, context.config()); } /** Returns the result of validating {@code profile} against the examples in {@code specification}. */ - public static Result validate(Profile profile, SchemaSpecification specification, Arguments args) { - var featureCollectorFactory = new FeatureCollector.Factory(PlanetilerConfig.from(args.silence()), Stats.inMemory()); + public static Result validate(Profile profile, SchemaSpecification specification, PlanetilerConfig config) { + var featureCollectorFactory = new FeatureCollector.Factory(config, Stats.inMemory()); return new Result(specification.examples().stream().map(example -> new ExampleResult(example, Try.apply(() -> { List issues = new ArrayList<>(); var input = example.input(); diff --git a/planetiler-custommap/src/main/resources/samples/shortbread.yml b/planetiler-custommap/src/main/resources/samples/shortbread.yml index 848b5c1e..ddf02018 100644 --- a/planetiler-custommap/src/main/resources/samples/shortbread.yml +++ b/planetiler-custommap/src/main/resources/samples/shortbread.yml @@ -2,6 +2,13 @@ schema_name: Shortbread schema_description: A basic, lean, general-purpose vector tile schema for OpenStreetMap data. See https://shortbread.geofabrik.de/ attribution: © OpenStreetMap contributors examples: shortbread.spec.yml +args: + area: + description: Geofabrik area to download + default: massachusetts + osm_url: + description: OSM URL to download + default: '${ "geofabrik:" + args.area }' sources: ocean: type: shapefile @@ -11,7 +18,7 @@ sources: url: https://shortbread.geofabrik.de/shapefiles/admin-points-4326.zip osm: type: osm - url: geofabrik:massachusetts + url: '${ args.osm_url }' definitions: # TODO let attribute definitions set multiple keys so you can just use `- *names` attributes: diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java index d17094b2..b7dec4a8 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java @@ -3,7 +3,6 @@ package com.onthegomap.planetiler.custommap; import static com.onthegomap.planetiler.expression.Expression.*; import static org.junit.jupiter.api.Assertions.assertEquals; -import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment; import com.onthegomap.planetiler.expression.Expression; import java.util.Map; import org.junit.jupiter.api.Test; @@ -13,7 +12,7 @@ class BooleanExpressionParserTest { private static void assertParse(String yaml, Expression parsed) { Object expression = YAML.load(yaml, Object.class); - var actual = BooleanExpressionParser.parse(expression, TVP, ScriptEnvironment.root()); + var actual = BooleanExpressionParser.parse(expression, TVP, TestContexts.ROOT_CONTEXT); assertEquals( parsed.simplify().generateJavaCode(), actual.simplify().generateJavaCode() diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java index 50fc457e..78f8804e 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java @@ -1,5 +1,6 @@ package com.onthegomap.planetiler.custommap; +import static com.onthegomap.planetiler.custommap.TestContexts.PROCESS_FEATURE; import static com.onthegomap.planetiler.custommap.expression.ConfigExpression.*; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -16,7 +17,7 @@ import org.junit.jupiter.params.provider.ValueSource; class ConfigExpressionParserTest { private static final TagValueProducer TVP = new TagValueProducer(Map.of()); private static final ConfigExpression.Signature FEATURE_SIGNATURE = - signature(Contexts.ProcessFeature.DESCRIPTION, Object.class); + signature(PROCESS_FEATURE, Object.class); private static void assertParse(String yaml, ConfigExpression parsed, Class clazz) { Object expression = YAML.load(yaml, Object.class); diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java index 227412d0..725ddb7e 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java @@ -9,13 +9,16 @@ import static org.junit.jupiter.api.Assertions.*; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.FeatureCollector.Feature; import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.custommap.configschema.DataSourceType; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import com.onthegomap.planetiler.custommap.util.TestConfigurableUtils; import com.onthegomap.planetiler.reader.SimpleFeature; import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.stats.Stats; import java.nio.file.Path; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -26,6 +29,7 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; class ConfiguredFeatureTest { + private PlanetilerConfig planetilerConfig = PlanetilerConfig.defaults(); private static final Function TEST_RESOURCE = TestConfigurableUtils::pathToTestResource; private static final Function SAMPLE_RESOURCE = TestConfigurableUtils::pathToSample; @@ -74,32 +78,35 @@ class ConfiguredFeatureTest { "bridge", "yes" ); - private static Profile loadConfig(Function pathFunction, String filename) { + private Profile loadConfig(Function pathFunction, String filename) { var staticAttributeConfig = pathFunction.apply(filename); var schema = SchemaConfig.load(staticAttributeConfig); - return new ConfiguredProfile(schema); + var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args()); + planetilerConfig = root.config(); + return new ConfiguredProfile(schema, root); } - private static Profile loadConfig(String config) { + private ConfiguredProfile loadConfig(String config) { var schema = SchemaConfig.load(config); - return new ConfiguredProfile(schema); + var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args()); + planetilerConfig = root.config(); + return new ConfiguredProfile(schema, root); } - private static void testFeature(Function pathFunction, String schemaFilename, SourceFeature sf, + private void testFeature(Function pathFunction, String schemaFilename, SourceFeature sf, Consumer test, int expectedMatchCount) { var profile = loadConfig(pathFunction, schemaFilename); testFeature(sf, test, expectedMatchCount, profile); } - private static void testFeature(String config, SourceFeature sf, Consumer test, int expectedMatchCount) { + private void testFeature(String config, SourceFeature sf, Consumer test, int expectedMatchCount) { var profile = loadConfig(config); testFeature(sf, test, expectedMatchCount, profile); } - private static void testFeature(SourceFeature sf, Consumer test, int expectedMatchCount, Profile profile) { - var config = PlanetilerConfig.defaults(); - var factory = new FeatureCollector.Factory(config, Stats.inMemory()); + private void testFeature(SourceFeature sf, Consumer test, int expectedMatchCount, Profile profile) { + var factory = new FeatureCollector.Factory(planetilerConfig, Stats.inMemory()); var fc = factory.get(sf); profile.processFeature(sf, fc); @@ -114,14 +121,14 @@ class ConfiguredFeatureTest { assertEquals(expectedMatchCount, length.get(), "Wrong number of features generated"); } - private static void testPolygon(String config, Map tags, + private void testPolygon(String config, Map tags, Consumer test, int expectedMatchCount) { var sf = SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList()); testFeature(config, sf, test, expectedMatchCount); } - private static void testPoint(String config, Map tags, + private void testPoint(String config, Map tags, Consumer test, int expectedMatchCount) { var sf = SimpleFeature.createFakeOsmFeature(newPoint(0, 0), tags, "osm", null, 1, emptyList()); @@ -129,21 +136,21 @@ class ConfiguredFeatureTest { } - private static void testLinestring(String config, + private void testLinestring(String config, Map tags, Consumer test, int expectedMatchCount) { var sf = SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList()); testFeature(config, sf, test, expectedMatchCount); } - private static void testPolygon(Function pathFunction, String schemaFilename, Map tags, + private void testPolygon(Function pathFunction, String schemaFilename, Map tags, Consumer test, int expectedMatchCount) { var sf = SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList()); testFeature(pathFunction, schemaFilename, sf, test, expectedMatchCount); } - private static void testLinestring(Function pathFunction, String schemaFilename, + private void testLinestring(Function pathFunction, String schemaFilename, Map tags, Consumer test, int expectedMatchCount) { var sf = SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList()); @@ -784,4 +791,235 @@ class ConfiguredFeatureTest { private void testInvalidSchema(String filename, String message) { assertThrows(RuntimeException.class, () -> loadConfig(TEST_INVALID_RESOURCE, filename), message); } + + @ParameterizedTest + @ValueSource(strings = { + "arg_value: argument", + "value: '${ args.argument }'", + "value: '${ args[\"argument\"] }'", + }) + void testUseArgumentNotDefined(String string) { + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of( + "argument", "value" + ))); + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + layers: + - id: testLayer + features: + - source: osm + geometry: point + include_when: + natural: water + attributes: + - key: key + %s + """.formatted(string); + + testPoint(config, Map.of( + "natural", "water" + ), feature -> { + assertEquals("value", feature.getAttrsAtZoom(14).get("key")); + }, 1); + } + + @ParameterizedTest + @ValueSource(strings = { + "arg_value: threads", + "value: '${ args.threads }'", + "value: '${ args[\"threads\"] }'", + }) + void testOverrideArgument(String string) { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + args: + threads: 2 + layers: + - id: testLayer + features: + - source: osm + geometry: point + include_when: + natural: water + attributes: + - key: key + type: integer + %s + """.formatted(string); + + testPoint(config, Map.of( + "natural", "water" + ), feature -> { + assertEquals(2, feature.getAttrsAtZoom(14).get("key")); + }, 1); + } + + @Test + void testDefineArgument() { + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of( + "custom_overridden_arg", "test2", + "custom_simple_overridden_int_arg", "3" + ))); + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + args: + custom_int_arg: + type: integer + description: test arg out + default: 12 + custom_boolean_arg: + type: boolean + description: test boolean arg out + default: true + custom_overridden_arg: + default: test + custom_simple_string_arg: value + custom_simple_int_arg: 1 + custom_simple_double_arg: 1.5 + custom_simple_bool_arg: true + custom_simple_overridden_int_arg: 2 + layers: + - id: testLayer + features: + - source: osm + geometry: point + include_when: + natural: water + attributes: + - key: int + value: '${ args.custom_int_arg }' + - key: bool + value: '${ args["custom_boolean_arg"] }' + - key: overridden + arg_value: custom_overridden_arg + - key: custom_simple_string_arg + arg_value: custom_simple_string_arg + - key: custom_simple_int_arg + arg_value: custom_simple_int_arg + - key: custom_simple_bool_arg + arg_value: custom_simple_bool_arg + - key: custom_simple_overridden_int_arg + arg_value: custom_simple_overridden_int_arg + - key: custom_simple_double_arg + arg_value: custom_simple_double_arg + - key: custom_simple_int_arg_as_string + arg_value: custom_simple_int_arg + type: string + """; + + testPoint(config, Map.of( + "natural", "water" + ), feature -> { + assertEquals(12L, feature.getAttrsAtZoom(14).get("int")); + assertEquals(true, feature.getAttrsAtZoom(14).get("bool")); + assertEquals("test2", feature.getAttrsAtZoom(14).get("overridden")); + + assertEquals("value", feature.getAttrsAtZoom(14).get("custom_simple_string_arg")); + assertEquals(1, feature.getAttrsAtZoom(14).get("custom_simple_int_arg")); + assertEquals("1", feature.getAttrsAtZoom(14).get("custom_simple_int_arg_as_string")); + assertEquals(1.5, feature.getAttrsAtZoom(14).get("custom_simple_double_arg")); + assertEquals(true, feature.getAttrsAtZoom(14).get("custom_simple_bool_arg")); + assertEquals(3, feature.getAttrsAtZoom(14).get("custom_simple_overridden_int_arg")); + }, 1); + } + + @Test + void testDefineArgumentsUsingExpressions() { + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of( + "custom_overridden_arg", "test2", + "custom_simple_overridden_int_arg", "3" + ))); + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + args: + arg1: + type: long + description: test arg out + default: '${ 2 - 1 }' + arg2: '${ 2 - 1 }' + arg3: + default: '${ 2 - 1 }' + arg4: ${ args.arg3 + 1 } + layers: + - id: testLayer + features: + - source: osm + geometry: point + attributes: + - key: arg1 + arg_value: arg1 + - key: arg2 + arg_value: arg2 + - key: arg3 + arg_value: arg3 + - key: arg4 + arg_value: arg4 + """; + + testPoint(config, Map.of(), feature -> { + assertEquals(1L, feature.getAttrsAtZoom(14).get("arg1")); + assertEquals(1L, feature.getAttrsAtZoom(14).get("arg2")); + assertEquals(1L, feature.getAttrsAtZoom(14).get("arg3")); + assertEquals(2L, feature.getAttrsAtZoom(14).get("arg4")); + }, 1); + } + + @Test + void testUseArgumentInSourceUrlPath() { + var config = """ + args: + area: rhode-island + url: '${ "geofabrik:" + args.area }' + sources: + osm: + type: osm + url: '${ args.url }' + layers: + - id: testLayer + features: + - source: osm + geometry: point + """; + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of( + "area", "boston" + ))); + assertEquals(List.of(new Source( + "osm", + DataSourceType.OSM, + "geofabrik:boston", + null + )), loadConfig(config).sources()); + + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of())); + assertEquals(List.of(new Source( + "osm", + DataSourceType.OSM, + "geofabrik:rhode-island", + null + )), loadConfig(config).sources()); + + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of())); + assertEquals("geofabrik_rhode_island.osm.pbf", loadConfig(config).sources().get(0).defaultFileUrl()); + + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of( + "url", "https://example.com/file.osm.pbf" + ))); + assertEquals("example.com_file.osm.pbf", loadConfig(config).sources().get(0).defaultFileUrl()); + } } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java new file mode 100644 index 00000000..d44a5637 --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java @@ -0,0 +1,228 @@ +package com.onthegomap.planetiler.custommap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.custommap.expression.ParseException; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ContextsTest { + + @Test + void testParseEmptyArgs() { + Map cliArgs = Map.of(); + Map schemaArgs = Map.of(); + Contexts.buildRootContext(Arguments.of(cliArgs), schemaArgs); + } + + @Test + void setPlanetilerConfigThroughCli() { + Map cliArgs = Map.of("threads", "9999"); + Map schemaArgs = Map.of("threads", "8888"); + var result = Contexts.buildRootContext(Arguments.of(cliArgs), schemaArgs); + assertEquals(9999, result.config().threads()); + } + + @Test + void setPlanetilerConfigThroughSchemaArgs() { + assertEquals(8888, Contexts.buildRootContext( + Arguments.of(Map.of()), + Map.of("threads", "8888") + ).config().threads()); + assertEquals(8888, Contexts.buildRootContext( + Arguments.of(Map.of()), + Map.of("threads", 8888) + ).config().threads()); + } + + @Test + void configDefinesDefaultArgumentValue() { + assertEquals(8888, Contexts.buildRootContext( + Arguments.of(Map.of()), + Map.of( + "arg", 8888 + ) + ).argument("arg")); + } + + @Test + void overrideDefaultArgFromConfig() { + assertEquals(9999, Contexts.buildRootContext( + Arguments.of(Map.of("arg", "9999")), + Map.of( + "arg", 8888 + ) + ).argument("arg")); + } + + @Test + void defineArgTypeInConfig() { + assertEquals(8888, Contexts.buildRootContext( + Arguments.of(), + Map.of( + "arg", Map.of( + "default", "8888", + "type", "integer" + ) + ) + ).argument("arg")); + } + + @Test + void implyTypeFromDefaultValue() { + assertEquals("8888", Contexts.buildRootContext( + Arguments.of(), + Map.of( + "arg", "8888" + ) + ).argument("arg")); + assertEquals("9999", Contexts.buildRootContext( + Arguments.of("arg", "9999"), + Map.of( + "arg", "8888" + ) + ).argument("arg")); + + assertEquals(8888, Contexts.buildRootContext( + Arguments.of(), + Map.of( + "arg", 8888 + ) + ).argument("arg")); + assertEquals(9999, Contexts.buildRootContext( + Arguments.of("arg", "9999"), + Map.of( + "arg", 8888 + ) + ).argument("arg")); + } + + @Test + void computeDefaultValueUsingExpression() { + assertEquals(2, Contexts.buildRootContext( + Arguments.of(), + Map.of( + "arg", Map.of( + "default", "${ 1+1 }", + "type", "integer" + ) + ) + ).argument("arg")); + } + + @Test + void implyTypeFromExpressionValue() { + assertEquals(2L, Contexts.buildRootContext( + Arguments.of(), + Map.of( + "arg", "${ 1+1 }" + ) + ).argument("arg")); + } + + @Test + void referenceOtherArgInExpression() { + Map configArgs = Map.of( + "arg1", 1, + "arg2", "${ args.arg1 + 1}" + ); + assertEquals(2L, Contexts.buildRootContext( + Arguments.of(), + configArgs + ).argument("arg2")); + assertEquals(3L, Contexts.buildRootContext( + Arguments.of(Map.of("arg1", "2")), + configArgs + ).argument("arg2")); + assertEquals(10L, Contexts.buildRootContext( + Arguments.of(Map.of("arg2", "10")), + configArgs + ).argument("arg2")); + } + + @Test + void referenceOtherArgInExpressionTwice() { + Map configArgs = Map.of( + "arg1", 1, + "arg2", "${ args.arg1 + 1}", + "arg3", "${ args.arg2 + 1}" + ); + assertEquals(3L, Contexts.buildRootContext( + Arguments.of(), + configArgs + ).argument("arg3")); + } + + @Test + void failOnInfiniteLoop() { + Map configArgs = Map.of( + "arg1", Map.of( + "default", "${ args.arg3 + 1 }", + "type", "long" + ), + "arg2", "${ args.arg1 + 1}", + "arg3", "${ args.arg2 + 1}" + ); + var empty = Arguments.of(); + assertThrows(ParseException.class, () -> Contexts.buildRootContext( + empty, + configArgs + )); + // but if you break the chain it's OK? + assertEquals(3L, Contexts.buildRootContext( + Arguments.of(Map.of("arg1", "1")), + configArgs + ).argument("arg3")); + } + + @Test + void setPlanetilerConfigFromOtherArg() { + assertEquals(8888, Contexts.buildRootContext( + Arguments.of(Map.of()), + Map.of( + "other", "8888", + "threads", "${ args.other }" + ) + ).config().threads()); + } + + @Test + void testCantRedefineBuiltin() { + var fromCli = Arguments.of(Map.of()); + Map fromConfig = Map.of( + "threads", Map.of( + "default", 4, + "type", "string" + ) + ); + assertThrows(ParseException.class, () -> Contexts.buildRootContext(fromCli, fromConfig)); + } + + @Test + void testDefineRequiredArg() { + var argsWithoutValue = Arguments.of(Map.of()); + var argsWithValue = Arguments.of(Map.of("arg", "3")); + Map fromConfig = Map.of( + "arg", Map.of( + "type", "integer", + "description", "desc" + ) + ); + assertThrows(ParseException.class, () -> Contexts.buildRootContext(argsWithoutValue, fromConfig)); + assertEquals(3, Contexts.buildRootContext(argsWithValue, fromConfig).argument("arg")); + } + + @Test + void setPlanetilerConfigFromOtherPlanetilerConfig() { + var root = Contexts.buildRootContext( + Arguments.of(Map.of()), + Map.of( + "mmap", "${ args.threads < 1 }" + ) + ); + assertTrue(root.config().mmapTempStorage()); + } +} diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java index 9a6a802b..3400fe66 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java @@ -2,7 +2,6 @@ package com.onthegomap.planetiler.custommap; import static org.junit.jupiter.api.DynamicTest.dynamicTest; -import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import com.onthegomap.planetiler.custommap.validator.SchemaSpecification; import com.onthegomap.planetiler.custommap.validator.SchemaValidator; @@ -21,8 +20,7 @@ class SchemaTests { var base = Path.of("src", "main", "resources", "samples"); var result = SchemaValidator.validate( SchemaConfig.load(base.resolve(schema)), - SchemaSpecification.load(base.resolve(spec)), - Arguments.of() + SchemaSpecification.load(base.resolve(spec)) ); return result.results().stream() .map(test -> dynamicTest(test.example().name(), () -> { diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java index 699606c4..ade195cb 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java @@ -3,6 +3,7 @@ package com.onthegomap.planetiler.custommap; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import java.io.IOException; import java.nio.file.Files; @@ -33,8 +34,9 @@ class SchemaYAMLLoadTest { for (Path schemaFile : schemaFiles) { var schemaConfig = SchemaConfig.load(schemaFile); + var root = Contexts.buildRootContext(Arguments.of(), schemaConfig.args()); assertNotNull(schemaConfig, () -> "Failed to unmarshall " + schemaFile.toString()); - assertNotNull(new ConfiguredProfile(schemaConfig), () -> "Failed to load profile from " + schemaFile.toString()); + new ConfiguredProfile(schemaConfig, root); } } } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java index 1c6d0e4d..ac217aee 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java @@ -1,5 +1,6 @@ package com.onthegomap.planetiler.custommap; +import static com.onthegomap.planetiler.custommap.TestContexts.ROOT; import static org.junit.jupiter.api.Assertions.assertEquals; import com.onthegomap.planetiler.geo.GeoUtils; @@ -16,7 +17,7 @@ class TagValueProducerTest { assertEquals(expected, tvp.valueForKey(wrapped, key)); assertEquals(expected, tvp.valueGetterForKey(key).apply(wrapped, key)); assertEquals(expected, tvp.valueProducerForKey(key) - .apply(new Contexts.ProcessFeature(SimpleFeature.create(GeoUtils.EMPTY_GEOMETRY, tags), tvp) + .apply(ROOT.createProcessFeatureContext(SimpleFeature.create(GeoUtils.EMPTY_GEOMETRY, tags), tvp) .createPostMatchContext(List.of()))); } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java new file mode 100644 index 00000000..246db9fc --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java @@ -0,0 +1,14 @@ +package com.onthegomap.planetiler.custommap; + +import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment; + +public class TestContexts { + public static final Contexts.Root ROOT = Contexts.emptyRoot(); + public static final ScriptEnvironment ROOT_CONTEXT = ROOT.description(); + public static final ScriptEnvironment PROCESS_FEATURE = + Contexts.ProcessFeature.description(ROOT); + public static final ScriptEnvironment FEATURE_POST_MATCH = + Contexts.FeaturePostMatch.description(ROOT); + public static final ScriptEnvironment FEATURE_ATTRIBUTE = + Contexts.FeatureAttribute.description(ROOT); +} diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java index 95e93b1c..29d72730 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java @@ -4,7 +4,7 @@ import static com.onthegomap.planetiler.expression.Expression.and; import static com.onthegomap.planetiler.expression.Expression.or; import static org.junit.jupiter.api.Assertions.assertEquals; -import com.onthegomap.planetiler.custommap.Contexts; +import com.onthegomap.planetiler.custommap.TestContexts; import com.onthegomap.planetiler.expression.Expression; import org.junit.jupiter.api.Test; @@ -12,11 +12,20 @@ class BooleanExpressionScriptTest { @Test void testSimplify() { assertEquals(Expression.TRUE, - and(or(BooleanExpressionScript.script("1+1<3", Contexts.Root.DESCRIPTION))).simplify()); + and(or(BooleanExpressionScript.script("1+1<3", TestContexts.ROOT_CONTEXT))).simplify()); assertEquals(Expression.FALSE, - and(or(BooleanExpressionScript.script("1+1>3", Contexts.Root.DESCRIPTION))).simplify()); + and(or(BooleanExpressionScript.script("1+1>3", TestContexts.ROOT_CONTEXT))).simplify()); - var other = BooleanExpressionScript.script("feature.tags.natural", Contexts.ProcessFeature.DESCRIPTION); + var other = + BooleanExpressionScript.script("feature.tags.natural", TestContexts.PROCESS_FEATURE); assertEquals(other, and(or(other)).simplify()); } + + @Test + void testSimplifyInlinesArguments() { + assertEquals(Expression.TRUE, + and(or(BooleanExpressionScript.script("args.threads > 0", TestContexts.ROOT_CONTEXT))).simplify()); + assertEquals(Expression.FALSE, + and(or(BooleanExpressionScript.script("args.threads < 0", TestContexts.ROOT_CONTEXT))).simplify()); + } } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java index 39cfc20a..4d0bdad1 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java @@ -1,6 +1,9 @@ package com.onthegomap.planetiler.custommap.expression; import static com.onthegomap.planetiler.TestUtils.newPoint; +import static com.onthegomap.planetiler.custommap.TestContexts.FEATURE_POST_MATCH; +import static com.onthegomap.planetiler.custommap.TestContexts.PROCESS_FEATURE; +import static com.onthegomap.planetiler.custommap.TestContexts.ROOT_CONTEXT; import static com.onthegomap.planetiler.custommap.expression.ConfigExpression.*; import static com.onthegomap.planetiler.expression.Expression.matchAny; import static com.onthegomap.planetiler.expression.Expression.or; @@ -8,8 +11,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.custommap.Contexts; import com.onthegomap.planetiler.custommap.TagValueProducer; +import com.onthegomap.planetiler.custommap.TestContexts; import com.onthegomap.planetiler.expression.DataType; import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.expression.MultiExpression; @@ -19,10 +24,9 @@ import java.util.Map; import org.junit.jupiter.api.Test; class ConfigExpressionTest { - private static final ConfigExpression.Signature ROOT = - signature(Contexts.Root.DESCRIPTION, Integer.class); + private static final ConfigExpression.Signature ROOT = signature(ROOT_CONTEXT, Integer.class); private static final ConfigExpression.Signature FEATURE_SIGNATURE = - signature(Contexts.ProcessFeature.DESCRIPTION, Integer.class); + signature(PROCESS_FEATURE, Integer.class); @Test void testConst() { @@ -32,7 +36,7 @@ class ConfigExpressionTest { @Test void testVariable() { var feature = SimpleFeature.create(newPoint(0, 0), Map.of("a", "b", "c", 1), "source", "source_layer", 99); - var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of())); + var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of())); // simple match assertEquals("source", variable(FEATURE_SIGNATURE.withOutput(String.class), "feature.source").apply(context)); assertEquals("source_layer", @@ -45,32 +49,32 @@ class ConfigExpressionTest { @Test void testCoalesce() { - assertNull(coalesce(List.of()).apply(Contexts.root())); + assertNull(coalesce(List.of()).apply(TestContexts.ROOT)); assertNull(coalesce( List.of( constOf(null) - )).apply(Contexts.root())); + )).apply(TestContexts.ROOT)); assertEquals(2, coalesce( List.of( constOf(null), constOf(2) - )).apply(Contexts.root())); + )).apply(TestContexts.ROOT)); assertEquals(1, coalesce( List.of( constOf(1), constOf(2) - )).apply(Contexts.root())); + )).apply(TestContexts.ROOT)); } @Test void testDynamic() { - assertEquals(1, script(ROOT, "5 - 4").apply(Contexts.root())); + assertEquals(1, script(ROOT, "5 - 4").apply(TestContexts.ROOT)); } @Test void testMatch() { var feature = SimpleFeature.create(newPoint(0, 0), Map.of("a", "b", "c", 1)); - var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of())); + var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of())); // simple match assertEquals(2, match(FEATURE_SIGNATURE, MultiExpression.of(List.of( MultiExpression.entry(constOf(1), @@ -280,33 +284,30 @@ class ConfigExpressionTest { assertEquals( "123", getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")).apply( - new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()))) + TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()))) ); assertEquals( 123, getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")) - .apply(new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of("abc", "integer")))) + .apply(TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of("abc", "integer")))) ); assertEquals( 123, - getTag(signature(Contexts.FeaturePostMatch.DESCRIPTION, Object.class), constOf("abc")) - .apply(new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of("abc", "integer"))) + getTag(signature(FEATURE_POST_MATCH, Object.class), constOf("abc")) + .apply(TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of("abc", "integer"))) .createPostMatchContext(List.of())) ); - assertEquals( - null, - getTag(signature(Contexts.Root.DESCRIPTION, Object.class), constOf("abc")) - .apply(Contexts.root()) - ); + assertNull(getTag(signature(ROOT_CONTEXT, Object.class), constOf("abc")) + .apply(TestContexts.ROOT)); } @Test void testCastGetTag() { var feature = SimpleFeature.create(newPoint(0, 0), Map.of("abc", "123"), "source", "source_layer", 99); - var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of())); + var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of())); var expression = cast( FEATURE_SIGNATURE.withOutput(Integer.class), getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")), @@ -328,7 +329,7 @@ class ConfigExpressionTest { constOf("123"), DataType.GET_INT ); - assertEquals(123, expression.apply(Contexts.root())); + assertEquals(123, expression.apply(TestContexts.ROOT)); } @Test @@ -341,4 +342,31 @@ class ConfigExpressionTest { ).simplify() ); } + + @Test + void testSimplifyGetArg() { + var args = Arguments.of(Map.of("arg", "12")); + var root = Contexts.buildRootContext(args, Map.of()); + var context = signature(root.description(), Integer.class); + assertEquals(constOf(12), + getArg( + context, + constOf("arg") + ).simplify() + ); + + assertEquals(constOf(12), + script( + context, + "args.arg" + ).simplify() + ); + + assertEquals(constOf(12), + script( + context, + "args['arg']" + ).simplify() + ); + } } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java index 19117111..c25ebc46 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java @@ -63,4 +63,13 @@ class DataTypeTest { assertEquals(0, DataType.from("direction").convertFrom("no")); assertEquals(0, DataType.from("direction").convertFrom("false")); } + + @Test + void testTypeOf() { + assertEquals(DataType.GET_DOUBLE, DataType.typeOf(1.5)); + assertEquals(DataType.GET_LONG, DataType.typeOf(1L)); + assertEquals(DataType.GET_INT, DataType.typeOf(1)); + assertEquals(DataType.GET_BOOLEAN, DataType.typeOf(true)); + assertEquals(DataType.GET_STRING, DataType.typeOf("string")); + } } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java index ce62fd5a..92c9e4af 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java @@ -3,6 +3,9 @@ package com.onthegomap.planetiler.custommap.expression; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.custommap.Contexts; +import java.util.Map; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -52,10 +55,25 @@ class ExpressionTests { "min([1.1, 2.2, 3.3])|1.1|double", "max([1])|1|long", "min([1])|1|long", + + "args.arg_from_config|1|long", + "args.arg_from_config + 1|2|long", + "args[\"arg_from_config\"]|1|long", + "args[\"arg_from_config\"] + 1|2|long", + "args.overridden_arg_from_config|4|long", + "args[\"overridden_arg_from_config\"] + 1|5|long", + "args.arg_from_cli|2|string", + "args[\"arg_from_cli\"]|2|string", }, delimiter = '|') void testExpression(String in, String expected, String type) { - var expression = ConfigExpressionScript.parse(in, ScriptEnvironment.root()); - var result = expression.apply(ScriptContext.empty()); + Map configFromSchema = Map.of("arg_from_config", 1, "overridden_arg_from_config", 3); + Map configFromCli = Map.of("arg_from_cli", "2", "overridden_arg_from_config", "4"); + var context = Contexts.buildRootContext( + Arguments.of(configFromCli), + configFromSchema + ); + var expression = ConfigExpressionScript.parse(in, context.description()); + var result = expression.apply(context); switch (type) { case "long" -> assertEquals(Long.valueOf(expected), result); case "double" -> assertEquals(Double.valueOf(expected), result); diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java index 97501f80..5c38b6ed 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,12 +24,10 @@ class SchemaValidatorTest { record Result(SchemaValidator.Result output, String cliOutput) {} - Result validate(String schema, String spec) throws IOException { - var args = Arguments.of(); + private Result validate(String schema, String spec) throws IOException { var result = SchemaValidator.validate( SchemaConfig.load(schema), - SchemaSpecification.load(spec), - args + SchemaSpecification.load(spec) ); for (var example : result.results()) { if (example.issues().isFailure()) { @@ -39,30 +36,29 @@ class SchemaValidatorTest { } // also exercise the cli writer and return what it would have printed to stdout var cliOutput = validateCli(Files.writeString(tmpDir.resolve("schema"), - schema + "\nexamples: " + Files.writeString(tmpDir.resolve("spec.yml"), spec)), args); + schema + "\nexamples: " + Files.writeString(tmpDir.resolve("spec.yml"), spec))); // also test the case where the examples are embedded in the schema itself assertEquals( cliOutput, - validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\n" + spec), args) + validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\n" + spec)) ); // also test where examples points to a relative path (written in previous step) assertEquals( cliOutput, - validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\nexamples: spec.yml"), args) + validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\nexamples: spec.yml")) ); return new Result(result, cliOutput); } - private String validateCli(Path path, Arguments args) { + private String validateCli(Path path) { try ( var baos = new ByteArrayOutputStream(); var printStream = new PrintStream(baos, true, StandardCharsets.UTF_8) ) { SchemaValidator.validateFromCli( path, - args, printStream ); return baos.toString(StandardCharsets.UTF_8); @@ -180,4 +176,45 @@ class SchemaValidatorTest { ); assertFalse(results.output.ok(), results.toString()); } + + @Test + void testValidationWiresInArguments() throws IOException { + var results = validate( + """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + args: + key: default_value + layers: + - id: water + features: + - source: osm + geometry: polygon + include_when: + natural: water + attributes: + - key: from_arg + arg_value: key + - key: threads + value: '${ args.threads + 1 }' + """, + """ + examples: + - name: test output + input: + source: osm + geometry: polygon + tags: + natural: water + output: + layer: water + tags: + from_arg: default_value + threads: %s + """.formatted(1 + Math.max(Runtime.getRuntime().availableProcessors(), 2)) + ); + assertTrue(results.output.ok(), results.toString()); + } } diff --git a/quickstart.sh b/quickstart.sh index a544b3e8..0c298ab6 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -16,6 +16,7 @@ DRY_RUN="" VERSION="latest" DOCKER_DIR="$(pwd)/data" TASK="openmaptiles" +BUILD="true" # Parse args into env vars while [[ $# -gt 0 ]]; do @@ -25,6 +26,7 @@ while [[ $# -gt 0 ]]; do --dockerdir) DOCKER_DIR="$2"; shift ;; --jar) METHOD="jar" ;; --build|--source) METHOD="build" ;; + --skipbuild) METHOD="build"; BUILD="false" ;; --version=*) VERSION="${1#*=}" ;; --version) VERSION="$2"; shift ;; @@ -134,8 +136,10 @@ case $METHOD in run "$JAVA" "${JVM_ARGS}" -jar planetiler.jar "${PLANETILER_ARGS[@]}" ;; build) - echo "Building planetiler..." - run ./mvnw -q -DskipTests --projects planetiler-dist -am clean package + if [ "$BUILD" == "true" ]; then + echo "Building planetiler..." + run ./mvnw -q -DskipTests --projects planetiler-dist -am clean package + fi run "$JAVA" "${JVM_ARGS}" -jar planetiler-dist/target/*with-deps.jar "${PLANETILER_ARGS[@]}" ;; esac