sources = new ArrayList<>();
+ schema.sources().forEach((key, value) -> {
+ var url = ConfigExpressionParser.tryStaticEvaluate(rootContext, value.url(), String.class).get();
+ var path = ConfigExpressionParser.tryStaticEvaluate(rootContext, value.localPath(), String.class).get();
+ sources.add(new Source(key, value.type(), url, path == null ? null : Path.of(path)));
+ });
+ return sources;
}
}
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
index 04ed7384..73a9e74d 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
@@ -1,14 +1,34 @@
package com.onthegomap.planetiler.custommap;
+import com.google.api.expr.v1alpha1.Constant;
+import com.google.api.expr.v1alpha1.Decl;
+import com.google.api.expr.v1alpha1.Type;
+import com.google.protobuf.NullValue;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.custommap.expression.ParseException;
import com.onthegomap.planetiler.custommap.expression.ScriptContext;
import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
+import com.onthegomap.planetiler.expression.DataType;
import com.onthegomap.planetiler.reader.SourceFeature;
import com.onthegomap.planetiler.reader.WithGeometryType;
import com.onthegomap.planetiler.reader.WithTags;
+import com.onthegomap.planetiler.util.Try;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.projectnessie.cel.checker.Decls;
import org.projectnessie.cel.common.types.NullT;
+import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry;
+import org.projectnessie.cel.common.types.ref.TypeAdapter;
+import org.projectnessie.cel.common.types.ref.Val;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Wrapper objects that provide all available inputs to different parts of planetiler schema configs at runtime.
@@ -17,35 +37,256 @@ import org.projectnessie.cel.common.types.NullT;
* that all global variables from a parent context are also available to its child context.
*/
public class Contexts {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Contexts.class);
private static Object wrapNullable(Object nullable) {
return nullable == null ? NullT.NullValue : nullable;
}
- public static Root root() {
- return new Root();
+ public static Root emptyRoot() {
+ return new Root(Arguments.of().silence(), Map.of());
+ }
+
+ /**
+ * Returns a {@link Root} context built from {@code schemaArgs} argument definitions and {@code origArguments}
+ * arguments provided from the command-line/environment.
+ *
+ * Arguments may depend on the value of other arguments so this iteratively evaluates the arguments until their values
+ * settle.
+ *
+ * @throws ParseException if the argument definitions are malformed, or if there's an infinite loop
+ */
+ public static Contexts.Root buildRootContext(Arguments origArguments, Map schemaArgs) {
+ boolean loggingEnabled = !origArguments.silenced();
+ origArguments.silence();
+ Map argDescriptions = new LinkedHashMap<>();
+ Map unparsedSchemaArgs = new HashMap<>(schemaArgs);
+ Map parsedSchemaArgs = new HashMap<>(origArguments.toMap());
+ Contexts.Root result = new Root(origArguments, parsedSchemaArgs);
+ Arguments arguments = origArguments;
+ int iters = 0;
+ // arguments may reference the value of other arguments, so continue parsing until they all succeed...
+ while (!unparsedSchemaArgs.isEmpty()) {
+ final var root = result;
+ final var args = arguments;
+ Map failures = new HashMap<>();
+
+ Map.copyOf(unparsedSchemaArgs).forEach((key, value) -> {
+ boolean builtin = root.builtInArgs.contains(key);
+ String description;
+ Object defaultValueObject;
+ DataType type = null;
+ if (value instanceof Map, ?> map) {
+ if (builtin) {
+ throw new ParseException("Cannot override built-in argument: " + key);
+ }
+ var typeObject = map.get("type");
+ if (typeObject != null) {
+ type = DataType.from(Objects.toString(typeObject));
+ }
+ var descriptionObject = map.get("description");
+ description = descriptionObject == null ? "no description provided" : descriptionObject.toString();
+ defaultValueObject = map.get("default");
+ if (type != null) {
+ var fromArgs = args.getString(key, description, null);
+ if (fromArgs != null) {
+ parsedSchemaArgs.put(key, type.convertFrom(fromArgs));
+ }
+ }
+ } else {
+ defaultValueObject = value;
+ description = "no description provided";
+ }
+ argDescriptions.put(key, description);
+ Try defaultValue = ConfigExpressionParser.tryStaticEvaluate(root, defaultValueObject, Object.class);
+ if (defaultValue.isSuccess()) {
+ Object raw = defaultValue.get();
+ String asString = Objects.toString(raw);
+ if (type == null) {
+ type = DataType.typeOf(raw);
+ }
+ var stringResult = args.getString(key, description, asString);
+ Object castedResult = type.convertFrom(stringResult);
+ if (stringResult == null) {
+ throw new ParseException("Missing required parameter: " + key + "(" + description + ")");
+ } else if (castedResult == null) {
+ throw new ParseException("Cannot convert value for " + key + " to " + type.id() + ": " + stringResult);
+ }
+ parsedSchemaArgs.put(key, castedResult);
+ unparsedSchemaArgs.remove(key);
+ } else {
+ failures.put(key, defaultValue.exception());
+ }
+ });
+
+ arguments = origArguments.orElse(Arguments.of(parsedSchemaArgs.entrySet().stream().collect(Collectors.toMap(
+ Map.Entry::getKey,
+ e -> Objects.toString(e.getValue()))
+ )));
+ result = new Root(arguments, parsedSchemaArgs);
+ if (iters++ > 100) {
+ failures
+ .forEach(
+ (key, failure) -> LOGGER.error("Error computing {}:\n{}", key,
+ ExceptionUtils.getRootCause(failure).toString().indent(4)));
+ throw new ParseException("Infinite loop while processing arguments: " + unparsedSchemaArgs.keySet());
+ }
+ }
+ var finalArguments = loggingEnabled ? arguments.withExactlyOnceLogging() : arguments.silence();
+ if (loggingEnabled) {
+ argDescriptions.forEach((key, description) -> finalArguments.getString(key, description, null));
+ }
+ return new Root(finalArguments, parsedSchemaArgs);
}
/**
* Root context available everywhere in a planetiler schema config.
+ *
+ * Holds argument values parsed from the schema config and command-line args.
*/
- public record Root() implements ScriptContext {
+ public static final class Root implements ScriptContext {
+ private static final TypeAdapter TYPE_ADAPTER = ProtoTypeRegistry.newRegistry();
+ private final Arguments arguments;
+ private final PlanetilerConfig config;
+ private final ScriptEnvironment description;
+ private final Map bindings = new HashMap<>();
+ private final Map argumentValues = new HashMap<>();
+ public final Set builtInArgs;
- // TODO add argument parsing
- public static final ScriptEnvironment DESCRIPTION =
- ScriptEnvironment.root().forInput(Root.class);
+ public Arguments arguments() {
+ return arguments;
+ }
+
+ public PlanetilerConfig config() {
+ return config;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return this == o || (o instanceof Root root && argumentValues.equals(root.argumentValues));
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(argumentValues);
+ }
+
+ @Override
+ public Object argument(String key) {
+ return argumentValues.get(key);
+ }
+
+ private Root(Arguments arguments, Map schemaArgs) {
+ this.arguments = arguments;
+ this.config = PlanetilerConfig.from(arguments);
+ argumentValues.put("threads", config.threads());
+ argumentValues.put("write_threads", config.featureWriteThreads());
+ argumentValues.put("process_threads", config.featureProcessThreads());
+ argumentValues.put("feature_read_threads", config.featureReadThreads());
+ // args.put("loginterval", config.logInterval());
+ argumentValues.put("minzoom", config.minzoom());
+ argumentValues.put("maxzoom", config.maxzoom());
+ argumentValues.put("render_maxzoom", config.maxzoomForRendering());
+ argumentValues.put("skip_mbtiles_index_creation", config.skipIndexCreation());
+ argumentValues.put("optimize_db", config.optimizeDb());
+ argumentValues.put("emit_tiles_in_order", config.emitTilesInOrder());
+ argumentValues.put("force", config.force());
+ argumentValues.put("gzip_temp", config.gzipTempStorage());
+ argumentValues.put("mmap_temp", config.mmapTempStorage());
+ argumentValues.put("sort_max_readers", config.sortMaxReaders());
+ argumentValues.put("sort_max_writers", config.sortMaxWriters());
+ argumentValues.put("nodemap_type", config.nodeMapType());
+ argumentValues.put("nodemap_storage", config.nodeMapStorage());
+ argumentValues.put("nodemap_madvise", config.nodeMapMadvise());
+ argumentValues.put("multipolygon_geometry_storage", config.multipolygonGeometryStorage());
+ argumentValues.put("multipolygon_geometry_madvise", config.multipolygonGeometryMadvise());
+ argumentValues.put("http_user_agent", config.httpUserAgent());
+ // args.put("http_timeout", config.httpTimeout());
+ argumentValues.put("http_retries", config.httpRetries());
+ argumentValues.put("download_chunk_size_mb", config.downloadChunkSizeMB());
+ argumentValues.put("download_threads", config.downloadThreads());
+ argumentValues.put("min_feature_size_at_max_zoom", config.minFeatureSizeAtMaxZoom());
+ argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom());
+ argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());
+ argumentValues.put("simplify_tolerance", config.simplifyToleranceBelowMaxZoom());
+ argumentValues.put("compact_db", config.compactDb());
+ argumentValues.put("skip_filled_tiles", config.skipFilledTiles());
+ argumentValues.put("tile_warning_size_mb", config.tileWarningSizeBytes());
+ builtInArgs = Set.copyOf(argumentValues.keySet());
+
+ schemaArgs.forEach(argumentValues::putIfAbsent);
+ config.arguments().toMap().forEach(argumentValues::putIfAbsent);
+
+ argumentValues.forEach((k, v) -> bindings.put("args." + k, TYPE_ADAPTER.nativeToValue(v)));
+ bindings.put("args", TYPE_ADAPTER.nativeToValue(this.argumentValues));
+
+ description = ScriptEnvironment.root(this).forInput(Root.class)
+ .withDeclarations(
+ argumentValues.entrySet().stream()
+ .map(entry -> decl(entry.getKey(), entry.getValue()))
+ .toList()
+ ).withDeclarations(
+ Decls.newVar("args", Decls.newMapType(Decls.String, Decls.Any))
+ );
+ }
+
+ private Decl decl(String name, Object value) {
+ Type type;
+ var builder = Constant.newBuilder();
+ if (value instanceof String s) {
+ builder.setStringValue(s);
+ type = Decls.String;
+ } else if (value instanceof Boolean b) {
+ builder.setBoolValue(b);
+ type = Decls.Bool;
+ } else if (value instanceof Long || value instanceof Integer) {
+ builder.setInt64Value(((Number) value).longValue());
+ type = Decls.Int;
+ } else if (value instanceof Double || value instanceof Float) {
+ builder.setDoubleValue(((Number) value).doubleValue());
+ type = Decls.Double;
+ } else if (value == null) {
+ builder.setNullValue(NullValue.NULL_VALUE);
+ type = Decls.Null;
+ } else {
+ throw new IllegalArgumentException(
+ "Unrecognized constant type: " + value + " (" + value.getClass().getName() + ")");
+ }
+ return Decls.newConst("args." + name, type, builder.build());
+ }
+
+ public ScriptEnvironment description() {
+ return description;
+ }
@Override
public Object apply(String input) {
+ return bindings.get(input);
+ }
+
+ public ProcessFeature createProcessFeatureContext(SourceFeature sourceFeature, TagValueProducer tagValueProducer) {
+ return new ProcessFeature(this, sourceFeature, tagValueProducer);
+ }
+ }
+
+ private interface NestedContext extends ScriptContext {
+
+ default Root root() {
return null;
}
+
+ @Override
+ default Object argument(String key) {
+ return root().argument(key);
+ }
}
/**
* Makes nested contexts adhere to {@link WithTags} and {@link WithGeometryType} by recursively fetching source
* feature from the root context.
*/
- private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType {
+ private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType, NestedContext {
+
default FeatureContext parent() {
return null;
}
@@ -54,6 +295,11 @@ public class Contexts {
return parent().feature();
}
+ @Override
+ default Root root() {
+ return parent().root();
+ }
+
@Override
default Map tags() {
return feature().tags();
@@ -87,7 +333,10 @@ public class Contexts {
* @param feature The input feature being processed
* @param tagValueProducer Common parsing for input feature tags
*/
- public record ProcessFeature(@Override SourceFeature feature, @Override TagValueProducer tagValueProducer)
+ public record ProcessFeature(
+ @Override Root root, @Override SourceFeature feature,
+ @Override TagValueProducer tagValueProducer
+ )
implements FeatureContext {
private static final String FEATURE_TAGS = "feature.tags";
@@ -95,14 +344,16 @@ public class Contexts {
private static final String FEATURE_SOURCE = "feature.source";
private static final String FEATURE_SOURCE_LAYER = "feature.source_layer";
- public static final ScriptEnvironment DESCRIPTION = ScriptEnvironment.root()
- .forInput(ProcessFeature.class)
- .withDeclarations(
- Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)),
- Decls.newVar(FEATURE_ID, Decls.Int),
- Decls.newVar(FEATURE_SOURCE, Decls.String),
- Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String)
- );
+ public static ScriptEnvironment description(Root root) {
+ return root.description()
+ .forInput(ProcessFeature.class)
+ .withDeclarations(
+ Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)),
+ Decls.newVar(FEATURE_ID, Decls.Int),
+ Decls.newVar(FEATURE_SOURCE, Decls.String),
+ Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String)
+ );
+ }
@Override
public Object apply(String key) {
@@ -127,7 +378,7 @@ public class Contexts {
/**
* Context available after a feature has been matched.
- *
+ *
* Adds {@code match_key} and {@code match_value} variables that capture which tag key/value caused the feature to be
* included.
*
@@ -139,12 +390,14 @@ public class Contexts {
private static final String MATCH_KEY = "match_key";
private static final String MATCH_VALUE = "match_value";
- public static final ScriptEnvironment DESCRIPTION = ProcessFeature.DESCRIPTION
- .forInput(FeaturePostMatch.class)
- .withDeclarations(
- Decls.newVar(MATCH_KEY, Decls.String),
- Decls.newVar(MATCH_VALUE, Decls.Any)
- );
+ public static ScriptEnvironment description(Root root) {
+ return ProcessFeature.description(root)
+ .forInput(FeaturePostMatch.class)
+ .withDeclarations(
+ Decls.newVar(MATCH_KEY, Decls.String),
+ Decls.newVar(MATCH_VALUE, Decls.Any)
+ );
+ }
@Override
public Object apply(String key) {
@@ -182,10 +435,14 @@ public class Contexts {
* @param value Value of the attribute
*/
public record FeatureAttribute(@Override FeaturePostMatch parent, Object value) implements FeatureContext {
+
private static final String VALUE = "value";
- public static final ScriptEnvironment DESCRIPTION = FeaturePostMatch.DESCRIPTION
- .forInput(FeatureAttribute.class)
- .withDeclarations(Decls.newVar(VALUE, Decls.Any));
+
+ public static ScriptEnvironment description(Root root) {
+ return FeaturePostMatch.description(root)
+ .forInput(FeatureAttribute.class)
+ .withDeclarations(Decls.newVar(VALUE, Decls.Any));
+ }
@Override
public Object apply(String key) {
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java
new file mode 100644
index 00000000..102e1209
--- /dev/null
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Source.java
@@ -0,0 +1,23 @@
+package com.onthegomap.planetiler.custommap;
+
+import com.onthegomap.planetiler.custommap.configschema.DataSourceType;
+import java.nio.file.Path;
+
+/** A parsed source definition from a config file. */
+public record Source(
+ String id,
+ DataSourceType type,
+ String url,
+ Path localPath
+) {
+
+ public String defaultFileUrl() {
+ String result = url
+ .replaceFirst("^https?://", "")
+ .replaceAll("[\\W&&[^.]]+", "_");
+ if (type == DataSourceType.OSM && !result.endsWith(".pbf")) {
+ result = result + ".osm.pbf";
+ }
+ return result;
+ }
+}
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java
index 3f6d1cbd..6e29b15e 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/AttributeDefinition.java
@@ -14,6 +14,7 @@ public record AttributeDefinition(
// pass-through to value expression
@JsonProperty("value") Object value,
@JsonProperty("tag_value") String tagValue,
+ @JsonProperty("arg_value") String argValue,
Object type,
Object coalesce
) {}
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java
index 193e5958..6296a64d 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSource.java
@@ -1,10 +1,9 @@
package com.onthegomap.planetiler.custommap.configschema;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.nio.file.Path;
public record DataSource(
DataSourceType type,
- String url,
- @JsonProperty("local_path") Path localPath
+ Object url,
+ @JsonProperty("local_path") Object localPath
) {}
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java
index 9b1f1e44..c4b70912 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java
@@ -17,7 +17,8 @@ public record SchemaConfig(
Object definitions,
@JsonProperty("tag_mappings") Map inputMappings,
Collection layers,
- Object examples
+ Object examples,
+ Map args
) {
private static final String DEFAULT_ATTRIBUTION = """
@@ -29,6 +30,11 @@ public record SchemaConfig(
return attribution == null ? DEFAULT_ATTRIBUTION : attribution;
}
+ @Override
+ public Map args() {
+ return args == null ? Map.of() : args;
+ }
+
public static SchemaConfig load(Path path) {
return YAML.load(path, SchemaConfig.class);
}
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
index f8882411..1ccf4994 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
@@ -45,6 +45,11 @@ public interface ConfigExpression
return new GetTag<>(signature, tag);
}
+ static ConfigExpression getArg(Signature signature,
+ ConfigExpression tag) {
+ return new GetArg<>(signature, tag);
+ }
+
static ConfigExpression cast(Signature signature,
ConfigExpression input, DataType dataType) {
return new Cast<>(signature, input, dataType);
@@ -163,13 +168,16 @@ public interface ConfigExpression
default -> {
var result = children.stream()
.flatMap(
- child -> child instanceof Coalesce childCoalesce ? childCoalesce.children.stream() :
+ child -> child instanceof Coalesce, ?> childCoalesce ? childCoalesce.children.stream() :
Stream.of(child))
.filter(child -> !child.equals(constOf(null)))
.distinct()
.toList();
- var indexOfFirstConst = result.stream().takeWhile(d -> !(d instanceof ConfigExpression.Const)).count();
- yield coalesce(result.stream().limit(indexOfFirstConst + 1).toList());
+ var indexOfFirstConst = result.stream().takeWhile(d -> !(d instanceof ConfigExpression.Const, ?>)).count();
+ yield coalesce(result.stream().map(d -> {
+ @SuppressWarnings("unchecked") ConfigExpression casted = (ConfigExpression) d;
+ return casted;
+ }).limit(indexOfFirstConst + 1).toList());
}
};
}
@@ -210,6 +218,29 @@ public interface ConfigExpression
}
}
+ /** An expression that returns the value associated a given argument at runtime. */
+ record GetArg (
+ Signature signature,
+ ConfigExpression arg
+ ) implements ConfigExpression {
+
+ @Override
+ public O apply(I i) {
+ return TypeConversion.convert(i.argument(arg.apply(i)), signature.out);
+ }
+
+ @Override
+ public ConfigExpression simplifyOnce() {
+ var key = arg.simplifyOnce();
+ if (key instanceof ConfigExpression.Const constKey) {
+ var rawResult = signature.in.root().argument(constKey.value);
+ return constOf(TypeConversion.convert(rawResult, signature.out));
+ } else {
+ return new GetArg<>(signature, key);
+ }
+ }
+ }
+
/** An expression that converts the input to a desired output {@link DataType} at runtime. */
record Cast (
Signature signature,
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java
index 52c3818c..1aa77b01 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionScript.java
@@ -1,12 +1,10 @@
package com.onthegomap.planetiler.custommap.expression;
-import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.TypeConversion;
import com.onthegomap.planetiler.custommap.expression.stdlib.PlanetilerStdLib;
+import com.onthegomap.planetiler.util.Memoized;
import com.onthegomap.planetiler.util.Try;
-import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import org.projectnessie.cel.extension.StringsLib;
import org.projectnessie.cel.tools.Script;
@@ -23,6 +21,8 @@ import org.projectnessie.cel.tools.ScriptHost;
public class ConfigExpressionScript implements ConfigExpression {
private static final Pattern EXPRESSION_PATTERN = Pattern.compile("^\\s*\\$\\{(.*)}\\s*$");
private static final Pattern ESCAPED_EXPRESSION_PATTERN = Pattern.compile("^\\s*\\\\+\\$\\{(.*)}\\s*$");
+ private static final Memoized, ?> staticEvaluationCache =
+ Memoized.memoize(ConfigExpressionScript::doStaticEvaluate);
private final Script script;
private final Class returnType;
private final String scriptText;
@@ -107,9 +107,6 @@ public class ConfigExpressionScript implements Confi
if (!description.declarations().isEmpty()) {
scriptBuilder.withDeclarations(description.declarations());
}
- if (!description.types().isEmpty()) {
- scriptBuilder.withTypes(description.types());
- }
var script = scriptBuilder.build();
return new ConfigExpressionScript<>(string, script, description, expected);
@@ -132,17 +129,16 @@ public class ConfigExpressionScript implements Confi
// ignore the parsed script object
return this == o || (o instanceof ConfigExpressionScript, ?> config &&
returnType.equals(config.returnType) &&
- scriptText.equals(config.scriptText));
+ scriptText.equals(config.scriptText) &&
+ descriptor.equals(config.descriptor));
}
@Override
public int hashCode() {
// ignore the parsed script object
- return Objects.hash(returnType, scriptText);
+ return Objects.hash(returnType, scriptText, descriptor);
}
- private static final Map, Boolean> staticEvaluationCache = new ConcurrentHashMap<>();
-
/**
* Attempts to parse and evaluate this script in an environment with no variables.
*
@@ -152,19 +148,12 @@ public class ConfigExpressionScript implements Confi
public Try tryStaticEvaluate() {
// type checking can be expensive when run hundreds of times simplifying expressions iteratively and it never
// changes for a given script and input environment, so cache results between calls.
- boolean canStaticEvaluate =
- staticEvaluationCache.computeIfAbsent(this, config -> config.doTryStaticEvaluate().isSuccess());
- if (canStaticEvaluate) {
- return doTryStaticEvaluate();
- } else {
- return Try.failure(new IllegalStateException());
- }
+ return staticEvaluationCache.tryApply(this, returnType);
}
- private Try doTryStaticEvaluate() {
- return Try
- .apply(
- () -> ConfigExpressionScript.parse(scriptText, Contexts.Root.DESCRIPTION, returnType).apply(Contexts.root()));
+ private O doStaticEvaluate() {
+ return ConfigExpressionScript.parse(scriptText, descriptor.root().description(), returnType)
+ .apply(descriptor.root());
}
@Override
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java
index 9dd8f22a..84bec706 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptContext.java
@@ -22,4 +22,8 @@ public interface ScriptContext extends Function, WithTags {
default TagValueProducer tagValueProducer() {
return TagValueProducer.EMPTY;
}
+
+ default Object argument(String key) {
+ return null;
+ }
}
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
index 089bfdc0..244a4122 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
@@ -1,35 +1,44 @@
package com.onthegomap.planetiler.custommap.expression;
import com.google.api.expr.v1alpha1.Decl;
+import com.onthegomap.planetiler.custommap.Contexts;
import java.util.List;
import java.util.stream.Stream;
/**
* Type definitions for the environment that a script expression runs in.
*
- * @param types Additional types available.
* @param declarations Global variable types
* @param clazz Class of the input context type
* @param The runtime expression context type
*/
-public record ScriptEnvironment (List types, List declarations, Class clazz) {
+public record ScriptEnvironment (List declarations, Class clazz, Contexts.Root root) {
+ private static List concat(List a, List b) {
+ return Stream.concat(a.stream(), b.stream()).toList();
+ }
+
private static List concat(List a, T[] b) {
return Stream.concat(a.stream(), Stream.of(b)).toList();
}
/** Returns a copy of this environment with a new input type {@code U}. */
public ScriptEnvironment forInput(Class newClazz) {
- return new ScriptEnvironment<>(types, declarations, newClazz);
+ return new ScriptEnvironment<>(declarations, newClazz, root);
}
/** Returns a copy of this environment with a list of variable declarations appended to the global environment. */
public ScriptEnvironment withDeclarations(Decl... others) {
- return new ScriptEnvironment<>(types, concat(declarations, others), clazz);
+ return new ScriptEnvironment<>(concat(declarations, others), clazz, root);
}
- /** Returns an empty environment with no variables defined. */
- public static ScriptEnvironment root() {
- return new ScriptEnvironment<>(List.of(), List.of(), ScriptContext.class);
+ /** Returns a copy of this environment with a list of variable declarations appended to the global environment. */
+ public ScriptEnvironment withDeclarations(List others) {
+ return new ScriptEnvironment<>(concat(declarations, others), clazz, root);
+ }
+
+ /** Returns an empty environment with only static global variables (like command-line args) defined. */
+ public static ScriptEnvironment root(Contexts.Root root) {
+ return new ScriptEnvironment<>(List.of(), ScriptContext.class, root);
}
/** Returns true if this contains a variable declaration for {@code variable}. */
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
index a1f4416c..1964b963 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
@@ -6,6 +6,7 @@ import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.custommap.ConfiguredProfile;
+import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.YAML;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.geo.GeometryType;
@@ -55,13 +56,13 @@ public class SchemaValidator {
PrintStream output = System.out;
output.println("OK");
- var paths = validateFromCli(schema, arguments, output);
+ var paths = validateFromCli(schema, output);
if (watch) {
output.println();
output.println("Watching filesystem for changes...");
var watcher = FileWatcher.newWatcher(paths.toArray(Path[]::new));
- watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, arguments, output));
+ watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, output));
}
}
@@ -69,32 +70,32 @@ public class SchemaValidator {
return t != null && (cause.isInstance(t) || hasCause(t.getCause(), cause));
}
- static Set validateFromCli(Path schema, Arguments args, PrintStream output) {
+ static Set validateFromCli(Path schemaPath, PrintStream output) {
Set pathsToWatch = new HashSet<>();
- pathsToWatch.add(schema);
+ pathsToWatch.add(schemaPath);
output.println();
output.println("Validating...");
output.println();
SchemaValidator.Result result;
try {
- var parsedSchema = SchemaConfig.load(schema);
- var examples = parsedSchema.examples();
+ var schema = SchemaConfig.load(schemaPath);
+ var examples = schema.examples();
// examples can either be embedded in the yaml file, or referenced
SchemaSpecification spec;
if (examples instanceof String s) {
var path = Path.of(s);
if (!path.isAbsolute()) {
- path = schema.resolveSibling(path);
+ path = schemaPath.resolveSibling(path);
}
// if referenced, make sure we watch that file for changes
pathsToWatch.add(path);
spec = SchemaSpecification.load(path);
} else if (examples != null) {
- spec = YAML.convertValue(parsedSchema, SchemaSpecification.class);
+ spec = YAML.convertValue(schema, SchemaSpecification.class);
} else {
spec = new SchemaSpecification(List.of());
}
- result = validate(parsedSchema, spec, args);
+ result = validate(schema, spec);
} catch (Exception exception) {
Throwable rootCause = ExceptionUtils.getRootCause(exception);
if (hasCause(exception, com.onthegomap.planetiler.custommap.expression.ParseException.class)) {
@@ -175,13 +176,14 @@ public class SchemaValidator {
* Returns the result of validating the profile defined by {@code schema} against the examples in
* {@code specification}.
*/
- public static Result validate(SchemaConfig schema, SchemaSpecification specification, Arguments args) {
- return validate(new ConfiguredProfile(schema), specification, args);
+ public static Result validate(SchemaConfig schema, SchemaSpecification specification) {
+ var context = Contexts.buildRootContext(Arguments.of().silence(), schema.args());
+ return validate(new ConfiguredProfile(schema, context), specification, context.config());
}
/** Returns the result of validating {@code profile} against the examples in {@code specification}. */
- public static Result validate(Profile profile, SchemaSpecification specification, Arguments args) {
- var featureCollectorFactory = new FeatureCollector.Factory(PlanetilerConfig.from(args.silence()), Stats.inMemory());
+ public static Result validate(Profile profile, SchemaSpecification specification, PlanetilerConfig config) {
+ var featureCollectorFactory = new FeatureCollector.Factory(config, Stats.inMemory());
return new Result(specification.examples().stream().map(example -> new ExampleResult(example, Try.apply(() -> {
List issues = new ArrayList<>();
var input = example.input();
diff --git a/planetiler-custommap/src/main/resources/samples/shortbread.yml b/planetiler-custommap/src/main/resources/samples/shortbread.yml
index 848b5c1e..ddf02018 100644
--- a/planetiler-custommap/src/main/resources/samples/shortbread.yml
+++ b/planetiler-custommap/src/main/resources/samples/shortbread.yml
@@ -2,6 +2,13 @@ schema_name: Shortbread
schema_description: A basic, lean, general-purpose vector tile schema for OpenStreetMap data. See https://shortbread.geofabrik.de/
attribution: © OpenStreetMap contributors
examples: shortbread.spec.yml
+args:
+ area:
+ description: Geofabrik area to download
+ default: massachusetts
+ osm_url:
+ description: OSM URL to download
+ default: '${ "geofabrik:" + args.area }'
sources:
ocean:
type: shapefile
@@ -11,7 +18,7 @@ sources:
url: https://shortbread.geofabrik.de/shapefiles/admin-points-4326.zip
osm:
type: osm
- url: geofabrik:massachusetts
+ url: '${ args.osm_url }'
definitions:
# TODO let attribute definitions set multiple keys so you can just use `- *names`
attributes:
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java
index d17094b2..b7dec4a8 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java
@@ -3,7 +3,6 @@ package com.onthegomap.planetiler.custommap;
import static com.onthegomap.planetiler.expression.Expression.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
import com.onthegomap.planetiler.expression.Expression;
import java.util.Map;
import org.junit.jupiter.api.Test;
@@ -13,7 +12,7 @@ class BooleanExpressionParserTest {
private static void assertParse(String yaml, Expression parsed) {
Object expression = YAML.load(yaml, Object.class);
- var actual = BooleanExpressionParser.parse(expression, TVP, ScriptEnvironment.root());
+ var actual = BooleanExpressionParser.parse(expression, TVP, TestContexts.ROOT_CONTEXT);
assertEquals(
parsed.simplify().generateJavaCode(),
actual.simplify().generateJavaCode()
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java
index 50fc457e..78f8804e 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java
@@ -1,5 +1,6 @@
package com.onthegomap.planetiler.custommap;
+import static com.onthegomap.planetiler.custommap.TestContexts.PROCESS_FEATURE;
import static com.onthegomap.planetiler.custommap.expression.ConfigExpression.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -16,7 +17,7 @@ import org.junit.jupiter.params.provider.ValueSource;
class ConfigExpressionParserTest {
private static final TagValueProducer TVP = new TagValueProducer(Map.of());
private static final ConfigExpression.Signature FEATURE_SIGNATURE =
- signature(Contexts.ProcessFeature.DESCRIPTION, Object.class);
+ signature(PROCESS_FEATURE, Object.class);
private static void assertParse(String yaml, ConfigExpression, ?> parsed, Class clazz) {
Object expression = YAML.load(yaml, Object.class);
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
index 227412d0..725ddb7e 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
@@ -9,13 +9,16 @@ import static org.junit.jupiter.api.Assertions.*;
import com.onthegomap.planetiler.FeatureCollector;
import com.onthegomap.planetiler.FeatureCollector.Feature;
import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.custommap.configschema.DataSourceType;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.custommap.util.TestConfigurableUtils;
import com.onthegomap.planetiler.reader.SimpleFeature;
import com.onthegomap.planetiler.reader.SourceFeature;
import com.onthegomap.planetiler.stats.Stats;
import java.nio.file.Path;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
@@ -26,6 +29,7 @@ import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
class ConfiguredFeatureTest {
+ private PlanetilerConfig planetilerConfig = PlanetilerConfig.defaults();
private static final Function TEST_RESOURCE = TestConfigurableUtils::pathToTestResource;
private static final Function SAMPLE_RESOURCE = TestConfigurableUtils::pathToSample;
@@ -74,32 +78,35 @@ class ConfiguredFeatureTest {
"bridge", "yes"
);
- private static Profile loadConfig(Function pathFunction, String filename) {
+ private Profile loadConfig(Function pathFunction, String filename) {
var staticAttributeConfig = pathFunction.apply(filename);
var schema = SchemaConfig.load(staticAttributeConfig);
- return new ConfiguredProfile(schema);
+ var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args());
+ planetilerConfig = root.config();
+ return new ConfiguredProfile(schema, root);
}
- private static Profile loadConfig(String config) {
+ private ConfiguredProfile loadConfig(String config) {
var schema = SchemaConfig.load(config);
- return new ConfiguredProfile(schema);
+ var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args());
+ planetilerConfig = root.config();
+ return new ConfiguredProfile(schema, root);
}
- private static void testFeature(Function pathFunction, String schemaFilename, SourceFeature sf,
+ private void testFeature(Function pathFunction, String schemaFilename, SourceFeature sf,
Consumer test, int expectedMatchCount) {
var profile = loadConfig(pathFunction, schemaFilename);
testFeature(sf, test, expectedMatchCount, profile);
}
- private static void testFeature(String config, SourceFeature sf, Consumer test, int expectedMatchCount) {
+ private void testFeature(String config, SourceFeature sf, Consumer test, int expectedMatchCount) {
var profile = loadConfig(config);
testFeature(sf, test, expectedMatchCount, profile);
}
- private static void testFeature(SourceFeature sf, Consumer test, int expectedMatchCount, Profile profile) {
- var config = PlanetilerConfig.defaults();
- var factory = new FeatureCollector.Factory(config, Stats.inMemory());
+ private void testFeature(SourceFeature sf, Consumer test, int expectedMatchCount, Profile profile) {
+ var factory = new FeatureCollector.Factory(planetilerConfig, Stats.inMemory());
var fc = factory.get(sf);
profile.processFeature(sf, fc);
@@ -114,14 +121,14 @@ class ConfiguredFeatureTest {
assertEquals(expectedMatchCount, length.get(), "Wrong number of features generated");
}
- private static void testPolygon(String config, Map tags,
+ private void testPolygon(String config, Map tags,
Consumer test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList());
testFeature(config, sf, test, expectedMatchCount);
}
- private static void testPoint(String config, Map tags,
+ private void testPoint(String config, Map tags,
Consumer test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newPoint(0, 0), tags, "osm", null, 1, emptyList());
@@ -129,21 +136,21 @@ class ConfiguredFeatureTest {
}
- private static void testLinestring(String config,
+ private void testLinestring(String config,
Map tags, Consumer test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList());
testFeature(config, sf, test, expectedMatchCount);
}
- private static void testPolygon(Function pathFunction, String schemaFilename, Map tags,
+ private void testPolygon(Function pathFunction, String schemaFilename, Map tags,
Consumer test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList());
testFeature(pathFunction, schemaFilename, sf, test, expectedMatchCount);
}
- private static void testLinestring(Function pathFunction, String schemaFilename,
+ private void testLinestring(Function pathFunction, String schemaFilename,
Map tags, Consumer test, int expectedMatchCount) {
var sf =
SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList());
@@ -784,4 +791,235 @@ class ConfiguredFeatureTest {
private void testInvalidSchema(String filename, String message) {
assertThrows(RuntimeException.class, () -> loadConfig(TEST_INVALID_RESOURCE, filename), message);
}
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "arg_value: argument",
+ "value: '${ args.argument }'",
+ "value: '${ args[\"argument\"] }'",
+ })
+ void testUseArgumentNotDefined(String string) {
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
+ "argument", "value"
+ )));
+ var config = """
+ sources:
+ osm:
+ type: osm
+ url: geofabrik:rhode-island
+ local_path: data/rhode-island.osm.pbf
+ layers:
+ - id: testLayer
+ features:
+ - source: osm
+ geometry: point
+ include_when:
+ natural: water
+ attributes:
+ - key: key
+ %s
+ """.formatted(string);
+
+ testPoint(config, Map.of(
+ "natural", "water"
+ ), feature -> {
+ assertEquals("value", feature.getAttrsAtZoom(14).get("key"));
+ }, 1);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "arg_value: threads",
+ "value: '${ args.threads }'",
+ "value: '${ args[\"threads\"] }'",
+ })
+ void testOverrideArgument(String string) {
+ var config = """
+ sources:
+ osm:
+ type: osm
+ url: geofabrik:rhode-island
+ local_path: data/rhode-island.osm.pbf
+ args:
+ threads: 2
+ layers:
+ - id: testLayer
+ features:
+ - source: osm
+ geometry: point
+ include_when:
+ natural: water
+ attributes:
+ - key: key
+ type: integer
+ %s
+ """.formatted(string);
+
+ testPoint(config, Map.of(
+ "natural", "water"
+ ), feature -> {
+ assertEquals(2, feature.getAttrsAtZoom(14).get("key"));
+ }, 1);
+ }
+
+ @Test
+ void testDefineArgument() {
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
+ "custom_overridden_arg", "test2",
+ "custom_simple_overridden_int_arg", "3"
+ )));
+ var config = """
+ sources:
+ osm:
+ type: osm
+ url: geofabrik:rhode-island
+ local_path: data/rhode-island.osm.pbf
+ args:
+ custom_int_arg:
+ type: integer
+ description: test arg out
+ default: 12
+ custom_boolean_arg:
+ type: boolean
+ description: test boolean arg out
+ default: true
+ custom_overridden_arg:
+ default: test
+ custom_simple_string_arg: value
+ custom_simple_int_arg: 1
+ custom_simple_double_arg: 1.5
+ custom_simple_bool_arg: true
+ custom_simple_overridden_int_arg: 2
+ layers:
+ - id: testLayer
+ features:
+ - source: osm
+ geometry: point
+ include_when:
+ natural: water
+ attributes:
+ - key: int
+ value: '${ args.custom_int_arg }'
+ - key: bool
+ value: '${ args["custom_boolean_arg"] }'
+ - key: overridden
+ arg_value: custom_overridden_arg
+ - key: custom_simple_string_arg
+ arg_value: custom_simple_string_arg
+ - key: custom_simple_int_arg
+ arg_value: custom_simple_int_arg
+ - key: custom_simple_bool_arg
+ arg_value: custom_simple_bool_arg
+ - key: custom_simple_overridden_int_arg
+ arg_value: custom_simple_overridden_int_arg
+ - key: custom_simple_double_arg
+ arg_value: custom_simple_double_arg
+ - key: custom_simple_int_arg_as_string
+ arg_value: custom_simple_int_arg
+ type: string
+ """;
+
+ testPoint(config, Map.of(
+ "natural", "water"
+ ), feature -> {
+ assertEquals(12L, feature.getAttrsAtZoom(14).get("int"));
+ assertEquals(true, feature.getAttrsAtZoom(14).get("bool"));
+ assertEquals("test2", feature.getAttrsAtZoom(14).get("overridden"));
+
+ assertEquals("value", feature.getAttrsAtZoom(14).get("custom_simple_string_arg"));
+ assertEquals(1, feature.getAttrsAtZoom(14).get("custom_simple_int_arg"));
+ assertEquals("1", feature.getAttrsAtZoom(14).get("custom_simple_int_arg_as_string"));
+ assertEquals(1.5, feature.getAttrsAtZoom(14).get("custom_simple_double_arg"));
+ assertEquals(true, feature.getAttrsAtZoom(14).get("custom_simple_bool_arg"));
+ assertEquals(3, feature.getAttrsAtZoom(14).get("custom_simple_overridden_int_arg"));
+ }, 1);
+ }
+
+ @Test
+ void testDefineArgumentsUsingExpressions() {
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
+ "custom_overridden_arg", "test2",
+ "custom_simple_overridden_int_arg", "3"
+ )));
+ var config = """
+ sources:
+ osm:
+ type: osm
+ url: geofabrik:rhode-island
+ local_path: data/rhode-island.osm.pbf
+ args:
+ arg1:
+ type: long
+ description: test arg out
+ default: '${ 2 - 1 }'
+ arg2: '${ 2 - 1 }'
+ arg3:
+ default: '${ 2 - 1 }'
+ arg4: ${ args.arg3 + 1 }
+ layers:
+ - id: testLayer
+ features:
+ - source: osm
+ geometry: point
+ attributes:
+ - key: arg1
+ arg_value: arg1
+ - key: arg2
+ arg_value: arg2
+ - key: arg3
+ arg_value: arg3
+ - key: arg4
+ arg_value: arg4
+ """;
+
+ testPoint(config, Map.of(), feature -> {
+ assertEquals(1L, feature.getAttrsAtZoom(14).get("arg1"));
+ assertEquals(1L, feature.getAttrsAtZoom(14).get("arg2"));
+ assertEquals(1L, feature.getAttrsAtZoom(14).get("arg3"));
+ assertEquals(2L, feature.getAttrsAtZoom(14).get("arg4"));
+ }, 1);
+ }
+
+ @Test
+ void testUseArgumentInSourceUrlPath() {
+ var config = """
+ args:
+ area: rhode-island
+ url: '${ "geofabrik:" + args.area }'
+ sources:
+ osm:
+ type: osm
+ url: '${ args.url }'
+ layers:
+ - id: testLayer
+ features:
+ - source: osm
+ geometry: point
+ """;
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
+ "area", "boston"
+ )));
+ assertEquals(List.of(new Source(
+ "osm",
+ DataSourceType.OSM,
+ "geofabrik:boston",
+ null
+ )), loadConfig(config).sources());
+
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+ assertEquals(List.of(new Source(
+ "osm",
+ DataSourceType.OSM,
+ "geofabrik:rhode-island",
+ null
+ )), loadConfig(config).sources());
+
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+ assertEquals("geofabrik_rhode_island.osm.pbf", loadConfig(config).sources().get(0).defaultFileUrl());
+
+ this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of(
+ "url", "https://example.com/file.osm.pbf"
+ )));
+ assertEquals("example.com_file.osm.pbf", loadConfig(config).sources().get(0).defaultFileUrl());
+ }
}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java
new file mode 100644
index 00000000..d44a5637
--- /dev/null
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ContextsTest.java
@@ -0,0 +1,228 @@
+package com.onthegomap.planetiler.custommap;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.custommap.expression.ParseException;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+
+class ContextsTest {
+
+ @Test
+ void testParseEmptyArgs() {
+ Map cliArgs = Map.of();
+ Map schemaArgs = Map.of();
+ Contexts.buildRootContext(Arguments.of(cliArgs), schemaArgs);
+ }
+
+ @Test
+ void setPlanetilerConfigThroughCli() {
+ Map cliArgs = Map.of("threads", "9999");
+ Map schemaArgs = Map.of("threads", "8888");
+ var result = Contexts.buildRootContext(Arguments.of(cliArgs), schemaArgs);
+ assertEquals(9999, result.config().threads());
+ }
+
+ @Test
+ void setPlanetilerConfigThroughSchemaArgs() {
+ assertEquals(8888, Contexts.buildRootContext(
+ Arguments.of(Map.of()),
+ Map.of("threads", "8888")
+ ).config().threads());
+ assertEquals(8888, Contexts.buildRootContext(
+ Arguments.of(Map.of()),
+ Map.of("threads", 8888)
+ ).config().threads());
+ }
+
+ @Test
+ void configDefinesDefaultArgumentValue() {
+ assertEquals(8888, Contexts.buildRootContext(
+ Arguments.of(Map.of()),
+ Map.of(
+ "arg", 8888
+ )
+ ).argument("arg"));
+ }
+
+ @Test
+ void overrideDefaultArgFromConfig() {
+ assertEquals(9999, Contexts.buildRootContext(
+ Arguments.of(Map.of("arg", "9999")),
+ Map.of(
+ "arg", 8888
+ )
+ ).argument("arg"));
+ }
+
+ @Test
+ void defineArgTypeInConfig() {
+ assertEquals(8888, Contexts.buildRootContext(
+ Arguments.of(),
+ Map.of(
+ "arg", Map.of(
+ "default", "8888",
+ "type", "integer"
+ )
+ )
+ ).argument("arg"));
+ }
+
+ @Test
+ void implyTypeFromDefaultValue() {
+ assertEquals("8888", Contexts.buildRootContext(
+ Arguments.of(),
+ Map.of(
+ "arg", "8888"
+ )
+ ).argument("arg"));
+ assertEquals("9999", Contexts.buildRootContext(
+ Arguments.of("arg", "9999"),
+ Map.of(
+ "arg", "8888"
+ )
+ ).argument("arg"));
+
+ assertEquals(8888, Contexts.buildRootContext(
+ Arguments.of(),
+ Map.of(
+ "arg", 8888
+ )
+ ).argument("arg"));
+ assertEquals(9999, Contexts.buildRootContext(
+ Arguments.of("arg", "9999"),
+ Map.of(
+ "arg", 8888
+ )
+ ).argument("arg"));
+ }
+
+ @Test
+ void computeDefaultValueUsingExpression() {
+ assertEquals(2, Contexts.buildRootContext(
+ Arguments.of(),
+ Map.of(
+ "arg", Map.of(
+ "default", "${ 1+1 }",
+ "type", "integer"
+ )
+ )
+ ).argument("arg"));
+ }
+
+ @Test
+ void implyTypeFromExpressionValue() {
+ assertEquals(2L, Contexts.buildRootContext(
+ Arguments.of(),
+ Map.of(
+ "arg", "${ 1+1 }"
+ )
+ ).argument("arg"));
+ }
+
+ @Test
+ void referenceOtherArgInExpression() {
+ Map configArgs = Map.of(
+ "arg1", 1,
+ "arg2", "${ args.arg1 + 1}"
+ );
+ assertEquals(2L, Contexts.buildRootContext(
+ Arguments.of(),
+ configArgs
+ ).argument("arg2"));
+ assertEquals(3L, Contexts.buildRootContext(
+ Arguments.of(Map.of("arg1", "2")),
+ configArgs
+ ).argument("arg2"));
+ assertEquals(10L, Contexts.buildRootContext(
+ Arguments.of(Map.of("arg2", "10")),
+ configArgs
+ ).argument("arg2"));
+ }
+
+ @Test
+ void referenceOtherArgInExpressionTwice() {
+ Map configArgs = Map.of(
+ "arg1", 1,
+ "arg2", "${ args.arg1 + 1}",
+ "arg3", "${ args.arg2 + 1}"
+ );
+ assertEquals(3L, Contexts.buildRootContext(
+ Arguments.of(),
+ configArgs
+ ).argument("arg3"));
+ }
+
+ @Test
+ void failOnInfiniteLoop() {
+ Map configArgs = Map.of(
+ "arg1", Map.of(
+ "default", "${ args.arg3 + 1 }",
+ "type", "long"
+ ),
+ "arg2", "${ args.arg1 + 1}",
+ "arg3", "${ args.arg2 + 1}"
+ );
+ var empty = Arguments.of();
+ assertThrows(ParseException.class, () -> Contexts.buildRootContext(
+ empty,
+ configArgs
+ ));
+ // but if you break the chain it's OK?
+ assertEquals(3L, Contexts.buildRootContext(
+ Arguments.of(Map.of("arg1", "1")),
+ configArgs
+ ).argument("arg3"));
+ }
+
+ @Test
+ void setPlanetilerConfigFromOtherArg() {
+ assertEquals(8888, Contexts.buildRootContext(
+ Arguments.of(Map.of()),
+ Map.of(
+ "other", "8888",
+ "threads", "${ args.other }"
+ )
+ ).config().threads());
+ }
+
+ @Test
+ void testCantRedefineBuiltin() {
+ var fromCli = Arguments.of(Map.of());
+ Map fromConfig = Map.of(
+ "threads", Map.of(
+ "default", 4,
+ "type", "string"
+ )
+ );
+ assertThrows(ParseException.class, () -> Contexts.buildRootContext(fromCli, fromConfig));
+ }
+
+ @Test
+ void testDefineRequiredArg() {
+ var argsWithoutValue = Arguments.of(Map.of());
+ var argsWithValue = Arguments.of(Map.of("arg", "3"));
+ Map fromConfig = Map.of(
+ "arg", Map.of(
+ "type", "integer",
+ "description", "desc"
+ )
+ );
+ assertThrows(ParseException.class, () -> Contexts.buildRootContext(argsWithoutValue, fromConfig));
+ assertEquals(3, Contexts.buildRootContext(argsWithValue, fromConfig).argument("arg"));
+ }
+
+ @Test
+ void setPlanetilerConfigFromOtherPlanetilerConfig() {
+ var root = Contexts.buildRootContext(
+ Arguments.of(Map.of()),
+ Map.of(
+ "mmap", "${ args.threads < 1 }"
+ )
+ );
+ assertTrue(root.config().mmapTempStorage());
+ }
+}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java
index 9a6a802b..3400fe66 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java
@@ -2,7 +2,6 @@ package com.onthegomap.planetiler.custommap;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
-import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import com.onthegomap.planetiler.custommap.validator.SchemaSpecification;
import com.onthegomap.planetiler.custommap.validator.SchemaValidator;
@@ -21,8 +20,7 @@ class SchemaTests {
var base = Path.of("src", "main", "resources", "samples");
var result = SchemaValidator.validate(
SchemaConfig.load(base.resolve(schema)),
- SchemaSpecification.load(base.resolve(spec)),
- Arguments.of()
+ SchemaSpecification.load(base.resolve(spec))
);
return result.results().stream()
.map(test -> dynamicTest(test.example().name(), () -> {
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java
index 699606c4..ade195cb 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaYAMLLoadTest.java
@@ -3,6 +3,7 @@ package com.onthegomap.planetiler.custommap;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import java.io.IOException;
import java.nio.file.Files;
@@ -33,8 +34,9 @@ class SchemaYAMLLoadTest {
for (Path schemaFile : schemaFiles) {
var schemaConfig = SchemaConfig.load(schemaFile);
+ var root = Contexts.buildRootContext(Arguments.of(), schemaConfig.args());
assertNotNull(schemaConfig, () -> "Failed to unmarshall " + schemaFile.toString());
- assertNotNull(new ConfiguredProfile(schemaConfig), () -> "Failed to load profile from " + schemaFile.toString());
+ new ConfiguredProfile(schemaConfig, root);
}
}
}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java
index 1c6d0e4d..ac217aee 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TagValueProducerTest.java
@@ -1,5 +1,6 @@
package com.onthegomap.planetiler.custommap;
+import static com.onthegomap.planetiler.custommap.TestContexts.ROOT;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.geo.GeoUtils;
@@ -16,7 +17,7 @@ class TagValueProducerTest {
assertEquals(expected, tvp.valueForKey(wrapped, key));
assertEquals(expected, tvp.valueGetterForKey(key).apply(wrapped, key));
assertEquals(expected, tvp.valueProducerForKey(key)
- .apply(new Contexts.ProcessFeature(SimpleFeature.create(GeoUtils.EMPTY_GEOMETRY, tags), tvp)
+ .apply(ROOT.createProcessFeatureContext(SimpleFeature.create(GeoUtils.EMPTY_GEOMETRY, tags), tvp)
.createPostMatchContext(List.of())));
}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java
new file mode 100644
index 00000000..246db9fc
--- /dev/null
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/TestContexts.java
@@ -0,0 +1,14 @@
+package com.onthegomap.planetiler.custommap;
+
+import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment;
+
+public class TestContexts {
+ public static final Contexts.Root ROOT = Contexts.emptyRoot();
+ public static final ScriptEnvironment ROOT_CONTEXT = ROOT.description();
+ public static final ScriptEnvironment PROCESS_FEATURE =
+ Contexts.ProcessFeature.description(ROOT);
+ public static final ScriptEnvironment FEATURE_POST_MATCH =
+ Contexts.FeaturePostMatch.description(ROOT);
+ public static final ScriptEnvironment FEATURE_ATTRIBUTE =
+ Contexts.FeatureAttribute.description(ROOT);
+}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java
index 95e93b1c..29d72730 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScriptTest.java
@@ -4,7 +4,7 @@ import static com.onthegomap.planetiler.expression.Expression.and;
import static com.onthegomap.planetiler.expression.Expression.or;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import com.onthegomap.planetiler.custommap.Contexts;
+import com.onthegomap.planetiler.custommap.TestContexts;
import com.onthegomap.planetiler.expression.Expression;
import org.junit.jupiter.api.Test;
@@ -12,11 +12,20 @@ class BooleanExpressionScriptTest {
@Test
void testSimplify() {
assertEquals(Expression.TRUE,
- and(or(BooleanExpressionScript.script("1+1<3", Contexts.Root.DESCRIPTION))).simplify());
+ and(or(BooleanExpressionScript.script("1+1<3", TestContexts.ROOT_CONTEXT))).simplify());
assertEquals(Expression.FALSE,
- and(or(BooleanExpressionScript.script("1+1>3", Contexts.Root.DESCRIPTION))).simplify());
+ and(or(BooleanExpressionScript.script("1+1>3", TestContexts.ROOT_CONTEXT))).simplify());
- var other = BooleanExpressionScript.script("feature.tags.natural", Contexts.ProcessFeature.DESCRIPTION);
+ var other =
+ BooleanExpressionScript.script("feature.tags.natural", TestContexts.PROCESS_FEATURE);
assertEquals(other, and(or(other)).simplify());
}
+
+ @Test
+ void testSimplifyInlinesArguments() {
+ assertEquals(Expression.TRUE,
+ and(or(BooleanExpressionScript.script("args.threads > 0", TestContexts.ROOT_CONTEXT))).simplify());
+ assertEquals(Expression.FALSE,
+ and(or(BooleanExpressionScript.script("args.threads < 0", TestContexts.ROOT_CONTEXT))).simplify());
+ }
}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java
index 39cfc20a..4d0bdad1 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ConfigExpressionTest.java
@@ -1,6 +1,9 @@
package com.onthegomap.planetiler.custommap.expression;
import static com.onthegomap.planetiler.TestUtils.newPoint;
+import static com.onthegomap.planetiler.custommap.TestContexts.FEATURE_POST_MATCH;
+import static com.onthegomap.planetiler.custommap.TestContexts.PROCESS_FEATURE;
+import static com.onthegomap.planetiler.custommap.TestContexts.ROOT_CONTEXT;
import static com.onthegomap.planetiler.custommap.expression.ConfigExpression.*;
import static com.onthegomap.planetiler.expression.Expression.matchAny;
import static com.onthegomap.planetiler.expression.Expression.or;
@@ -8,8 +11,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.Contexts;
import com.onthegomap.planetiler.custommap.TagValueProducer;
+import com.onthegomap.planetiler.custommap.TestContexts;
import com.onthegomap.planetiler.expression.DataType;
import com.onthegomap.planetiler.expression.Expression;
import com.onthegomap.planetiler.expression.MultiExpression;
@@ -19,10 +24,9 @@ import java.util.Map;
import org.junit.jupiter.api.Test;
class ConfigExpressionTest {
- private static final ConfigExpression.Signature ROOT =
- signature(Contexts.Root.DESCRIPTION, Integer.class);
+ private static final ConfigExpression.Signature ROOT = signature(ROOT_CONTEXT, Integer.class);
private static final ConfigExpression.Signature FEATURE_SIGNATURE =
- signature(Contexts.ProcessFeature.DESCRIPTION, Integer.class);
+ signature(PROCESS_FEATURE, Integer.class);
@Test
void testConst() {
@@ -32,7 +36,7 @@ class ConfigExpressionTest {
@Test
void testVariable() {
var feature = SimpleFeature.create(newPoint(0, 0), Map.of("a", "b", "c", 1), "source", "source_layer", 99);
- var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()));
+ var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()));
// simple match
assertEquals("source", variable(FEATURE_SIGNATURE.withOutput(String.class), "feature.source").apply(context));
assertEquals("source_layer",
@@ -45,32 +49,32 @@ class ConfigExpressionTest {
@Test
void testCoalesce() {
- assertNull(coalesce(List.of()).apply(Contexts.root()));
+ assertNull(coalesce(List.of()).apply(TestContexts.ROOT));
assertNull(coalesce(
List.of(
constOf(null)
- )).apply(Contexts.root()));
+ )).apply(TestContexts.ROOT));
assertEquals(2, coalesce(
List.of(
constOf(null),
constOf(2)
- )).apply(Contexts.root()));
+ )).apply(TestContexts.ROOT));
assertEquals(1, coalesce(
List.of(
constOf(1),
constOf(2)
- )).apply(Contexts.root()));
+ )).apply(TestContexts.ROOT));
}
@Test
void testDynamic() {
- assertEquals(1, script(ROOT, "5 - 4").apply(Contexts.root()));
+ assertEquals(1, script(ROOT, "5 - 4").apply(TestContexts.ROOT));
}
@Test
void testMatch() {
var feature = SimpleFeature.create(newPoint(0, 0), Map.of("a", "b", "c", 1));
- var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()));
+ var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()));
// simple match
assertEquals(2, match(FEATURE_SIGNATURE, MultiExpression.of(List.of(
MultiExpression.entry(constOf(1),
@@ -280,33 +284,30 @@ class ConfigExpressionTest {
assertEquals(
"123",
getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")).apply(
- new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of())))
+ TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of())))
);
assertEquals(
123,
getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc"))
- .apply(new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of("abc", "integer"))))
+ .apply(TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of("abc", "integer"))))
);
assertEquals(
123,
- getTag(signature(Contexts.FeaturePostMatch.DESCRIPTION, Object.class), constOf("abc"))
- .apply(new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of("abc", "integer")))
+ getTag(signature(FEATURE_POST_MATCH, Object.class), constOf("abc"))
+ .apply(TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of("abc", "integer")))
.createPostMatchContext(List.of()))
);
- assertEquals(
- null,
- getTag(signature(Contexts.Root.DESCRIPTION, Object.class), constOf("abc"))
- .apply(Contexts.root())
- );
+ assertNull(getTag(signature(ROOT_CONTEXT, Object.class), constOf("abc"))
+ .apply(TestContexts.ROOT));
}
@Test
void testCastGetTag() {
var feature = SimpleFeature.create(newPoint(0, 0), Map.of("abc", "123"), "source", "source_layer", 99);
- var context = new Contexts.ProcessFeature(feature, new TagValueProducer(Map.of()));
+ var context = TestContexts.ROOT.createProcessFeatureContext(feature, new TagValueProducer(Map.of()));
var expression = cast(
FEATURE_SIGNATURE.withOutput(Integer.class),
getTag(FEATURE_SIGNATURE.withOutput(Object.class), constOf("abc")),
@@ -328,7 +329,7 @@ class ConfigExpressionTest {
constOf("123"),
DataType.GET_INT
);
- assertEquals(123, expression.apply(Contexts.root()));
+ assertEquals(123, expression.apply(TestContexts.ROOT));
}
@Test
@@ -341,4 +342,31 @@ class ConfigExpressionTest {
).simplify()
);
}
+
+ @Test
+ void testSimplifyGetArg() {
+ var args = Arguments.of(Map.of("arg", "12"));
+ var root = Contexts.buildRootContext(args, Map.of());
+ var context = signature(root.description(), Integer.class);
+ assertEquals(constOf(12),
+ getArg(
+ context,
+ constOf("arg")
+ ).simplify()
+ );
+
+ assertEquals(constOf(12),
+ script(
+ context,
+ "args.arg"
+ ).simplify()
+ );
+
+ assertEquals(constOf(12),
+ script(
+ context,
+ "args['arg']"
+ ).simplify()
+ );
+ }
}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java
index 19117111..c25ebc46 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/DataTypeTest.java
@@ -63,4 +63,13 @@ class DataTypeTest {
assertEquals(0, DataType.from("direction").convertFrom("no"));
assertEquals(0, DataType.from("direction").convertFrom("false"));
}
+
+ @Test
+ void testTypeOf() {
+ assertEquals(DataType.GET_DOUBLE, DataType.typeOf(1.5));
+ assertEquals(DataType.GET_LONG, DataType.typeOf(1L));
+ assertEquals(DataType.GET_INT, DataType.typeOf(1));
+ assertEquals(DataType.GET_BOOLEAN, DataType.typeOf(true));
+ assertEquals(DataType.GET_STRING, DataType.typeOf("string"));
+ }
}
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java
index ce62fd5a..92c9e4af 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/expression/ExpressionTests.java
@@ -3,6 +3,9 @@ package com.onthegomap.planetiler.custommap.expression;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.custommap.Contexts;
+import java.util.Map;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
@@ -52,10 +55,25 @@ class ExpressionTests {
"min([1.1, 2.2, 3.3])|1.1|double",
"max([1])|1|long",
"min([1])|1|long",
+
+ "args.arg_from_config|1|long",
+ "args.arg_from_config + 1|2|long",
+ "args[\"arg_from_config\"]|1|long",
+ "args[\"arg_from_config\"] + 1|2|long",
+ "args.overridden_arg_from_config|4|long",
+ "args[\"overridden_arg_from_config\"] + 1|5|long",
+ "args.arg_from_cli|2|string",
+ "args[\"arg_from_cli\"]|2|string",
}, delimiter = '|')
void testExpression(String in, String expected, String type) {
- var expression = ConfigExpressionScript.parse(in, ScriptEnvironment.root());
- var result = expression.apply(ScriptContext.empty());
+ Map configFromSchema = Map.of("arg_from_config", 1, "overridden_arg_from_config", 3);
+ Map configFromCli = Map.of("arg_from_cli", "2", "overridden_arg_from_config", "4");
+ var context = Contexts.buildRootContext(
+ Arguments.of(configFromCli),
+ configFromSchema
+ );
+ var expression = ConfigExpressionScript.parse(in, context.description());
+ var result = expression.apply(context);
switch (type) {
case "long" -> assertEquals(Long.valueOf(expected), result);
case "double" -> assertEquals(Double.valueOf(expected), result);
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java
index 97501f80..5c38b6ed 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java
@@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.custommap.configschema.SchemaConfig;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -25,12 +24,10 @@ class SchemaValidatorTest {
record Result(SchemaValidator.Result output, String cliOutput) {}
- Result validate(String schema, String spec) throws IOException {
- var args = Arguments.of();
+ private Result validate(String schema, String spec) throws IOException {
var result = SchemaValidator.validate(
SchemaConfig.load(schema),
- SchemaSpecification.load(spec),
- args
+ SchemaSpecification.load(spec)
);
for (var example : result.results()) {
if (example.issues().isFailure()) {
@@ -39,30 +36,29 @@ class SchemaValidatorTest {
}
// also exercise the cli writer and return what it would have printed to stdout
var cliOutput = validateCli(Files.writeString(tmpDir.resolve("schema"),
- schema + "\nexamples: " + Files.writeString(tmpDir.resolve("spec.yml"), spec)), args);
+ schema + "\nexamples: " + Files.writeString(tmpDir.resolve("spec.yml"), spec)));
// also test the case where the examples are embedded in the schema itself
assertEquals(
cliOutput,
- validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\n" + spec), args)
+ validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\n" + spec))
);
// also test where examples points to a relative path (written in previous step)
assertEquals(
cliOutput,
- validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\nexamples: spec.yml"), args)
+ validateCli(Files.writeString(tmpDir.resolve("schema"), schema + "\nexamples: spec.yml"))
);
return new Result(result, cliOutput);
}
- private String validateCli(Path path, Arguments args) {
+ private String validateCli(Path path) {
try (
var baos = new ByteArrayOutputStream();
var printStream = new PrintStream(baos, true, StandardCharsets.UTF_8)
) {
SchemaValidator.validateFromCli(
path,
- args,
printStream
);
return baos.toString(StandardCharsets.UTF_8);
@@ -180,4 +176,45 @@ class SchemaValidatorTest {
);
assertFalse(results.output.ok(), results.toString());
}
+
+ @Test
+ void testValidationWiresInArguments() throws IOException {
+ var results = validate(
+ """
+ sources:
+ osm:
+ type: osm
+ url: geofabrik:rhode-island
+ args:
+ key: default_value
+ layers:
+ - id: water
+ features:
+ - source: osm
+ geometry: polygon
+ include_when:
+ natural: water
+ attributes:
+ - key: from_arg
+ arg_value: key
+ - key: threads
+ value: '${ args.threads + 1 }'
+ """,
+ """
+ examples:
+ - name: test output
+ input:
+ source: osm
+ geometry: polygon
+ tags:
+ natural: water
+ output:
+ layer: water
+ tags:
+ from_arg: default_value
+ threads: %s
+ """.formatted(1 + Math.max(Runtime.getRuntime().availableProcessors(), 2))
+ );
+ assertTrue(results.output.ok(), results.toString());
+ }
}
diff --git a/quickstart.sh b/quickstart.sh
index a544b3e8..0c298ab6 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -16,6 +16,7 @@ DRY_RUN=""
VERSION="latest"
DOCKER_DIR="$(pwd)/data"
TASK="openmaptiles"
+BUILD="true"
# Parse args into env vars
while [[ $# -gt 0 ]]; do
@@ -25,6 +26,7 @@ while [[ $# -gt 0 ]]; do
--dockerdir) DOCKER_DIR="$2"; shift ;;
--jar) METHOD="jar" ;;
--build|--source) METHOD="build" ;;
+ --skipbuild) METHOD="build"; BUILD="false" ;;
--version=*) VERSION="${1#*=}" ;;
--version) VERSION="$2"; shift ;;
@@ -134,8 +136,10 @@ case $METHOD in
run "$JAVA" "${JVM_ARGS}" -jar planetiler.jar "${PLANETILER_ARGS[@]}"
;;
build)
- echo "Building planetiler..."
- run ./mvnw -q -DskipTests --projects planetiler-dist -am clean package
+ if [ "$BUILD" == "true" ]; then
+ echo "Building planetiler..."
+ run ./mvnw -q -DskipTests --projects planetiler-dist -am clean package
+ fi
run "$JAVA" "${JVM_ARGS}" -jar planetiler-dist/target/*with-deps.jar "${PLANETILER_ARGS[@]}"
;;
esac