kopia lustrzana https://github.com/onthegomap/planetiler
Retry wikidata requests (#115)
rodzic
e5acfbd17c
commit
1ce2b5f76c
|
@ -23,6 +23,7 @@ public record PlanetilerConfig(
|
|||
String nodeMapStorage,
|
||||
String httpUserAgent,
|
||||
Duration httpTimeout,
|
||||
int httpRetries,
|
||||
long downloadChunkSizeMB,
|
||||
int downloadThreads,
|
||||
double minFeatureSizeAtMaxZoom,
|
||||
|
@ -45,6 +46,9 @@ public record PlanetilerConfig(
|
|||
if (maxzoom > MAX_MAXZOOM) {
|
||||
throw new IllegalArgumentException("Max zoom must be <= " + MAX_MAXZOOM + ", was " + maxzoom);
|
||||
}
|
||||
if (httpRetries < 0) {
|
||||
throw new IllegalArgumentException("HTTP Retries must be >= 0, was " + httpRetries);
|
||||
}
|
||||
}
|
||||
|
||||
public static PlanetilerConfig defaults() {
|
||||
|
@ -74,6 +78,7 @@ public record PlanetilerConfig(
|
|||
arguments.getString("http_user_agent", "User-Agent header to set when downloading files over HTTP",
|
||||
"Planetiler downloader (https://github.com/onthegomap/planetiler)"),
|
||||
arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
|
||||
arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
|
||||
arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
|
||||
arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
|
||||
arguments.getDouble("min_feature_size_at_max_zoom",
|
||||
|
|
|
@ -281,7 +281,7 @@ public class Wikidata {
|
|||
SELECT ?id ?label where {
|
||||
VALUES ?id { %s } ?id (owl:sameAs* / rdfs:label) ?label
|
||||
}
|
||||
""".formatted(qidList).replaceAll("\\s+", " ");
|
||||
""".formatted(qidList).replaceAll("\\s+", " ").trim();
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create("https://query.wikidata.org/bigdata/namespace/wdq/sparql"))
|
||||
.timeout(config.httpTimeout())
|
||||
|
@ -290,10 +290,28 @@ public class Wikidata {
|
|||
.header(CONTENT_TYPE, "application/sparql-query")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8))
|
||||
.build();
|
||||
InputStream response = client.send(request);
|
||||
|
||||
try (var bis = new BufferedInputStream(response)) {
|
||||
return parseResults(bis);
|
||||
InputStream response = null;
|
||||
for (int i = 0; i <= config.httpRetries() && response == null; i++) {
|
||||
try {
|
||||
response = client.send(request);
|
||||
} catch (IOException e) {
|
||||
boolean lastTry = i == config.httpRetries();
|
||||
if (!lastTry) {
|
||||
LOGGER.warn("sparql query failed, retrying: " + e);
|
||||
} else {
|
||||
LOGGER.error("sparql query failed, exhausted retries: " + e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (response != null) {
|
||||
try (var bis = new BufferedInputStream(response)) {
|
||||
return parseResults(bis);
|
||||
}
|
||||
} else {
|
||||
throw new IllegalStateException("No response or exception"); // should never happen
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,10 +1,13 @@
|
|||
package com.onthegomap.planetiler.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
|
||||
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
|
@ -30,8 +33,48 @@ import org.mockito.Mockito;
|
|||
|
||||
public class WikidataTest {
|
||||
|
||||
final PlanetilerConfig config = PlanetilerConfig.defaults();
|
||||
final PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromArgs("--http-retries=1"));
|
||||
final Profile profile = new Profile.NullProfile();
|
||||
final String response = """
|
||||
{
|
||||
"head" : {
|
||||
"vars" : [ "id", "label" ]
|
||||
},
|
||||
"results" : {
|
||||
"bindings" : [ {
|
||||
"id" : {
|
||||
"type" : "uri",
|
||||
"value" : "http://www.wikidata.org/entity/Q1"
|
||||
},
|
||||
"label" : {
|
||||
"xml:lang" : "en",
|
||||
"type" : "literal",
|
||||
"value" : "en name"
|
||||
}
|
||||
}, {
|
||||
"id" : {
|
||||
"type" : "uri",
|
||||
"value" : "http://www.wikidata.org/entity/Q1"
|
||||
},
|
||||
"label" : {
|
||||
"xml:lang" : "es",
|
||||
"type" : "literal",
|
||||
"value" : "es name"
|
||||
}
|
||||
}, {
|
||||
"id" : {
|
||||
"type" : "uri",
|
||||
"value" : "http://www.wikidata.org/entity/Q2"
|
||||
},
|
||||
"label" : {
|
||||
"xml:lang" : "es",
|
||||
"type" : "literal",
|
||||
"value" : "es name2"
|
||||
}
|
||||
} ]
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
@Test
|
||||
public void testWikidataTranslations() {
|
||||
|
@ -56,46 +99,8 @@ public class WikidataTest {
|
|||
Wikidata fixture = new Wikidata(writer, client, 2, profile, config);
|
||||
fixture.fetch(1L);
|
||||
Mockito.verifyNoInteractions(client);
|
||||
Mockito.when(client.send(Mockito.any())).thenReturn(new ByteArrayInputStream("""
|
||||
{
|
||||
"head" : {
|
||||
"vars" : [ "id", "label" ]
|
||||
},
|
||||
"results" : {
|
||||
"bindings" : [ {
|
||||
"id" : {
|
||||
"type" : "uri",
|
||||
"value" : "http://www.wikidata.org/entity/Q1"
|
||||
},
|
||||
"label" : {
|
||||
"xml:lang" : "en",
|
||||
"type" : "literal",
|
||||
"value" : "en name"
|
||||
}
|
||||
}, {
|
||||
"id" : {
|
||||
"type" : "uri",
|
||||
"value" : "http://www.wikidata.org/entity/Q1"
|
||||
},
|
||||
"label" : {
|
||||
"xml:lang" : "es",
|
||||
"type" : "literal",
|
||||
"value" : "es name"
|
||||
}
|
||||
}, {
|
||||
"id" : {
|
||||
"type" : "uri",
|
||||
"value" : "http://www.wikidata.org/entity/Q2"
|
||||
},
|
||||
"label" : {
|
||||
"xml:lang" : "es",
|
||||
"type" : "literal",
|
||||
"value" : "es name2"
|
||||
}
|
||||
} ]
|
||||
}
|
||||
}
|
||||
""".getBytes(StandardCharsets.UTF_8)));
|
||||
Mockito.when(client.send(Mockito.any()))
|
||||
.thenReturn(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)));
|
||||
fixture.fetch(2L);
|
||||
|
||||
return List.of(
|
||||
|
@ -133,6 +138,29 @@ public class WikidataTest {
|
|||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetryFailedRequestOnce() throws IOException, InterruptedException {
|
||||
StringWriter writer = new StringWriter();
|
||||
Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
|
||||
Wikidata fixture = new Wikidata(writer, client, 1, profile, config);
|
||||
Mockito.when(client.send(Mockito.any()))
|
||||
// fail once then succeed
|
||||
.thenThrow(IOException.class)
|
||||
.thenReturn(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)));
|
||||
fixture.fetch(1L);
|
||||
var translations = Wikidata.load(new BufferedReader(new StringReader(writer.toString())));
|
||||
assertEquals(Map.of("en", "en name", "es", "es name"), translations.get(1));
|
||||
assertEquals(Map.of("es", "es name2"), translations.get(2));
|
||||
|
||||
Mockito.reset(client);
|
||||
Mockito.when(client.send(Mockito.any()))
|
||||
// fail all subsequent requests
|
||||
.thenThrow(IOException.class);
|
||||
var outerException = assertThrows(RuntimeException.class, () -> fixture.fetch(2L));
|
||||
var innerException = outerException.getCause();
|
||||
assertInstanceOf(IOException.class, innerException);
|
||||
}
|
||||
|
||||
private static void assertEqualsIgnoringWhitespace(String expected, String actual) {
|
||||
assertEquals(ignoreWhitespace(expected), ignoreWhitespace(actual));
|
||||
}
|
||||
|
|
Ładowanie…
Reference in New Issue