mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Improve DB Indexes and Fix Deferred Index Logging in Compact DB Mode (#245)

pull/248/head
Björn Bilger 2022-06-02 03:29:59 +02:00 zatwierdzone przez GitHub
rodzic f93e5221f8
commit 97642fc096
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
7 zmienionych plików z 107 dodań i 57 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
planetiler-basemap/src/test/java/com/onthegomap/planetiler/basemap/util/VerifyMonacoTest.java
Wyświetl plik

@ -37,13 +37,13 @@ class VerifyMonacoTest {
@Test
void testEmptyTablesInvalid() {
mbtiles.createTables().addTileIndex();
mbtiles.createTablesWithIndexes();
assertInvalid(mbtiles);
}
@Test
void testStilInvalidWithOneTile() throws IOException {
mbtiles.createTables().addTileIndex();
mbtiles.createTablesWithIndexes();
mbtiles.metadata().setName("name");
try (var writer = mbtiles.newBatchedTileWriter()) {
VectorTile tile = new VectorTile();

9
planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java
Wyświetl plik

@ -68,9 +68,10 @@ public class BenchmarkMbtilesWriter {
Path outputPath = getTempOutputPath();
try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.compactDb())) {
mbtiles.createTables();
if (!config.deferIndexCreation()) {
mbtiles.addTileIndex();
if (config.skipIndexCreation()) {
mbtiles.createTablesWithoutIndexes();
} else {
mbtiles.createTablesWithIndexes();
}
try (var writer = mbtiles.newBatchedTileWriter()) {
@ -101,7 +102,7 @@ public class BenchmarkMbtilesWriter {
for (int z = 0; z <= 14; z++) {
int maxCoord = 1 << z;
for (int x = 0; x < maxCoord; x++) {
for (int y = 0; y < maxCoord; y++) {
for (int y = maxCoord - 1; y >= 0; y--) {
TileCoord coord = TileCoord.ofXYZ(x, y, z);
TileEncodingResult toWrite;

6
planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
Wyświetl plik

@ -18,7 +18,7 @@ public record PlanetilerConfig(
Duration logInterval,
int minzoom,
int maxzoom,
boolean deferIndexCreation,
boolean skipIndexCreation,
boolean optimizeDb,
boolean emitTilesInOrder,
boolean force,
@ -98,7 +98,7 @@ public record PlanetilerConfig(
arguments.getDuration("loginterval", "time between logs", "10s"),
arguments.getInteger("minzoom", "minimum zoom level", MIN_MINZOOM),
arguments.getInteger("maxzoom", "maximum zoom level (limit 14)", MAX_MAXZOOM),
arguments.getBoolean("defer_mbtiles_index_creation", "skip adding index to mbtiles file", false),
arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false),
arguments.getBoolean("optimize_db", "optimize mbtiles after writing", false),
arguments.getBoolean("emit_tiles_in_order", "emit tiles in index order", true),
arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
@ -142,7 +142,7 @@ public record PlanetilerConfig(
false),
arguments.getBoolean("compact_db",
"Reduce the DB size by separating and deduping the tile data",
false)
true)
);
}

81
planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
Wyświetl plik

@ -53,12 +53,6 @@ public final class Mbtiles implements Closeable {
private static final String TILES_COL_X = "tile_column";
private static final String TILES_COL_Y = "tile_row";
private static final String TILES_COL_Z = "zoom_level";
public static final String ADD_TILE_INDEX_SQL = "create unique index tile_index on %s (%s, %s, %s)".formatted(
TILES_TABLE,
TILES_COL_Z,
TILES_COL_X,
TILES_COL_Y
);
private static final String TILES_COL_DATA = "tile_data";
private static final String TILES_DATA_TABLE = "tiles_data";
@ -70,13 +64,6 @@ public final class Mbtiles implements Closeable {
private static final String TILES_SHALLOW_COL_Y = TILES_COL_Y;
private static final String TILES_SHALLOW_COL_Z = TILES_COL_Z;
private static final String TILES_SHALLOW_COL_DATA_ID = TILES_DATA_COL_DATA_ID;
private static final String ADD_TILES_SHALLOW_INDEX_SQL =
"create unique index tiles_shallow_index on %s (%s, %s, %s)".formatted(
TILES_SHALLOW_TABLE,
TILES_SHALLOW_COL_Z,
TILES_SHALLOW_COL_X,
TILES_SHALLOW_COL_Y
);
private static final String METADATA_TABLE = "metadata";
private static final String METADATA_COL_NAME = "name";
@ -118,7 +105,7 @@ public final class Mbtiles implements Closeable {
/** @see {@link #newInMemoryDatabase(boolean)} */
public static Mbtiles newInMemoryDatabase() {
return newInMemoryDatabase(false);
return newInMemoryDatabase(true);
}
/** Returns a new connection to an mbtiles file optimized for fast bulk writes. */
@ -181,15 +168,20 @@ public final class Mbtiles implements Closeable {
return execute(Arrays.asList(queries));
}
public Mbtiles addTileIndex() {
if (compactDb) {
return execute(ADD_TILES_SHALLOW_INDEX_SQL);
} else {
return execute(ADD_TILE_INDEX_SQL);
}
/**
* Creates the required tables (and views) but skips index creation on some tables. Those indexes should be added
* later manually as described in {@code #getManualIndexCreationStatements()}.
*/
public Mbtiles createTablesWithoutIndexes() {
return createTables(true);
}
public Mbtiles createTables() {
/** Creates the required tables (and views) including all indexes. */
public Mbtiles createTablesWithIndexes() {
return createTables(false);
}
private Mbtiles createTables(boolean skipIndexCreation) {
List<String> ddlStatements = new ArrayList<>();
@ -199,6 +191,13 @@ public final class Mbtiles implements Closeable {
.add("create unique index name on " + METADATA_TABLE + " (" + METADATA_COL_NAME + ");");
if (compactDb) {
/*
* "primary key without rowid" results in a clustered index which is much more compact and performant (r/w)
* than "unique" which results in a non-clustered index
*/
String tilesShallowPrimaryKeyAddition = skipIndexCreation ? "" : """
, primary key(%s,%s,%s)
""".formatted(TILES_SHALLOW_COL_Z, TILES_SHALLOW_COL_X, TILES_SHALLOW_COL_Y);
ddlStatements
.add("""
create table %s (
@ -206,9 +205,14 @@ public final class Mbtiles implements Closeable {
%s integer,
%s integer,
%s integer
)
%s
) %s
""".formatted(TILES_SHALLOW_TABLE,
TILES_SHALLOW_COL_Z, TILES_SHALLOW_COL_X, TILES_SHALLOW_COL_Y, TILES_SHALLOW_COL_DATA_ID));
TILES_SHALLOW_COL_Z, TILES_SHALLOW_COL_X, TILES_SHALLOW_COL_Y, TILES_SHALLOW_COL_DATA_ID,
tilesShallowPrimaryKeyAddition,
skipIndexCreation ? "" : "without rowid"));
// here it's not worth to skip the "primary key"/index - doing so even hurts write performance
ddlStatements.add("""
create table %s (
%s integer primary key,
@ -234,14 +238,39 @@ public final class Mbtiles implements Closeable {
TILES_DATA_TABLE, TILES_SHALLOW_TABLE, TILES_SHALLOW_COL_DATA_ID, TILES_DATA_TABLE, TILES_DATA_COL_DATA_ID
));
} else {
ddlStatements.add("create table " + TILES_TABLE + " (" + TILES_COL_Z + " integer, " + TILES_COL_X + " integer, " +
TILES_COL_Y + ", " + TILES_COL_DATA + " blob);");
// here "primary key (with rowid)" is much more compact than a "primary key without rowid" because the tile data is part of the table
String tilesUniqueAddition = skipIndexCreation ? "" : """
, primary key(%s,%s,%s)
""".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y);
ddlStatements.add("""
create table %s (
%s integer,
%s integer,
%s integer,
%s blob
%s
)
""".formatted(TILES_TABLE, TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, tilesUniqueAddition));
}
return execute(ddlStatements);
}
/** Returns the DDL statements to create the indexes manually when the option to skip index creation was chosen. */
public List<String> getManualIndexCreationStatements() {
if (compactDb) {
return List.of(
"create unique index tiles_shallow_index on %s (%s, %s, %s)"
.formatted(TILES_SHALLOW_TABLE, TILES_SHALLOW_COL_Z, TILES_SHALLOW_COL_X, TILES_SHALLOW_COL_Y)
);
} else {
return List.of(
"create unique index tile_index on %s (%s, %s, %s)"
.formatted(TILES_TABLE, TILES_COL_Z, TILES_COL_X, TILES_COL_Y)
);
}
}
public Mbtiles vacuumAnalyze() {
return execute(
"VACUUM;",

12
planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java
Wyświetl plik

@ -309,11 +309,15 @@ public class MbtilesWriter {
}
private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionException, InterruptedException {
db.createTables();
if (!config.deferIndexCreation()) {
db.addTileIndex();
if (config.skipIndexCreation()) {
db.createTablesWithoutIndexes();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Skipping index creation. Add later by executing: {}",
String.join(" ; ", db.getManualIndexCreationStatements()));
}
} else {
LOGGER.info("Deferring index creation. Add later by executing: {}", Mbtiles.ADD_TILE_INDEX_SQL);
db.createTablesWithIndexes();
}
db.metadata()

46
planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java
Wyświetl plik

@ -1,9 +1,7 @@
package com.onthegomap.planetiler.mbtiles;
import static com.onthegomap.planetiler.TestUtils.assertSameJson;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.*;
import com.google.common.math.IntMath;
import com.onthegomap.planetiler.TestUtils;
@ -12,7 +10,9 @@ import com.onthegomap.planetiler.geo.TileCoord;
import java.io.IOException;
import java.math.RoundingMode;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.OptionalInt;
import java.util.Set;
@ -34,13 +34,15 @@ class MbtilesTest {
private static final
void testWriteTiles(int howMany, boolean deferIndexCreation, boolean optimize, boolean compactDb)
void testWriteTiles(int howMany, boolean skipIndexCreation, boolean optimize, boolean compactDb)
throws IOException, SQLException {
try (Mbtiles db = Mbtiles.newInMemoryDatabase(compactDb)) {
db.createTables();
if (!deferIndexCreation) {
db.addTileIndex();
if (skipIndexCreation) {
db.createTablesWithoutIndexes();
} else {
db.createTablesWithIndexes();
}
assertNull(db.getTile(0, 0, 0));
Set<Mbtiles.TileEntry> expected = new TreeSet<>();
try (var writer = db.newBatchedTileWriter()) {
@ -57,9 +59,7 @@ class MbtilesTest {
expected.add(entry);
}
}
if (deferIndexCreation) {
db.addTileIndex();
}
if (optimize) {
db.vacuumAnalyze();
}
@ -94,7 +94,7 @@ class MbtilesTest {
}
@Test
void testDeferIndexCreation() throws IOException, SQLException {
void testSkipIndexCreation() throws IOException, SQLException {
testWriteTiles(10, true, false, false);
}
@ -103,11 +103,27 @@ class MbtilesTest {
testWriteTiles(10, false, true, false);
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testManualIndexCreationStatements(boolean compactDb) throws IOException, SQLException {
try (Mbtiles db = Mbtiles.newInMemoryDatabase(compactDb)) {
db.createTablesWithoutIndexes();
List<String> indexCreationStmts = db.getManualIndexCreationStatements();
assertFalse(indexCreationStmts.isEmpty());
for (String indexCreationStmt : indexCreationStmts) {
try (Statement stmt = db.connection().createStatement()) {
assertDoesNotThrow(() -> stmt.execute(indexCreationStmt));
}
}
}
}
@Test
void testAddMetadata() throws IOException {
Map<String, String> expected = new TreeMap<>();
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
var metadata = db.createTables().metadata();
var metadata = db.createTablesWithoutIndexes().metadata();
metadata.setName("name value");
expected.put("name", "name value");
@ -144,7 +160,7 @@ class MbtilesTest {
void testAddMetadataWorldBounds() throws IOException {
Map<String, String> expected = new TreeMap<>();
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
var metadata = db.createTables().metadata();
var metadata = db.createTablesWithoutIndexes().metadata();
metadata.setBoundsAndCenter(GeoUtils.WORLD_LAT_LON_BOUNDS);
expected.put("bounds", "-180,-85.05113,180,85.05113");
expected.put("center", "0,0,0");
@ -157,7 +173,7 @@ class MbtilesTest {
void testAddMetadataSmallBounds() throws IOException {
Map<String, String> expected = new TreeMap<>();
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
var metadata = db.createTables().metadata();
var metadata = db.createTablesWithoutIndexes().metadata();
metadata.setBoundsAndCenter(new Envelope(-73.6632, -69.7598, 41.1274, 43.0185));
expected.put("bounds", "-73.6632,41.1274,-69.7598,43.0185");
expected.put("center", "-71.7115,42.07295,7");
@ -168,7 +184,7 @@ class MbtilesTest {
private void testMetadataJson(Mbtiles.MetadataJson object, String expected) throws IOException {
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
var metadata = db.createTables().metadata();
var metadata = db.createTablesWithoutIndexes().metadata();
metadata.setJson(object);
var actual = metadata.getAll().get("json");
assertSameJson(expected, actual);

6
planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java
Wyświetl plik

@ -37,13 +37,13 @@ class VerifyTest {
@Test
void testEmptyTablesInvalid() {
mbtiles.createTables().addTileIndex();
mbtiles.createTablesWithIndexes();
assertInvalid(mbtiles);
}
@Test
void testValidWithNameAndOneTile() throws IOException {
mbtiles.createTables().addTileIndex();
mbtiles.createTablesWithIndexes();
mbtiles.metadata().setName("name");
try (var writer = mbtiles.newBatchedTileWriter()) {
VectorTile tile = new VectorTile();
@ -60,7 +60,7 @@ class VerifyTest {
@Test
void testInvalidGeometry() throws IOException {
mbtiles.createTables().addTileIndex();
mbtiles.createTablesWithIndexes();
mbtiles.metadata().setName("name");
try (var writer = mbtiles.newBatchedTileWriter()) {
VectorTile tile = new VectorTile();