Memory-mapped lazy reads by default (#300)

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java

@@ -111,7 +111,7 @@ class ExternalMergeSort implements FeatureSort {
     }
     this.gzip = gzip;
     this.mmapIO = mmap;
-    long memLimit = ProcessInfo.getMaxMemoryBytes() / 2;
+    long memLimit = ProcessInfo.getMaxMemoryBytes() / 3;
     if (chunkSizeLimit > memLimit) {
       throw new IllegalStateException("Not enough memory for chunkSize=" + chunkSizeLimit + " limit=" + memLimit);
     }

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -143,7 +143,7 @@ public record PlanetilerConfig(
         0.1d),
       arguments.getBoolean("osm_lazy_reads",
         "Read OSM blocks from disk in worker threads",
-        false),
+        true),
       arguments.getBoolean("compact_db",
         "Reduce the DB size by separating and deduping the tile data",
         true),

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmInputFile.java

@@ -2,6 +2,7 @@ package com.onthegomap.planetiler.reader.osm;
 
 import com.onthegomap.planetiler.config.Bounds;
 import com.onthegomap.planetiler.reader.FileFormatException;
+import com.onthegomap.planetiler.util.ByteBufferUtil;
 import com.onthegomap.planetiler.util.DiskBacked;
 import com.onthegomap.planetiler.util.FileUtils;
 import java.io.IOException;
@@ -226,7 +227,10 @@ public class OsmInputFile implements Bounds.Provider, Supplier<OsmBlockSource>,
 
       public Iterable<OsmElement> decodeElements() {
         try {
-          return PbfDecoder.decode(readBytes(channel, offset, length));
+          var buffer = channel.map(FileChannel.MapMode.READ_ONLY, offset, length);
+          var result = PbfDecoder.decode(buffer);
+          ByteBufferUtil.free(buffer);
+          return result;
         } catch (IOException e) {
           throw new UncheckedIOException(e);
         }

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java

@@ -7,6 +7,7 @@ import com.google.common.collect.Iterators;
 import com.onthegomap.planetiler.reader.FileFormatException;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -39,8 +40,22 @@ public class PbfDecoder implements Iterable<OsmElement> {
     fieldDecoder = new PbfFieldDecoder(block);
   }
 
+  private PbfDecoder(ByteBuffer rawBlob) throws IOException {
+    byte[] data = readBlobContent(rawBlob);
+    block = Osmformat.PrimitiveBlock.parseFrom(data);
+    fieldDecoder = new PbfFieldDecoder(block);
+  }
+
+  private static byte[] readBlobContent(ByteBuffer input) throws IOException {
+    return readBlobContent(Fileformat.Blob.parseFrom(input));
+
+  }
+
   private static byte[] readBlobContent(byte[] input) throws IOException {
-    Fileformat.Blob blob = Fileformat.Blob.parseFrom(input);
+    return readBlobContent(Fileformat.Blob.parseFrom(input));
+  }
+
+  private static byte[] readBlobContent(Fileformat.Blob blob) {
     byte[] blobData;
 
     if (blob.hasRaw()) {
@@ -74,6 +89,15 @@ public class PbfDecoder implements Iterable<OsmElement> {
     }
   }
 
+  /** Decompresses and parses a block of primitive OSM elements. */
+  public static Iterable<OsmElement> decode(ByteBuffer raw) {
+    try {
+      return new PbfDecoder(raw);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Unable to process PBF blob", e);
+    }
+  }
+
   /** Decompresses and parses a header block of an OSM input file. */
   public static OsmHeader decodeHeader(byte[] raw) {
     try {

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -1608,7 +1608,7 @@ class PlanetilerTests {
     Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf");
     Files.copy(originalOsm, tempOsm);
     Planetiler.create(Arguments.fromArgs(
-      ("--tmpdir" + tempDir + " " + args).split("\\s+")
+      ("--tmpdir=" + tempDir.resolve("data") + " " + args).split("\\s+")
     ))
       .setProfile(new Profile.NullProfile() {
         @Override