Minimize number of temp feature chunk files (#199)

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java

@@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory;
 class ExternalMergeSort implements FeatureSort {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(ExternalMergeSort.class);
-  private static final long MAX_CHUNK_SIZE = 1_000_000_000; // 1GB
+  private static final long MAX_CHUNK_SIZE = 2_000_000_000; // 2GB
   private final Path dir;
   private final Stats stats;
   private final int chunkSizeLimit;
@@ -69,7 +69,7 @@ class ExternalMergeSort implements FeatureSort {
       config.threads(),
       (int) Math.min(
         MAX_CHUNK_SIZE,
-        (ProcessInfo.getMaxMemoryBytes() / 2) / config.threads()
+        ProcessInfo.getMaxMemoryBytes() / 3
       ),
       config.gzipTempStorage(),
       config,
@@ -83,15 +83,15 @@ class ExternalMergeSort implements FeatureSort {
     this.stats = stats;
     this.chunkSizeLimit = chunkSizeLimit;
     this.gzip = gzip;
-    long memory = ProcessInfo.getMaxMemoryBytes();
-    if (chunkSizeLimit > memory / 2) {
-      throw new IllegalStateException(
-        "Not enough memory to use chunk size " + chunkSizeLimit + " only have " + memory);
+    long memLimit = ProcessInfo.getMaxMemoryBytes() / 2;
+    if (chunkSizeLimit > memLimit) {
+      throw new IllegalStateException("Not enough memory for chunkSize=" + chunkSizeLimit + " limit=" + memLimit);
     }
-    this.workers = workers;
+    int maxWorkersBasedOnMemory = Math.max(1, (int) (memLimit / Math.max(1, chunkSizeLimit)));
+    this.workers = Math.min(workers, maxWorkersBasedOnMemory);
     this.readerLimit = Math.max(1, config.sortMaxReaders());
     this.writerLimit = Math.max(1, config.sortMaxWriters());
-    LOGGER.info("Using merge sort feature map, chunk size={}mb workers={}", chunkSizeLimit / 1_000_000, workers);
+    LOGGER.info("Using merge sort feature map, chunk size={}mb max workers={}", chunkSizeLimit / 1_000_000, workers);
     try {
       FileUtils.deleteDirectory(dir);
       Files.createDirectories(dir);
@@ -339,7 +339,7 @@ class ExternalMergeSort implements FeatureSort {
       }
 
       public SortableChunk sort() {
-        Arrays.sort(featuresToSort);
+        Arrays.parallelSort(featuresToSort);
         return this;
       }
 

sonar-project.properties

@@ -1,4 +1,4 @@
-sonar.issue.ignore.multicriteria=js1659,js3358,js1172,js106,js125,js2699,js3776
+sonar.issue.ignore.multicriteria=js1659,js3358,js1172,js106,js125,js2699,js3776,js1121
 # subjective
 sonar.issue.ignore.multicriteria.js1659.ruleKey=java:S1659
 sonar.issue.ignore.multicriteria.js1659.resourceKey=**/*.java
@@ -10,6 +10,8 @@ sonar.issue.ignore.multicriteria.js125.ruleKey=java:S125
 sonar.issue.ignore.multicriteria.js125.resourceKey=**/*.java
 sonar.issue.ignore.multicriteria.js3776.ruleKey=java:S3776
 sonar.issue.ignore.multicriteria.js3776.resourceKey=**/*.java
+sonar.issue.ignore.multicriteria.js1121.ruleKey=java:S1121
+sonar.issue.ignore.multicriteria.js1121.resourceKey=**/*.java
 
 # layer constructors need same signatures
 sonar.issue.ignore.multicriteria.js1172.ruleKey=java:S1172