diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java index b5560d86..97ce161b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java @@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory; class ExternalMergeSort implements FeatureSort { private static final Logger LOGGER = LoggerFactory.getLogger(ExternalMergeSort.class); - private static final long MAX_CHUNK_SIZE = 1_000_000_000; // 1GB + private static final long MAX_CHUNK_SIZE = 2_000_000_000; // 2GB private final Path dir; private final Stats stats; private final int chunkSizeLimit; @@ -69,7 +69,7 @@ class ExternalMergeSort implements FeatureSort { config.threads(), (int) Math.min( MAX_CHUNK_SIZE, - (ProcessInfo.getMaxMemoryBytes() / 2) / config.threads() + ProcessInfo.getMaxMemoryBytes() / 3 ), config.gzipTempStorage(), config, @@ -83,15 +83,15 @@ class ExternalMergeSort implements FeatureSort { this.stats = stats; this.chunkSizeLimit = chunkSizeLimit; this.gzip = gzip; - long memory = ProcessInfo.getMaxMemoryBytes(); - if (chunkSizeLimit > memory / 2) { - throw new IllegalStateException( - "Not enough memory to use chunk size " + chunkSizeLimit + " only have " + memory); + long memLimit = ProcessInfo.getMaxMemoryBytes() / 2; + if (chunkSizeLimit > memLimit) { + throw new IllegalStateException("Not enough memory for chunkSize=" + chunkSizeLimit + " limit=" + memLimit); } - this.workers = workers; + int maxWorkersBasedOnMemory = Math.max(1, (int) (memLimit / Math.max(1, chunkSizeLimit))); + this.workers = Math.min(workers, maxWorkersBasedOnMemory); this.readerLimit = Math.max(1, config.sortMaxReaders()); this.writerLimit = Math.max(1, config.sortMaxWriters()); - LOGGER.info("Using merge sort feature map, chunk size={}mb workers={}", chunkSizeLimit / 1_000_000, workers); + LOGGER.info("Using merge sort feature map, chunk size={}mb max workers={}", chunkSizeLimit / 1_000_000, workers); try { FileUtils.deleteDirectory(dir); Files.createDirectories(dir); @@ -339,7 +339,7 @@ class ExternalMergeSort implements FeatureSort { } public SortableChunk sort() { - Arrays.sort(featuresToSort); + Arrays.parallelSort(featuresToSort); return this; } diff --git a/sonar-project.properties b/sonar-project.properties index e29ebd08..6eb9c468 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,4 +1,4 @@ -sonar.issue.ignore.multicriteria=js1659,js3358,js1172,js106,js125,js2699,js3776 +sonar.issue.ignore.multicriteria=js1659,js3358,js1172,js106,js125,js2699,js3776,js1121 # subjective sonar.issue.ignore.multicriteria.js1659.ruleKey=java:S1659 sonar.issue.ignore.multicriteria.js1659.resourceKey=**/*.java @@ -10,6 +10,8 @@ sonar.issue.ignore.multicriteria.js125.ruleKey=java:S125 sonar.issue.ignore.multicriteria.js125.resourceKey=**/*.java sonar.issue.ignore.multicriteria.js3776.ruleKey=java:S3776 sonar.issue.ignore.multicriteria.js3776.resourceKey=**/*.java +sonar.issue.ignore.multicriteria.js1121.ruleKey=java:S1121 +sonar.issue.ignore.multicriteria.js1121.resourceKey=**/*.java # layer constructors need same signatures sonar.issue.ignore.multicriteria.js1172.ruleKey=java:S1172