Add job to automatically delete unused photos

config.js

@@ -94,5 +94,9 @@ config.cronjobs = [
 	{
 		script: 'updateSotaTrails',
 		schedule: '30 2 * * *'
+	},
+	{
+		script: 'deleteUnusedPhotos',
+		schedule: '10 3 * * *'
 	}
 ];

jobs/deleteUnusedPhotos.js

@@ -0,0 +1,61 @@
+const Minio = require('minio')
+const MongoClient = require('mongodb').MongoClient
+const assert = require('assert')
+const config = require('../config')
+
+let minAgeForDeletion = 7*86400*1000
+let minExpectedPhotos = 30000
+let maxDeletedPhotos = 1000
+
+const client = new MongoClient(config.mongodb.url)
+client.connect(async function (err) {
+  assert.equal(null, err)
+  let db = client.db(config.mongodb.dbName)
+
+  // Obtain list of all used filenames from database
+  let keepFilenames = new Set(await db.collection('summits').distinct('photos.filename', {'photos.filename': {'$ne': null}}))
+  if (keepFilenames.size < minExpectedPhotos) {
+    console.error(`Expected at least ${minExpectedPhotos} in DB, found only ${keepFilenames.size}`)
+    client.close()
+    return
+  }
+
+  // Check for unused files in regular storage
+  await deleteFilesNotIn(config.photos.storage, '', keepFilenames)
+
+  // Check for unused files in original storage
+  await deleteFilesNotIn(config.photos.originalStorage, 'original/', keepFilenames)
+
+  client.close()
+});
+
+async function deleteFilesNotIn(storage, prefix, keepFilenames) {
+  let allFiles = await listFiles(storage, prefix)
+  let filesToDelete = []
+  let now = new Date()
+  for (let file of allFiles) {
+    let baseName = file.name.split(/[\\/]/).pop()
+    if (!baseName.endsWith('.jpg') || (now.getTime() - file.lastModified.getTime()) < minAgeForDeletion) {
+      continue
+    }
+    if (!keepFilenames.has(baseName)) {
+      filesToDelete.push(file.name)
+    }
+  }
+  if (filesToDelete.length > maxDeletedPhotos) {
+    console.error(`Expected at most ${maxDeletedPhotos} files to delete, but found ${filesToDelete.length}`)
+    return
+  }
+  await deleteFiles(storage, filesToDelete)
+  console.log(`Deleted ${filesToDelete.length} files from ${storage.endPoint}:${storage.bucketName}`)
+}
+
+async function listFiles(storageConfig, targetPath) {
+  let minioClient = new Minio.Client(storageConfig)
+  return await minioClient.listObjectsV2(storageConfig.bucketName, targetPath, true).toArray()
+}
+
+async function deleteFiles(storageConfig, filePaths) {
+  let minioClient = new Minio.Client(storageConfig)
+  return await minioClient.removeObjects(storageConfig.bucketName, filePaths)
+}