From d299b20227f76231b548e28a6bdac6875b1ca9ec Mon Sep 17 00:00:00 2001
From: Manuel Kasper <mk@neon1.net>
Date: Thu, 18 Aug 2022 17:24:24 +0200
Subject: [PATCH] Add job to automatically delete unused photos

---
 config.js                  |  4 +++
 jobs/deleteUnusedPhotos.js | 61 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 jobs/deleteUnusedPhotos.js

diff --git a/config.js b/config.js
index a6c02b1..59c06fd 100644
--- a/config.js
+++ b/config.js
@@ -94,5 +94,9 @@ config.cronjobs = [
 	{
 		script: 'updateSotaTrails',
 		schedule: '30 2 * * *'
+	},
+	{
+		script: 'deleteUnusedPhotos',
+		schedule: '10 3 * * *'
 	}
 ];
diff --git a/jobs/deleteUnusedPhotos.js b/jobs/deleteUnusedPhotos.js
new file mode 100644
index 0000000..b4acbc2
--- /dev/null
+++ b/jobs/deleteUnusedPhotos.js
@@ -0,0 +1,61 @@
+const Minio = require('minio')
+const MongoClient = require('mongodb').MongoClient
+const assert = require('assert')
+const config = require('../config')
+
+let minAgeForDeletion = 7*86400*1000
+let minExpectedPhotos = 30000
+let maxDeletedPhotos = 1000
+
+const client = new MongoClient(config.mongodb.url)
+client.connect(async function (err) {
+  assert.equal(null, err)
+  let db = client.db(config.mongodb.dbName)
+
+  // Obtain list of all used filenames from database
+  let keepFilenames = new Set(await db.collection('summits').distinct('photos.filename', {'photos.filename': {'$ne': null}}))
+  if (keepFilenames.size < minExpectedPhotos) {
+    console.error(`Expected at least ${minExpectedPhotos} in DB, found only ${keepFilenames.size}`)
+    client.close()
+    return
+  }
+
+  // Check for unused files in regular storage
+  await deleteFilesNotIn(config.photos.storage, '', keepFilenames)
+
+  // Check for unused files in original storage
+  await deleteFilesNotIn(config.photos.originalStorage, 'original/', keepFilenames)
+
+  client.close()
+});
+
+async function deleteFilesNotIn(storage, prefix, keepFilenames) {
+  let allFiles = await listFiles(storage, prefix)
+  let filesToDelete = []
+  let now = new Date()
+  for (let file of allFiles) {
+    let baseName = file.name.split(/[\\/]/).pop()
+    if (!baseName.endsWith('.jpg') || (now.getTime() - file.lastModified.getTime()) < minAgeForDeletion) {
+      continue
+    }
+    if (!keepFilenames.has(baseName)) {
+      filesToDelete.push(file.name)
+    }
+  }
+  if (filesToDelete.length > maxDeletedPhotos) {
+    console.error(`Expected at most ${maxDeletedPhotos} files to delete, but found ${filesToDelete.length}`)
+    return
+  }
+  await deleteFiles(storage, filesToDelete)
+  console.log(`Deleted ${filesToDelete.length} files from ${storage.endPoint}:${storage.bucketName}`)
+}
+
+async function listFiles(storageConfig, targetPath) {
+  let minioClient = new Minio.Client(storageConfig)
+  return await minioClient.listObjectsV2(storageConfig.bucketName, targetPath, true).toArray()
+}
+
+async function deleteFiles(storageConfig, filePaths) {
+  let minioClient = new Minio.Client(storageConfig)
+  return await minioClient.removeObjects(storageConfig.bucketName, filePaths)
+}