mirror
/
sotlas-api
kopia lustrzana https://github.com/manuelkasper/sotlas-api
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Retry object storage uploads several times

master
Manuel Kasper 2022-10-01 14:03:17 +02:00
rodzic e3587c2394
commit 8e5e69fcb2
3 zmienionych plików z 56 dodań i 9 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

45
package-lock.json wygenerowano
Wyświetl plik

@ -32,6 +32,7 @@
"mongodb": "^4.8.1",
"multer": "^1.4.2",
"node-cron": "^3.0.1",
"promise-retry": "^2.0.1",
"reconnect-net": "^1.1.1",
"sharp": "^0.30.7",
"togeojson": "^0.16.0",
@ -1261,6 +1262,11 @@
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
},
"node_modules/err-code": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
},
"node_modules/es-abstract": {
"version": "1.20.1",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz",
@ -3014,6 +3020,18 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/promise-retry": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
"dependencies": {
"err-code": "^2.0.2",
"retry": "^0.12.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/proxy-addr": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
@ -3252,6 +3270,14 @@
"node": ">=8"
}
},
"node_modules/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
"engines": {
"node": ">= 4"
}
},
"node_modules/rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@ -5143,6 +5169,11 @@
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
},
"err-code": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
},
"es-abstract": {
"version": "1.20.1",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz",
@ -6484,6 +6515,15 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"promise-retry": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
"requires": {
"err-code": "^2.0.2",
"retry": "^0.12.0"
}
},
"proxy-addr": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
@ -6679,6 +6719,11 @@
}
}
},
"retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="
},
"rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",

1
package.json
Wyświetl plik

@ -32,6 +32,7 @@
"mongodb": "^4.8.1",
"multer": "^1.4.2",
"node-cron": "^3.0.1",
"promise-retry": "^2.0.1",
"reconnect-net": "^1.1.1",
"sharp": "^0.30.7",
"togeojson": "^0.16.0",

19
photos.js
Wyświetl plik

@ -6,6 +6,7 @@ const exif = require('exif-reader')
const path = require('path')
const hasha = require('hasha')
const minio = require('minio')
const promiseRetry = require('promise-retry')
const config = require('./config')
const db = require('./db')
@ -23,7 +24,12 @@ module.exports = {
// Upload original photo to Backblaze (don't wait for completion)
fsPromises.readFile(filename)
.then(buffer => {
uploadToCloud(config.photos.originalStorage, 'original/' + hashFilename, buffer)
promiseRetry((retry, number) => {
return uploadToCloud(config.photos.originalStorage, 'original/' + hashFilename, buffer).catch(retry)
}, {retries: 5})
.catch(() => {
console.error(`[ALERT] Cloud photo original upload failed for ${filename}`)
})
})
let photo = {
@ -87,7 +93,9 @@ module.exports = {
tasks.push(
makeResized(filename, config.photos.sizes[sizeDescr].width, config.photos.sizes[sizeDescr].height)
.then(buffer => {
return uploadToCloud(config.photos.storage, sizeDescr + '/' + hashFilename, buffer)
return promiseRetry((retry, number) => {
return uploadToCloud(config.photos.storage, sizeDescr + '/' + hashFilename, buffer).catch(retry)
}, {retries: 2})
})
)
})
@ -112,13 +120,6 @@ function uploadToCloud(storageConfig, targetPath, buffer) {
'x-amz-acl': 'public-read'
}
return minioClient.putObject(storageConfig.bucketName, targetPath, buffer, metadata)
.catch(err => {
// Try again
return minioClient.putObject(storageConfig.bucketName, targetPath, buffer, metadata)
.catch(err => {
console.error('[ALERT] Cloud photo upload failed: ' + err)
})
})
}
function getMetadata(src) {