From 5db511f6b6124ab7a7a6f084e87be3139fc0c7fe Mon Sep 17 00:00:00 2001 From: Manuel Kasper Date: Sat, 6 Aug 2022 10:50:43 +0200 Subject: [PATCH] Modernize update summits script --- package-lock.json | 85 ++------------ package.json | 3 +- tools/updateSotaSummits.js | 222 ++++++++++++++++++------------------- 3 files changed, 115 insertions(+), 195 deletions(-) diff --git a/package-lock.json b/package-lock.json index aa42694..65f057f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@turf/simplify": "^5.1.5", "axios": "^0.21.1", "carrier": "^0.3.0", - "csv-parse": "^4.6.3", + "csv-parse": "^5.3.0", "diacritics": "^1.3.0", "exif-parser": "^0.1.12", "exif-reader": "^1.0.3", @@ -33,7 +33,6 @@ "multer": "^1.4.2", "nodemailer": "^6.4.6", "reconnect-net": "^1.1.1", - "request": "^2.88.0", "sharp": "^0.26.3", "togeojson": "^0.16.0", "togpx": "^0.5.4", @@ -803,14 +802,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", - "engines": { - "node": ">=0.8" - } - }, "node_modules/clone-response": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", @@ -1056,12 +1047,9 @@ } }, "node_modules/csv-parse": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz", - "integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==", - "dependencies": { - "pad": "^3.2.0" - } + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz", + "integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==" }, "node_modules/dashdash": { "version": "1.14.1", @@ -1101,14 +1089,6 @@ "node": ">=4.0.0" } }, - "node_modules/defaults": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", - "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", - "dependencies": { - "clone": "^1.0.2" - } - }, "node_modules/defer-to-connect": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", @@ -3052,17 +3032,6 @@ "node": ">=12.20" } }, - "node_modules/pad": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz", - "integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==", - "dependencies": { - "wcwidth": "^1.0.1" - }, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", @@ -4165,14 +4134,6 @@ "resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz", "integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y=" }, - "node_modules/wcwidth": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", - "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=", - "dependencies": { - "defaults": "^1.0.3" - } - }, "node_modules/web-encoding": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", @@ -4981,11 +4942,6 @@ "safe-buffer": "^5.0.1" } }, - "clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=" - }, "clone-response": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", @@ -5203,12 +5159,9 @@ } }, "csv-parse": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz", - "integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==", - "requires": { - "pad": "^3.2.0" - } + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz", + "integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==" }, "dashdash": { "version": "1.14.1", @@ -5239,14 +5192,6 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, - "defaults": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", - "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", - "requires": { - "clone": "^1.0.2" - } - }, "defer-to-connect": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", @@ -6765,14 +6710,6 @@ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==" }, - "pad": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz", - "integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==", - "requires": { - "wcwidth": "^1.0.1" - } - }, "parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", @@ -7644,14 +7581,6 @@ "resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz", "integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y=" }, - "wcwidth": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", - "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=", - "requires": { - "defaults": "^1.0.3" - } - }, "web-encoding": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", diff --git a/package.json b/package.json index 8a6278a..5fc9c0d 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "@turf/simplify": "^5.1.5", "axios": "^0.21.1", "carrier": "^0.3.0", - "csv-parse": "^4.6.3", + "csv-parse": "^5.3.0", "diacritics": "^1.3.0", "exif-parser": "^0.1.12", "exif-reader": "^1.0.3", @@ -33,7 +33,6 @@ "multer": "^1.4.2", "nodemailer": "^6.4.6", "reconnect-net": "^1.1.1", - "request": "^2.88.0", "sharp": "^0.26.3", "togeojson": "^0.16.0", "togpx": "^0.5.4", diff --git a/tools/updateSotaSummits.js b/tools/updateSotaSummits.js index 7cbe6b2..e9f2160 100644 --- a/tools/updateSotaSummits.js +++ b/tools/updateSotaSummits.js @@ -1,141 +1,133 @@ -const request = require('request'); +const axios = require('axios'); const MongoClient = require('mongodb').MongoClient; const config = require('../config'); const assert = require('assert'); -const parse = require('csv-parse'); +const {parse} = require('csv-parse/sync'); const fs = require('fs'); const removeDiacritics = require('diacritics').remove; -const client = new MongoClient(config.mongodb.url, {useUnifiedTopology: true}); +const client = new MongoClient(config.mongodb.url); client.connect(function (err) { assert.equal(null, err); processSummitList(client.db(config.mongodb.dbName)); }); -function processSummitList(db) { +async function processSummitList(db) { let associations = new Map(); let now = new Date(); - let prefixToIsoCode = []; - parse(fs.readFileSync(__dirname + '/isocodes.txt'), function(err, isocodes) { - assert.equal(err, null); - prefixToIsoCode = isocodes; - }); + let prefixToIsoCode = parse(fs.readFileSync(__dirname + '/isocodes.txt')); - request(config.summitListUrl, (error, response, body) => { - assert.equal(error, null); + let response = await axios.get(config.summitListUrl); + let body = response.data.substring(response.data.indexOf("\n")+1, response.data.length); + + let summits = parse(body, {columns: true, relax_column_count: true}); + + if (summits.length < 100000) { + console.error("Bad number of summits, expecting more than 100000"); + client.close(); + return; + } + + let bulkWrites = []; + for (let summit of summits) { + summit.SummitCode = summit.SummitCode.trim(); //anomaly GW/NW-003 + summit.ValidFrom = dateToMongo(summit.ValidFrom); + summit.ValidTo = dateToMongo(summit.ValidTo, true); + if (summit.ActivationDate) { + summit.ActivationDate = dateToMongo(summit.ActivationDate); + } else { + summit.ActivationDate = null; + summit.ActivationCall = null; + } + + bulkWrites.push({updateOne: { + filter: {code: summit.SummitCode}, + update: { $set: { + code: summit.SummitCode, + name: summit.SummitName, + nameNd: removeDiacritics(summit.SummitName), + altitude: parseInt(summit.AltM), + points: parseInt(summit.Points), + bonusPoints: parseInt(summit.BonusPoints), + coordinates: { + longitude: Number(parseFloat(summit.Longitude).toFixed(5)), + latitude: Number(parseFloat(summit.Latitude).toFixed(5)) + }, + validFrom: summit.ValidFrom, + validTo: summit.ValidTo, + activationCount: parseInt(summit.ActivationCount), + activationCall: summit.ActivationCall, + activationDate: summit.ActivationDate + }}, + upsert: true + }}); + + if (bulkWrites.length >= config.mongodb.batchSize) { + await db.collection('summits').bulkWrite(bulkWrites); + bulkWrites = []; + } + + let SummitAssociation = getAssociation(summit.SummitCode); + let SummitRegion = getRegion(summit.SummitCode); - body = body.substring(body.indexOf("\n")+1, body.length); - - parse(body, {columns: true, relax_column_count: true}, function(err, summits) { - assert.equal(err, null); - - if (summits.length < 100000) { - console.error("Bad number of summits, expecting more than 100000"); - client.close(); - return; - } + let isValid = (summit.ValidFrom <= now && summit.ValidTo >= now); + let association = associations.get(SummitAssociation); + if (!association) { + let info = isoCodeForPrefix(SummitAssociation, prefixToIsoCode) + association = {code: SummitAssociation, name: summit.AssociationName, isoCode: info.isoCode, continent: info.continent, regions: new Map(), summitCount: 0}; + associations.set(SummitAssociation, association); + } + let region = association.regions.get(SummitRegion); + if (!region) { + region = {code: SummitRegion, name: summit.RegionName, summitCount: 0}; + association.regions.set(SummitRegion, region); + } + if (isValid) { + association.summitCount++; + region.summitCount++; + } - let bulkWrites = []; - for (let summit of summits) { - summit.SummitCode = summit.SummitCode.trim(); //anomaly GW/NW-003 - summit.ValidFrom = dateToMongo(summit.ValidFrom); - summit.ValidTo = dateToMongo(summit.ValidTo, true); - if (summit.ActivationDate) { - summit.ActivationDate = dateToMongo(summit.ActivationDate); - } else { - summit.ActivationDate = null; - summit.ActivationCall = null; - } + let lat = parseFloat(summit.Latitude); + let lon = parseFloat(summit.Longitude); - bulkWrites.push({updateOne: { - filter: {code: summit.SummitCode}, - update: { $set: { - code: summit.SummitCode, - name: summit.SummitName, - nameNd: removeDiacritics(summit.SummitName), - altitude: parseInt(summit.AltM), - points: parseInt(summit.Points), - bonusPoints: parseInt(summit.BonusPoints), - coordinates: { - longitude: Number(parseFloat(summit.Longitude).toFixed(5)), - latitude: Number(parseFloat(summit.Latitude).toFixed(5)) - }, - validFrom: summit.ValidFrom, - validTo: summit.ValidTo, - activationCount: parseInt(summit.ActivationCount), - activationCall: summit.ActivationCall, - activationDate: summit.ActivationDate - }}, - upsert: true - }}); + if (!region.bounds) { + region.bounds = [[lon, lat], [lon, lat]]; + } else { + region.bounds[0][0] = Math.min(region.bounds[0][0], lon); + region.bounds[0][1] = Math.min(region.bounds[0][1], lat); + region.bounds[1][0] = Math.max(region.bounds[1][0], lon); + region.bounds[1][1] = Math.max(region.bounds[1][1], lat); + } - if (bulkWrites.length >= config.mongodb.batchSize) { - db.collection('summits').bulkWrite(bulkWrites); - bulkWrites = []; - } + if (!association.bounds) { + association.bounds = [[lon, lat], [lon, lat]]; + } else { + association.bounds[0][0] = Math.min(association.bounds[0][0], lon); + association.bounds[0][1] = Math.min(association.bounds[0][1], lat); + association.bounds[1][0] = Math.max(association.bounds[1][0], lon); + association.bounds[1][1] = Math.max(association.bounds[1][1], lat); + } + } - let SummitAssociation = getAssociation(summit.SummitCode); - let SummitRegion = getRegion(summit.SummitCode); - - let isValid = (summit.ValidFrom <= now && summit.ValidTo >= now); - let association = associations.get(SummitAssociation); - if (!association) { - let info = isoCodeForPrefix(SummitAssociation, prefixToIsoCode) - association = {code: SummitAssociation, name: summit.AssociationName, isoCode: info.isoCode, continent: info.continent, regions: new Map(), summitCount: 0}; - associations.set(SummitAssociation, association); - } - let region = association.regions.get(SummitRegion); - if (!region) { - region = {code: SummitRegion, name: summit.RegionName, summitCount: 0}; - association.regions.set(SummitRegion, region); - } - if (isValid) { - association.summitCount++; - region.summitCount++; - } + if (bulkWrites.length > 0) { + await db.collection('summits').bulkWrite(bulkWrites); + } - let lat = parseFloat(summit.Latitude); - let lon = parseFloat(summit.Longitude); - - if (!region.bounds) { - region.bounds = [[lon, lat], [lon, lat]]; - } else { - region.bounds[0][0] = Math.min(region.bounds[0][0], lon); - region.bounds[0][1] = Math.min(region.bounds[0][1], lat); - region.bounds[1][0] = Math.max(region.bounds[1][0], lon); - region.bounds[1][1] = Math.max(region.bounds[1][1], lat); - } - - if (!association.bounds) { - association.bounds = [[lon, lat], [lon, lat]]; - } else { - association.bounds[0][0] = Math.min(association.bounds[0][0], lon); - association.bounds[0][1] = Math.min(association.bounds[0][1], lat); - association.bounds[1][0] = Math.max(association.bounds[1][0], lon); - association.bounds[1][1] = Math.max(association.bounds[1][1], lat); - } - } - - if (bulkWrites.length > 0) { - db.collection('summits').bulkWrite(bulkWrites); - } - - for (let association of associations.values()) { - association.regions = [...association.regions.values()]; - } - - let associationCollection = db.collection('associations'); - associationCollection.deleteMany({}, () => { - associationCollection.insertMany([...associations.values()], (err, r) => { - if (err) - console.error(err); - client.close(); - }); - }); - }); + // Update associations + for (let association of associations.values()) { + association.regions = [...association.regions.values()]; + } + + let session = client.startSession(); + await session.withTransaction(async () => { + let associationCollection = db.collection('associations'); + await associationCollection.deleteMany({}, { session }); + await associationCollection.insertMany([...associations.values()], { session }); }); + session.endSession(); + client.close(); } function dateToMongo(date, endOfDay = false) {