kopia lustrzana https://github.com/manuelkasper/sotlas-api
Modernize update summits script
Manuel Kasper
rodzic
8d7bbb4986
commit
5db511f6b6
|
|
@ -12,7 +12,7 @@
|
|||
"@turf/simplify": "^5.1.5",
|
||||
"axios": "^0.21.1",
|
||||
"carrier": "^0.3.0",
|
||||
"csv-parse": "^4.6.3",
|
||||
"csv-parse": "^5.3.0",
|
||||
"diacritics": "^1.3.0",
|
||||
"exif-parser": "^0.1.12",
|
||||
"exif-reader": "^1.0.3",
|
||||
|
|
@ -33,7 +33,6 @@
|
|||
"multer": "^1.4.2",
|
||||
"nodemailer": "^6.4.6",
|
||||
"reconnect-net": "^1.1.1",
|
||||
"request": "^2.88.0",
|
||||
"sharp": "^0.26.3",
|
||||
"togeojson": "^0.16.0",
|
||||
"togpx": "^0.5.4",
|
||||
|
|
@ -803,14 +802,6 @@
|
|||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/clone": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
|
||||
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/clone-response": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
|
||||
|
|
@ -1056,12 +1047,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "4.6.3",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz",
|
||||
"integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==",
|
||||
"dependencies": {
|
||||
"pad": "^3.2.0"
|
||||
}
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz",
|
||||
"integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ=="
|
||||
},
|
||||
"node_modules/dashdash": {
|
||||
"version": "1.14.1",
|
||||
|
|
@ -1101,14 +1089,6 @@
|
|||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/defaults": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
|
||||
"integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
|
||||
"dependencies": {
|
||||
"clone": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/defer-to-connect": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
|
||||
|
|
@ -3052,17 +3032,6 @@
|
|||
"node": ">=12.20"
|
||||
}
|
||||
},
|
||||
"node_modules/pad": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
|
||||
"integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
|
||||
"dependencies": {
|
||||
"wcwidth": "^1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/parse-asn1": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
|
||||
|
|
@ -4165,14 +4134,6 @@
|
|||
"resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz",
|
||||
"integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y="
|
||||
},
|
||||
"node_modules/wcwidth": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
|
||||
"integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
|
||||
"dependencies": {
|
||||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/web-encoding": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",
|
||||
|
|
@ -4981,11 +4942,6 @@
|
|||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"clone": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
|
||||
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4="
|
||||
},
|
||||
"clone-response": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
|
||||
|
|
@ -5203,12 +5159,9 @@
|
|||
}
|
||||
},
|
||||
"csv-parse": {
|
||||
"version": "4.6.3",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz",
|
||||
"integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==",
|
||||
"requires": {
|
||||
"pad": "^3.2.0"
|
||||
}
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz",
|
||||
"integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ=="
|
||||
},
|
||||
"dashdash": {
|
||||
"version": "1.14.1",
|
||||
|
|
@ -5239,14 +5192,6 @@
|
|||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
|
||||
},
|
||||
"defaults": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
|
||||
"integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
|
||||
"requires": {
|
||||
"clone": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"defer-to-connect": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
|
||||
|
|
@ -6765,14 +6710,6 @@
|
|||
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
|
||||
"integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="
|
||||
},
|
||||
"pad": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
|
||||
"integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
|
||||
"requires": {
|
||||
"wcwidth": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"parse-asn1": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
|
||||
|
|
@ -7644,14 +7581,6 @@
|
|||
"resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz",
|
||||
"integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y="
|
||||
},
|
||||
"wcwidth": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
|
||||
"integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
|
||||
"requires": {
|
||||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"web-encoding": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
"@turf/simplify": "^5.1.5",
|
||||
"axios": "^0.21.1",
|
||||
"carrier": "^0.3.0",
|
||||
"csv-parse": "^4.6.3",
|
||||
"csv-parse": "^5.3.0",
|
||||
"diacritics": "^1.3.0",
|
||||
"exif-parser": "^0.1.12",
|
||||
"exif-reader": "^1.0.3",
|
||||
|
|
@ -33,7 +33,6 @@
|
|||
"multer": "^1.4.2",
|
||||
"nodemailer": "^6.4.6",
|
||||
"reconnect-net": "^1.1.1",
|
||||
"request": "^2.88.0",
|
||||
"sharp": "^0.26.3",
|
||||
"togeojson": "^0.16.0",
|
||||
"togpx": "^0.5.4",
|
||||
|
|
|
|||
|
|
@ -1,141 +1,133 @@
|
|||
const request = require('request');
|
||||
const axios = require('axios');
|
||||
const MongoClient = require('mongodb').MongoClient;
|
||||
const config = require('../config');
|
||||
const assert = require('assert');
|
||||
const parse = require('csv-parse');
|
||||
const {parse} = require('csv-parse/sync');
|
||||
const fs = require('fs');
|
||||
const removeDiacritics = require('diacritics').remove;
|
||||
|
||||
const client = new MongoClient(config.mongodb.url, {useUnifiedTopology: true});
|
||||
const client = new MongoClient(config.mongodb.url);
|
||||
client.connect(function (err) {
|
||||
assert.equal(null, err);
|
||||
|
||||
processSummitList(client.db(config.mongodb.dbName));
|
||||
});
|
||||
|
||||
function processSummitList(db) {
|
||||
async function processSummitList(db) {
|
||||
let associations = new Map();
|
||||
let now = new Date();
|
||||
|
||||
let prefixToIsoCode = [];
|
||||
parse(fs.readFileSync(__dirname + '/isocodes.txt'), function(err, isocodes) {
|
||||
assert.equal(err, null);
|
||||
prefixToIsoCode = isocodes;
|
||||
});
|
||||
let prefixToIsoCode = parse(fs.readFileSync(__dirname + '/isocodes.txt'));
|
||||
|
||||
request(config.summitListUrl, (error, response, body) => {
|
||||
assert.equal(error, null);
|
||||
let response = await axios.get(config.summitListUrl);
|
||||
let body = response.data.substring(response.data.indexOf("\n")+1, response.data.length);
|
||||
|
||||
let summits = parse(body, {columns: true, relax_column_count: true});
|
||||
|
||||
if (summits.length < 100000) {
|
||||
console.error("Bad number of summits, expecting more than 100000");
|
||||
client.close();
|
||||
return;
|
||||
}
|
||||
|
||||
let bulkWrites = [];
|
||||
for (let summit of summits) {
|
||||
summit.SummitCode = summit.SummitCode.trim(); //anomaly GW/NW-003
|
||||
summit.ValidFrom = dateToMongo(summit.ValidFrom);
|
||||
summit.ValidTo = dateToMongo(summit.ValidTo, true);
|
||||
if (summit.ActivationDate) {
|
||||
summit.ActivationDate = dateToMongo(summit.ActivationDate);
|
||||
} else {
|
||||
summit.ActivationDate = null;
|
||||
summit.ActivationCall = null;
|
||||
}
|
||||
|
||||
bulkWrites.push({updateOne: {
|
||||
filter: {code: summit.SummitCode},
|
||||
update: { $set: {
|
||||
code: summit.SummitCode,
|
||||
name: summit.SummitName,
|
||||
nameNd: removeDiacritics(summit.SummitName),
|
||||
altitude: parseInt(summit.AltM),
|
||||
points: parseInt(summit.Points),
|
||||
bonusPoints: parseInt(summit.BonusPoints),
|
||||
coordinates: {
|
||||
longitude: Number(parseFloat(summit.Longitude).toFixed(5)),
|
||||
latitude: Number(parseFloat(summit.Latitude).toFixed(5))
|
||||
},
|
||||
validFrom: summit.ValidFrom,
|
||||
validTo: summit.ValidTo,
|
||||
activationCount: parseInt(summit.ActivationCount),
|
||||
activationCall: summit.ActivationCall,
|
||||
activationDate: summit.ActivationDate
|
||||
}},
|
||||
upsert: true
|
||||
}});
|
||||
|
||||
if (bulkWrites.length >= config.mongodb.batchSize) {
|
||||
await db.collection('summits').bulkWrite(bulkWrites);
|
||||
bulkWrites = [];
|
||||
}
|
||||
|
||||
let SummitAssociation = getAssociation(summit.SummitCode);
|
||||
let SummitRegion = getRegion(summit.SummitCode);
|
||||
|
||||
body = body.substring(body.indexOf("\n")+1, body.length);
|
||||
|
||||
parse(body, {columns: true, relax_column_count: true}, function(err, summits) {
|
||||
assert.equal(err, null);
|
||||
|
||||
if (summits.length < 100000) {
|
||||
console.error("Bad number of summits, expecting more than 100000");
|
||||
client.close();
|
||||
return;
|
||||
}
|
||||
let isValid = (summit.ValidFrom <= now && summit.ValidTo >= now);
|
||||
let association = associations.get(SummitAssociation);
|
||||
if (!association) {
|
||||
let info = isoCodeForPrefix(SummitAssociation, prefixToIsoCode)
|
||||
association = {code: SummitAssociation, name: summit.AssociationName, isoCode: info.isoCode, continent: info.continent, regions: new Map(), summitCount: 0};
|
||||
associations.set(SummitAssociation, association);
|
||||
}
|
||||
let region = association.regions.get(SummitRegion);
|
||||
if (!region) {
|
||||
region = {code: SummitRegion, name: summit.RegionName, summitCount: 0};
|
||||
association.regions.set(SummitRegion, region);
|
||||
}
|
||||
if (isValid) {
|
||||
association.summitCount++;
|
||||
region.summitCount++;
|
||||
}
|
||||
|
||||
let bulkWrites = [];
|
||||
for (let summit of summits) {
|
||||
summit.SummitCode = summit.SummitCode.trim(); //anomaly GW/NW-003
|
||||
summit.ValidFrom = dateToMongo(summit.ValidFrom);
|
||||
summit.ValidTo = dateToMongo(summit.ValidTo, true);
|
||||
if (summit.ActivationDate) {
|
||||
summit.ActivationDate = dateToMongo(summit.ActivationDate);
|
||||
} else {
|
||||
summit.ActivationDate = null;
|
||||
summit.ActivationCall = null;
|
||||
}
|
||||
let lat = parseFloat(summit.Latitude);
|
||||
let lon = parseFloat(summit.Longitude);
|
||||
|
||||
bulkWrites.push({updateOne: {
|
||||
filter: {code: summit.SummitCode},
|
||||
update: { $set: {
|
||||
code: summit.SummitCode,
|
||||
name: summit.SummitName,
|
||||
nameNd: removeDiacritics(summit.SummitName),
|
||||
altitude: parseInt(summit.AltM),
|
||||
points: parseInt(summit.Points),
|
||||
bonusPoints: parseInt(summit.BonusPoints),
|
||||
coordinates: {
|
||||
longitude: Number(parseFloat(summit.Longitude).toFixed(5)),
|
||||
latitude: Number(parseFloat(summit.Latitude).toFixed(5))
|
||||
},
|
||||
validFrom: summit.ValidFrom,
|
||||
validTo: summit.ValidTo,
|
||||
activationCount: parseInt(summit.ActivationCount),
|
||||
activationCall: summit.ActivationCall,
|
||||
activationDate: summit.ActivationDate
|
||||
}},
|
||||
upsert: true
|
||||
}});
|
||||
if (!region.bounds) {
|
||||
region.bounds = [[lon, lat], [lon, lat]];
|
||||
} else {
|
||||
region.bounds[0][0] = Math.min(region.bounds[0][0], lon);
|
||||
region.bounds[0][1] = Math.min(region.bounds[0][1], lat);
|
||||
region.bounds[1][0] = Math.max(region.bounds[1][0], lon);
|
||||
region.bounds[1][1] = Math.max(region.bounds[1][1], lat);
|
||||
}
|
||||
|
||||
if (bulkWrites.length >= config.mongodb.batchSize) {
|
||||
db.collection('summits').bulkWrite(bulkWrites);
|
||||
bulkWrites = [];
|
||||
}
|
||||
if (!association.bounds) {
|
||||
association.bounds = [[lon, lat], [lon, lat]];
|
||||
} else {
|
||||
association.bounds[0][0] = Math.min(association.bounds[0][0], lon);
|
||||
association.bounds[0][1] = Math.min(association.bounds[0][1], lat);
|
||||
association.bounds[1][0] = Math.max(association.bounds[1][0], lon);
|
||||
association.bounds[1][1] = Math.max(association.bounds[1][1], lat);
|
||||
}
|
||||
}
|
||||
|
||||
let SummitAssociation = getAssociation(summit.SummitCode);
|
||||
let SummitRegion = getRegion(summit.SummitCode);
|
||||
|
||||
let isValid = (summit.ValidFrom <= now && summit.ValidTo >= now);
|
||||
let association = associations.get(SummitAssociation);
|
||||
if (!association) {
|
||||
let info = isoCodeForPrefix(SummitAssociation, prefixToIsoCode)
|
||||
association = {code: SummitAssociation, name: summit.AssociationName, isoCode: info.isoCode, continent: info.continent, regions: new Map(), summitCount: 0};
|
||||
associations.set(SummitAssociation, association);
|
||||
}
|
||||
let region = association.regions.get(SummitRegion);
|
||||
if (!region) {
|
||||
region = {code: SummitRegion, name: summit.RegionName, summitCount: 0};
|
||||
association.regions.set(SummitRegion, region);
|
||||
}
|
||||
if (isValid) {
|
||||
association.summitCount++;
|
||||
region.summitCount++;
|
||||
}
|
||||
if (bulkWrites.length > 0) {
|
||||
await db.collection('summits').bulkWrite(bulkWrites);
|
||||
}
|
||||
|
||||
let lat = parseFloat(summit.Latitude);
|
||||
let lon = parseFloat(summit.Longitude);
|
||||
|
||||
if (!region.bounds) {
|
||||
region.bounds = [[lon, lat], [lon, lat]];
|
||||
} else {
|
||||
region.bounds[0][0] = Math.min(region.bounds[0][0], lon);
|
||||
region.bounds[0][1] = Math.min(region.bounds[0][1], lat);
|
||||
region.bounds[1][0] = Math.max(region.bounds[1][0], lon);
|
||||
region.bounds[1][1] = Math.max(region.bounds[1][1], lat);
|
||||
}
|
||||
|
||||
if (!association.bounds) {
|
||||
association.bounds = [[lon, lat], [lon, lat]];
|
||||
} else {
|
||||
association.bounds[0][0] = Math.min(association.bounds[0][0], lon);
|
||||
association.bounds[0][1] = Math.min(association.bounds[0][1], lat);
|
||||
association.bounds[1][0] = Math.max(association.bounds[1][0], lon);
|
||||
association.bounds[1][1] = Math.max(association.bounds[1][1], lat);
|
||||
}
|
||||
}
|
||||
|
||||
if (bulkWrites.length > 0) {
|
||||
db.collection('summits').bulkWrite(bulkWrites);
|
||||
}
|
||||
|
||||
for (let association of associations.values()) {
|
||||
association.regions = [...association.regions.values()];
|
||||
}
|
||||
|
||||
let associationCollection = db.collection('associations');
|
||||
associationCollection.deleteMany({}, () => {
|
||||
associationCollection.insertMany([...associations.values()], (err, r) => {
|
||||
if (err)
|
||||
console.error(err);
|
||||
client.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
// Update associations
|
||||
for (let association of associations.values()) {
|
||||
association.regions = [...association.regions.values()];
|
||||
}
|
||||
|
||||
let session = client.startSession();
|
||||
await session.withTransaction(async () => {
|
||||
let associationCollection = db.collection('associations');
|
||||
await associationCollection.deleteMany({}, { session });
|
||||
await associationCollection.insertMany([...associations.values()], { session });
|
||||
});
|
||||
session.endSession();
|
||||
client.close();
|
||||
}
|
||||
|
||||
function dateToMongo(date, endOfDay = false) {
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue