mirror
/
vizicities
kopia lustrzana https://github.com/robhawkes/vizicities
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Tidied up Web Worker feature loading

0.1
Rob Hawkes 2014-02-01 11:25:03 +00:00
rodzic 9d112088cb
commit a4fc0857bb
2 zmienionych plików z 15 dodań i 389 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
src/client/City.js
Wyświetl plik

@ -191,9 +191,7 @@
var buildingManager = new VIZI.BuildingManager();
buildingManager.load(url).then(function(value) {
VIZI.Log(value);
// buildingManager.processFeatures(value.features);
buildingManager.processFeaturesWorker(value.features).then(function(result) {
// buildingManager.processFeaturesWorkerInput("osm-buildings.js").then(function(result) {
VIZI.Log("Finished loading buildings in " + (Date.now() - startTime) + "ms");
deferred.resolve(buildingManager);
}, undefined, function(progress) {

402
src/client/objects/ObjectManager.js
Wyświetl plik

@ -53,11 +53,20 @@
return deferred.promise;
};
// ##########################################
// Web Worker Loader
// ##########################################
// - Features (as JSON) are passed to worker manager
// - Worker manager splits features into batches and passes each worker a batch
// - Each worker processes features and passes a reference back using transferable objects
// - Features are added to scene for each completed worker promise
// TODO: Should be possible if geo functionality can be performed before / after the worker task
// TODO: Try and get rid of lock-up that occurs at beginning and end of worker process (possibly due to size of data being sent back and forth)
// TODO: Build objects as BufferGeometry for very easy export and messaging out of worker
// http://stackoverflow.com/questions/18262868/transforming-geometry-to-buffergeometry
// https://github.com/mrdoob/three.js/blob/f396baf5876eb41bcd2ee34eb65b1f97bb92d530/examples/js/exporters/BufferGeometryExporter.js
VIZI.ObjectManager.prototype.processFeaturesWorker = function(features) {
VIZI.Log("Processing features using worker");
@ -80,24 +89,6 @@
// TODO: See if initialising this before calling processFeaturesWorker speeds things up
var worker = cw({
processDebug: function(features, callback) {
var inputSize = JSON.stringify(features).length;
var startTime = Date.now();
var count = 0;
var timeTaken = Date.now() - startTime;
var exportedGeom = {};
// The size of this seems to be the problem
// Work out how to reduce it
var outputSize = JSON.stringify(exportedGeom).length;
var timeSent = Date.now();
callback({json: exportedGeom, outputSize: outputSize, inputSize: inputSize, count: count, startTime: startTime, timeTaken: timeTaken, timeSent: timeSent});
},
process: function(features, callback) {
importScripts("worker/three.min.js", "worker/GeometryExporter.js", "worker/underscore.min.js");
@ -171,15 +162,16 @@
// Work out how to reduce it
var outputSize = JSON.stringify(exportedGeom).length;
// Convert exported geom into a typed array
var verticesArray = new Float64Array( exportedGeom.vertices );
var normalsArray = new Float64Array( exportedGeom.normals );
// var colorsArray = new Float64Array( exportedGeom.colors );
// Seems to be manually set to have 1 array in the uvs array
// https://github.com/mrdoob/three.js/blob/master/examples/js/exporters/GeometryExporter.js#L231
var uvsArray = new Float64Array( exportedGeom.uvs[0] );
var facesArray = new Float64Array( exportedGeom.faces );
// Store geom typed array as Three.js model object
var model = {
metadata: exportedGeom.metadata,
colors: exportedGeom.colors,
@ -191,13 +183,11 @@
var timeSent = Date.now();
// var data = exportedGeom;
// var data = {model: exportedGeom, outputSize: outputSize, inputSize: inputSize, count: count, startTime: startTime, timeTaken: timeTaken, timeSent: timeSent};
var data2 = {model: model, outputSize: outputSize, inputSize: inputSize, count: count, startTime: startTime, timeTaken: timeTaken, timeSent: timeSent};
var data = {model: model, outputSize: outputSize, inputSize: inputSize, count: count, startTime: startTime, timeTaken: timeTaken, timeSent: timeSent};
// callback(data);
callback(data2, [model.vertices.buffer, model.normals.buffer, model.uvs.buffer, model.faces.buffer]);
// callback(data2);
// Send exported geom back to worker manager
// Second parameter contains reference to typed arrays as transferable objects
callback(data, [model.vertices.buffer, model.normals.buffer, model.uvs.buffer, model.faces.buffer]);
}
});
@ -212,7 +202,6 @@
// 4 batches or below seems to stop the model.faces typed array from converting to a normal array
var batches = 8;
var featuresPerBatch = Math.ceil(features.length / batches);
var batchedMeshes = [];
var batchPromises = [];
var i = batches;
@ -230,73 +219,6 @@
var self = this;
// TODO: Update this to fire progress events for each completed batch
// Possibly by using promise sequences rather than waiting for all to complete
// https://github.com/kriskowal/q#sequences
// Handle promises
// var processedCount = 0;
// var totalReceivedTime = 0;
// var result = batchPromises[0];
// // TODO: Work out why this is causing some buildings not to get parsed
// // If I don't
// batchPromises.forEach(function (f) {
//result = result.then(function(value) {
// var data = value.data;
// VIZI.Log(data);
// // Not sure how reliable the send time is
// var timeToSend = value.timeToSend;
// var timeToArrive = value.timeToArrive;
// var timeTaken = data.timeTaken;
// var inputSize = data.inputSize;
// var outputSize = data.outputSize;
// var count = data.count;
// var model = data.model;
// // Convert typed data back to arrays
// // model.vertices = Array.apply( [], model.vertices );
// // model.normals = Array.apply( [], model.normals );
// // // Wrap UVs within an array
// // // https://github.com/mrdoob/three.js/blob/master/examples/js/exporters/GeometryExporter.js#L231
// // model.uvs = [ Array.apply( [], model.uvs ) ];
// // model.faces = Array.apply( [], model.faces );
// totalReceivedTime += timeToArrive;
// VIZI.Log("Worker input sent in " + timeToSend + "ms");
// VIZI.Log("Worker input size is " + inputSize);
// VIZI.Log("Worker output received in " + timeToArrive + "ms");
// VIZI.Log("Worker output size is " + outputSize);
// VIZI.Log("Processed " + count + " features in " + timeTaken + "ms");
// // VIZI.Log(data.json.uvs);
// // VIZI.Log(data.uvsArray);
// // VIZI.Log(model);
// // TODO: Stop this locking up the browser
// // No visible lock up at all when removed
// var geom = loader.parse(model);
// var mesh = new THREE.Mesh(geom.geometry, material);
// self.publish("addToScene", mesh);
// processedCount++;
// deferred.notify( processedCount / batches );
// if (processedCount === batches) {
// VIZI.Log("Average output received time is " + (totalReceivedTime / batches) + "ms");
// VIZI.Log("Overall worker time is " + (Date.now() - startTime) + "ms");
// worker.close();
// deferred.resolve();
// }
// return f;
//});
// });
Q.allSettled(batchPromises).then(function (promises) {
var totalReceivedTime = 0;
@ -305,8 +227,6 @@
var value = promise.value;
var data = value.data;
// VIZI.Log(data);
// Not sure how reliable the send time is
var timeToSend = value.timeToSend;
var timeToArrive = value.timeToArrive;
@ -332,298 +252,6 @@
VIZI.Log("Worker output size is " + outputSize);
VIZI.Log("Processed " + count + " features in " + timeTaken + "ms");
// VIZI.Log(data.json.uvs);
// VIZI.Log(data.uvsArray);
// VIZI.Log(model);
// TODO: Stop this locking up the browser
// No visible lock up at all when removed
var geom = loader.parse(model);
var mesh = new THREE.Mesh(geom.geometry, material);
self.publish("addToScene", mesh);
}
});
VIZI.Log("Average output received time is " + (totalReceivedTime / batches) + "ms");
VIZI.Log("Overall worker time is " + (Date.now() - startTime) + "ms");
}).done(function() {
worker.close();
deferred.resolve();
});
return deferred.promise;
};
// Likely not showing anything due to coordinates needed converting [line 370-376] (so actually is working but buildings are placed in a minute area)
// Coordinates need to be converted in the worker or within the osm-buildings.js file
VIZI.ObjectManager.prototype.processFeaturesWorkerInput = function(featuresUrl) {
VIZI.Log("Processing features using worker");
var deferred = Q.defer();
//var geo = VIZI.Geo.getInstance();
// Convert coordinates
//var coordinateTime = Date.now();
//_.each(features, function(feature) {
// var coords = feature.geometry.coordinates[0];
// feature.geometry.coordinatesConverted = [[]];
// _.each(coords, function(coord) {
// feature.geometry.coordinatesConverted[0].push(geo.projection(coord));
// });
//});
// VIZI.Log("Converting coordinates: " + (Date.now() - coordinateTime));
// TODO: See if initialising this before calling processFeaturesWorker speeds things up
var worker = cw({
process: function(featuresUrl, callback) {
importScripts("worker/three.min.js", "worker/GeometryExporter.js", "worker/underscore.min.js");
importScripts("osm-buildings.js");
features = features.features;
var inputSize = JSON.stringify(features).length;
var startTime = Date.now();
var exporter = new THREE.GeometryExporter();
var applyVertexColors = function( g, c ) {
g.faces.forEach( function( f ) {
var n = ( f instanceof THREE.Face3 ) ? 3 : 4;
for( var j = 0; j < n; j ++ ) {
f.vertexColors[ j ] = c;
}
} );
};
var colour = new THREE.Color(0xcccccc);
var combinedGeom = new THREE.Geometry();
var count = 0;
_.each(features, function(feature) {
var properties = feature.properties;
var area = properties.area;
// Skip if building area is too small
if (area < 200) {
return;
}
// var coords = feature.geometry.coordinatesConverted[0];
var coords = feature.geometry.coordinates[0];
var shape = new THREE.Shape();
_.each(coords, function(coord, index) {
// Move if first coordinate
if (index === 0) {
shape.moveTo( coord[0], coord[1] );
} else {
shape.lineTo( coord[0], coord[1] );
}
});
//var height = 10 * this.geo.pixelsPerMeter;
var height = 10;
var extrudeSettings = { amount: height, bevelEnabled: false };
var geom = new THREE.ExtrudeGeometry( shape, extrudeSettings );
applyVertexColors( geom, colour );
geom.computeFaceNormals();
var mesh = new THREE.Mesh(geom);
mesh.position.y = height;
// Flip buildings as they are up-side down
mesh.rotation.x = 90 * Math.PI / 180;
THREE.GeometryUtils.merge(combinedGeom, mesh);
count++;
});
var timeTaken = Date.now() - startTime;
var exportedGeom = exporter.parse(combinedGeom);
// The size of this seems to be the problem
// Work out how to reduce it
var outputSize = JSON.stringify(exportedGeom).length;
var verticesArray = new Float64Array( exportedGeom.vertices );
var normalsArray = new Float64Array( exportedGeom.normals );
// var colorsArray = new Float64Array( exportedGeom.colors );
// Seems to be manually set to have 1 array in the uvs array
// https://github.com/mrdoob/three.js/blob/master/examples/js/exporters/GeometryExporter.js#L231
var uvsArray = new Float64Array( exportedGeom.uvs[0] );
var facesArray = new Float64Array( exportedGeom.faces );
var model = {
metadata: exportedGeom.metadata,
colors: exportedGeom.colors,
vertices: verticesArray,
normals: normalsArray,
uvs: uvsArray,
faces: facesArray
};
var timeSent = Date.now();
// var data = exportedGeom;
// var data = {model: exportedGeom, outputSize: outputSize, inputSize: inputSize, count: count, startTime: startTime, timeTaken: timeTaken, timeSent: timeSent};
var data2 = {model: model, outputSize: outputSize, inputSize: inputSize, count: count, startTime: startTime, timeTaken: timeTaken, timeSent: timeSent};
// callback(data);
callback(data2, [model.vertices.buffer, model.normals.buffer, model.uvs.buffer, model.faces.buffer]);
// callback(data2);
}
});
var startTime = Date.now();
// TODO: Work out why this still locks up the browser (amount of data being transferred back from the worker? Is it quicker to create objects in the browser?)
// Solution: https://speakerdeck.com/mourner/high-performance-data-visualizations?slide=51
// TODO: See if simply batching objects and creating them in the browser is less sluggish for the browser
// TODO: Work out why not every feature is being returned in the promises (about 10–20 less than expected)
// Batch features
// 4 batches or below seems to stop the model.faces typed array from converting to a normal array
var batches = 1;
// var featuresPerBatch = Math.ceil(features.length / batches);
// var batchedMeshes = [];
var batchPromises = [];
// var i = batches;
// while (i--) {
// var startIndex = i * featuresPerBatch;
// startIndex = (startIndex < 0) ? 0 : startIndex;
// var featuresBatch = features.splice(startIndex, featuresPerBatch-1);
// batchPromises.push(this.workerPromise(worker, featuresBatch));
// }
batchPromises.push(this.workerPromise(worker, []));
var loader = new THREE.JSONLoader();
var material = new THREE.MeshLambertMaterial({vertexColors: THREE.VertexColors});
var self = this;
// TODO: Update this to fire progress events for each completed batch
// Possibly by using promise sequences rather than waiting for all to complete
// https://github.com/kriskowal/q#sequences
// Handle promises
// var processedCount = 0;
// var totalReceivedTime = 0;
// var result = batchPromises[0];
// // TODO: Work out why this is causing some buildings not to get parsed
// // If I don't
// batchPromises.forEach(function (f) {
//result = result.then(function(value) {
// var data = value.data;
// VIZI.Log(data);
// // Not sure how reliable the send time is
// var timeToSend = value.timeToSend;
// var timeToArrive = value.timeToArrive;
// var timeTaken = data.timeTaken;
// var inputSize = data.inputSize;
// var outputSize = data.outputSize;
// var count = data.count;
// var model = data.model;
// // Convert typed data back to arrays
// // model.vertices = Array.apply( [], model.vertices );
// // model.normals = Array.apply( [], model.normals );
// // // Wrap UVs within an array
// // // https://github.com/mrdoob/three.js/blob/master/examples/js/exporters/GeometryExporter.js#L231
// // model.uvs = [ Array.apply( [], model.uvs ) ];
// // model.faces = Array.apply( [], model.faces );
// totalReceivedTime += timeToArrive;
// VIZI.Log("Worker input sent in " + timeToSend + "ms");
// VIZI.Log("Worker input size is " + inputSize);
// VIZI.Log("Worker output received in " + timeToArrive + "ms");
// VIZI.Log("Worker output size is " + outputSize);
// VIZI.Log("Processed " + count + " features in " + timeTaken + "ms");
// // VIZI.Log(data.json.uvs);
// // VIZI.Log(data.uvsArray);
// // VIZI.Log(model);
// // TODO: Stop this locking up the browser
// // No visible lock up at all when removed
// var geom = loader.parse(model);
// var mesh = new THREE.Mesh(geom.geometry, material);
// self.publish("addToScene", mesh);
// processedCount++;
// deferred.notify( processedCount / batches );
// if (processedCount === batches) {
// VIZI.Log("Average output received time is " + (totalReceivedTime / batches) + "ms");
// VIZI.Log("Overall worker time is " + (Date.now() - startTime) + "ms");
// worker.close();
// deferred.resolve();
// }
// return f;
//});
// });
Q.allSettled(batchPromises).then(function (promises) {
var totalReceivedTime = 0;
_.each(promises, function (promise) {
if (promise.state === "fulfilled") {
var value = promise.value;
var data = value.data;
// VIZI.Log(data);
// Not sure how reliable the send time is
var timeToSend = value.timeToSend;
var timeToArrive = value.timeToArrive;
var timeTaken = data.timeTaken;
var inputSize = data.inputSize;
var outputSize = data.outputSize;
var count = data.count;
var model = data.model;
// Convert typed data back to arrays
model.vertices = Array.apply( [], model.vertices );
model.normals = Array.apply( [], model.normals );
// Wrap UVs within an array
// https://github.com/mrdoob/three.js/blob/master/examples/js/exporters/GeometryExporter.js#L231
model.uvs = [ Array.apply( [], model.uvs ) ];
model.faces = Array.apply( [], model.faces );
totalReceivedTime += timeToArrive;
VIZI.Log("Worker input sent in " + timeToSend + "ms");
VIZI.Log("Worker input size is " + inputSize);
VIZI.Log("Worker output received in " + timeToArrive + "ms");
VIZI.Log("Worker output size is " + outputSize);
VIZI.Log("Processed " + count + " features in " + timeTaken + "ms");
// VIZI.Log(data.json.uvs);
// VIZI.Log(data.uvsArray);
// VIZI.Log(model);
// TODO: Stop this locking up the browser
// No visible lock up at all when removed
var geom = loader.parse(model);