diff --git a/examples/web-workers-geojson/index.html b/examples/web-workers-geojson/index.html new file mode 100644 index 0000000..3f95039 --- /dev/null +++ b/examples/web-workers-geojson/index.html @@ -0,0 +1,18 @@ + + + + + Web Workers GeoJSON ViziCities Example + + + + +
+ + + + + + + + diff --git a/examples/web-workers-geojson/main.css b/examples/web-workers-geojson/main.css new file mode 100644 index 0000000..4f8aa0b --- /dev/null +++ b/examples/web-workers-geojson/main.css @@ -0,0 +1,4 @@ +* { margin: 0; padding: 0; } +html, body { height: 100%; overflow: hidden;} + +#world { height: 100%; } diff --git a/examples/web-workers-geojson/main.js b/examples/web-workers-geojson/main.js new file mode 100644 index 0000000..67d626e --- /dev/null +++ b/examples/web-workers-geojson/main.js @@ -0,0 +1,35 @@ +// Manhattan +var coords = [40.722282152, -73.992919922]; + +var world = VIZI.world('world', { + skybox: false, + postProcessing: false +}).setView(coords); + +// Add controls +VIZI.Controls.orbit().addTo(world); + +// Leave a single CPU for the main browser thread +world.createWorkers(7).then(() => { + console.log('Workers ready'); + + // CartoDB basemap + VIZI.imageTileLayer('http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png', { + attribution: '© OpenStreetMap contributors, © CartoDB' + }).addTo(world); + + // Mapzen GeoJSON tile including points, linestrings and polygons + VIZI.geoJSONWorkerLayer('http://vector.mapzen.com/osm/roads,pois,buildings/14/4824/6159.json', { + output: true, + style: { + color: '#ff0000', + lineColor: '#0000ff', + lineRenderOrder: 1, + pointColor: '#00cc00' + }, + pointGeometry: function(feature) { + var geometry = new THREE.SphereGeometry(2, 16, 16); + return geometry; + } + }).addTo(world); +}); \ No newline at end of file diff --git a/examples/web-workers-geojson/vizicities-worker.js b/examples/web-workers-geojson/vizicities-worker.js new file mode 100644 index 0000000..c32aed6 --- /dev/null +++ b/examples/web-workers-geojson/vizicities-worker.js @@ -0,0 +1,61 @@ +importScripts('../vendor/three.min.js'); + +// Special version of ViziCities without controls (no DOM errors) +importScripts('../../dist/vizicities-worker.min.js'); + +const DEBUG = false; + +if (DEBUG) { console.log('Worker started', performance.now()); } + +// Send startup message to main thread +postMessage({ + type: 'startup', + payload: performance.now() +}); + +// Recieve message from main thread +onmessage = (event) => { + if (!event.data.method) { + postMessage({ + type: 'error', + payload: 'No method provided' + }); + + return; + } + + var time = performance.now(); + if (DEBUG) { console.log('Message received from main thread', time, event.data); } + // if (DEBUG) console.log('Time to receive message', time - event.data); + + // Run method + // if (!methods[event.data.method]) { + // postMessage({ + // type: 'error', + // payload: 'Method not found' + // }); + + // return; + // } + + var methods = event.data.method.split('.'); + + var _method = VIZI[methods[0]]; + + if (methods.length > 1) { + for (var i = 1; i < methods.length; i++) { + _method = _method[methods[i]]; + } + } + + // Call method with given arguments + _method.apply(this, event.data.args).then((result) => { + console.log('Message sent from worker', performance.now()); + + // Return results + postMessage({ + type: 'result', + payload: result.data + }, result.transferrables); + }); +}; \ No newline at end of file diff --git a/gulpfile.babel.js b/gulpfile.babel.js index ba803d4..cd0351e 100644 --- a/gulpfile.babel.js +++ b/gulpfile.babel.js @@ -112,6 +112,51 @@ function build() { .pipe($.livereload()); } +function buildWorker() { + return gulp.src(path.join('src', config.entryFileName + '-worker.js')) + .pipe($.plumber()) + .pipe(webpackStream({ + output: { + filename: exportFileName + '-worker.js', + libraryTarget: 'umd', + library: config.mainVarName + }, + externals: { + // Proxy the global THREE variable to require('three') + 'three': 'THREE', + // Proxy the global THREE variable to require('TweenLite') + 'TweenLite': 'TweenLite', + // Proxy the global THREE variable to require('TweenMax') + 'TweenMax': 'TweenMax', + // Proxy the global THREE variable to require('TimelineLite') + 'TimelineLite': 'TimelineLite', + // Proxy the global THREE variable to require('TimelineMax') + 'TimelineMax': 'TimelineMax' + }, + module: { + loaders: [ + { test: /\.js$/, exclude: /node_modules/, loader: 'babel-loader' }, + ] + }, + devtool: 'source-map' + })) + .pipe(gulp.dest(destinationFolder)) + .pipe($.filter(['*', '!**/*.js.map'])) + .pipe($.rename(exportFileName + '-worker.min.js')) + .pipe($.sourcemaps.init({ loadMaps: true })) + + // Don't mangle class names so we can use them in the console + // jscs:disable + // .pipe($.uglify({ mangle: { keep_fnames: true }})) + // jscs:enable + + // Using the mangle option above breaks the sourcemap for some reason + .pipe($.uglify()) + + .pipe($.sourcemaps.write('./')) + .pipe(gulp.dest(destinationFolder)); +} + function moveCSS() { return gulp.src(path.join('src', config.entryFileName + '.css')) .pipe(gulp.dest(destinationFolder)); @@ -240,6 +285,9 @@ gulp.task('move-css', ['clean'], moveCSS); // Build two versions of the library gulp.task('build', ['lint', 'move-css'], build); +// Build two versions of the library +gulp.task('build-worker', [], buildWorker); + // Lint and run our tests gulp.task('test', ['lint'], test); diff --git a/src/World.js b/src/World.js index 90310e1..fc44c31 100644 --- a/src/World.js +++ b/src/World.js @@ -5,6 +5,7 @@ import {point as Point} from './geo/Point'; import {latLon as LatLon} from './geo/LatLon'; import Engine from './engine/Engine'; import EnvironmentLayer from './layer/environment/EnvironmentLayer'; +import Worker from './util/Worker'; // TODO: Make sure nothing is left behind in the heap after calling destroy() @@ -39,6 +40,10 @@ class World extends EventEmitter { }); } + createWorkers(maxWorkers, workerScript) { + return Worker.createWorkers(maxWorkers, workerScript); + } + _initContainer(domId) { this._container = document.getElementById(domId); } diff --git a/src/layer/GeoJSONWorkerLayer.js b/src/layer/GeoJSONWorkerLayer.js new file mode 100644 index 0000000..c34e5c6 --- /dev/null +++ b/src/layer/GeoJSONWorkerLayer.js @@ -0,0 +1,545 @@ +import Layer from './Layer'; +import extend from 'lodash.assign'; +import reqwest from 'reqwest'; +import GeoJSON from '../util/GeoJSON'; +import Worker from '../util/Worker'; +import Buffer from '../util/Buffer'; +import Stringify from '../util/Stringify'; +import PolygonLayer from './geometry/PolygonLayer'; +import {latLon as LatLon} from '../geo/LatLon'; +import {point as Point} from '../geo/Point'; +import Geo from '../geo/Geo'; +import PickingMaterial from '../engine/PickingMaterial'; + +class GeoJSONWorkerLayer extends Layer { + constructor(geojson, options) { + var defaults = { + topojson: false, + style: GeoJSON.defaultStyle, + onEachFeature: null, + onEachFeatureWorker: null, + onAddAttributes: null, + interactive: false, + onClick: null, + headers: {} + }; + + var _options = extend({}, defaults, options); + + if (typeof options.style === 'object') { + _options.style = extend({}, defaults.style, options.style); + } + + super(_options); + + this._geojson = geojson; + } + + _onAdd(world) { + if (this._options.interactive) { + // Worker layer always controls output to add a picking mesh + this._pickingMesh = new THREE.Object3D(); + } + + // Process GeoJSON + return this._process(this._geojson); + } + + // Use workers to request and process GeoJSON, returning data structure + // containing geometry and any supplementary data for output + _process(_geojson) { + return new Promise((resolve, reject) => { + var style = this._options.style; + + if (typeof this._options.style === 'function') { + style = Stringify.functionToString(this._options.style); + } + + var geojson = _geojson; + var transferrables = []; + + if (typeof geojson !== 'string') { + this._geojson = geojson = Buffer.stringToUint8Array(JSON.stringify(geojson)); + transferrables.push(geojson.buffer); + this._execWorker(geojson, this._options.topojson, this._world._originPoint, style, this._options.interactive, transferrables).then(() => { + resolve(); + }); + } else if (typeof this._options.onEachFeature === 'function') { + GeoJSONWorkerLayer.RequestGeoJSON(geojson).then((res) => { + var fc = GeoJSON.collectFeatures(res, this._options.topojson); + var features = fc.features; + + var feature; + for (var i = 0; i < features.length; i++) { + feature = features[i]; + this._options.onEachFeature(feature); + }; + + this._geojson = geojson = Buffer.stringToUint8Array(JSON.stringify(fc)); + transferrables.push(geojson.buffer); + + this._execWorker(geojson, false, this._options.headers, this._world._originPoint, style, this._options.interactive, transferrables).then(() => { + resolve(); + }); + }); + } else { + this._execWorker(geojson, this._options.topojson, this._options.headers, this._world._originPoint, style, this._options.interactive, transferrables).then(() => { + resolve(); + }); + } + }); + } + + _execWorker(geojson, topojson, headers, originPoint, style, interactive, transferrables) { + return new Promise((resolve, reject) => { + console.time('Worker round trip'); + + Worker.exec('GeoJSONWorkerLayer.Process', [geojson, topojson, headers, originPoint, style, interactive], transferrables).then((results) => { + console.timeEnd('Worker round trip'); + + var splitVertices = Buffer.splitFloat32Array(results.attributes.vertices); + var splitNormals = Buffer.splitFloat32Array(results.attributes.normals); + var splitColours = Buffer.splitFloat32Array(results.attributes.colours); + + var splitProperties; + if (results.properties) { + splitProperties = Buffer.splitUint8Array(results.properties); + } + + // var splitPickingIds; + // if (results.pickingIds) { + // splitPickingIds = Buffer.splitFloat32Array(results.attributes.pickingIds); + // } + + var flats = results.flats; + + var objects = []; + var obj; + var pickingId; + var pickingIds; + var properties; + + var polygonAttributeLengths = { + positions: 3, + normals: 3, + colors: 3 + }; + + for (var i = 0; i < splitVertices.length; i++) { + if (splitProperties && splitProperties[i]) { + properties = JSON.parse(Buffer.uint8ArrayToString(splitProperties[i])); + } else { + properties = {}; + } + + // WORKERS: obj.attributes should actually an array of polygons for + // the feature, though the current logic isn't aware of that + obj = { + attributes: [{ + positions: splitVertices[i], + normals: splitNormals[i], + colors: splitColours[i] + }], + properties: properties, + flat: flats[i] + }; + + // if (splitPickingIds && splitPickingIds[i]) { + // obj.attributes.pickingIds = splitPickingIds[i]; + // } + + // WORKERS: If interactive, generate unique ID for each feature, create + // the buffer attributes and set up event listeners + if (this._options.interactive) { + pickingId = this.getPickingId(); + + pickingIds = new Float32Array(splitVertices[i].length / 3); + pickingIds.fill(pickingId); + + obj.attributes[0].pickingIds = pickingIds; + + polygonAttributeLengths.pickingIds = 1; + + this._addPicking(pickingId, properties); + } + + if (typeof this._options.onAddAttributes === 'function') { + var customAttributes = this._options.onAddAttributes(obj.attributes[0], properties); + var customAttribute; + for (var key in customAttributes) { + customAttribute = customAttributes[key]; + obj.attributes[0][key] = customAttribute.value; + polygonAttributeLengths[key] = customAttribute.length; + } + } + + objects.push(obj); + } + + var polygonAttributes = []; + + var polygonFlat = true; + + var obj; + for (var i = 0; i < objects.length; i++) { + obj = objects[i]; + + if (polygonFlat && !obj.flat) { + polygonFlat = false; + } + + var bufferAttributes = Buffer.mergeAttributes(obj.attributes); + polygonAttributes.push(bufferAttributes); + }; + + // console.log(splitVertices, splitNormals, splitColours, splitPickingIds); + + // var layer; + + // var polygonAttributes = []; + // var polygonFlat = true; + + // objects.forEach((obj, index) => { + // layer = polygonWorkerLayers[index]; + // layer.createGeometry(obj); + + // if (layer.isOutput()) { + // return; + // } + + // polygonAttributes.push(layer.getBufferAttributes()); + + // if (polygonFlat && !layer.isFlat()) { + // polygonFlat = false; + // } + // }); + + if (polygonAttributes.length > 0) { + var mergedPolygonAttributes = Buffer.mergeAttributes(polygonAttributes); + + // TODO: Make this work when style is a function per feature + var style = (typeof this._options.style === 'function') ? this._options.style(objects[0]) : this._options.style; + style = extend({}, GeoJSON.defaultStyle, style); + + this._setPolygonMesh(mergedPolygonAttributes, polygonAttributeLengths, style, polygonFlat); + this.add(this._polygonMesh); + } + + resolve(); + }); + }); + } + + // TODO: At some point this needs to return all the features to the main thread + // so it can generate meshes and output to the scene, as well as perhaps creating + // individual layers / components for each feature to track things like picking + // and properties + // + // TODO: Find a way so the origin point isn't needed to be passed in as it + // feels a bit messy and against the idea of a static Geo class + static Process(geojson, topojson, headers, originPoint, _style, _properties) { + return new Promise((resolve, reject) => { + GeoJSONWorkerLayer.ProcessGeoJSON(geojson, headers).then((res) => { + // Collects features into a single FeatureCollection + // + // Also converts TopoJSON to GeoJSON if instructed + var geojson = GeoJSON.collectFeatures(res, topojson); + + // TODO: Check that GeoJSON is valid / usable + + var features = geojson.features; + + // TODO: Run filter, if provided (must be static) + + var pointScale; + var polygons = []; + + // Deserialise style function if provided + if (typeof _style === 'string') { + _style = Stringify.stringToFunction(_style); + } + + // Assume that a style won't be set per feature + var style = _style; + + var feature; + for (var i = 0; i < features.length; i++) { + feature = features[i]; + + var geometry = feature.geometry; + var coordinates = (geometry.coordinates) ? geometry.coordinates : null; + + if (!coordinates || !geometry) { + return; + } + + // Get per-feature style object, if provided + if (typeof _style === 'function') { + style = extend({}, GeoJSON.defaultStyle, _style(feature)); + // console.log(feature, style); + } + + if (geometry.type === 'Polygon' || geometry.type === 'MultiPolygon') { + coordinates = (PolygonLayer.isSingle(coordinates)) ? [coordinates] : coordinates; + + var converted = coordinates.map(_coordinates => { + return _coordinates.map(ring => { + return ring.map(coordinate => { + return LatLon(coordinate[1], coordinate[0]); + }); + }); + }); + + var point; + var projected = converted.map(_coordinates => { + return _coordinates.map(ring => { + return ring.map(latlon => { + point = Geo.latLonToPoint(latlon)._subtract(originPoint); + + if (!pointScale) { + pointScale = Geo.pointScale(latlon); + } + + return point; + }); + }); + }); + + var polygon = { + projected: projected, + options: { + pointScale: pointScale, + style: style + } + }; + + if (_properties) { + polygon.properties = feature.properties; + } + + polygons.push(polygon); + } + + if (geometry.type === 'LineString' || geometry.type === 'MultiLineString') {} + + if (geometry.type === 'Point' || geometry.type === 'MultiPoint') {} + }; + + var bufferPromises = []; + + var polygon; + for (var i = 0; i < polygons.length; i++) { + polygon = polygons[i]; + bufferPromises.push(PolygonLayer.SetBufferAttributes(polygon.projected, polygon.options)); + }; + + Promise.all(bufferPromises).then((results) => { + var transferrables = []; + var transferrablesSize = 0; + + var vertices = []; + var normals = []; + var colours = []; + // var pickingIds = []; + + var properties = []; + + var flats = []; + var polygon; + + var result; + for (var i = 0; i < results.length; i++) { + result = results[i]; + + polygon = polygons[i]; + + // WORKERS: Making this a typed array will speed up transfer time + // As things stand this adds on a few milliseconds + flats.push(result.flat); + + // WORKERS: result.attributes is actually an array of polygons for each + // feature, though the current logic isn't keeping these all together + + var attributes; + for (var j = 0; j < result.attributes.length; j++) { + attributes = result.attributes[j]; + + vertices.push(attributes.vertices); + normals.push(attributes.normals); + colours.push(attributes.colours); + + // WORKERS: Handle interaction back in the main thread + // if (attributes.pickingIds) { + // pickingIds.push(attributes.pickingIds); + // } + + if (_properties) { + properties.push(Buffer.stringToUint8Array(JSON.stringify(polygon.properties))); + } + }; + }; + + var mergedAttributes = { + vertices: Buffer.mergeFloat32Arrays(vertices), + normals: Buffer.mergeFloat32Arrays(normals), + colours: Buffer.mergeFloat32Arrays(colours) + }; + + transferrables.push(mergedAttributes.vertices[0].buffer); + transferrables.push(mergedAttributes.vertices[1].buffer); + + transferrables.push(mergedAttributes.normals[0].buffer); + transferrables.push(mergedAttributes.normals[1].buffer); + + transferrables.push(mergedAttributes.colours[0].buffer); + transferrables.push(mergedAttributes.colours[1].buffer); + + var mergedProperties; + if (_properties) { + mergedProperties = Buffer.mergeUint8Arrays(properties); + + transferrables.push(mergedProperties[0].buffer); + transferrables.push(mergedProperties[1].buffer); + } + + // WORKERS: Handle interaction back in the main thread + // if (pickingIds.length > 0) { + // mergedAttributes.pickingIds = Buffer.mergeFloat32Arrays(pickingIds); + // transferrables.push(mergedAttributes.pickingIds[0].buffer); + // transferrables.push(mergedAttributes.pickingIds[1].buffer); + // } + + var output = { + attributes: mergedAttributes, + flats: flats + }; + + if (_properties) { + output.properties = mergedProperties; + } + + // TODO: Also return GeoJSON features that can be mapped to objects on + // the main thread. Allow user to provide filter / toggles to only return + // properties from the GeoJSON that they need (eg. don't return geometry, + // or don't return properties.height) + resolve({ + data: output, + transferrables: transferrables + }); + }); + }); + }); + } + + static ProcessGeoJSON(geojson, headers) { + if (typeof geojson === 'string') { + return GeoJSONWorkerLayer.RequestGeoJSON(geojson, headers); + } else { + return Promise.resolve(JSON.parse(Buffer.uint8ArrayToString(geojson))); + } + } + + static RequestGeoJSON(path, headers) { + return reqwest({ + url: path, + type: 'json', + crossOrigin: true, + headers: headers + }); + } + + // Create and store mesh from buffer attributes + // + // Could make this an abstract method for each geometry layer + _setPolygonMesh(attributes, attributeLengths, style, flat) { + var geometry = new THREE.BufferGeometry(); + + for (var key in attributes) { + geometry.addAttribute(key.slice(0, -1), new THREE.BufferAttribute(attributes[key], attributeLengths[key])); + } + + geometry.computeBoundingBox(); + + // Temporary until the above style logic is fixed for workers + // var style = extend({}, GeoJSON.defaultStyle); + + var material; + if (this._options.polygonMaterial && this._options.polygonMaterial instanceof THREE.Material) { + material = this._options.polygonMaterial; + } else if (!this._world._environment._skybox) { + material = new THREE.MeshPhongMaterial({ + vertexColors: THREE.VertexColors, + side: THREE.BackSide, + transparent: style.transparent, + opacity: style.opacity, + blending: style.blending + }); + } else { + material = new THREE.MeshStandardMaterial({ + vertexColors: THREE.VertexColors, + side: THREE.BackSide, + transparent: style.transparent, + opacity: style.opacity, + blending: style.blending + }); + material.roughness = 1; + material.metalness = 0.1; + material.envMapIntensity = 3; + material.envMap = this._world._environment._skybox.getRenderTarget(); + } + + var mesh; + + // Pass mesh through callback, if defined + if (typeof this._options.onPolygonMesh === 'function') { + mesh = this._options.onPolygonMesh(geometry, material); + } else { + mesh = new THREE.Mesh(geometry, material); + + mesh.castShadow = true; + mesh.receiveShadow = true; + } + + if (flat) { + material.depthWrite = false; + mesh.renderOrder = 1; + } + + if (this._options.interactive && this._pickingMesh) { + material = new PickingMaterial(); + material.side = THREE.BackSide; + + var pickingMesh = new THREE.Mesh(geometry, material); + this._pickingMesh.add(pickingMesh); + + this.addToPicking(this._pickingMesh); + } + + this._polygonMesh = mesh; + } + + // Set up and re-emit interaction events + _addPicking(pickingId, properties) { + this._world.on('pick-click-' + pickingId, (pickingId, point2d, point3d, intersects) => { + this._world.emit('click', this, properties); + }); + + this._world.on('pick-hover-' + pickingId, (pickingId, point2d, point3d, intersects) => { + this._world.emit('hover', this, properties); + }); + } + + // TODO: Finish cleanup + destroy() { + // Run common destruction logic from parent + super.destroy(); + } +} + +export default GeoJSONWorkerLayer; + +var noNew = function(geojson, options) { + return new GeoJSONWorkerLayer(geojson, options); +}; + +export {noNew as geoJSONWorkerLayer}; diff --git a/src/layer/TopoJSONWorkerLayer.js b/src/layer/TopoJSONWorkerLayer.js new file mode 100644 index 0000000..d439d2a --- /dev/null +++ b/src/layer/TopoJSONWorkerLayer.js @@ -0,0 +1,23 @@ +import GeoJSONWorkerLayer from './GeoJSONWorkerLayer'; +import extend from 'lodash.assign'; + +class TopoJSONWorkerLayer extends GeoJSONWorkerLayer { + constructor(topojson, options) { + var defaults = { + topojson: true + }; + + options = extend({}, defaults, options); + + super(topojson, options); + } +} + +export default TopoJSONWorkerLayer; + +var noNew = function(topojson, options) { + return new TopoJSONWorkerLayer(topojson, options); +}; + +// Initialise without requiring new keyword +export {noNew as topoJSONWorkerLayer}; diff --git a/src/layer/geometry/PolygonLayer.js b/src/layer/geometry/PolygonLayer.js index e783b4a..b0edc68 100644 --- a/src/layer/geometry/PolygonLayer.js +++ b/src/layer/geometry/PolygonLayer.js @@ -116,6 +116,8 @@ class PolygonLayer extends Layer { } // Create and store reference to THREE.BufferAttribute data for this layer + // + // TODO: Remove this and instead use the SetBufferAttributes static method _setBufferAttributes() { var attributes; @@ -142,10 +144,10 @@ class PolygonLayer extends Layer { // For each polygon attributes = this._projectedCoordinates.map(_projectedCoordinates => { // Convert coordinates to earcut format - var _earcut = this._toEarcut(_projectedCoordinates); + var _earcut = PolygonLayer.ToEarcut(_projectedCoordinates); // Triangulate faces using earcut - var faces = this._triangulate(_earcut.vertices, _earcut.holes, _earcut.dimensions); + var faces = PolygonLayer.Triangulate(_earcut.vertices, _earcut.holes, _earcut.dimensions); var groupedVertices = []; for (i = 0, il = _earcut.vertices.length; i < il; i += _earcut.dimensions) { @@ -219,7 +221,7 @@ class PolygonLayer extends Layer { } // Convert polygon representation to proper attribute arrays - return this._toAttributes(polygon); + return PolygonLayer.ToAttributes(polygon); }); } @@ -229,6 +231,113 @@ class PolygonLayer extends Layer { attributes = null; } + // TODO: Ensure that this has feature parity with the non-static method + static SetBufferAttributes(coordinates, options) { + return new Promise((resolve, reject) => { + var height = 0; + + // Convert height into world units + if (options.style.height && options.style.height !== 0) { + height = Geo.metresToWorld(options.style.height, options.pointScale); + } + + var colour = new THREE.Color(); + colour.set(options.style.color); + + // Light and dark colours used for poor-mans AO gradient on object sides + var light = new THREE.Color(0xffffff); + var shadow = new THREE.Color(0x666666); + + var flat = true; + + // For each polygon + var attributes = coordinates.map(_coordinates => { + // Convert coordinates to earcut format + var _earcut = PolygonLayer.ToEarcut(_coordinates); + + // Triangulate faces using earcut + var faces = PolygonLayer.Triangulate(_earcut.vertices, _earcut.holes, _earcut.dimensions); + + var groupedVertices = []; + for (i = 0, il = _earcut.vertices.length; i < il; i += _earcut.dimensions) { + groupedVertices.push(_earcut.vertices.slice(i, i + _earcut.dimensions)); + } + + var extruded = extrudePolygon(groupedVertices, faces, { + bottom: 0, + top: height + }); + + var topColor = colour.clone().multiply(light); + var bottomColor = colour.clone().multiply(shadow); + + var _vertices = extruded.positions; + var _faces = []; + var _colours = []; + + var _colour; + extruded.top.forEach((face, fi) => { + _colour = []; + + _colour.push([colour.r, colour.g, colour.b]); + _colour.push([colour.r, colour.g, colour.b]); + _colour.push([colour.r, colour.g, colour.b]); + + _faces.push(face); + _colours.push(_colour); + }); + + if (extruded.sides) { + flat = false; + + // Set up colours for every vertex with poor-mans AO on the sides + extruded.sides.forEach((face, fi) => { + _colour = []; + + // First face is always bottom-bottom-top + if (fi % 2 === 0) { + _colour.push([bottomColor.r, bottomColor.g, bottomColor.b]); + _colour.push([bottomColor.r, bottomColor.g, bottomColor.b]); + _colour.push([topColor.r, topColor.g, topColor.b]); + // Reverse winding for the second face + // top-top-bottom + } else { + _colour.push([topColor.r, topColor.g, topColor.b]); + _colour.push([topColor.r, topColor.g, topColor.b]); + _colour.push([bottomColor.r, bottomColor.g, bottomColor.b]); + } + + _faces.push(face); + _colours.push(_colour); + }); + } + + // Skip bottom as there's no point rendering it + // allFaces.push(extruded.faces); + + var polygon = { + vertices: _vertices, + faces: _faces, + colours: _colours, + facesCount: _faces.length + }; + + if (options.interactive && options.pickingId) { + // Inject picking ID + polygon.pickingId = options.pickingId; + } + + // Convert polygon representation to proper attribute arrays + return PolygonLayer.ToAttributes(polygon); + }); + + resolve({ + attributes: attributes, + flat: flat + }); + }); + } + getBufferAttributes() { return this._bufferAttributes; } @@ -378,7 +487,7 @@ class PolygonLayer extends Layer { } // Convert coordinates array to something earcut can understand - _toEarcut(coordinates) { + static ToEarcut(coordinates) { var dim = 2; var result = {vertices: [], holes: [], dimensions: dim}; var holeIndex = 0; @@ -400,7 +509,7 @@ class PolygonLayer extends Layer { } // Triangulate earcut-based input using earcut - _triangulate(contour, holes, dim) { + static Triangulate(contour, holes, dim) { // console.time('earcut'); var faces = earcut(contour, holes, dim); @@ -419,7 +528,7 @@ class PolygonLayer extends Layer { // THREE.BufferGeometry // // TODO: Can this be simplified? It's messy and huge - _toAttributes(polygon) { + static ToAttributes(polygon) { // Three components per vertex per face (3 x 3 = 9) var vertices = new Float32Array(polygon.facesCount * 9); var normals = new Float32Array(polygon.facesCount * 9); diff --git a/src/util/Buffer.js b/src/util/Buffer.js index d36cf7a..aa92f7c 100644 --- a/src/util/Buffer.js +++ b/src/util/Buffer.js @@ -5,6 +5,100 @@ import THREE from 'three'; var Buffer = (function() { + // Merge TypedArrays of the same type + // Returns merged array as well as indexes for splitting the array + var mergeFloat32Arrays = function(arrays) { + var size = 0; + var map = new Int32Array(arrays.length * 2); + + var lastIndex = 0; + var length; + + // Find size of each array + arrays.forEach((_array, index) => { + length = _array.length; + size += length; + map.set([lastIndex, lastIndex + length], index * 2); + lastIndex += length; + }); + + // Create a new array of total size + var mergedArray = new Float32Array(size); + + // Add each array to the new array + arrays.forEach((_array, index) => { + mergedArray.set(_array, map[index * 2]); + }); + + return [ + mergedArray, + map + ]; + }; + + var splitFloat32Array = function(data) { + var arr = data[0]; + var map = data[1]; + + var start; + var arrays = []; + + // Iterate over map + for (var i = 0; i < map.length / 2; i++) { + start = i * 2; + arrays.push(arr.subarray(map[start], map[start + 1])); + } + + return arrays; + }; + + // TODO: Create a generic method that can work for any typed array + var mergeUint8Arrays = function(arrays) { + var size = 0; + var map = new Int32Array(arrays.length * 2); + + var lastIndex = 0; + var length; + + // Find size of each array + arrays.forEach((_array, index) => { + length = _array.length; + size += length; + map.set([lastIndex, lastIndex + length], index * 2); + lastIndex += length; + }); + + // Create a new array of total size + var mergedArray = new Uint8Array(size); + + // Add each array to the new array + arrays.forEach((_array, index) => { + mergedArray.set(_array, map[index * 2]); + }); + + return [ + mergedArray, + map + ]; + }; + + // TODO: Dedupe with splitFloat32Array + var splitUint8Array = function(data) { + var arr = data[0]; + var map = data[1]; + + var start; + var arrays = []; + + // Iterate over map + for (var i = 0; i < map.length / 2; i++) { + start = i * 2; + arrays.push(arr.subarray(map[start], map[start + 1])); + } + + return arrays; + }; + // Merge multiple attribute objects into a single attribute object // // Attribute objects must all use the same attribute keys @@ -247,10 +341,27 @@ var Buffer = (function() { return geometry; }; + var textEncoder = new TextEncoder('utf-8'); + var textDecoder = new TextDecoder('utf-8'); + + var stringToUint8Array = function(str) { + return textEncoder.encode(str); + }; + + var uint8ArrayToString = function(ab) { + return textDecoder.decode(ab); + }; + return { + mergeFloat32Arrays: mergeFloat32Arrays, + splitFloat32Array: splitFloat32Array, + mergeUint8Arrays: mergeUint8Arrays, + splitUint8Array: splitUint8Array, mergeAttributes: mergeAttributes, createLineGeometry: createLineGeometry, - createGeometry: createGeometry + createGeometry: createGeometry, + stringToUint8Array: stringToUint8Array, + uint8ArrayToString: uint8ArrayToString }; })(); diff --git a/src/util/Stringify.js b/src/util/Stringify.js new file mode 100644 index 0000000..6363b85 --- /dev/null +++ b/src/util/Stringify.js @@ -0,0 +1,26 @@ +var Stringify = (function() { + var functionToString = function(f) { + return f.toString(); + }; + + // Based on https://github.com/tangrams/tangram/blob/2a31893c814cf15d5077f87ffa10af20160716b9/src/utils/utils.js#L245 + var stringToFunction = function(str) { + if (typeof str === 'string' && str.match(/^\s*function\s*\w*\s*\([\s\S]*\)\s*\{[\s\S]*\}/m) != null) { + var f; + + try { + eval('f = ' + str); + return f; + } catch (err) { + return str; + } + } + }; + + return { + functionToString: functionToString, + stringToFunction: stringToFunction + }; +})(); + +export default Stringify; diff --git a/src/util/Worker.js b/src/util/Worker.js new file mode 100644 index 0000000..0d43ddb --- /dev/null +++ b/src/util/Worker.js @@ -0,0 +1,26 @@ +import WorkerPool from './WorkerPool'; + +var Worker = (function() { + var _maxWorkers = 2; + var pool; + + var createWorkers = function(maxWorkers, workerScript) { + pool = new WorkerPool({ + numThreads: (maxWorkers) ? maxWorkers : _maxWorkers, + workerScript: (workerScript) ? workerScript : 'vizicities-worker.js' + }); + + return pool.createWorkers(); + }; + + var exec = function(method, args, transferrables) { + return pool.exec(method, args, transferrables); + }; + + return { + createWorkers: createWorkers, + exec: exec + }; +})(); + +export default Worker; diff --git a/src/util/WorkerPool.js b/src/util/WorkerPool.js new file mode 100644 index 0000000..ee52fc1 --- /dev/null +++ b/src/util/WorkerPool.js @@ -0,0 +1,117 @@ +import WorkerPoolWorker from './WorkerPoolWorker'; + +const DEBUG = false; + +class WorkerPool { + constructor(options) { + this.numThreads = options.numThreads || 2; + this.workerScript = options.workerScript; + + this.workers = []; + this.tasks = []; + } + + createWorkers() { + return new Promise((resolve, reject) => { + var workerPromises = []; + + for (var i = 0; i < this.numThreads; i++) { + workerPromises.push(this.createWorker()); + } + + Promise.all(workerPromises).then(() => { + if (DEBUG) { console.log('All workers ready', performance.now()); } + resolve(); + }); + }); + } + + createWorker() { + return new Promise((resolve, reject) => { + // Initialise worker + var worker = new WorkerPoolWorker({ + workerScript: this.workerScript + }); + + // Start worker and wait for it to be ready + return worker.start().then(() => { + if (DEBUG) { console.log('Worker ready', performance.now()); } + + // Add worker to pool + this.workers.push(worker); + + resolve(); + }); + }); + } + + getFreeWorker() { + return this.workers.find((worker) => { + return !worker.busy; + }); + } + + // Execute task on a worker + exec(method, args, transferrables) { + var deferred = Promise.deferred(); + + // Create task + var task = { + method: method, + args: args, + transferrables: transferrables, + deferred: deferred + }; + + // Add task to queue + this.tasks.push(task); + + // Trigger task processing + this.processTasks(); + + // Return task promise + return task.deferred.promise; + } + + processTasks() { + if (DEBUG) { console.log('Processing tasks'); } + + if (this.tasks.length === 0) { + return; + } + + // Find free worker + var worker = this.getFreeWorker(); + + if (!worker) { + if (DEBUG) { console.log('No workers free'); } + return; + } + + // Get oldest task + var task = this.tasks.shift(); + + // Execute task on worker + worker.exec(task.method, task.args, task.transferrables).then((result) => { + // Trigger task processing + this.processTasks(); + + // Return result in deferred task promise + task.deferred.resolve(result); + }); + } +} + +export default WorkerPool; + +// Quick shim to create deferred native promises +Promise.deferred = function() { + var result = {}; + + result.promise = new Promise((resolve, reject) => { + result.resolve = resolve; + result.reject = reject; + }); + + return result; +}; diff --git a/src/util/WorkerPoolWorker.js b/src/util/WorkerPoolWorker.js new file mode 100644 index 0000000..d039c75 --- /dev/null +++ b/src/util/WorkerPoolWorker.js @@ -0,0 +1,83 @@ +const DEBUG = false; + +class WorkerPoolWorker { + constructor(options) { + this.workerScript = options.workerScript; + + this.ready = false; + this.busy = false; + this.deferred = null; + } + + start() { + return new Promise((resolve, reject) => { + this.worker = new Worker(this.workerScript); + + var onStartup = (event) => { + if (!event.data || event.data.type !== 'startup') { + reject(); + return; + } + + this.ready = true; + + // Remove temporary message handler + this.worker.removeEventListener('message', onStartup); + + // Set up listener to respond to normal events now + this.worker.addEventListener('message', (event) => { + this.onMessage(event); + }); + + // Resolve once worker is ready + resolve(); + }; + + // Set up temporary event listener for warmup + this.worker.addEventListener('message', onStartup); + }); + } + + exec(method, args, transferrables) { + if (DEBUG) { console.log('Execute', method, args, transferrables); } + + var deferred = Promise.deferred(); + + this.busy = true; + this.deferred = deferred; + + this.worker.postMessage({ + method: method, + args: args + }, transferrables); + + return deferred.promise; + } + + onMessage(event) { + console.log('Message received from worker', performance.now()); + + this.busy = false; + + if (!event.data || event.data.type === 'error' || event.data.type !== 'result') { + this.deferred.reject(event.data.payload); + return; + } + + this.deferred.resolve(event.data.payload); + } +} + +export default WorkerPoolWorker; + +// Quick shim to create deferred native promises +Promise.deferred = function() { + var result = {}; + + result.promise = new Promise((resolve, reject) => { + result.resolve = resolve; + result.reject = reject; + }); + + return result; +}; diff --git a/src/util/index.js b/src/util/index.js index af4d17a..111a8a6 100644 --- a/src/util/index.js +++ b/src/util/index.js @@ -4,6 +4,8 @@ import wrapNum from './wrapNum'; import extrudePolygon from './extrudePolygon'; import GeoJSON from './GeoJSON'; import Buffer from './Buffer'; +import Worker from './Worker'; +import Stringify from './Stringify'; const Util = {}; @@ -11,5 +13,7 @@ Util.wrapNum = wrapNum; Util.extrudePolygon = extrudePolygon; Util.GeoJSON = GeoJSON; Util.Buffer = Buffer; +Util.Worker = Worker; +Util.Stringify = Stringify; export default Util; diff --git a/src/vizicities-worker.js b/src/vizicities-worker.js new file mode 100644 index 0000000..c3ae6e6 --- /dev/null +++ b/src/vizicities-worker.js @@ -0,0 +1,28 @@ +import Geo from './geo/Geo.js'; +import Layer, {layer} from './layer/Layer'; +import GeoJSONWorkerLayer, {geoJSONWorkerLayer} from './layer/GeoJSONWorkerLayer'; +import PolygonLayer, {polygonLayer} from './layer/geometry/PolygonLayer'; + +import Point, {point} from './geo/Point'; +import LatLon, {latLon} from './geo/LatLon'; + +import Util from './util/index'; + +const VIZI = { + version: '0.3', + + Geo: Geo, + Layer: Layer, + layer: layer, + GeoJSONWorkerLayer: GeoJSONWorkerLayer, + geoJSONWorkerLayer: geoJSONWorkerLayer, + PolygonLayer: PolygonLayer, + polygonLayer: polygonLayer, + Point: Point, + point: point, + LatLon: LatLon, + latLon: latLon, + Util: Util +}; + +export default VIZI; diff --git a/src/vizicities.js b/src/vizicities.js index 60ebc9c..876c88d 100644 --- a/src/vizicities.js +++ b/src/vizicities.js @@ -10,6 +10,8 @@ import GeoJSONTileLayer, {geoJSONTileLayer} from './layer/tile/GeoJSONTileLayer' import TopoJSONTileLayer, {topoJSONTileLayer} from './layer/tile/TopoJSONTileLayer'; import GeoJSONLayer, {geoJSONLayer} from './layer/GeoJSONLayer'; import TopoJSONLayer, {topoJSONLayer} from './layer/TopoJSONLayer'; +import GeoJSONWorkerLayer, {geoJSONWorkerLayer} from './layer/GeoJSONWorkerLayer'; +import TopoJSONWorkerLayer, {topoJSONWorkerLayer} from './layer/TopoJSONWorkerLayer'; import PolygonLayer, {polygonLayer} from './layer/geometry/PolygonLayer'; import PolylineLayer, {polylineLayer} from './layer/geometry/PolylineLayer'; import PointLayer, {pointLayer} from './layer/geometry/PointLayer'; @@ -43,6 +45,10 @@ const VIZI = { geoJSONLayer: geoJSONLayer, TopoJSONLayer: TopoJSONLayer, topoJSONLayer: topoJSONLayer, + GeoJSONWorkerLayer: GeoJSONWorkerLayer, + geoJSONWorkerLayer: geoJSONWorkerLayer, + TopoJSONWorkerLayer: TopoJSONWorkerLayer, + topoJSONWorkerLayer: topoJSONWorkerLayer, PolygonLayer: PolygonLayer, polygonLayer: polygonLayer, PolylineLayer: PolylineLayer,