Merge branch 'master' of https://github.com/OpenDroneMap/OpenDroneMap into update-vtk

# Conflicts:
#	opendm/tasks.py
#	scripts/resize.py


Former-commit-id: dda829f74e

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "src/bundler"]
-	path = src/bundler
-	url = https://github.com/chris-cooper/bundler_sfm

CNAME

@@ -0,0 +1 @@
+opendronemap.org

Dockerfile

@@ -14,12 +14,12 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent
 libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \
 libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \
 libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \
-jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev
+jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libjsoncpp-dev
 
 RUN apt-get remove libdc1394-22-dev
 RUN pip install --upgrade pip
 RUN pip install setuptools
-RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings
+RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz
 
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages"
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm"
@@ -44,6 +44,7 @@ COPY /SuperBuild/CMakeLists.txt /code/SuperBuild/CMakeLists.txt
 COPY docker.settings.yaml /code/settings.yaml
 COPY VERSION /code/VERSION
 
+
 #Compile code in SuperBuild and root directories
 
 RUN cd SuperBuild && mkdir build && cd build && cmake  .. && make -j$(nproc)     && cd ../.. && mkdir build && cd build && cmake .. && make -j$(nproc)

README.md

@@ -20,15 +20,22 @@ In a word, OpenDroneMap is a toolchain for processing raw civilian UAS imagery t
 
 Open Drone Map now includes state-of-the-art 3D reconstruction work by Michael Waechter, Nils Moehrle, and Michael Goesele. See their publication at http://www.gcc.tu-darmstadt.de/media/gcc/papers/Waechter-2014-LTB.pdf.
 
-
 ## QUICKSTART
 
-OpenDroneMap can run natively on Ubuntu 14.04 or later, see [Build and Run Using Docker](#build-and-run-using-docker) for running on Windows / MacOS. A [Vagrant VM](https://github.com/OpenDroneMap/odm_vagrant) is also available.
+### Docker (All platforms)
 
-*Support for Ubuntu 12.04 is currently BROKEN with the addition of OpenSfM and Ceres-Solver. It is likely to remain broken unless a champion is found to fix it.*
+The easiest way to run ODM is through Docker. If you don't have it installed,
+see the [Docker Ubuntu installation tutorial](https://docs.docker.com/engine/installation/linux/ubuntulinux/) and follow the
+instructions through "Create a Docker group". The Docker image workflow 
+has equivalent procedures for Mac OS X and Windows found at [docs.docker.com](docs.docker.com). Then run the following command which will build a pre-built image and run on images found in `$(pwd)/images` (you can change this if you need to, see the [wiki](https://github.com/OpenDroneMap/OpenDroneMap/wiki/Docker) for more detailed instructions. 
 
-**[Download the latest release here](https://github.com/OpenDroneMap/OpenDroneMap/releases)**
+```
+docker run -it --rm -v $(pwd)/images:/code/images -v $(pwd)/odm_orthophoto:/code/odm_orthophoto -v $(pwd)/odm_texturing:/code/odm_texturing opendronemap/opendronemap
+```
 
+### Native Install (Ubuntu 14.04 or later)
+
+**[Download the latest release here](https://github.com/OpenDroneMap/OpenDroneMap/releases)**  
 Current version: 0.3.1 (this software is in beta)
 
 1. Extract and enter the OpenDroneMap directory
@@ -37,11 +44,13 @@ Current version: 0.3.1 (this software is in beta)
 3. Download a sample dataset from [here](https://github.com/OpenDroneMap/odm_data_aukerman/archive/master.zip) (about 550MB) and extract it as a subdirectory in your project directory.
 4. Run `./run.sh odm_data_aukerman` 
 5. Enter dataset directory to view results: 
-  - orthophoto: odm_orthophoto/odm_orthophoto.tif
+    - orthophoto: odm_orthophoto/odm_orthophoto.tif
-  - textured mesh model: odm_texturing/odm_textured_model_geo.obj
+    - textured mesh model: odm_texturing/odm_textured_model_geo.obj
-  - point cloud (georeferenced): odm_georeferencing/odm_georeferenced_model.ply
+    - point cloud (georeferenced): odm_georeferencing/odm_georeferenced_model.ply
   
-See [here](https://github.com/OpenDroneMap/OpenDroneMap/blob/3964f21377e27c261c305b30537f699853ac2004/README.md#installation) for more detailed installation instructions. 
+See below for more detailed installation instructions. 
+
+## Diving Deeper
 
 ### Installation
 

SuperBuild/CMakeLists.txt

@@ -98,6 +98,10 @@ option(ODM_BUILD_CGAL "Force to build CGAL library" OFF)
 
 SETUP_EXTERNAL_PROJECT(CGAL ${ODM_CGAL_Version} ${ODM_BUILD_CGAL})
 
+# ---------------------------------------------------------------------------------------------
+# Hexer
+#
+SETUP_EXTERNAL_PROJECT(Hexer 1.4 ON)
 
 # ---------------------------------------------------------------------------------------------
 # Open Geometric Vision (OpenGV)
@@ -114,6 +118,7 @@ set(custom_libs OpenGV
 				Ecto
 				PDAL
 				MvsTexturing
+				Lidar2dems
 )
 
 # Dependencies of the SLAM module

SuperBuild/cmake/External-Hexer.cmake

@@ -0,0 +1,27 @@
+set(_proj_name hexer)
+set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
+
+ExternalProject_Add(${_proj_name}
+  DEPENDS           
+  PREFIX            ${_SB_BINARY_DIR}
+  TMP_DIR           ${_SB_BINARY_DIR}/tmp
+  STAMP_DIR         ${_SB_BINARY_DIR}/stamp
+  #--Download step--------------
+  DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}
+  URL               https://github.com/hobu/hexer/archive/2898b96b1105991e151696391b9111610276258f.tar.gz
+  URL_MD5           e8f2788332ad212cf78efa81a82e95dd
+  #--Update/Patch step----------
+  UPDATE_COMMAND    ""
+  #--Configure step-------------
+  SOURCE_DIR        ${SB_SOURCE_DIR}/${_proj_name}
+  CMAKE_ARGS
+    -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
+  #--Build step-----------------
+  BINARY_DIR        ${_SB_BINARY_DIR}
+  #--Install step---------------
+  INSTALL_DIR       ${SB_INSTALL_DIR}
+  #--Output logging-------------
+  LOG_DOWNLOAD      OFF
+  LOG_CONFIGURE     OFF
+  LOG_BUILD         OFF
+)

SuperBuild/cmake/External-Lidar2dems.cmake

@@ -0,0 +1,24 @@
+set(_proj_name lidar2dems)
+set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
+
+ExternalProject_Add(${_proj_name}
+  PREFIX            ${_SB_BINARY_DIR}
+  TMP_DIR           ${_SB_BINARY_DIR}/tmp
+  STAMP_DIR         ${_SB_BINARY_DIR}/stamp
+  #--Download step--------------
+  DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}/${_proj_name}
+  URL               https://github.com/OpenDroneMap/lidar2dems/archive/master.zip
+  #--Update/Patch step----------
+  UPDATE_COMMAND    ""
+  #--Configure step-------------
+  SOURCE_DIR        ${SB_SOURCE_DIR}/${_proj_name}
+  CONFIGURE_COMMAND ""
+  #--Build step-----------------
+  BUILD_COMMAND     ""
+  #--Install step---------------
+  INSTALL_COMMAND   "${SB_SOURCE_DIR}/${_proj_name}/install.sh" "${SB_INSTALL_DIR}"
+  #--Output logging-------------
+  LOG_DOWNLOAD      OFF
+  LOG_CONFIGURE     OFF
+  LOG_BUILD         OFF
+)

SuperBuild/cmake/External-PDAL.cmake

@@ -2,13 +2,14 @@ set(_proj_name pdal)
 set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
 
 ExternalProject_Add(${_proj_name}
+  DEPENDS           hexer
   PREFIX            ${_SB_BINARY_DIR}
   TMP_DIR           ${_SB_BINARY_DIR}/tmp
   STAMP_DIR         ${_SB_BINARY_DIR}/stamp
   #--Download step--------------
   DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}
-  URL               https://github.com/PDAL/PDAL/archive/aea5bb0cacc64b91d626eff491fbdbb5668c06d7.tar.gz
+  URL               https://github.com/PDAL/PDAL/archive/e881b581e3b91a928105d67db44c567f3b6d1afe.tar.gz
-  URL_MD5           726933f63f661e11e13775d6ce4e5ed0 
+  URL_MD5           438acbb736ba01fbe8f9ca7cdbf113bf
   #--Update/Patch step----------
   UPDATE_COMMAND    ""
   #--Configure step-------------
@@ -19,7 +20,7 @@ ExternalProject_Add(${_proj_name}
     -BUILD_PLUGIN_PGPOINTCLOUD=ON
     -DBUILD_PLUGIN_CPD=OFF
 	-DBUILD_PLUGIN_GREYHOUND=OFF
-	-DBUILD_PLUGIN_HEXBIN=OFF
+	-DBUILD_PLUGIN_HEXBIN=ON
 	-DBUILD_PLUGIN_ICEBRIDGE=OFF
 	-DBUILD_PLUGIN_MRSID=OFF
 	-DBUILD_PLUGIN_NITF=OFF

code_of_conduct.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
+reported by contacting the project team at `svm at clevelandmetroparks dot com`. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

configure.sh

@@ -21,7 +21,8 @@ install() {
                          libgdal-dev \
                          gdal-bin \
                          libgeotiff-dev \
-                         pkg-config
+                         pkg-config \
+                         libjsoncpp-dev
 
     echo "Getting CMake 3.1 for MVS-Texturing"
     sudo apt-get install -y software-properties-common python-software-properties
@@ -72,7 +73,7 @@ install() {
                         appsettings
 
     echo "Installing CGAL dependencies"
-    sudo apt-get install libgmp-dev libmpfr-dev
+    sudo apt-get install -y -qq libgmp-dev libmpfr-dev
 
     echo "Installing Ecto Dependencies"
     sudo pip install -U catkin-pkg
@@ -86,6 +87,13 @@ install() {
                          jhead \
                          liblas-bin
 
+    echo "Installing lidar2dems Dependencies"
+    sudo apt-get install -y -qq swig2.0 \
+                         python-wheel \
+                         libboost-log-dev
+
+    sudo pip install -U https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz
+
     echo "Compiling SuperBuild"
     cd ${RUNPATH}/SuperBuild
     mkdir -p build && cd build
@@ -134,4 +142,4 @@ else
     echo "Invalid instructions." >&2
     usage
     exit 1
-fi
+fi

contrib/visveg/readme.md

@@ -0,0 +1,31 @@
+# Visible Vegetation Indexes
+
+This script produces a Vegetation Index raster from a RGB orthophoto (odm_orthophoto.tif in your project)
+
+## Requirements
+* rasterio (pip install rasterio)
+* numpy python package (included in ODM build)
+
+## Usage
+```
+vegind.py <orthophoto.tif> index
+
+positional arguments:
+  <orthophoto.tif>  The RGB orthophoto. Must be a GeoTiff.
+  index     Index identifier. Allowed values: ngrdi, tgi, vari
+```
+Output will be generated with index suffix in the same directory as input.
+
+## Examples
+
+`python vegind.py /path/to/odm_orthophoto.tif tgi`
+
+Orthophoto photo of Koniaków grass field and forest in QGIS: ![](http://imgur.com/K6x3nB2.jpg)
+The Triangular Greenness Index output in QGIS (with a spectral pseudocolor): ![](http://i.imgur.com/f9TzISU.jpg)
+Visible Atmospheric Resistant Index: ![](http://imgur.com/Y7BHzLs.jpg)
+Normalized green-red difference index: ![](http://imgur.com/v8cmaPS.jpg)
+
+## Bibliography
+
+1. Hunt, E. Raymond, et al. "A Visible Band Index for Remote Sensing Leaf Chlorophyll Content At the Canopy Scale." ITC journal 21(2013): 103-112. doi: 10.1016/j.jag.2012.07.020
+(https://doi.org/10.1016/j.jag.2012.07.020)

contrib/visveg/vegind.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import rasterio, os, sys
+import numpy as np
+
+class bcolors:
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+try:
+    file = sys.argv[1]
+    typ = sys.argv[2]
+    (fileRoot, fileExt) = os.path.splitext(file)
+    outFileName = fileRoot + "_" + typ + fileExt
+    if typ not in ['vari', 'tgi', 'ngrdi']:
+        raise IndexError
+except (TypeError, IndexError, NameError):
+    print bcolors.FAIL + 'Arguments messed up. Check arguments order and index name' + bcolors.ENDC
+    print 'Usage: ./vegind.py orto index'
+    print '       orto - filepath to RGB orthophoto'
+    print '       index - Vegetation Index'
+    print bcolors.OKGREEN + 'Available indexes: vari, ngrdi, tgi' + bcolors.ENDC
+    sys.exit()
+
+
+def calcNgrdi(red, green):
+    """
+    Normalized green red difference index
+    Tucker,C.J.,1979.
+    Red and photographic infrared linear combinations for monitoring vegetation.
+    Remote Sensing of Environment 8, 127–150
+    :param red: red visible channel
+    :param green: green visible channel
+    :return: ngrdi index array
+    """
+    mask = np.not_equal(np.add(red,green), 0.0)
+    return np.choose(mask, (-9999.0, np.true_divide(
+    np.subtract(green,red),
+    np.add(red,green))))
+
+def calcVari(red,green,blue):
+    """
+    Calculates Visible Atmospheric Resistant Index
+    Gitelson, A.A., Kaufman, Y.J., Stark, R., Rundquist, D., 2002.
+    Novel algorithms for remote estimation of vegetation fraction.
+    Remote Sensing of Environment 80, 76–87.
+    :param red: red visible channel
+    :param green: green visible channel
+    :param blue: blue visible channel
+    :return: vari index array, that will be saved to tiff
+    """
+    mask = np.not_equal(np.subtract(np.add(green,red),blue), 0.0)
+    return np.choose(mask, (-9999.0, np.true_divide(np.subtract(green,red),np.subtract(np.add(green,red),blue))))
+
+def calcTgi(red,green,blue):
+    """
+    Calculates Triangular Greenness Index
+    Hunt, E. Raymond Jr.; Doraiswamy, Paul C.; McMurtrey, James E.; Daughtry, Craig S.T.; Perry, Eileen M.; and Akhmedov, Bakhyt,
+    A visible band index for remote sensing leaf chlorophyll content at the canopy scale (2013).
+    Publications from USDA-ARS / UNL Faculty. Paper 1156.
+    http://digitalcommons.unl.edu/usdaarsfacpub/1156
+    :param red: red channel
+    :param green: green channel
+    :param blue: blue channel
+    :return: tgi index array, that will be saved to tiff
+    """
+    mask = np.not_equal(green-red+blue-255.0, 0.0)
+    return np.choose(mask, (-9999.0, np.subtract(green, np.multiply(0.39,red), np.multiply(0.61, blue))))
+
+try:
+    with rasterio.Env():
+        ds = rasterio.open(file)
+        profile = ds.profile
+        profile.update(dtype=rasterio.float32, count=1, nodata=-9999)
+        red = np.float32(ds.read(1))
+        green = np.float32(ds.read(2))
+        blue = np.float32(ds.read(3))
+        np.seterr(divide='ignore', invalid='ignore')
+        if typ == 'ngrdi':
+            indeks = calcNgrdi(red,green)
+        elif typ == 'vari':
+            indeks = calcVari(red, green, blue)
+        elif typ == 'tgi':
+            indeks = calcTgi(red, green, blue)
+
+        with rasterio.open(outFileName, 'w', **profile) as dst:
+            dst.write(indeks.astype(rasterio.float32), 1)
+except rasterio.errors.RasterioIOError:
+    print bcolors.FAIL + 'Orthophoto file not found or access denied' + bcolors.ENDC
+    sys.exit()

core2.Dockerfile

@@ -14,12 +14,12 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent
 libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \
 libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \
 libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \
-jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev
+jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libjsoncpp-dev
 
 RUN apt-get remove libdc1394-22-dev
 RUN pip install --upgrade pip
 RUN pip install setuptools
-RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings
+RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz
 
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages"
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm"

docker.settings.yaml

@@ -43,10 +43,15 @@ project_path: '/' #DO NOT CHANGE THIS OR DOCKER WILL NOT WORK. It should be '/'
 #texturing_keep_unseen_faces: False
 #texturing_tone_mapping: 'none'
 #gcp: !!null  # YAML tag for None
-#dem: False
+#dtm: False      # Use this tag to build a DTM (Digital Terrain Model
-#dem_sample_radius: 1.0
+#dsm: False      # Use this tag to build a DSM (Digital Surface Model
-#dem_resolution: 2
+#dem-gapfill-steps: 4
-#dem_radius: 0.5
+#dem-resolution: 0.1
+#dem-maxangle:20
+#dem-maxsd: 2.5
+#dem-approximate: False
+#dem-decimation: 1
+#dem-terrain-type: ComplexForest
 #use_exif: False # Set to True if you have a GCP file (it auto-detects) and want to use EXIF
 #orthophoto_resolution: 20.0 # Pixels/meter
 #orthophoto_target_srs: !!null # Currently does nothing

opendm/config.py

@@ -10,7 +10,7 @@ import sys
 # parse arguments
 processopts = ['resize', 'opensfm', 'slam', 'cmvs', 'pmvs',
                'odm_meshing', 'odm_25dmeshing', 'mvs_texturing', 'odm_georeferencing',
-               'odm_orthophoto']
+               'odm_dem', 'odm_orthophoto']
 
 with open(io.join_paths(context.root_path, 'VERSION')) as version_file:
     __version__ = version_file.read().strip()
@@ -111,21 +111,6 @@ def config():
                               'More features leads to better results but slower '
                               'execution. Default: %(default)s'))
 
-    parser.add_argument('--matcher-threshold',
-                        metavar='<percent>',
-                        default=2.0,
-                        type=float,
-                        help=('Ignore matched keypoints if the two images share '
-                              'less than <float> percent of keypoints. Default:'
-                              ' %(default)s'))
-
-    parser.add_argument('--matcher-ratio',
-                        metavar='<float>',
-                        default=0.6,
-                        type=float,
-                        help=('Ratio of the distance to the next best matched '
-                              'keypoint. Default: %(default)s'))
-
     parser.add_argument('--matcher-neighbors',
                         type=int,
                         metavar='<integer>',
@@ -342,34 +327,87 @@ def config():
                         help=('Use this tag if you have a gcp_list.txt but '
                               'want to use the exif geotags instead'))
 
-    parser.add_argument('--dem',
+    parser.add_argument('--dtm',
                         action='store_true',
                         default=False,
-                        help='Use this tag to build a DEM using a progressive '
+                        help='Use this tag to build a DTM (Digital Terrain Model, ground only) using a progressive '
-                             'morphological filter in PDAL.')
+                             'morphological filter. Check the --dem* parameters for fine tuning.')
+    
+    parser.add_argument('--dsm',
+                        action='store_true',
+                        default=False,
+                        help='Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive '
+                             'morphological filter. Check the --dem* parameters for fine tuning.')
 
-    parser.add_argument('--dem-sample-radius',
+    parser.add_argument('--dem-gapfill-steps',
-                        metavar='<float>',
+                        metavar='<positive integer>',
-                        default=1.0,
+                        default=4,
-                        type=float,
+                        type=int,
-                        help='Minimum distance between samples for DEM '
+                        help='Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. '
-                             'generation.\nDefault=%(default)s')
+                             'Starting with a radius equal to the output resolution, N different DEMs are generated with '
+                             'progressively bigger radius using the inverse distance weighted (IDW) algorithm '
+                             'and merged together. Remaining gaps are then merged using nearest neighbor interpolation. '
+                             '\nDefault=%(default)s')
 
     parser.add_argument('--dem-resolution',
                         metavar='<float>',
                         type=float,
-                        default=2,
+                        default=0.1,
-                        help='Length of raster cell edges in X/Y units.'
+                        help='Length of raster cell edges in meters.'
                              '\nDefault: %(default)s')
 
-    parser.add_argument('--dem-radius',
+    parser.add_argument('--dem-maxangle',
-                        metavar='<float>',
+                        metavar='<positive float>',
                         type=float,
-                        default=0.5,
+                        default=20,
-                        help='Radius about cell center bounding points to '
+                        help='Points that are more than maxangle degrees off-nadir are discarded. '
-                             'use to calculate a cell value.\nDefault: '
+                             '\nDefault: '
                              '%(default)s')
 
+    parser.add_argument('--dem-maxsd',
+                        metavar='<positive float>',
+                        type=float,
+                        default=2.5,
+                        help='Points that deviate more than maxsd standard deviations from the local mean '
+                             'are discarded. \nDefault: '
+                             '%(default)s')
+
+    parser.add_argument('--dem-initial-distance',
+                        metavar='<positive float>',
+                        type=float,
+                        default=0.15,
+                        help='Used to classify ground vs non-ground points. Set this value to account for Z noise in meters. '
+                             'If you have an uncertainty of around 15 cm, set this value large enough to not exclude these points. '
+                             'Too small of a value will exclude valid ground points, while too large of a value will misclassify non-ground points for ground ones. '
+                             '\nDefault: '
+                             '%(default)s')
+
+    parser.add_argument('--dem-approximate',
+                        action='store_true',
+                        default=False,
+                        help='Use this tag use the approximate progressive  '
+                             'morphological filter, which computes DEMs faster '
+                             'but is not as accurate.')
+
+    parser.add_argument('--dem-decimation',
+                        metavar='<positive integer>',
+                        default=1,
+                        type=int,
+                        help='Decimate the points before generating the DEM. 1 is no decimation (full quality). '
+                             '100 decimates ~99%% of the points. Useful for speeding up '
+                             'generation.\nDefault=%(default)s')
+
+    parser.add_argument('--dem-terrain-type',
+                        metavar='<string>',
+                        choices=['FlatNonForest', 'FlatForest', 'ComplexNonForest', 'ComplexForest'],
+                        default='ComplexForest',
+                        help='One of: %(choices)s. Specifies the type of terrain. This mainly helps reduce processing time. '
+                             '\nFlatNonForest: Relatively flat region with little to no vegetation'
+                             '\nFlatForest: Relatively flat region that is forested'
+                             '\nComplexNonForest: Varied terrain with little to no vegetation'
+                             '\nComplexForest: Varied terrain that is forested'
+                             '\nDefault=%(default)s')
+
     parser.add_argument('--orthophoto-resolution',
                         metavar='<float > 0.0>',
                         default=20.0,

opendm/context.py

@@ -8,6 +8,7 @@ scripts_path = os.path.abspath(os.path.dirname(__file__))
 root_path, _ = os.path.split(scripts_path)
 
 superbuild_path = os.path.join(root_path, 'SuperBuild')
+superbuild_bin_path = os.path.join(superbuild_path, 'install', 'bin')
 tests_path = os.path.join(root_path, 'tests')
 tests_data_path = os.path.join(root_path, 'tests/test_data')
 

opendm/system.py

@@ -17,10 +17,16 @@ def get_ccd_widths():
     return dict(zip(map(string.lower, sensor_data.keys()), sensor_data.values()))
 
 
-def run(cmd):
+def run(cmd, env_paths=[]):
     """Run a system command"""
     log.ODM_DEBUG('running %s' % cmd)
-    retcode = subprocess.call(cmd, shell=True)
+
+    env = None
+    if len(env_paths) > 0:
+        env = os.environ.copy()
+        env["PATH"] = env["PATH"] + ":" + ":".join(env_paths)
+
+    retcode = subprocess.call(cmd, shell=True, env=env)
 
     if retcode < 0:
         raise Exception("Child was terminated by signal {}".format(-retcode))

opendm/tasks.py

@@ -13,8 +13,9 @@ tasks_dict = {'1': 'opensfm',
               '4': 'odm_meshing',
               '5': 'mvs_texturing',
               '6': 'odm_georeferencing',
-              '7': 'odm_orthophoto',
+              '7': 'odm_dem',
-              '8': 'zip_results'}
+              '8': 'odm_orthophoto',
+              '9': 'zip_results'}
 
 
 class ODMTaskManager(object):

opendm/types.py

@@ -221,56 +221,6 @@ class ODM_GeoRef(object):
         system.run('{bin}/pdal pipeline -i {json} --readers.ply.filename={f_in} '
                    '--writers.las.filename={f_out}'.format(**kwargs))
 
-    def convert_to_dem(self, _file, _file_out, pdalJSON, sample_radius, gdal_res, gdal_radius):
-        # Check if exists f_in
-        if not io.file_exists(_file):
-            log.ODM_ERROR('LAS file does not exist')
-            return False
-
-        kwargs = {
-            'bin': context.pdal_path,
-            'f_in': _file,
-            'sample_radius': sample_radius,
-            'gdal_res': gdal_res,
-            'gdal_radius': gdal_radius,
-            'f_out': _file_out,
-            'json': pdalJSON
-        }
-
-        pipelineJSON = '{{' \
-                       '    "pipeline":[' \
-                       '        "input.las",' \
-                       '    {{' \
-                       '        "type":"filters.sample",' \
-                       '        "radius":"{sample_radius}"' \
-                       '    }},' \
-                       '    {{' \
-                       '        "type":"filters.pmf"' \
-                       '    }},' \
-                       '    {{' \
-                       '      "type":"filters.range",' \
-                       '      "limits":"Classification[2:2]"' \
-                       '    }},' \
-                       '    {{' \
-                       '        "resolution": {gdal_res},' \
-                       '        "radius": {gdal_radius},' \
-                       '        "output_type":"idw",' \
-                       '        "filename":"outputfile.tif"' \
-                       '    }}' \
-                       '    ]' \
-                       '}}'.format(**kwargs)
-
-        with open(pdalJSON, 'w') as f:
-            f.write(pipelineJSON)
-
-        system.run('{bin}/pdal pipeline {json} --readers.las.filename={f_in} '
-                   '--writers.gdal.filename={f_out}'.format(**kwargs))
-
-        if io.file_exists(kwargs['f_out']):
-            return True
-        else:
-            return False
-
     def utm_to_latlon(self, _file, _photo, idx):
 
         gcp = self.gcps[idx]
@@ -478,8 +428,6 @@ class ODM_Tree(object):
             self.odm_georeferencing, 'odm_georeferenced_model.las')
         self.odm_georeferencing_dem = io.join_paths(
             self.odm_georeferencing, 'odm_georeferencing_model_dem.tif')
-        self.odm_georeferencing_dem_json = io.join_paths(
-            self.odm_georeferencing, 'dem.json')
 
         # odm_orthophoto
         self.odm_orthophoto_file = io.join_paths(self.odm_orthophoto, 'odm_orthophoto.png')
@@ -488,3 +436,6 @@ class ODM_Tree(object):
         self.odm_orthophoto_log = io.join_paths(self.odm_orthophoto, 'odm_orthophoto_log.txt')
         self.odm_orthophoto_tif_log = io.join_paths(self.odm_orthophoto, 'gdal_translate_log.txt')
         self.odm_orthophoto_gdaladdo_log = io.join_paths(self.odm_orthophoto, 'gdaladdo_log.txt')
+
+    def path(self, *args):
+        return io.join_paths(self.root_path, *args)

scripts/odm_app.py

@@ -15,6 +15,7 @@ from odm_meshing import ODMeshingCell
 from mvstex import ODMMvsTexCell
 from odm_georeferencing import ODMGeoreferencingCell
 from odm_orthophoto import ODMOrthoPhotoCell
+from odm_dem import ODMDEMCell
 
 
 class ODMApp(ecto.BlackBox):
@@ -71,11 +72,8 @@ class ODMApp(ecto.BlackBox):
                  'georeferencing': ODMGeoreferencingCell(img_size=p.args.resize_to,
                                                          gcp_file=p.args.gcp,
                                                          use_exif=p.args.use_exif,
-                                                         dem=p.args.dem,
-                                                         sample_radius=p.args.dem_sample_radius,
-                                                         gdal_res=p.args.dem_resolution,
-                                                         gdal_radius=p.args.dem_radius,
                                                          verbose=p.args.verbose),
+                 'dem': ODMDEMCell(verbose=p.args.verbose),
                  'orthophoto': ODMOrthoPhotoCell(resolution=p.args.orthophoto_resolution,
                                                  t_srs=p.args.orthophoto_target_srs,
                                                  no_tiled=p.args.orthophoto_no_tiled,
@@ -148,7 +146,12 @@ class ODMApp(ecto.BlackBox):
                         self.args[:] >> self.georeferencing['args'],
                         self.dataset['photos'] >> self.georeferencing['photos'],
                         self.texturing['reconstruction'] >> self.georeferencing['reconstruction']]
-
+        
+        # create odm dem
+        connections += [self.tree[:] >> self.dem['tree'],
+                        self.args[:] >> self.dem['args'],
+                        self.georeferencing['reconstruction'] >> self.dem['reconstruction']]
+        
         # create odm orthophoto
         connections += [self.tree[:] >> self.orthophoto['tree'],
                         self.args[:] >> self.orthophoto['args'],

scripts/odm_dem.py

@@ -0,0 +1,184 @@
+import ecto, os, json
+from shutil import copyfile
+
+from opendm import io
+from opendm import log
+from opendm import system
+from opendm import context
+from opendm import types
+
+
+class ODMDEMCell(ecto.Cell):
+    def declare_params(self, params):
+        params.declare("verbose", 'print additional messages to console', False)
+
+    def declare_io(self, params, inputs, outputs):
+        inputs.declare("tree", "Struct with paths", [])
+        inputs.declare("args", "The application arguments.", {})
+        inputs.declare("reconstruction", "list of ODMReconstructions", [])
+
+    def process(self, inputs, outputs):
+        # Benchmarking
+        start_time = system.now_raw()
+
+        log.ODM_INFO('Running ODM DEM Cell')
+
+        # get inputs
+        args = self.inputs.args
+        tree = self.inputs.tree
+        las_model_found = io.file_exists(tree.odm_georeferencing_model_las)
+        env_paths = [context.superbuild_bin_path]
+
+        # Just to make sure
+        l2d_module_installed = True
+        try:
+            system.run('l2d_classify --help > /dev/null', env_paths)
+        except:
+            log.ODM_WARNING('lidar2dems is not installed properly')
+            l2d_module_installed = False
+
+        log.ODM_INFO('Create DSM: ' + str(args.dsm))
+        log.ODM_INFO('Create DTM: ' + str(args.dtm))
+        log.ODM_INFO('DEM input file {0} found: {1}'.format(tree.odm_georeferencing_model_las, str(las_model_found)))
+
+        # Do we need to process anything here?
+        if (args.dsm or args.dtm) and las_model_found and l2d_module_installed:
+
+            # define paths and create working directories
+            odm_dem_root = tree.path('odm_dem')
+            system.mkdir_p(odm_dem_root)
+
+            dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif')
+            dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif')
+
+            # check if we rerun cell or not
+            rerun_cell = (args.rerun is not None and
+                          args.rerun == 'odm_dem') or \
+                         (args.rerun_all) or \
+                         (args.rerun_from is not None and
+                          'odm_dem' in args.rerun_from)
+
+            if (args.dtm and not io.file_exists(dtm_output_filename)) or \
+                (args.dsm and not io.file_exists(dsm_output_filename)) or \
+                rerun_cell:
+
+                # Extract boundaries and srs of point cloud
+                summary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.summary.json')
+                boundary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.boundary.json')
+
+                system.run('pdal info --summary {0} > {1}'.format(tree.odm_georeferencing_model_las, summary_file_path), env_paths)
+                system.run('pdal info --boundary {0} > {1}'.format(tree.odm_georeferencing_model_las,  boundary_file_path), env_paths)
+
+                pc_proj4 = ""
+                pc_geojson_bounds_feature = None
+
+                with open(summary_file_path, 'r') as f:
+                    json_f = json.loads(f.read())
+                    pc_proj4 = json_f['summary']['srs']['proj4']
+
+                with open(boundary_file_path, 'r') as f:
+                    json_f = json.loads(f.read())
+                    pc_geojson_boundary_feature = json_f['boundary']['boundary_json']
+
+                # Write bounds to GeoJSON
+                bounds_geojson_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.bounds.geojson')
+                with open(bounds_geojson_path, "w") as f:
+                    f.write(json.dumps({
+                        "type": "FeatureCollection",
+                        "features": [{
+                            "type": "Feature",
+                            "geometry": pc_geojson_boundary_feature
+                        }]
+                    }))
+
+                bounds_shapefile_path = os.path.join(odm_dem_root, 'bounds.shp')
+
+                # Convert bounds to Shapefile
+                kwargs = {
+                    'input': bounds_geojson_path,
+                    'output': bounds_shapefile_path,
+                    'proj4': pc_proj4
+                }
+                system.run('ogr2ogr -overwrite -a_srs "{proj4}" {output} {input}'.format(**kwargs))
+
+                # Process with lidar2dems
+                terrain_params_map = {
+                    'flatnonforest': (1, 3), 
+                    'flatforest': (1, 2), 
+                    'complexnonforest': (5, 2), 
+                    'complexforest': (10, 2)
+                }
+                terrain_params = terrain_params_map[args.dem_terrain_type.lower()]             
+
+                kwargs = {
+                    'verbose': '-v' if self.params.verbose else '',
+                    'slope': terrain_params[0],
+                    'cellsize': terrain_params[1],
+                    'outdir': odm_dem_root,
+                    'site': bounds_shapefile_path
+                }
+
+                l2d_params = '--slope {slope} --cellsize {cellsize} ' \
+                             '{verbose} ' \
+                             '-o -s {site} ' \
+                             '--outdir {outdir}'.format(**kwargs)
+
+                approximate = '--approximate' if args.dem_approximate else ''
+
+                # Classify only if we need a DTM
+                run_classification = args.dtm
+
+                if run_classification:
+                    system.run('l2d_classify {0} --decimation {1} '
+                               '{2} --initialDistance {3} {4}'.format(
+                        l2d_params, args.dem_decimation, approximate, 
+                        args.dem_initial_distance, tree.odm_georeferencing), env_paths)
+                else:
+                    log.ODM_INFO("Will skip classification, only DSM is needed")
+                    copyfile(tree.odm_georeferencing_model_las, os.path.join(odm_dem_root, 'bounds-0_l2d_s{slope}c{cellsize}.las'.format(**kwargs)))
+
+                products = []
+                if args.dsm: products.append('dsm') 
+                if args.dtm: products.append('dtm')
+
+                radius_steps = [args.dem_resolution]
+                for _ in range(args.dem_gapfill_steps - 1):
+                    radius_steps.append(radius_steps[-1] * 3) # 3 is arbitrary, maybe there's a better value?
+
+                for product in products:
+                    demargs = {
+                        'product': product,
+                        'indir': odm_dem_root,
+                        'l2d_params': l2d_params,
+                        'maxsd': args.dem_maxsd,
+                        'maxangle': args.dem_maxangle,
+                        'resolution': args.dem_resolution,
+                        'radius_steps': ' '.join(map(str, radius_steps)),
+                        'gapfill': '--gapfill' if args.dem_gapfill_steps > 0 else '',
+                        
+                        # If we didn't run a classification, we should pass the decimate parameter here
+                        'decimation': '--decimation {0}'.format(args.dem_decimation) if not run_classification else ''
+                    }
+
+                    system.run('l2d_dems {product} {indir} {l2d_params} '
+                               '--maxsd {maxsd} --maxangle {maxangle} '
+                               '--resolution {resolution} --radius {radius_steps} '
+                               '{decimation} '
+                               '{gapfill} '.format(**demargs), env_paths)
+
+                    # Rename final output
+                    if product == 'dsm':
+                        os.rename(os.path.join(odm_dem_root, 'bounds-0_dsm.idw.tif'), dsm_output_filename)
+                    elif product == 'dtm':
+                        os.rename(os.path.join(odm_dem_root, 'bounds-0_dtm.idw.tif'), dtm_output_filename)
+
+            else:
+                log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
+        else:
+            log.ODM_WARNING('DEM will not be generated')
+
+        if args.time:
+            system.benchmark(start_time, tree.benchmarking, 'Dem')
+
+        log.ODM_INFO('Running ODM DEM Cell - Finished')
+        return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT

scripts/odm_georeferencing.py

@@ -17,10 +17,6 @@ class ODMGeoreferencingCell(ecto.Cell):
                             'northing height pixelrow pixelcol imagename', 'gcp_list.txt')
         params.declare("img_size", 'image size used in calibration', 2400)
         params.declare("use_exif", 'use exif', False)
-        params.declare("dem", 'Generate a dem', False)
-        params.declare("sample_radius", "Minimum distance between samples for DEM gen", 3)
-        params.declare("gdal_res", "Length of raster cell edges in X/Y units ", 2)
-        params.declare("gdal_radius", "Radius about cell center bounding points to use to calculate a cell value", 0.5)
         params.declare("verbose", 'print additional messages to console', False)
 
     def declare_io(self, params, inputs, outputs):
@@ -175,19 +171,6 @@ class ODMGeoreferencingCell(ecto.Cell):
                                        tree.odm_georeferencing_model_las,
                                        tree.odm_georeferencing_las_json)
 
-                # If --dem, create a DEM
-                if args.dem:
-                    demcreated = geo_ref.convert_to_dem(tree.odm_georeferencing_model_las,
-                                                        tree.odm_georeferencing_dem,
-                                                        tree.odm_georeferencing_dem_json,
-                                                        self.params.sample_radius,
-                                                        self.params.gdal_res,
-                                                        self.params.gdal_radius)
-                    if not demcreated:
-                        log.ODM_WARNING('Something went wrong. Check the logs in odm_georeferencing.')
-                    else:
-                        log.ODM_INFO('DEM created at {0}'.format(tree.odm_georeferencing_dem))
-
                 # XYZ point cloud output
                 log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)")
                 with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile:

settings.yaml

@@ -44,10 +44,15 @@ project_path: '' # Example: '/home/user/ODMProjects
 #texturing_tone_mapping: 'none'
 #gcp: !!null  # YAML tag for None
 #use_exif: False # Set to True if you have a GCP file (it auto-detects) and want to use EXIF
-#dem: False
+#dtm: False      # Use this tag to build a DTM (Digital Terrain Model
-#dem_sample_radius: 1.0
+#dsm: False      # Use this tag to build a DSM (Digital Surface Model
-#dem_resolution: 2
+#dem-gapfill-steps: 4
-#dem_radius: 0.5
+#dem-resolution: 0.1
+#dem-maxangle:20
+#dem-maxsd: 2.5
+#dem-approximate: False
+#dem-decimation: 1
+#dem-terrain-type: ComplexForest
 #orthophoto_resolution: 20.0 # Pixels/meter
 #orthophoto_target_srs: !!null # Currently does nothing
 #orthophoto_no_tiled: False