End-to-end pipeline runs

opendm/cropper.py

@@ -37,7 +37,7 @@ class Cropper:
         # ext = .tif
 
         original_geotiff = os.path.join(path, "{}.original{}".format(basename, ext))
-        os.rename(geotiff_path, original_geotiff)
+        os.replace(geotiff_path, original_geotiff)
 
         try:
             kwargs = {
@@ -64,7 +64,7 @@ class Cropper:
             log.ODM_WARNING('Something went wrong while cropping: {}'.format(e))
             
             # Revert rename
-            os.rename(original_geotiff, geotiff_path)
+            os.replace(original_geotiff, geotiff_path)
 
         return geotiff_path
 
@@ -159,8 +159,8 @@ class Cropper:
         if pc_geojson_boundary_feature is None: raise RuntimeError("Could not determine point cloud boundaries")
 
         # Write bounds to GeoJSON
-        bounds_geojson_path = self.path('bounds.geojson')
+        tmp_bounds_geojson_path = self.path('tmp-bounds.geojson')
-        with open(bounds_geojson_path, "w") as f:
+        with open(tmp_bounds_geojson_path, "w") as f:
             f.write(json.dumps({
                 "type": "FeatureCollection",
                 "features": [{
@@ -172,7 +172,7 @@ class Cropper:
         # Create a convex hull around the boundary
         # as to encompass the entire area (no holes)    
         driver = ogr.GetDriverByName('GeoJSON')
-        ds = driver.Open(bounds_geojson_path, 0) # ready-only
+        ds = driver.Open(tmp_bounds_geojson_path, 0) # ready-only
         layer = ds.GetLayer()
 
         # Collect all Geometry
@@ -202,7 +202,7 @@ class Cropper:
         # Save to a new file
         bounds_geojson_path = self.path('bounds.geojson')
         if os.path.exists(bounds_geojson_path):
-            driver.DeleteDataSource(bounds_geojson_path)
+            os.remove(bounds_geojson_path)
 
         out_ds = driver.CreateDataSource(bounds_geojson_path)
         layer = out_ds.CreateLayer("convexhull", geom_type=ogr.wkbPolygon)
@@ -219,6 +219,10 @@ class Cropper:
         # Remove decimated point cloud
         if os.path.exists(decimated_pointcloud_path):
             os.remove(decimated_pointcloud_path)
+        
+        # Remove tmp bounds
+        if os.path.exists(tmp_bounds_geojson_path):
+            os.remove(tmp_bounds_geojson_path)
 
         return bounds_geojson_path
 

opendm/dem/commands.py

@@ -258,13 +258,13 @@ def create_dem(input_point_cloud, dem_type, output_type='max', radiuses=['0.56']
         median_smoothing(geotiff_path, output_path)
         os.remove(geotiff_path)
     else:
-        os.rename(geotiff_path, output_path)
+        os.replace(geotiff_path, output_path)
 
     if os.path.exists(geotiff_tmp_path):
         if not keep_unfilled_copy: 
             os.remove(geotiff_tmp_path)
         else:
-            os.rename(geotiff_tmp_path, io.related_file_path(output_path, postfix=".unfilled"))
+            os.replace(geotiff_tmp_path, io.related_file_path(output_path, postfix=".unfilled"))
     
     for cleanup_file in [tiles_vrt_path, merged_vrt_path, geotiff_small_path, geotiff_small_filled_path]:
         if os.path.exists(cleanup_file): os.remove(cleanup_file)

opendm/dem/pdal.py

@@ -36,9 +36,9 @@ import json as jsonlib
 import tempfile
 from opendm import system
 from opendm import log
+from opendm.utils import double_quote
 
 from datetime import datetime
-from pipes import quote
 
 
 """ JSON Functions """
@@ -191,7 +191,7 @@ def merge_point_clouds(input_files, output_file, verbose=False):
     cmd = [
         'pdal',
         'merge',
-        ' '.join(map(quote, input_files + [output_file])),
+        ' '.join(map(double_quote, input_files + [output_file])),
     ]
 
     if verbose:

opendm/osfm.py

@@ -349,7 +349,7 @@ class OSFMContext:
             # (containing only the primary band)
             if os.path.exists(self.recon_file()):
                 os.remove(self.recon_file())
-            os.rename(self.recon_backup_file(), self.recon_file())
+            os.replace(self.recon_backup_file(), self.recon_file())
             log.ODM_INFO("Restored reconstruction.json")
 
     def backup_reconstruction(self):

opendm/point_cloud.py

@@ -6,7 +6,7 @@ from opendm.system import run
 from opendm import entwine
 from opendm import io
 from opendm.concurrency import parallel_map
-from pipes import quote
+from opendm.utils import double_quote
 
 def ply_info(input_ply):
     if not os.path.exists(input_ply):
@@ -240,7 +240,7 @@ def merge(input_point_cloud_files, output_file, rerun=False):
         os.remove(output_file)
 
     kwargs = {
-        'all_inputs': " ".join(map(quote, input_point_cloud_files)),
+        'all_inputs': " ".join(map(double_quote, input_point_cloud_files)),
         'output': output_file
     }
 
@@ -312,7 +312,7 @@ def merge_ply(input_point_cloud_files, output_file, dims=None):
         '--writers.ply.sized_types=false',
         '--writers.ply.storage_mode="little endian"',
         ('--writers.ply.dims="%s"' % dims) if dims is not None else '',
-        ' '.join(map(quote, input_point_cloud_files + [output_file])),
+        ' '.join(map(double_quote, input_point_cloud_files + [output_file])),
     ]
 
     system.run(' '.join(cmd))

opendm/tiles/tiler.py

@@ -1,11 +1,12 @@
 import os
+import sys
 from opendm import log
 from opendm import system
 from opendm import io
 
 def generate_tiles(geotiff, output_dir, max_concurrency):
     gdal2tiles = os.path.join(os.path.dirname(__file__), "gdal2tiles.py")
-    system.run('python3 "%s" --processes %s -z 5-21 -n -w none "%s" "%s"' % (gdal2tiles, max_concurrency, geotiff, output_dir))
+    system.run('%s "%s" --processes %s -z 5-21 -n -w none "%s" "%s"' % (sys.executable, gdal2tiles, max_concurrency, geotiff, output_dir))
 
 def generate_orthophoto_tiles(geotiff, output_dir, max_concurrency):
     try:
@@ -29,7 +30,7 @@ def generate_colored_hillshade(geotiff):
 
         system.run('gdaldem color-relief "%s" "%s" "%s" -alpha -co ALPHA=YES' % (geotiff, relief_file, colored_dem))
         system.run('gdaldem hillshade "%s" "%s" -z 1.0 -s 1.0 -az 315.0 -alt 45.0' % (geotiff, hillshade_dem))
-        system.run('python3 "%s" "%s" "%s" "%s"' % (hsv_merge_script, colored_dem, hillshade_dem, colored_hillshade_dem))
+        system.run('%s "%s" "%s" "%s" "%s"' % (sys.executable, hsv_merge_script, colored_dem, hillshade_dem, colored_hillshade_dem))
         
         return outputs
     except Exception as e:

opendm/utils.py

@@ -1,6 +1,7 @@
 from opendm import log
 from opendm.photo import find_largest_photo_dim
 from osgeo import gdal
+from shlex import _find_unsafe
 
 def get_depthmap_resolution(args, photos):
     if 'depthmap_resolution_is_set' in args:
@@ -39,4 +40,15 @@ def get_raster_stats(geotiff):
             'stddev': s[3]
         })
             
-    return stats
+    return stats
+
+def double_quote(s):
+    """Return a shell-escaped version of the string *s*."""
+    if not s:
+        return '""'
+    if _find_unsafe(s) is None:
+        return s
+
+    # use double quotes, and prefix double quotes with a \
+    # the string $"b is then quoted as "$\"b"
+    return '"' + s.replace('"', '\\\"') + '"'

run.py

@@ -11,9 +11,9 @@ from opendm import config
 from opendm import system
 from opendm import io
 from opendm.progress import progressbc
+from opendm.utils import double_quote
 
 import os
-from pipes import quote
 
 from stages.odm_app import ODMApp
 
@@ -50,18 +50,18 @@ if __name__ == '__main__':
         log.ODM_INFO("Rerun all -- Removing old data")
         os.system("rm -rf " + 
                     " ".join([
-                        quote(os.path.join(args.project_path, "odm_georeferencing")),
+                        double_quote(os.path.join(args.project_path, "odm_georeferencing")),
-                        quote(os.path.join(args.project_path, "odm_meshing")),
+                        double_quote(os.path.join(args.project_path, "odm_meshing")),
-                        quote(os.path.join(args.project_path, "odm_orthophoto")),
+                        double_quote(os.path.join(args.project_path, "odm_orthophoto")),
-                        quote(os.path.join(args.project_path, "odm_dem")),
+                        double_quote(os.path.join(args.project_path, "odm_dem")),
-                        quote(os.path.join(args.project_path, "odm_report")),
+                        double_quote(os.path.join(args.project_path, "odm_report")),
-                        quote(os.path.join(args.project_path, "odm_texturing")),
+                        double_quote(os.path.join(args.project_path, "odm_texturing")),
-                        quote(os.path.join(args.project_path, "opensfm")),
+                        double_quote(os.path.join(args.project_path, "opensfm")),
-                        quote(os.path.join(args.project_path, "odm_filterpoints")),
+                        double_quote(os.path.join(args.project_path, "odm_filterpoints")),
-                        quote(os.path.join(args.project_path, "odm_texturing_25d")),
+                        double_quote(os.path.join(args.project_path, "odm_texturing_25d")),
-                        quote(os.path.join(args.project_path, "openmvs")),
+                        double_quote(os.path.join(args.project_path, "openmvs")),
-                        quote(os.path.join(args.project_path, "entwine_pointcloud")),
+                        double_quote(os.path.join(args.project_path, "entwine_pointcloud")),
-                        quote(os.path.join(args.project_path, "submodels")),
+                        double_quote(os.path.join(args.project_path, "submodels")),
                     ]))
 
     app = ODMApp(args)

stages/odm_orthophoto.py

@@ -9,7 +9,7 @@ from opendm import gsd
 from opendm import orthophoto
 from opendm.concurrency import get_max_memory
 from opendm.cutline import compute_cutline
-from pipes import quote
+from opendm.utils import double_quote
 from opendm import pseudogeo
 from opendm.multispectral import get_primary_band_name
 
@@ -63,11 +63,11 @@ class ODMOrthoPhotoStage(types.ODM_Stage):
                     if not primary:
                         subdir = band['name'].lower()
                     models.append(os.path.join(base_dir, subdir, model_file))
-                kwargs['bands'] = '-bands %s' % (','.join([quote(b['name']) for b in reconstruction.multi_camera]))
+                kwargs['bands'] = '-bands %s' % (','.join([double_quote(b['name']) for b in reconstruction.multi_camera]))
             else:
                 models.append(os.path.join(base_dir, model_file))
 
-            kwargs['models'] = ','.join(map(quote, models))
+            kwargs['models'] = ','.join(map(double_quote, models))
 
             # run odm_orthophoto
             system.run('{odm_ortho_bin} -inputFiles {models} '
@@ -148,7 +148,7 @@ class ODMOrthoPhotoStage(types.ODM_Stage):
                 if io.file_exists(tree.odm_orthophoto_render):
                     pseudogeo.add_pseudo_georeferencing(tree.odm_orthophoto_render)
                     log.ODM_INFO("Renaming %s --> %s" % (tree.odm_orthophoto_render, tree.odm_orthophoto_tif))
-                    os.rename(tree.odm_orthophoto_render, tree.odm_orthophoto_tif)
+                    os.replace(tree.odm_orthophoto_render, tree.odm_orthophoto_tif)
                 else:
                     log.ODM_WARNING("Could not generate an orthophoto (it did not render)")
         else:

stages/openmvs.py

@@ -144,7 +144,7 @@ class ODMOpenMVSStage(types.ODM_Stage):
                     log.ODM_ERROR("Could not compute dense point cloud (no PLY files available).")
                 if len(scene_ply_files) == 1:
                     # Simply rename
-                    os.rename(scene_ply_files[0], tree.openmvs_model)
+                    os.replace(scene_ply_files[0], tree.openmvs_model)
                     log.ODM_INFO("%s --> %s"% (scene_ply_files[0], tree.openmvs_model))
                 else:
                     # Merge

stages/splitmerge.py

@@ -17,7 +17,7 @@ from opendm.concurrency import get_max_memory
 from opendm.remote import LocalRemoteExecutor
 from opendm.shots import merge_geojson_shots
 from opendm import point_cloud
-from pipes import quote
+from opendm.utils import double_quote
 from opendm.tiles.tiler import generate_dem_tiles
 
 class ODMSplitStage(types.ODM_Stage):
@@ -219,7 +219,7 @@ class ODMSplitStage(types.ODM_Stage):
                         argv = get_submodel_argv(args, tree.submodels_path, sp_octx.name())
 
                         # Re-run the ODM toolchain on the submodel
-                        system.run(" ".join(map(quote, map(str, argv))), env_vars=os.environ.copy())
+                        system.run(" ".join(map(double_quote, map(str, argv))), env_vars=os.environ.copy())
                 else:
                     lre.set_projects([os.path.abspath(os.path.join(p, "..")) for p in submodel_paths])
                     lre.run_toolchain()