From d91f4db580837178b4a0f93ba73a13b5d46e5d11 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 10 Dec 2020 19:23:21 +0000 Subject: [PATCH 1/3] Entwine/untwine, life's better together --- Dockerfile | 1 + SuperBuild/CMakeLists.txt | 1 + SuperBuild/cmake/External-Entwine.cmake | 31 +++++++++++ opendm/entwine.py | 68 ++++++++++++++++++++++--- portable.Dockerfile | 1 + snap/snapcraft.yaml | 1 + 6 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 SuperBuild/cmake/External-Entwine.cmake diff --git a/Dockerfile b/Dockerfile index 603bae42..c247d8ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ RUN rm -rf \ /code/SuperBuild/download \ /code/SuperBuild/src/ceres \ /code/SuperBuild/src/untwine \ + /code/SuperBuild/src/entwine \ /code/SuperBuild/src/gflags \ /code/SuperBuild/src/hexer \ /code/SuperBuild/src/lastools \ diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index f0b0906f..0a5b9ab9 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -109,6 +109,7 @@ set(custom_libs OpenSfM Zstd PDAL Untwine + Entwine MvsTexturing OpenMVS ) diff --git a/SuperBuild/cmake/External-Entwine.cmake b/SuperBuild/cmake/External-Entwine.cmake new file mode 100644 index 00000000..45c6a7ae --- /dev/null +++ b/SuperBuild/cmake/External-Entwine.cmake @@ -0,0 +1,31 @@ +set(_proj_name entwine) +set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") + +ExternalProject_Add(${_proj_name} + DEPENDS pdal + PREFIX ${_SB_BINARY_DIR} + TMP_DIR ${_SB_BINARY_DIR}/tmp + STAMP_DIR ${_SB_BINARY_DIR}/stamp + #--Download step-------------- + DOWNLOAD_DIR ${SB_DOWNLOAD_DIR} + GIT_REPOSITORY https://github.com/connormanning/entwine/ + GIT_TAG 2.1.0 + #--Update/Patch step---------- + UPDATE_COMMAND "" + #--Configure step------------- + SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name} + CMAKE_ARGS + -DCMAKE_CXX_FLAGS=-isystem\ ${SB_SOURCE_DIR}/pdal + -DADDITIONAL_LINK_DIRECTORIES_PATHS=${SB_INSTALL_DIR}/lib + -DWITH_TESTS=OFF + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR} + #--Build step----------------- + BINARY_DIR ${_SB_BINARY_DIR} + #--Install step--------------- + INSTALL_DIR ${SB_INSTALL_DIR} + #--Output logging------------- + LOG_DOWNLOAD OFF + LOG_CONFIGURE OFF + LOG_BUILD OFF +) diff --git a/opendm/entwine.py b/opendm/entwine.py index d0989cfd..b1f8875c 100644 --- a/opendm/entwine.py +++ b/opendm/entwine.py @@ -6,6 +6,22 @@ from opendm import log from opendm import system from opendm import concurrency +import os +import shutil +from pipes import quote +from opendm import io +from opendm import log +from opendm import system +from opendm import concurrency + +import os +import shutil +from pipes import quote +from opendm import io +from opendm import log +from opendm import system +from opendm import concurrency + def build(input_point_cloud_files, output_path, max_concurrency=8, rerun=False): num_files = len(input_point_cloud_files) if num_files == 0: @@ -14,10 +30,52 @@ def build(input_point_cloud_files, output_path, max_concurrency=8, rerun=False): tmpdir = io.related_file_path(output_path, postfix="-tmp") - if rerun and io.dir_exists(output_path): - log.ODM_WARNING("Removing previous EPT directory: %s" % output_path) - shutil.rmtree(output_path) + def dir_cleanup(): + if io.dir_exists(output_path): + log.ODM_WARNING("Removing previous EPT directory: %s" % output_path) + shutil.rmtree(output_path) + + if io.dir_exists(tmpdir): + log.ODM_WARNING("Removing previous EPT temp directory: %s" % tmpdir) + shutil.rmtree(tmpdir) + + if rerun: + dir_cleanup() + # Attempt with entwine (faster, more memory hungry) + try: + build_entwine(input_point_cloud_files, tmpdir, output_path, max_concurrency=max_concurrency) + except Exception as e: + log.ODM_WARNING("Cannot build EPT using entwine (%s), attempting with untwine..." % str(e)) + dir_cleanup() + build_untwine(input_point_cloud_files, tmpdir, output_path, max_concurrency=max_concurrency) + + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + + +def build_entwine(input_point_cloud_files, tmpdir, output_path, max_concurrency=8): + kwargs = { + 'threads': max_concurrency, + 'tmpdir': tmpdir, + 'all_inputs': "-i " + " ".join(map(quote, input_point_cloud_files)), + 'outputdir': output_path + } + + # Run scan to compute dataset bounds + system.run('entwine scan --threads {threads} --tmp "{tmpdir}" {all_inputs} -o "{outputdir}"'.format(**kwargs)) + scan_json = os.path.join(output_path, "scan.json") + + if os.path.exists(scan_json): + kwargs['input'] = scan_json + for _ in range(len(input_point_cloud_files)): + # One at a time + system.run('entwine build --threads {threads} --tmp "{tmpdir}" -i "{input}" -o "{outputdir}" --run 1'.format(**kwargs)) + else: + log.ODM_WARNING("%s does not exist, no point cloud will be built." % scan_json) + + +def build_untwine(input_point_cloud_files, tmpdir, output_path, max_concurrency=8, rerun=False): kwargs = { # 'threads': max_concurrency, 'tmpdir': tmpdir, @@ -27,7 +85,3 @@ def build(input_point_cloud_files, output_path, max_concurrency=8, rerun=False): # Run untwine system.run('untwine --temp_dir "{tmpdir}" {files} --output_dir "{outputdir}"'.format(**kwargs)) - - # Cleanup - if os.path.exists(tmpdir): - shutil.rmtree(tmpdir) \ No newline at end of file diff --git a/portable.Dockerfile b/portable.Dockerfile index 03ba2ac7..2d8a4c9d 100644 --- a/portable.Dockerfile +++ b/portable.Dockerfile @@ -20,6 +20,7 @@ RUN rm -rf \ /code/SuperBuild/download \ /code/SuperBuild/src/ceres \ /code/SuperBuild/src/untwine \ + /code/SuperBuild/src/entwine \ /code/SuperBuild/src/gflags \ /code/SuperBuild/src/hexer \ /code/SuperBuild/src/lastools \ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 44525576..c0fb9fea 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -197,6 +197,7 @@ parts: - -odm/SuperBuild/download - -odm/SuperBuild/src/ceres - -odm/SuperBuild/src/untwine + - -odm/SuperBuild/src/entwine - -odm/SuperBuild/src/gflags - -odm/SuperBuild/src/hexer - -odm/SuperBuild/src/lastools From bb9a2061c167c973548d559d03d1e92703a1d610 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 10 Dec 2020 19:28:58 +0000 Subject: [PATCH 2/3] Remove dup imports --- opendm/entwine.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/opendm/entwine.py b/opendm/entwine.py index b1f8875c..644046d2 100644 --- a/opendm/entwine.py +++ b/opendm/entwine.py @@ -6,21 +6,6 @@ from opendm import log from opendm import system from opendm import concurrency -import os -import shutil -from pipes import quote -from opendm import io -from opendm import log -from opendm import system -from opendm import concurrency - -import os -import shutil -from pipes import quote -from opendm import io -from opendm import log -from opendm import system -from opendm import concurrency def build(input_point_cloud_files, output_path, max_concurrency=8, rerun=False): num_files = len(input_point_cloud_files) From b726465e47df6add749b79cc61e174ac5542d5f7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 10 Dec 2020 14:29:41 -0500 Subject: [PATCH 3/3] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2bf1c1cc..f90b1afc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.1 +2.3.2