diff --git a/app/admin.py b/app/admin.py index c4c61a04..029dd3cb 100644 --- a/app/admin.py +++ b/app/admin.py @@ -16,7 +16,7 @@ from app.models import Preset from app.models import Plugin from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin, valid_plugin, \ get_plugins_persistent_path, clear_plugins_cache, init_plugins -from .models import Project, Task, ImageUpload, Setting, Theme +from .models import Project, Task, Setting, Theme from django import forms from codemirror2.widgets import CodeMirrorEditor from webodm import settings @@ -37,12 +37,6 @@ class TaskAdmin(admin.ModelAdmin): admin.site.register(Task, TaskAdmin) - -class ImageUploadAdmin(admin.ModelAdmin): - readonly_fields = ('image',) - -admin.site.register(ImageUpload, ImageUploadAdmin) - admin.site.register(Preset, admin.ModelAdmin) diff --git a/app/api/imageuploads.py b/app/api/imageuploads.py index 6fa2d8a0..64efabd5 100644 --- a/app/api/imageuploads.py +++ b/app/api/imageuploads.py @@ -4,7 +4,6 @@ import math from .tasks import TaskNestedView from rest_framework import exceptions -from app.models import ImageUpload from app.models.task import assets_directory_path from PIL import Image, ImageDraw, ImageOps from django.http import HttpResponse @@ -33,12 +32,7 @@ class Thumbnail(TaskNestedView): Generate a thumbnail on the fly for a particular task's image """ task = self.get_and_check_task(request, pk) - image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first() - - if image is None: - raise exceptions.NotFound() - - image_path = image.path() + image_path = task.get_image_path(image_filename) if not os.path.isfile(image_path): raise exceptions.NotFound() @@ -146,12 +140,7 @@ class ImageDownload(TaskNestedView): Download a task's image """ task = self.get_and_check_task(request, pk) - image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first() - - if image is None: - raise exceptions.NotFound() - - image_path = image.path() + image_path = task.get_image_path(image_filename) if not os.path.isfile(image_path): raise exceptions.NotFound() diff --git a/app/api/tasks.py b/app/api/tasks.py index 7e6e21b3..f3761dfb 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -179,7 +179,7 @@ class TaskViewSet(viewsets.ViewSet): raise exceptions.NotFound() task.partial = False - task.images_count = models.ImageUpload.objects.filter(task=task).count() + task.images_count = len(task.scan_images()) if task.images_count < 2: raise exceptions.ValidationError(detail=_("You need to upload at least 2 images before commit")) @@ -206,11 +206,8 @@ class TaskViewSet(viewsets.ViewSet): if len(files) == 0: raise exceptions.ValidationError(detail=_("No files uploaded")) - with transaction.atomic(): - for image in files: - models.ImageUpload.objects.create(task=task, image=image) - - task.images_count = models.ImageUpload.objects.filter(task=task).count() + task.handle_images_upload(files) + task.images_count = len(task.scan_images()) # Update other parameters such as processing node, task name, etc. serializer = TaskSerializer(task, data=request.data, partial=True) serializer.is_valid(raise_exception=True) @@ -256,9 +253,8 @@ class TaskViewSet(viewsets.ViewSet): task = models.Task.objects.create(project=project, pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None) - for image in files: - models.ImageUpload.objects.create(task=task, image=image) - task.images_count = len(files) + task.handle_images_upload(files) + task.images_count = len(task.scan_images()) # Update other parameters such as processing node, task name, etc. serializer = TaskSerializer(task, data=request.data, partial=True) diff --git a/app/migrations/0012_public_task_uuids.py b/app/migrations/0012_public_task_uuids.py index 7b8f7956..73919178 100644 --- a/app/migrations/0012_public_task_uuids.py +++ b/app/migrations/0012_public_task_uuids.py @@ -8,17 +8,14 @@ import uuid, os, pickle, tempfile from webodm import settings tasks = [] -imageuploads = [] task_ids = {} # map old task IDs --> new task IDs def dump(apps, schema_editor): - global tasks, imageuploads, task_ids + global tasks, task_ids Task = apps.get_model('app', 'Task') - ImageUpload = apps.get_model('app', 'ImageUpload') tasks = list(Task.objects.all().values('id', 'project')) - imageuploads = list(ImageUpload.objects.all().values('id', 'task')) # Generate UUIDs for task in tasks: @@ -31,9 +28,9 @@ def dump(apps, schema_editor): task_ids[task['id']] = new_id tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle") - pickle.dump((tasks, imageuploads, task_ids), open(tmp_path, 'wb')) + pickle.dump((tasks, task_ids), open(tmp_path, 'wb')) - if len(tasks) > 0: print("Dumped tasks and imageuploads") + if len(tasks) > 0: print("Dumped tasks") class Migration(migrations.Migration): diff --git a/app/migrations/0013_public_task_uuids.py b/app/migrations/0013_public_task_uuids.py index 321ecfc3..7486a82e 100644 --- a/app/migrations/0013_public_task_uuids.py +++ b/app/migrations/0013_public_task_uuids.py @@ -8,7 +8,6 @@ import uuid, os, pickle, tempfile from webodm import settings tasks = [] -imageuploads = [] task_ids = {} # map old task IDs --> new task IDs def task_path(project_id, task_id): @@ -44,10 +43,10 @@ def create_uuids(apps, schema_editor): def restore(apps, schema_editor): - global tasks, imageuploads, task_ids + global tasks, task_ids tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle") - tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb')) + tasks, task_ids = pickle.load(open(tmp_path, 'rb')) class Migration(migrations.Migration): diff --git a/app/migrations/0015_public_task_uuids.py b/app/migrations/0015_public_task_uuids.py deleted file mode 100644 index df60ae54..00000000 --- a/app/migrations/0015_public_task_uuids.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-11-30 15:41 -from __future__ import unicode_literals - -from django.db import migrations, models -import os, pickle, tempfile - -from webodm import settings - -tasks = [] -imageuploads = [] -task_ids = {} # map old task IDs --> new task IDs - - -def restoreImageUploadFks(apps, schema_editor): - global imageuploads, task_ids - - ImageUpload = apps.get_model('app', 'ImageUpload') - Task = apps.get_model('app', 'Task') - - for img in imageuploads: - i = ImageUpload.objects.get(pk=img['id']) - old_image_path = i.image.name - task_id = task_ids[img['task']] - - # project/2/task/5/DJI_0032.JPG --> project/2/task//DJI_0032.JPG - dirs, filename = os.path.split(old_image_path) - head, tail = os.path.split(dirs) - new_image_path = os.path.join(head, str(task_id), filename) - - i.task = Task.objects.get(id=task_id) - i.image.name = new_image_path - i.save() - - print("{} --> {} (Task {})".format(old_image_path, new_image_path, str(task_id))) - - -def restore(apps, schema_editor): - global tasks, imageuploads, task_ids - - tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle") - tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb')) - - -class Migration(migrations.Migration): - - dependencies = [ - ('app', '0014_public_task_uuids'), - ] - - operations = [ - migrations.RunPython(restore), - migrations.RunPython(restoreImageUploadFks), - ] diff --git a/app/migrations/0016_public_task_uuids.py b/app/migrations/0016_public_task_uuids.py index 7022496c..cc34e4e4 100644 --- a/app/migrations/0016_public_task_uuids.py +++ b/app/migrations/0016_public_task_uuids.py @@ -9,7 +9,7 @@ from webodm import settings class Migration(migrations.Migration): dependencies = [ - ('app', '0015_public_task_uuids'), + ('app', '0014_public_task_uuids'), ] operations = [ diff --git a/app/migrations/0026_update_images_count.py b/app/migrations/0026_update_images_count.py index 6c55b8f8..8cde8ebf 100644 --- a/app/migrations/0026_update_images_count.py +++ b/app/migrations/0026_update_images_count.py @@ -10,7 +10,7 @@ def update_images_count(apps, schema_editor): for t in Task.objects.all(): print("Updating {}".format(t)) - t.images_count = t.imageupload_set.count() + t.images_count = len(t.scan_images()) t.save() diff --git a/app/migrations/0029_auto_20190907_1348.py b/app/migrations/0029_auto_20190907_1348.py index 09b6e0a1..9977a4d2 100644 --- a/app/migrations/0029_auto_20190907_1348.py +++ b/app/migrations/0029_auto_20190907_1348.py @@ -1,6 +1,6 @@ # Generated by Django 2.1.11 on 2019-09-07 13:48 -import app.models.image_upload +import app.models from django.db import migrations, models @@ -14,6 +14,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='imageupload', name='image', - field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path), + field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_directory_path), ), ] diff --git a/app/migrations/0031_auto_20210610_1850.py b/app/migrations/0031_auto_20210610_1850.py index 4d351f81..9ae9c16c 100644 --- a/app/migrations/0031_auto_20210610_1850.py +++ b/app/migrations/0031_auto_20210610_1850.py @@ -1,7 +1,7 @@ # Generated by Django 2.1.15 on 2021-06-10 18:50 -import app.models.image_upload import app.models.task +from app.models import image_directory_path import colorfield.fields from django.conf import settings import django.contrib.gis.db.models.fields @@ -60,7 +60,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='imageupload', name='image', - field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path, verbose_name='Image'), + field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=image_directory_path, verbose_name='Image'), ), migrations.AlterField( model_name='imageupload', diff --git a/app/migrations/0034_delete_imageupload.py b/app/migrations/0034_delete_imageupload.py new file mode 100644 index 00000000..d2227fee --- /dev/null +++ b/app/migrations/0034_delete_imageupload.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.27 on 2023-03-23 17:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0033_auto_20230307_1532'), + ] + + operations = [ + migrations.DeleteModel( + name='ImageUpload', + ), + ] diff --git a/app/models/__init__.py b/app/models/__init__.py index dc0ababc..b7434b5d 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,4 +1,3 @@ -from .image_upload import ImageUpload, image_directory_path from .project import Project from .task import Task, validate_task_options, gcp_directory_path from .preset import Preset @@ -7,3 +6,6 @@ from .setting import Setting from .plugin_datum import PluginDatum from .plugin import Plugin +# deprecated +def image_directory_path(image_upload, filename): + raise Exception("Deprecated") \ No newline at end of file diff --git a/app/models/image_upload.py b/app/models/image_upload.py deleted file mode 100644 index 8ccbdb79..00000000 --- a/app/models/image_upload.py +++ /dev/null @@ -1,21 +0,0 @@ -from .task import Task, assets_directory_path -from django.db import models -from django.utils.translation import gettext_lazy as _ - -def image_directory_path(image_upload, filename): - return assets_directory_path(image_upload.task.id, image_upload.task.project.id, filename) - - -class ImageUpload(models.Model): - task = models.ForeignKey(Task, on_delete=models.CASCADE, help_text=_("Task this image belongs to"), verbose_name=_("Task")) - image = models.ImageField(upload_to=image_directory_path, help_text=_("File uploaded by a user"), max_length=512, verbose_name=_("Image")) - - def __str__(self): - return self.image.name - - def path(self): - return self.image.path - - class Meta: - verbose_name = _("Image Upload") - verbose_name_plural = _("Image Uploads") \ No newline at end of file diff --git a/app/models/task.py b/app/models/task.py index 79e4092a..471b56a5 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -21,6 +21,7 @@ from django.contrib.gis.gdal import GDALRaster from django.contrib.gis.gdal import OGRGeometry from django.contrib.gis.geos import GEOSGeometry from django.contrib.postgres import fields +from django.core.files.uploadedfile import InMemoryUploadedFile from django.core.exceptions import ValidationError, SuspiciousFileOperation from django.db import models from django.db import transaction @@ -310,15 +311,6 @@ class Task(models.Model): shutil.move(old_task_folder, new_task_folder_parent) logger.info("Moved task folder from {} to {}".format(old_task_folder, new_task_folder)) - - with transaction.atomic(): - for img in self.imageupload_set.all(): - prev_name = img.image.name - img.image.name = assets_directory_path(self.id, new_project_id, - os.path.basename(img.image.name)) - logger.info("Changing {} to {}".format(prev_name, img)) - img.save() - else: logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self, old_task_folder, @@ -430,16 +422,6 @@ class Task(models.Model): logger.info("Duplicating {} to {}".format(self, task)) - for img in self.imageupload_set.all(): - img.pk = None - img.task = task - - prev_name = img.image.name - img.image.name = assets_directory_path(task.id, task.project.id, - os.path.basename(img.image.name)) - - img.save() - if os.path.isdir(self.task_path()): try: # Try to use hard links first @@ -629,7 +611,8 @@ class Task(models.Model): if not self.uuid and self.pending_action is None and self.status is None: logger.info("Processing... {}".format(self)) - images = [image.path() for image in self.imageupload_set.all()] + images_path = self.task_path() + images = [os.path.join(images_path, i) for i in self.scan_images()] # Track upload progress, but limit the number of DB updates # to every 2 seconds (and always record the 100% progress) @@ -1122,3 +1105,34 @@ class Task(models.Model): pass else: raise + + def scan_images(self): + tp = self.task_path() + try: + return [e.name for e in os.scandir(tp) if e.is_file()] + except: + return [] + + def get_image_path(self, filename): + p = self.task_path(filename) + return path_traversal_check(p, self.task_path()) + + def handle_images_upload(self, files): + for file in files: + name = file.name + if name is None: + continue + + tp = self.task_path() + if not os.path.exists(tp): + os.makedirs(tp, exist_ok=True) + + dst_path = self.get_image_path(name) + + with open(dst_path, 'wb+') as fd: + if isinstance(file, InMemoryUploadedFile): + for chunk in file.chunks(): + fd.write(chunk) + else: + with open(file.temporary_file_path(), 'rb') as f: + copyfileobj(f, fd) \ No newline at end of file diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 256b766a..36881c5a 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -22,7 +22,7 @@ from app import pending_actions from app.api.formulas import algos, get_camera_filters_for from app.api.tiler import ZOOM_EXTRA_LEVELS from app.cogeo import valid_cogeo -from app.models import Project, Task, ImageUpload +from app.models import Project, Task from app.models.task import task_directory_path, full_task_directory_path, TaskInterruptedException from app.plugins.signals import task_completed, task_removed, task_removing from app.tests.classes import BootTransactionTestCase @@ -239,7 +239,7 @@ class TestApiTask(BootTransactionTestCase): self.assertEqual(task.running_progress, 0.0) # Two images should have been uploaded - self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2) + self.assertEqual(len(task.scan_images()), 2) # Can_rerun_from should be an empty list self.assertTrue(len(res.data['can_rerun_from']) == 0) @@ -797,7 +797,7 @@ class TestApiTask(BootTransactionTestCase): # Has been removed along with assets self.assertFalse(Task.objects.filter(pk=task.id).exists()) - self.assertFalse(ImageUpload.objects.filter(task=task).exists()) + self.assertEqual(len(task.scan_images()), 0) task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id)) self.assertFalse(os.path.exists(task_assets_path)) @@ -881,9 +881,7 @@ class TestApiTask(BootTransactionTestCase): # Reassigning the task to another project should move its assets self.assertTrue(os.path.exists(full_task_directory_path(task.id, project.id))) - self.assertTrue(len(task.imageupload_set.all()) == 2) - for image in task.imageupload_set.all(): - self.assertTrue('project/{}/'.format(project.id) in image.image.path) + self.assertTrue(len(task.scan_images()) == 2) task.project = other_project task.save() @@ -891,9 +889,6 @@ class TestApiTask(BootTransactionTestCase): self.assertFalse(os.path.exists(full_task_directory_path(task.id, project.id))) self.assertTrue(os.path.exists(full_task_directory_path(task.id, other_project.id))) - for image in task.imageupload_set.all(): - self.assertTrue('project/{}/'.format(other_project.id) in image.image.path) - # Restart node-odm as to not generate orthophotos testWatch.clear() with start_processing_node(["--test_skip_orthophotos"]): @@ -953,7 +948,7 @@ class TestApiTask(BootTransactionTestCase): new_task = Task.objects.get(pk=new_task_id) # New task has same number of image uploads - self.assertEqual(task.imageupload_set.count(), new_task.imageupload_set.count()) + self.assertEqual(len(task.scan_images()), len(new_task.scan_images())) # Directories have been created self.assertTrue(os.path.exists(new_task.task_path())) diff --git a/contrib/Hard_Recovery_Guide.md b/contrib/Hard_Recovery_Guide.md index ba87784f..7f97d6e2 100644 --- a/contrib/Hard_Recovery_Guide.md +++ b/contrib/Hard_Recovery_Guide.md @@ -25,7 +25,7 @@ python manage.py shell ```python # START COPY FIRST PART from django.contrib.auth.models import User -from app.models import Project, Task, ImageUpload +from app.models import Project, Task import os from django.contrib.gis.gdal import GDALRaster from django.contrib.gis.gdal import OGRGeometry @@ -89,17 +89,7 @@ def create_project(project_id, user): project.owner = user project.id = int(project_id) return project -def reindex_shots(projectID, taskID): - project_and_task_path = f'project/{projectID}/task/{taskID}' - try: - with open(f"/webodm/app/media/{project_and_task_path}/assets/images.json", 'r') as file: - camera_shots = json.load(file) - for image_shot in camera_shots: - ImageUpload.objects.update_or_create(task=Task.objects.get(pk=taskID), - image=f"{project_and_task_path}/{image_shot['filename']}") - print(f"Succesfully indexed file {image_shot['filename']}") - except Exception as e: - print(e) + # END COPY FIRST PART ``` @@ -110,7 +100,7 @@ user = User.objects.get(username="YOUR NEW CREATED ADMIN USERNAME HERE") # END COPY COPY SECOND PART ``` -## Step 3. This is the main part of script which make the main magic of the project. It will read media dir and create tasks and projects from the sources, also it will reindex photo sources, if avaliable +## Step 3. This is the main part of script which make the main magic of the project. It will read media dir and create tasks and projects from the sources ```python # START COPY THIRD PART for project_id in os.listdir("/webodm/app/media/project"): @@ -124,7 +114,6 @@ for project_id in os.listdir("/webodm/app/media/project"): task = Task(project=project) task.id = task_id process_task(task) - reindex_shots(project_id, task_id) # END COPY THIRD PART ``` ## Step 4. You must update project ID sequence for new created tasks diff --git a/coreplugins/cloudimport/api_views.py b/coreplugins/cloudimport/api_views.py index b456aaf6..b1fa2dbb 100644 --- a/coreplugins/cloudimport/api_views.py +++ b/coreplugins/cloudimport/api_views.py @@ -4,6 +4,7 @@ import os from os import path from app import models, pending_actions +from app.security import path_traversal_check from app.plugins.views import TaskView from app.plugins.worker import run_function_async from app.plugins import get_current_plugin @@ -105,15 +106,13 @@ def import_files(task_id, files): from app.plugins import logger def download_file(task, file): - path = task.task_path(file['name']) + path = path_traversal_check(task.task_path(file['name']), task.task_path()) download_stream = requests.get(file['url'], stream=True, timeout=60) with open(path, 'wb') as fd: for chunk in download_stream.iter_content(4096): fd.write(chunk) - models.ImageUpload.objects.create(task=task, image=path) - logger.info("Will import {} files".format(len(files))) task = models.Task.objects.get(pk=task_id) task.create_task_directories() @@ -134,4 +133,5 @@ def import_files(task_id, files): task.pending_action = None task.processing_time = 0 task.partial = False + task.images_count = len(task.scan_images()) task.save() diff --git a/coreplugins/dronedb/api_views.py b/coreplugins/dronedb/api_views.py index 1b4323bc..dec635fe 100644 --- a/coreplugins/dronedb/api_views.py +++ b/coreplugins/dronedb/api_views.py @@ -8,10 +8,10 @@ import os from os import listdir, path from app import models, pending_actions +from app.security import path_traversal_check from app.plugins.views import TaskView from app.plugins.worker import run_function_async, task from app.plugins import get_current_plugin -from app.models import ImageUpload from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals from coreplugins.dronedb.ddb import DEFAULT_HUB_URL, DroneDB, parse_url, verify_url @@ -218,7 +218,7 @@ def import_files(task_id, carrier): headers['Authorization'] = 'Bearer ' + carrier['token'] def download_file(task, file): - path = task.task_path(file['name']) + path = path_traversal_check(task.task_path(file['name']), task.task_path()) logger.info("Downloading file: " + file['url']) download_stream = requests.get(file['url'], stream=True, timeout=60, headers=headers) @@ -226,8 +226,6 @@ def import_files(task_id, carrier): for chunk in download_stream.iter_content(4096): fd.write(chunk) - models.ImageUpload.objects.create(task=task, image=path) - logger.info("Will import {} files".format(len(files))) task = models.Task.objects.get(pk=task_id) task.create_task_directories() diff --git a/coreplugins/openaerialmap/api.py b/coreplugins/openaerialmap/api.py index 5bf45f66..2b640e6a 100644 --- a/coreplugins/openaerialmap/api.py +++ b/coreplugins/openaerialmap/api.py @@ -9,7 +9,6 @@ from rest_framework import serializers from rest_framework import status from rest_framework.response import Response -from app.models import ImageUpload from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals from app.plugins.views import TaskView from app.plugins.worker import task @@ -58,9 +57,10 @@ class Info(TaskView): task_info = get_task_info(task.id) # Populate fields from first image in task - img = ImageUpload.objects.filter(task=task).exclude(image__iendswith='.txt').first() - if img is not None: - img_path = os.path.join(settings.MEDIA_ROOT, img.path()) + imgs = [f for f in task.scan_images() if not f.lower().endswith(".txt")] + if len(imgs) > 0: + img = imgs[0] + img_path = task.get_image_path(img) im = Image.open(img_path) # TODO: for better data we could look over all images