Drop ImageUpload model

app/admin.py

@@ -16,7 +16,7 @@ from app.models import Preset
 from app.models import Plugin
 from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin, valid_plugin, \
     get_plugins_persistent_path, clear_plugins_cache, init_plugins
-from .models import Project, Task, ImageUpload, Setting, Theme
+from .models import Project, Task, Setting, Theme
 from django import forms
 from codemirror2.widgets import CodeMirrorEditor
 from webodm import settings
@@ -37,12 +37,6 @@ class TaskAdmin(admin.ModelAdmin):
 
 admin.site.register(Task, TaskAdmin)
 
-
-class ImageUploadAdmin(admin.ModelAdmin):
-    readonly_fields = ('image',)
-
-admin.site.register(ImageUpload, ImageUploadAdmin)
-
 admin.site.register(Preset, admin.ModelAdmin)
 
 

app/api/imageuploads.py

@@ -4,7 +4,6 @@ import math
 
 from .tasks import TaskNestedView
 from rest_framework import exceptions
-from app.models import ImageUpload
 from app.models.task import assets_directory_path
 from PIL import Image, ImageDraw, ImageOps
 from django.http import HttpResponse
@@ -33,12 +32,7 @@ class Thumbnail(TaskNestedView):
         Generate a thumbnail on the fly for a particular task's image
         """
         task = self.get_and_check_task(request, pk)
-        image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first()
+        image_path = task.get_image_path(image_filename)
-
-        if image is None:
-            raise exceptions.NotFound()
-
-        image_path = image.path()
         if not os.path.isfile(image_path):
             raise exceptions.NotFound()
 
@@ -146,12 +140,7 @@ class ImageDownload(TaskNestedView):
         Download a task's image
         """
         task = self.get_and_check_task(request, pk)
-        image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first()
+        image_path = task.get_image_path(image_filename)
-
-        if image is None:
-            raise exceptions.NotFound()
-
-        image_path = image.path()
         if not os.path.isfile(image_path):
             raise exceptions.NotFound()
 

app/api/tasks.py

@@ -179,7 +179,7 @@ class TaskViewSet(viewsets.ViewSet):
             raise exceptions.NotFound()
 
         task.partial = False
-        task.images_count = models.ImageUpload.objects.filter(task=task).count()
+        task.images_count = len(task.scan_images())
 
         if task.images_count < 2:
             raise exceptions.ValidationError(detail=_("You need to upload at least 2 images before commit"))
@@ -206,11 +206,8 @@ class TaskViewSet(viewsets.ViewSet):
         if len(files) == 0:
             raise exceptions.ValidationError(detail=_("No files uploaded"))
 
-        with transaction.atomic():
+        task.handle_images_upload(files)
-            for image in files:
+        task.images_count = len(task.scan_images())
-                models.ImageUpload.objects.create(task=task, image=image)
-
-        task.images_count = models.ImageUpload.objects.filter(task=task).count()
         # Update other parameters such as processing node, task name, etc.
         serializer = TaskSerializer(task, data=request.data, partial=True)
         serializer.is_valid(raise_exception=True)
@@ -256,9 +253,8 @@ class TaskViewSet(viewsets.ViewSet):
                 task = models.Task.objects.create(project=project,
                                                   pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
 
-                for image in files:
+                task.handle_images_upload(files)
-                    models.ImageUpload.objects.create(task=task, image=image)
+                task.images_count = len(task.scan_images())
-                task.images_count = len(files)
 
                 # Update other parameters such as processing node, task name, etc.
                 serializer = TaskSerializer(task, data=request.data, partial=True)

app/migrations/0012_public_task_uuids.py

@@ -8,17 +8,14 @@ import uuid, os, pickle, tempfile
 from webodm import settings
 
 tasks = []
-imageuploads = []
 task_ids = {} # map old task IDs --> new task IDs
 
 def dump(apps, schema_editor):
-    global tasks, imageuploads, task_ids
+    global tasks, task_ids
 
     Task = apps.get_model('app', 'Task')
-    ImageUpload = apps.get_model('app', 'ImageUpload')
 
     tasks = list(Task.objects.all().values('id', 'project'))
-    imageuploads = list(ImageUpload.objects.all().values('id', 'task'))
 
     # Generate UUIDs
     for task in tasks:
@@ -31,9 +28,9 @@ def dump(apps, schema_editor):
         task_ids[task['id']] = new_id
 
     tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
-    pickle.dump((tasks, imageuploads, task_ids), open(tmp_path, 'wb'))
+    pickle.dump((tasks, task_ids), open(tmp_path, 'wb'))
 
-    if len(tasks) > 0: print("Dumped tasks and imageuploads")
+    if len(tasks) > 0: print("Dumped tasks")
 
 
 class Migration(migrations.Migration):

app/migrations/0013_public_task_uuids.py

@@ -8,7 +8,6 @@ import uuid, os, pickle, tempfile
 from webodm import settings
 
 tasks = []
-imageuploads = []
 task_ids = {} # map old task IDs --> new task IDs
 
 def task_path(project_id, task_id):
@@ -44,10 +43,10 @@ def create_uuids(apps, schema_editor):
 
 
 def restore(apps, schema_editor):
-    global tasks, imageuploads, task_ids
+    global tasks, task_ids
     
     tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
-    tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb'))
+    tasks, task_ids = pickle.load(open(tmp_path, 'rb'))
 
 
 class Migration(migrations.Migration):

app/migrations/0015_public_task_uuids.py

@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.1 on 2017-11-30 15:41
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import os, pickle, tempfile
-
-from webodm import settings
-
-tasks = []
-imageuploads = []
-task_ids = {} # map old task IDs --> new task IDs
-
-
-def restoreImageUploadFks(apps, schema_editor):
-    global imageuploads, task_ids
-
-    ImageUpload = apps.get_model('app', 'ImageUpload')
-    Task = apps.get_model('app', 'Task')
-
-    for img in imageuploads:
-        i = ImageUpload.objects.get(pk=img['id'])
-        old_image_path = i.image.name
-        task_id = task_ids[img['task']]
-
-        # project/2/task/5/DJI_0032.JPG --> project/2/task/<NEW_TASK_ID>/DJI_0032.JPG
-        dirs, filename = os.path.split(old_image_path)
-        head, tail = os.path.split(dirs)
-        new_image_path = os.path.join(head, str(task_id), filename)
-
-        i.task = Task.objects.get(id=task_id)
-        i.image.name = new_image_path
-        i.save()
-
-        print("{} --> {} (Task {})".format(old_image_path, new_image_path, str(task_id)))
-
-
-def restore(apps, schema_editor):
-    global tasks, imageuploads, task_ids
-    
-    tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
-    tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb'))
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('app', '0014_public_task_uuids'),
-    ]
-
-    operations = [
-        migrations.RunPython(restore),
-        migrations.RunPython(restoreImageUploadFks),      
-    ]

app/migrations/0016_public_task_uuids.py

@@ -9,7 +9,7 @@ from webodm import settings
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('app', '0015_public_task_uuids'),
+        ('app', '0014_public_task_uuids'),
     ]
 
     operations = [

app/migrations/0026_update_images_count.py

@@ -10,7 +10,7 @@ def update_images_count(apps, schema_editor):
 
     for t in Task.objects.all():
         print("Updating {}".format(t))
-        t.images_count = t.imageupload_set.count()
+        t.images_count = len(t.scan_images())
         t.save()
 
 

app/migrations/0029_auto_20190907_1348.py

@@ -1,6 +1,6 @@
 # Generated by Django 2.1.11 on 2019-09-07 13:48
 
-import app.models.image_upload
+import app.models
 from django.db import migrations, models
 
 
@@ -14,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='imageupload',
             name='image',
-            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path),
+            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_directory_path),
         ),
     ]

app/migrations/0031_auto_20210610_1850.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.1.15 on 2021-06-10 18:50
 
-import app.models.image_upload
 import app.models.task
+from app.models import image_directory_path
 import colorfield.fields
 from django.conf import settings
 import django.contrib.gis.db.models.fields
@@ -60,7 +60,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='imageupload',
             name='image',
-            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path, verbose_name='Image'),
+            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=image_directory_path, verbose_name='Image'),
         ),
         migrations.AlterField(
             model_name='imageupload',

app/migrations/0034_delete_imageupload.py

@@ -0,0 +1,16 @@
+# Generated by Django 2.2.27 on 2023-03-23 17:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0033_auto_20230307_1532'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='ImageUpload',
+        ),
+    ]

app/models/__init__.py

@@ -1,4 +1,3 @@
-from .image_upload import ImageUpload, image_directory_path
 from .project import Project
 from .task import Task, validate_task_options, gcp_directory_path
 from .preset import Preset
@@ -7,3 +6,6 @@ from .setting import Setting
 from .plugin_datum import PluginDatum
 from .plugin import Plugin
 
+# deprecated
+def image_directory_path(image_upload, filename):
+    raise Exception("Deprecated")

app/models/image_upload.py

@@ -1,21 +0,0 @@
-from .task import Task, assets_directory_path
-from django.db import models
-from django.utils.translation import gettext_lazy as _
-
-def image_directory_path(image_upload, filename):
-    return assets_directory_path(image_upload.task.id, image_upload.task.project.id, filename)
-
-
-class ImageUpload(models.Model):
-    task = models.ForeignKey(Task, on_delete=models.CASCADE, help_text=_("Task this image belongs to"), verbose_name=_("Task"))
-    image = models.ImageField(upload_to=image_directory_path, help_text=_("File uploaded by a user"), max_length=512, verbose_name=_("Image"))
-
-    def __str__(self):
-        return self.image.name
-
-    def path(self):
-        return self.image.path
-
-    class Meta:
-        verbose_name = _("Image Upload")
-        verbose_name_plural = _("Image Uploads")

app/models/task.py

@@ -21,6 +21,7 @@ from django.contrib.gis.gdal import GDALRaster
 from django.contrib.gis.gdal import OGRGeometry
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.postgres import fields
+from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.core.exceptions import ValidationError, SuspiciousFileOperation
 from django.db import models
 from django.db import transaction
@@ -310,15 +311,6 @@ class Task(models.Model):
                 shutil.move(old_task_folder, new_task_folder_parent)
 
                 logger.info("Moved task folder from {} to {}".format(old_task_folder, new_task_folder))
-
-                with transaction.atomic():
-                    for img in self.imageupload_set.all():
-                        prev_name = img.image.name
-                        img.image.name = assets_directory_path(self.id, new_project_id,
-                                                               os.path.basename(img.image.name))
-                        logger.info("Changing {} to {}".format(prev_name, img))
-                        img.save()
-
             else:
                 logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
                                                                                                              old_task_folder,
@@ -430,16 +422,6 @@ class Task(models.Model):
 
                 logger.info("Duplicating {} to {}".format(self, task))
 
-                for img in self.imageupload_set.all():
-                    img.pk = None
-                    img.task = task
-
-                    prev_name = img.image.name
-                    img.image.name = assets_directory_path(task.id, task.project.id,
-                                                            os.path.basename(img.image.name))
-                    
-                    img.save()
-
                 if os.path.isdir(self.task_path()):
                     try:
                         # Try to use hard links first
@@ -629,7 +611,8 @@ class Task(models.Model):
                 if not self.uuid and self.pending_action is None and self.status is None:
                     logger.info("Processing... {}".format(self))
 
-                    images = [image.path() for image in self.imageupload_set.all()]
+                    images_path = self.task_path()
+                    images = [os.path.join(images_path, i) for i in self.scan_images()]
 
                     # Track upload progress, but limit the number of DB updates
                     # to every 2 seconds (and always record the 100% progress)
@@ -1122,3 +1105,34 @@ class Task(models.Model):
                 pass
             else:
                 raise
+
+    def scan_images(self):
+        tp = self.task_path()
+        try:
+            return [e.name for e in os.scandir(tp) if e.is_file()]
+        except:
+            return []
+
+    def get_image_path(self, filename):
+        p = self.task_path(filename)
+        return path_traversal_check(p, self.task_path())
+    
+    def handle_images_upload(self, files):
+        for file in files:
+            name = file.name
+            if name is None:
+                continue
+
+            tp = self.task_path()
+            if not os.path.exists(tp):
+                os.makedirs(tp, exist_ok=True)
+
+            dst_path = self.get_image_path(name)
+
+            with open(dst_path, 'wb+') as fd:
+                if isinstance(file, InMemoryUploadedFile):
+                    for chunk in file.chunks():
+                        fd.write(chunk)
+                else:
+                    with open(file.temporary_file_path(), 'rb') as f:
+                        copyfileobj(f, fd)

app/tests/test_api_task.py

@@ -22,7 +22,7 @@ from app import pending_actions
 from app.api.formulas import algos, get_camera_filters_for
 from app.api.tiler import ZOOM_EXTRA_LEVELS
 from app.cogeo import valid_cogeo
-from app.models import Project, Task, ImageUpload
+from app.models import Project, Task
 from app.models.task import task_directory_path, full_task_directory_path, TaskInterruptedException
 from app.plugins.signals import task_completed, task_removed, task_removing
 from app.tests.classes import BootTransactionTestCase
@@ -239,7 +239,7 @@ class TestApiTask(BootTransactionTestCase):
             self.assertEqual(task.running_progress, 0.0)
 
             # Two images should have been uploaded
-            self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
+            self.assertEqual(len(task.scan_images()), 2)
 
             # Can_rerun_from should be an empty list
             self.assertTrue(len(res.data['can_rerun_from']) == 0)
@@ -797,7 +797,7 @@ class TestApiTask(BootTransactionTestCase):
 
             # Has been removed along with assets
             self.assertFalse(Task.objects.filter(pk=task.id).exists())
-            self.assertFalse(ImageUpload.objects.filter(task=task).exists())
+            self.assertEqual(len(task.scan_images()), 0)
 
             task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
             self.assertFalse(os.path.exists(task_assets_path))
@@ -881,9 +881,7 @@ class TestApiTask(BootTransactionTestCase):
 
             # Reassigning the task to another project should move its assets
             self.assertTrue(os.path.exists(full_task_directory_path(task.id, project.id)))
-            self.assertTrue(len(task.imageupload_set.all()) == 2)
+            self.assertTrue(len(task.scan_images()) == 2)
-            for image in task.imageupload_set.all():
-                self.assertTrue('project/{}/'.format(project.id) in image.image.path)
 
             task.project = other_project
             task.save()
@@ -891,9 +889,6 @@ class TestApiTask(BootTransactionTestCase):
             self.assertFalse(os.path.exists(full_task_directory_path(task.id, project.id)))
             self.assertTrue(os.path.exists(full_task_directory_path(task.id, other_project.id)))
 
-            for image in task.imageupload_set.all():
-                self.assertTrue('project/{}/'.format(other_project.id) in image.image.path)
-
         # Restart node-odm as to not generate orthophotos
         testWatch.clear()
         with start_processing_node(["--test_skip_orthophotos"]):
@@ -953,7 +948,7 @@ class TestApiTask(BootTransactionTestCase):
         new_task = Task.objects.get(pk=new_task_id)
 
         # New task has same number of image uploads
-        self.assertEqual(task.imageupload_set.count(), new_task.imageupload_set.count())
+        self.assertEqual(len(task.scan_images()), len(new_task.scan_images()))
         
         # Directories have been created
         self.assertTrue(os.path.exists(new_task.task_path()))

contrib/Hard_Recovery_Guide.md

@@ -25,7 +25,7 @@ python manage.py shell
 ```python
 # START COPY FIRST PART
 from django.contrib.auth.models import User
-from app.models import Project, Task, ImageUpload
+from app.models import Project, Task
 import os
 from django.contrib.gis.gdal import GDALRaster
 from django.contrib.gis.gdal import OGRGeometry
@@ -89,17 +89,7 @@ def create_project(project_id, user):
     project.owner = user
     project.id = int(project_id)
     return project
-def reindex_shots(projectID, taskID):
+
-    project_and_task_path = f'project/{projectID}/task/{taskID}'
-    try:
-        with open(f"/webodm/app/media/{project_and_task_path}/assets/images.json", 'r') as file:
-            camera_shots = json.load(file)
-            for image_shot in camera_shots:
-                ImageUpload.objects.update_or_create(task=Task.objects.get(pk=taskID),
-                                                 image=f"{project_and_task_path}/{image_shot['filename']}")
-                print(f"Succesfully indexed file {image_shot['filename']}")
-    except Exception as e:
-        print(e)
 
 # END COPY FIRST PART
 ```
@@ -110,7 +100,7 @@ user = User.objects.get(username="YOUR NEW CREATED ADMIN USERNAME HERE")
 # END COPY COPY SECOND PART
 ```
 
-## Step 3. This is the main part of script which make the main magic of the project. It will read media dir and create tasks and projects from the sources, also it will reindex photo sources, if avaliable
+## Step 3. This is the main part of script which make the main magic of the project. It will read media dir and create tasks and projects from the sources
 ```python
 # START COPY THIRD PART
 for project_id in os.listdir("/webodm/app/media/project"):
@@ -124,7 +114,6 @@ for project_id in os.listdir("/webodm/app/media/project"):
             task = Task(project=project)
             task.id = task_id
             process_task(task)
-            reindex_shots(project_id, task_id)
 # END COPY THIRD PART
 ```
 ## Step 4. You must update project ID sequence for new created tasks

coreplugins/cloudimport/api_views.py

@@ -4,6 +4,7 @@ import os
 from os import path
 
 from app import models, pending_actions
+from app.security import path_traversal_check
 from app.plugins.views import TaskView
 from app.plugins.worker import run_function_async
 from app.plugins import get_current_plugin
@@ -105,15 +106,13 @@ def import_files(task_id, files):
     from app.plugins import logger
 
     def download_file(task, file):
-        path = task.task_path(file['name'])
+        path = path_traversal_check(task.task_path(file['name']), task.task_path())
         download_stream = requests.get(file['url'], stream=True, timeout=60)
 
         with open(path, 'wb') as fd:
             for chunk in download_stream.iter_content(4096):
                 fd.write(chunk)
         
-        models.ImageUpload.objects.create(task=task, image=path)
-
     logger.info("Will import {} files".format(len(files)))
     task = models.Task.objects.get(pk=task_id)
     task.create_task_directories()
@@ -134,4 +133,5 @@ def import_files(task_id, files):
     task.pending_action = None
     task.processing_time = 0
     task.partial = False
+    task.images_count = len(task.scan_images())
     task.save()

coreplugins/dronedb/api_views.py

@@ -8,10 +8,10 @@ import os
 from os import listdir, path
 
 from app import models, pending_actions
+from app.security import path_traversal_check
 from app.plugins.views import TaskView
 from app.plugins.worker import run_function_async, task
 from app.plugins import get_current_plugin
-from app.models import ImageUpload
 from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals
 
 from coreplugins.dronedb.ddb import DEFAULT_HUB_URL, DroneDB, parse_url, verify_url
@@ -218,7 +218,7 @@ def import_files(task_id, carrier):
         headers['Authorization'] = 'Bearer ' + carrier['token']
 
     def download_file(task, file):
-        path = task.task_path(file['name'])
+        path = path_traversal_check(task.task_path(file['name']), task.task_path())
         logger.info("Downloading file: " + file['url'])
         download_stream = requests.get(file['url'], stream=True, timeout=60, headers=headers)
 
@@ -226,8 +226,6 @@ def import_files(task_id, carrier):
             for chunk in download_stream.iter_content(4096):
                 fd.write(chunk)
         
-        models.ImageUpload.objects.create(task=task, image=path)
-
     logger.info("Will import {} files".format(len(files)))
     task = models.Task.objects.get(pk=task_id)
     task.create_task_directories()

coreplugins/openaerialmap/api.py

@@ -9,7 +9,6 @@ from rest_framework import serializers
 from rest_framework import status
 from rest_framework.response import Response
 
-from app.models import ImageUpload
 from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals
 from app.plugins.views import TaskView
 from app.plugins.worker import task
@@ -58,9 +57,10 @@ class Info(TaskView):
         task_info = get_task_info(task.id)
 
         # Populate fields from first image in task
-        img = ImageUpload.objects.filter(task=task).exclude(image__iendswith='.txt').first()
+        imgs = [f for f in task.scan_images() if not f.lower().endswith(".txt")]
-        if img is not None:
+        if len(imgs) > 0:
-            img_path = os.path.join(settings.MEDIA_ROOT, img.path())
+            img = imgs[0]
+            img_path = task.get_image_path(img)
             im = Image.open(img_path)
 
             # TODO: for better data we could look over all images