mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Drop ImageUpload model

pull/1310/head
Piero Toffanin 2023-03-23 13:31:07 -04:00
rodzic 6258626102
commit ed5ac98d06
19 zmienionych plików z 89 dodań i 175 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

8
app/admin.py
Wyświetl plik

@ -16,7 +16,7 @@ from app.models import Preset
from app.models import Plugin
from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin, valid_plugin, \
get_plugins_persistent_path, clear_plugins_cache, init_plugins
from .models import Project, Task, ImageUpload, Setting, Theme
from .models import Project, Task, Setting, Theme
from django import forms
from codemirror2.widgets import CodeMirrorEditor
from webodm import settings
@ -37,12 +37,6 @@ class TaskAdmin(admin.ModelAdmin):
admin.site.register(Task, TaskAdmin)
class ImageUploadAdmin(admin.ModelAdmin):
readonly_fields = ('image',)
admin.site.register(ImageUpload, ImageUploadAdmin)
admin.site.register(Preset, admin.ModelAdmin)

15
app/api/imageuploads.py
Wyświetl plik

@ -4,7 +4,6 @@ import math
from .tasks import TaskNestedView
from rest_framework import exceptions
from app.models import ImageUpload
from app.models.task import assets_directory_path
from PIL import Image, ImageDraw, ImageOps
from django.http import HttpResponse
@ -33,12 +32,7 @@ class Thumbnail(TaskNestedView):
Generate a thumbnail on the fly for a particular task's image
"""
task = self.get_and_check_task(request, pk)
image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first()
if image is None:
raise exceptions.NotFound()
image_path = image.path()
image_path = task.get_image_path(image_filename)
if not os.path.isfile(image_path):
raise exceptions.NotFound()
@ -146,12 +140,7 @@ class ImageDownload(TaskNestedView):
Download a task's image
"""
task = self.get_and_check_task(request, pk)
image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first()
if image is None:
raise exceptions.NotFound()
image_path = image.path()
image_path = task.get_image_path(image_filename)
if not os.path.isfile(image_path):
raise exceptions.NotFound()

14
app/api/tasks.py
Wyświetl plik

@ -179,7 +179,7 @@ class TaskViewSet(viewsets.ViewSet):
raise exceptions.NotFound()
task.partial = False
task.images_count = models.ImageUpload.objects.filter(task=task).count()
task.images_count = len(task.scan_images())
if task.images_count < 2:
raise exceptions.ValidationError(detail=_("You need to upload at least 2 images before commit"))
@ -206,11 +206,8 @@ class TaskViewSet(viewsets.ViewSet):
if len(files) == 0:
raise exceptions.ValidationError(detail=_("No files uploaded"))
with transaction.atomic():
for image in files:
models.ImageUpload.objects.create(task=task, image=image)
task.images_count = models.ImageUpload.objects.filter(task=task).count()
task.handle_images_upload(files)
task.images_count = len(task.scan_images())
# Update other parameters such as processing node, task name, etc.
serializer = TaskSerializer(task, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
@ -256,9 +253,8 @@ class TaskViewSet(viewsets.ViewSet):
task = models.Task.objects.create(project=project,
pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
for image in files:
models.ImageUpload.objects.create(task=task, image=image)
task.images_count = len(files)
task.handle_images_upload(files)
task.images_count = len(task.scan_images())
# Update other parameters such as processing node, task name, etc.
serializer = TaskSerializer(task, data=request.data, partial=True)

9
app/migrations/0012_public_task_uuids.py
Wyświetl plik

@ -8,17 +8,14 @@ import uuid, os, pickle, tempfile
from webodm import settings
tasks = []
imageuploads = []
task_ids = {} # map old task IDs --> new task IDs
def dump(apps, schema_editor):
global tasks, imageuploads, task_ids
global tasks, task_ids
Task = apps.get_model('app', 'Task')
ImageUpload = apps.get_model('app', 'ImageUpload')
tasks = list(Task.objects.all().values('id', 'project'))
imageuploads = list(ImageUpload.objects.all().values('id', 'task'))
# Generate UUIDs
for task in tasks:
@ -31,9 +28,9 @@ def dump(apps, schema_editor):
task_ids[task['id']] = new_id
tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
pickle.dump((tasks, imageuploads, task_ids), open(tmp_path, 'wb'))
pickle.dump((tasks, task_ids), open(tmp_path, 'wb'))
if len(tasks) > 0: print("Dumped tasks and imageuploads")
if len(tasks) > 0: print("Dumped tasks")
class Migration(migrations.Migration):

5
app/migrations/0013_public_task_uuids.py
Wyświetl plik

@ -8,7 +8,6 @@ import uuid, os, pickle, tempfile
from webodm import settings
tasks = []
imageuploads = []
task_ids = {} # map old task IDs --> new task IDs
def task_path(project_id, task_id):
@ -44,10 +43,10 @@ def create_uuids(apps, schema_editor):
def restore(apps, schema_editor):
global tasks, imageuploads, task_ids
global tasks, task_ids
tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb'))
tasks, task_ids = pickle.load(open(tmp_path, 'rb'))
class Migration(migrations.Migration):

54
app/migrations/0015_public_task_uuids.py
Wyświetl plik

@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.1 on 2017-11-30 15:41
from __future__ import unicode_literals
from django.db import migrations, models
import os, pickle, tempfile
from webodm import settings
tasks = []
imageuploads = []
task_ids = {} # map old task IDs --> new task IDs
def restoreImageUploadFks(apps, schema_editor):
global imageuploads, task_ids
ImageUpload = apps.get_model('app', 'ImageUpload')
Task = apps.get_model('app', 'Task')
for img in imageuploads:
i = ImageUpload.objects.get(pk=img['id'])
old_image_path = i.image.name
task_id = task_ids[img['task']]
# project/2/task/5/DJI_0032.JPG --> project/2/task/<NEW_TASK_ID>/DJI_0032.JPG
dirs, filename = os.path.split(old_image_path)
head, tail = os.path.split(dirs)
new_image_path = os.path.join(head, str(task_id), filename)
i.task = Task.objects.get(id=task_id)
i.image.name = new_image_path
i.save()
print("{} --> {} (Task {})".format(old_image_path, new_image_path, str(task_id)))
def restore(apps, schema_editor):
global tasks, imageuploads, task_ids
tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb'))
class Migration(migrations.Migration):
dependencies = [
('app', '0014_public_task_uuids'),
]
operations = [
migrations.RunPython(restore),
migrations.RunPython(restoreImageUploadFks),
]

2
app/migrations/0016_public_task_uuids.py
Wyświetl plik

@ -9,7 +9,7 @@ from webodm import settings
class Migration(migrations.Migration):
dependencies = [
('app', '0015_public_task_uuids'),
('app', '0014_public_task_uuids'),
]
operations = [

2
app/migrations/0026_update_images_count.py
Wyświetl plik

@ -10,7 +10,7 @@ def update_images_count(apps, schema_editor):
for t in Task.objects.all():
print("Updating {}".format(t))
t.images_count = t.imageupload_set.count()
t.images_count = len(t.scan_images())
t.save()

4
app/migrations/0029_auto_20190907_1348.py
Wyświetl plik

@ -1,6 +1,6 @@
# Generated by Django 2.1.11 on 2019-09-07 13:48
import app.models.image_upload
import app.models
from django.db import migrations, models
@ -14,6 +14,6 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='imageupload',
name='image',
field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path),
field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_directory_path),
),
]

4
app/migrations/0031_auto_20210610_1850.py
Wyświetl plik

@ -1,7 +1,7 @@
# Generated by Django 2.1.15 on 2021-06-10 18:50
import app.models.image_upload
import app.models.task
from app.models import image_directory_path
import colorfield.fields
from django.conf import settings
import django.contrib.gis.db.models.fields
@ -60,7 +60,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='imageupload',
name='image',
field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path, verbose_name='Image'),
field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=image_directory_path, verbose_name='Image'),
),
migrations.AlterField(
model_name='imageupload',

16
app/migrations/0034_delete_imageupload.py 100644
Wyświetl plik

@ -0,0 +1,16 @@
# Generated by Django 2.2.27 on 2023-03-23 17:10
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('app', '0033_auto_20230307_1532'),
]
operations = [
migrations.DeleteModel(
name='ImageUpload',
),
]

4
app/models/__init__.py
Wyświetl plik

@ -1,4 +1,3 @@
from .image_upload import ImageUpload, image_directory_path
from .project import Project
from .task import Task, validate_task_options, gcp_directory_path
from .preset import Preset
@ -7,3 +6,6 @@ from .setting import Setting
from .plugin_datum import PluginDatum
from .plugin import Plugin
# deprecated
def image_directory_path(image_upload, filename):
raise Exception("Deprecated")

21
app/models/image_upload.py
Wyświetl plik

@ -1,21 +0,0 @@
from .task import Task, assets_directory_path
from django.db import models
from django.utils.translation import gettext_lazy as _
def image_directory_path(image_upload, filename):
return assets_directory_path(image_upload.task.id, image_upload.task.project.id, filename)
class ImageUpload(models.Model):
task = models.ForeignKey(Task, on_delete=models.CASCADE, help_text=_("Task this image belongs to"), verbose_name=_("Task"))
image = models.ImageField(upload_to=image_directory_path, help_text=_("File uploaded by a user"), max_length=512, verbose_name=_("Image"))
def __str__(self):
return self.image.name
def path(self):
return self.image.path
class Meta:
verbose_name = _("Image Upload")
verbose_name_plural = _("Image Uploads")

54
app/models/task.py
Wyświetl plik

@ -21,6 +21,7 @@ from django.contrib.gis.gdal import GDALRaster
from django.contrib.gis.gdal import OGRGeometry
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.postgres import fields
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.exceptions import ValidationError, SuspiciousFileOperation
from django.db import models
from django.db import transaction
@ -310,15 +311,6 @@ class Task(models.Model):
shutil.move(old_task_folder, new_task_folder_parent)
logger.info("Moved task folder from {} to {}".format(old_task_folder, new_task_folder))
with transaction.atomic():
for img in self.imageupload_set.all():
prev_name = img.image.name
img.image.name = assets_directory_path(self.id, new_project_id,
os.path.basename(img.image.name))
logger.info("Changing {} to {}".format(prev_name, img))
img.save()
else:
logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
old_task_folder,
@ -430,16 +422,6 @@ class Task(models.Model):
logger.info("Duplicating {} to {}".format(self, task))
for img in self.imageupload_set.all():
img.pk = None
img.task = task
prev_name = img.image.name
img.image.name = assets_directory_path(task.id, task.project.id,
os.path.basename(img.image.name))
img.save()
if os.path.isdir(self.task_path()):
try:
# Try to use hard links first
@ -629,7 +611,8 @@ class Task(models.Model):
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
images_path = self.task_path()
images = [os.path.join(images_path, i) for i in self.scan_images()]
# Track upload progress, but limit the number of DB updates
# to every 2 seconds (and always record the 100% progress)
@ -1122,3 +1105,34 @@ class Task(models.Model):
pass
else:
raise
def scan_images(self):
tp = self.task_path()
try:
return [e.name for e in os.scandir(tp) if e.is_file()]
except:
return []
def get_image_path(self, filename):
p = self.task_path(filename)
return path_traversal_check(p, self.task_path())
def handle_images_upload(self, files):
for file in files:
name = file.name
if name is None:
continue
tp = self.task_path()
if not os.path.exists(tp):
os.makedirs(tp, exist_ok=True)
dst_path = self.get_image_path(name)
with open(dst_path, 'wb+') as fd:
if isinstance(file, InMemoryUploadedFile):
for chunk in file.chunks():
fd.write(chunk)
else:
with open(file.temporary_file_path(), 'rb') as f:
copyfileobj(f, fd)

15
app/tests/test_api_task.py
Wyświetl plik

@ -22,7 +22,7 @@ from app import pending_actions
from app.api.formulas import algos, get_camera_filters_for
from app.api.tiler import ZOOM_EXTRA_LEVELS
from app.cogeo import valid_cogeo
from app.models import Project, Task, ImageUpload
from app.models import Project, Task
from app.models.task import task_directory_path, full_task_directory_path, TaskInterruptedException
from app.plugins.signals import task_completed, task_removed, task_removing
from app.tests.classes import BootTransactionTestCase
@ -239,7 +239,7 @@ class TestApiTask(BootTransactionTestCase):
self.assertEqual(task.running_progress, 0.0)
# Two images should have been uploaded
self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
self.assertEqual(len(task.scan_images()), 2)
# Can_rerun_from should be an empty list
self.assertTrue(len(res.data['can_rerun_from']) == 0)
@ -797,7 +797,7 @@ class TestApiTask(BootTransactionTestCase):
# Has been removed along with assets
self.assertFalse(Task.objects.filter(pk=task.id).exists())
self.assertFalse(ImageUpload.objects.filter(task=task).exists())
self.assertEqual(len(task.scan_images()), 0)
task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
self.assertFalse(os.path.exists(task_assets_path))
@ -881,9 +881,7 @@ class TestApiTask(BootTransactionTestCase):
# Reassigning the task to another project should move its assets
self.assertTrue(os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(len(task.imageupload_set.all()) == 2)
for image in task.imageupload_set.all():
self.assertTrue('project/{}/'.format(project.id) in image.image.path)
self.assertTrue(len(task.scan_images()) == 2)
task.project = other_project
task.save()
@ -891,9 +889,6 @@ class TestApiTask(BootTransactionTestCase):
self.assertFalse(os.path.exists(full_task_directory_path(task.id, project.id)))
self.assertTrue(os.path.exists(full_task_directory_path(task.id, other_project.id)))
for image in task.imageupload_set.all():
self.assertTrue('project/{}/'.format(other_project.id) in image.image.path)
# Restart node-odm as to not generate orthophotos
testWatch.clear()
with start_processing_node(["--test_skip_orthophotos"]):
@ -953,7 +948,7 @@ class TestApiTask(BootTransactionTestCase):
new_task = Task.objects.get(pk=new_task_id)
# New task has same number of image uploads
self.assertEqual(task.imageupload_set.count(), new_task.imageupload_set.count())
self.assertEqual(len(task.scan_images()), len(new_task.scan_images()))
# Directories have been created
self.assertTrue(os.path.exists(new_task.task_path()))

17
contrib/Hard_Recovery_Guide.md
Wyświetl plik

@ -25,7 +25,7 @@ python manage.py shell
```python
# START COPY FIRST PART
from django.contrib.auth.models import User
from app.models import Project, Task, ImageUpload
from app.models import Project, Task
import os
from django.contrib.gis.gdal import GDALRaster
from django.contrib.gis.gdal import OGRGeometry
@ -89,17 +89,7 @@ def create_project(project_id, user):
project.owner = user
project.id = int(project_id)
return project
def reindex_shots(projectID, taskID):
project_and_task_path = f'project/{projectID}/task/{taskID}'
try:
with open(f"/webodm/app/media/{project_and_task_path}/assets/images.json", 'r') as file:
camera_shots = json.load(file)
for image_shot in camera_shots:
ImageUpload.objects.update_or_create(task=Task.objects.get(pk=taskID),
image=f"{project_and_task_path}/{image_shot['filename']}")
print(f"Succesfully indexed file {image_shot['filename']}")
except Exception as e:
print(e)
# END COPY FIRST PART
```
@ -110,7 +100,7 @@ user = User.objects.get(username="YOUR NEW CREATED ADMIN USERNAME HERE")
# END COPY COPY SECOND PART
```
## Step 3. This is the main part of script which make the main magic of the project. It will read media dir and create tasks and projects from the sources, also it will reindex photo sources, if avaliable
## Step 3. This is the main part of script which make the main magic of the project. It will read media dir and create tasks and projects from the sources
```python
# START COPY THIRD PART
for project_id in os.listdir("/webodm/app/media/project"):
@ -124,7 +114,6 @@ for project_id in os.listdir("/webodm/app/media/project"):
task = Task(project=project)
task.id = task_id
process_task(task)
reindex_shots(project_id, task_id)
# END COPY THIRD PART
```
## Step 4. You must update project ID sequence for new created tasks

6
coreplugins/cloudimport/api_views.py
Wyświetl plik

@ -4,6 +4,7 @@ import os
from os import path
from app import models, pending_actions
from app.security import path_traversal_check
from app.plugins.views import TaskView
from app.plugins.worker import run_function_async
from app.plugins import get_current_plugin
@ -105,15 +106,13 @@ def import_files(task_id, files):
from app.plugins import logger
def download_file(task, file):
path = task.task_path(file['name'])
path = path_traversal_check(task.task_path(file['name']), task.task_path())
download_stream = requests.get(file['url'], stream=True, timeout=60)
with open(path, 'wb') as fd:
for chunk in download_stream.iter_content(4096):
fd.write(chunk)
models.ImageUpload.objects.create(task=task, image=path)
logger.info("Will import {} files".format(len(files)))
task = models.Task.objects.get(pk=task_id)
task.create_task_directories()
@ -134,4 +133,5 @@ def import_files(task_id, files):
task.pending_action = None
task.processing_time = 0
task.partial = False
task.images_count = len(task.scan_images())
task.save()

6
coreplugins/dronedb/api_views.py
Wyświetl plik

@ -8,10 +8,10 @@ import os
from os import listdir, path
from app import models, pending_actions
from app.security import path_traversal_check
from app.plugins.views import TaskView
from app.plugins.worker import run_function_async, task
from app.plugins import get_current_plugin
from app.models import ImageUpload
from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals
from coreplugins.dronedb.ddb import DEFAULT_HUB_URL, DroneDB, parse_url, verify_url
@ -218,7 +218,7 @@ def import_files(task_id, carrier):
headers['Authorization'] = 'Bearer ' + carrier['token']
def download_file(task, file):
path = task.task_path(file['name'])
path = path_traversal_check(task.task_path(file['name']), task.task_path())
logger.info("Downloading file: " + file['url'])
download_stream = requests.get(file['url'], stream=True, timeout=60, headers=headers)
@ -226,8 +226,6 @@ def import_files(task_id, carrier):
for chunk in download_stream.iter_content(4096):
fd.write(chunk)
models.ImageUpload.objects.create(task=task, image=path)
logger.info("Will import {} files".format(len(files)))
task = models.Task.objects.get(pk=task_id)
task.create_task_directories()

8
coreplugins/openaerialmap/api.py
Wyświetl plik

@ -9,7 +9,6 @@ from rest_framework import serializers
from rest_framework import status
from rest_framework.response import Response
from app.models import ImageUpload
from app.plugins import GlobalDataStore, get_site_settings, signals as plugin_signals
from app.plugins.views import TaskView
from app.plugins.worker import task
@ -58,9 +57,10 @@ class Info(TaskView):
task_info = get_task_info(task.id)
# Populate fields from first image in task
img = ImageUpload.objects.filter(task=task).exclude(image__iendswith='.txt').first()
if img is not None:
img_path = os.path.join(settings.MEDIA_ROOT, img.path())
imgs = [f for f in task.scan_images() if not f.lower().endswith(".txt")]
if len(imgs) > 0:
img = imgs[0]
img_path = task.get_image_path(img)
im = Image.open(img_path)
# TODO: for better data we could look over all images