feat(api): Add inplace_to_s3 management command

This command allows to update Uploads that originally were imported
using --in_place but are moved to s3. This command does not copy any
file, it just makes sure the files are read from S3 after they have been
moved.

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2506>

api/funkwhale_api/common/management/commands/inplace_to_s3.py

@@ -0,0 +1,93 @@
+import pathlib
+from argparse import RawTextHelpFormatter
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from funkwhale_api.music import models
+
+
+class Command(BaseCommand):
+    help = """
+    Update the reference for Uploads that have been imported with --in-place and are now moved to s3.
+
+    Please note: This does not move any file! Make sure you already moved the files to your s3 bucket.
+
+    Specify --source to filter the reference to update to files from a specific in-place directory. If no
+    --source is given, all in-place imported track references will be updated.
+
+    Specify --target to specify a subdirectory in the S3 bucket where you moved the files. If no --target is
+    given, the file is expected to be stored in the same path as before.
+
+    Examples:
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target unset
+
+    All files imported from /music will be updated and expected to be in the same folder structure in the bucket
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target: /in_place
+
+    The music file is expected to be stored in the bucket in the directory /in_place/Artist/Album/track.ogg
+    """
+
+    def create_parser(self, *args, **kwargs):
+        parser = super().create_parser(*args, **kwargs)
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            default=True,
+            help="Disable dry run mode and apply updates for real on the database",
+        )
+        parser.add_argument(
+            "--source",
+            type=pathlib.Path,
+            required=True,
+            help="Specify the path of the directory where the files originally were stored to update their reference.",
+        )
+        parser.add_argument(
+            "--target",
+            type=pathlib.Path,
+            help="Specify a subdirectory in the S3 bucket where you moved the files to.",
+        )
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        if options["dry_run"]:
+            self.stdout.write("Dry-run on, will not touch the database")
+        else:
+            self.stdout.write("Dry-run off, *changing the database*")
+        self.stdout.write("")
+
+        prefix = f"file://{options['source']}"
+
+        to_change = models.Upload.objects.filter(source__startswith=prefix)
+
+        self.stdout.write(f"Found {to_change.count()} uploads to update.")
+
+        target = options["target"] if options["target"] else options["source"]
+
+        for upl in to_change:
+            upl.audio_file = str(upl.source).replace(str(prefix), str(target))
+            upl.source = None
+            self.stdout.write(f"Upload expected in {upl.audio_file}")
+            if not options["dry_run"]:
+                upl.save()
+
+        self.stdout.write("")
+        if options["dry_run"]:
+            self.stdout.write(
+                "Nothing was updated, rerun this command with --no-dry-run to apply the changes"
+            )
+        else:
+            self.stdout.write("Updating completed!")
+
+        self.stdout.write("")

api/tests/common/test_commands.py

@@ -1,4 +1,5 @@
 import os
+from io import StringIO
 
 import pytest
 from django.core.management import call_command
@@ -116,3 +117,56 @@ def test_unblocked_commands(command, mocker):
     mocker.patch.dict(os.environ, {"FORCE": "1"})
 
     call_command(command)
+
+
+def test_inplace_to_s3_without_source():
+    with pytest.raises(CommandError):
+        call_command("inplace_to_s3")
+
+
+def test_inplace_to_s3_dryrun(factories):
+    upload = factories["music.Upload"](in_place=True, source="file:///music/music.mp3")
+    call_command("inplace_to_s3", "--source", "/music")
+    assert upload.source == "file:///music/music.mp3"
+    assert upload.audio_file is None
+
+
+data = [
+    {
+        "file": "/music/test.mp3",
+        "source": "/",
+        "target": None,
+        "expected": "/music/test.mp3",
+    },
+    {
+        "file": "/music/test.mp3",
+        "source": "/music",
+        "target": "/in-place",
+        "expected": "/in-place/test.mp3",
+    },
+    {
+        "file": "/music/test.mp3",
+        "source": "/music",
+        "target": "/in-place/music",
+        "expected": "/in-place/music/test.mp3",
+    },
+    {"file": "/music/test.mp3", "source": "/abcd", "target": "/music", "expected": "0"},
+]
+
+
+@pytest.mark.parametrize("data", data)
+def test_inplace_to_s3(factories, data):
+    out = StringIO()
+    factories["music.Upload"](in_place=True, source=f"file://{data['file']}")
+    if data["target"]:
+        call_command(
+            "inplace_to_s3",
+            "--source",
+            data["source"],
+            "--target",
+            data["target"],
+            stdout=out,
+        )
+    else:
+        call_command("inplace_to_s3", "--source", data["source"], stdout=out)
+    assert data["expected"] in out.getvalue()

changes/changelog.d/2156.feature

@@ -0,0 +1,2 @@
+New management command to update Uploads which have been imported using --in-place and are now
+stored in s3 (#2156)