funkwhale/api/funkwhale_api/music/models.py

import datetime
import logging
import os
import tempfile
import urllib.parse
import uuid

from django.db.models.expressions import OuterRef, Subquery
from django.db.models.query_utils import Q

import arrow
import pydub
from django.conf import settings
from django.contrib.contenttypes.fields import GenericRelation
from django.db.models import JSONField
from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.base import ContentFile
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models, transaction
from django.db.models.signals import post_save, pre_save
from django.dispatch import receiver
from django.urls import reverse
from django.utils import timezone
from django.db.models import Prefetch, Count

from funkwhale_api import musicbrainz
from funkwhale_api.common import fields
from funkwhale_api.common import models as common_models
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.tags import models as tags_models
from . import importers, metadata, utils

logger = logging.getLogger(__name__)

MAX_LENGTHS = {
    "ARTIST_NAME": 255,
    "ALBUM_TITLE": 255,
    "TRACK_TITLE": 255,
    "COPYRIGHT": 500,
}


ARTIST_CONTENT_CATEGORY_CHOICES = [
    ("music", "music"),
    ("podcast", "podcast"),
    ("other", "other"),
]


def empty_dict():
    return {}


class APIModelMixin(models.Model):
    fid = models.URLField(unique=True, max_length=500, db_index=True, null=True)
    mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
    from_activity = models.ForeignKey(
        "federation.Activity", null=True, blank=True, on_delete=models.SET_NULL
    )
    api_includes = []
    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
    import_hooks = []
    body_text = SearchVectorField(blank=True)

    class Meta:
        abstract = True
        ordering = ["-creation_date"]
        indexes = [
            GinIndex(fields=["body_text"]),
        ]

    @classmethod
    def get_or_create_from_api(cls, mbid):
        try:
            return cls.objects.get(mbid=mbid), False
        except cls.DoesNotExist:
            return cls.create_from_api(id=mbid), True

    def get_api_data(self):
        return self.__class__.api.get(id=self.mbid, includes=self.api_includes)[
            self.musicbrainz_model
        ]

    @classmethod
    def create_from_api(cls, **kwargs):
        if kwargs.get("id"):
            raw_data = cls.api.get(id=kwargs["id"], includes=cls.api_includes)[
                cls.musicbrainz_model
            ]
        else:
            raw_data = cls.api.search(**kwargs)[
                "{0}-list".format(cls.musicbrainz_model)
            ][0]
        cleaned_data = cls.clean_musicbrainz_data(raw_data)
        return importers.load(cls, cleaned_data, raw_data, cls.import_hooks)

    @classmethod
    def clean_musicbrainz_data(cls, data):
        cleaned_data = {}
        mapping = importers.Mapping(cls.musicbrainz_mapping)
        for key, value in data.items():
            try:
                cleaned_key, cleaned_value = mapping.from_musicbrainz(key, value)
                cleaned_data[cleaned_key] = cleaned_value
            except KeyError:
                pass
        return cleaned_data

    @property
    def musicbrainz_url(self):
        if self.mbid:
            return "https://musicbrainz.org/{}/{}".format(
                self.musicbrainz_model, self.mbid
            )

    def get_federation_id(self):
        if self.fid:
            return self.fid

        return federation_utils.full_url(
            reverse(
                "federation:music:{}-detail".format(self.federation_namespace),
                kwargs={"uuid": self.uuid},
            )
        )

    def save(self, **kwargs):
        if not self.pk and not self.fid:
            self.fid = self.get_federation_id()

        return super().save(**kwargs)

    @property
    def is_local(self) -> bool:
        return federation_utils.is_local(self.fid)

    @property
    def domain_name(self):
        if not self.fid:
            return

        parsed = urllib.parse.urlparse(self.fid)
        return parsed.hostname

    def get_tags(self):
        return list(sorted(self.tagged_items.values_list("tag__name", flat=True)))


class License(models.Model):
    code = models.CharField(primary_key=True, max_length=100)
    url = models.URLField(max_length=500)

    # if true, license is a copyleft license, meaning that derivative
    # work must be shared under the same license
    copyleft = models.BooleanField()
    # if true, commercial use of the work is allowed
    commercial = models.BooleanField()
    # if true, attribution to the original author is required when reusing
    # the work
    attribution = models.BooleanField()
    # if true, derivative work are allowed
    derivative = models.BooleanField()
    # if true, redistribution of the wor is allowed
    redistribute = models.BooleanField()

    @property
    def conf(self):
        from . import licenses

        for row in licenses.LICENSES:
            if self.code == row["code"]:
                return row
        logger.warning("%s do not match any registered license", self.code)


class ArtistQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
    def with_albums_count(self):
        return self.annotate(_albums_count=models.Count("albums"))

    def with_albums(self):
        return self.prefetch_related(
            models.Prefetch(
                "albums",
                queryset=Album.objects.with_tracks_count().select_related(
                    "attachment_cover", "attributed_to"
                ),
            )
        )

    def annotate_playable_by_actor(self, actor):
        tracks = (
            Upload.objects.playable_by(actor)
            .filter(track__artist=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(tracks)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
        tracks = Track.objects.playable_by(actor)
        matches = self.filter(pk__in=tracks.values("artist_id")).values_list("pk")
        if include:
            return self.filter(pk__in=matches)
        else:
            return self.exclude(pk__in=matches)


class Artist(APIModelMixin):
    name = models.CharField(max_length=MAX_LENGTHS["ARTIST_NAME"])
    federation_namespace = "artists"
    musicbrainz_model = "artist"
    musicbrainz_mapping = {
        "mbid": {"musicbrainz_field_name": "id"},
        "name": {"musicbrainz_field_name": "name"},
    }
    # Music entities are attributed to actors, to validate that updates occur
    # from an authorized account. On top of that, we consider the instance actor
    # can update anything under it's own domain
    attributed_to = models.ForeignKey(
        "federation.Actor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="attributed_artists",
    )
    tagged_items = GenericRelation(tags_models.TaggedItem)
    fetches = GenericRelation(
        "federation.Fetch",
        content_type_field="object_content_type",
        object_id_field="object_id",
    )
    description = models.ForeignKey(
        "common.Content", null=True, blank=True, on_delete=models.SET_NULL
    )
    attachment_cover = models.ForeignKey(
        "common.Attachment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="covered_artist",
    )
    content_category = models.CharField(
        max_length=30,
        db_index=True,
        default="music",
        choices=ARTIST_CONTENT_CATEGORY_CHOICES,
        null=False,
    )
    modification_date = models.DateTimeField(default=timezone.now, db_index=True)
    api = musicbrainz.api.artists
    objects = ArtistQuerySet.as_manager()

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return "/library/artists/{}".format(self.pk)

    def get_moderation_url(self):
        return "/manage/library/artists/{}".format(self.pk)

    @classmethod
    def get_or_create_from_name(cls, name, **kwargs):
        kwargs.update({"name": name})
        return cls.objects.get_or_create(name__iexact=name, defaults=kwargs)

    @property
    def cover(self):
        return self.attachment_cover

    def get_channel(self):
        try:
            return self.channel
        except ObjectDoesNotExist:
            return None


def import_artist(v):
    a = Artist.get_or_create_from_api(mbid=v[0]["artist"]["id"])[0]
    return a


def parse_date(v):
    d = arrow.get(v).date()
    return d


def import_tracks(instance, cleaned_data, raw_data):
    for track_data in raw_data["medium-list"][0]["track-list"]:
        track_cleaned_data = Track.clean_musicbrainz_data(track_data["recording"])
        track_cleaned_data["album"] = instance
        track_cleaned_data["position"] = int(track_data["position"])
        importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)


class AlbumQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
    def with_tracks_count(self):
        return self.annotate(_tracks_count=models.Count("tracks"))

    def annotate_playable_by_actor(self, actor):
        tracks = (
            Upload.objects.playable_by(actor)
            .filter(track__album=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(tracks)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
        tracks = Track.objects.playable_by(actor)
        matches = self.filter(pk__in=tracks.values("album_id")).values_list("pk")
        if include:
            return self.filter(pk__in=matches)
        else:
            return self.exclude(pk__in=matches)

    def with_duration(self):
        # takes one upload per track
        subquery = Subquery(
            Upload.objects.filter(track_id=OuterRef("tracks"))
            .order_by("id")
            .values("id")[:1]
        )
        return self.annotate(
            duration=models.Sum(
                "tracks__uploads__duration",
                filter=Q(tracks__uploads=subquery),
            )
        )


class Album(APIModelMixin):
    title = models.CharField(max_length=MAX_LENGTHS["ALBUM_TITLE"])
    artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
    release_date = models.DateField(null=True, blank=True, db_index=True)
    release_group_id = models.UUIDField(null=True, blank=True)
    attachment_cover = models.ForeignKey(
        "common.Attachment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="covered_album",
    )
    TYPE_CHOICES = (("album", "Album"),)
    type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album")

    # Music entities are attributed to actors, to validate that updates occur
    # from an authorized account. On top of that, we consider the instance actor
    # can update anything under it's own domain
    attributed_to = models.ForeignKey(
        "federation.Actor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="attributed_albums",
    )
    tagged_items = GenericRelation(tags_models.TaggedItem)
    fetches = GenericRelation(
        "federation.Fetch",
        content_type_field="object_content_type",
        object_id_field="object_id",
    )

    description = models.ForeignKey(
        "common.Content", null=True, blank=True, on_delete=models.SET_NULL
    )

    api_includes = ["artist-credits", "recordings", "media", "release-groups"]
    api = musicbrainz.api.releases
    federation_namespace = "albums"
    musicbrainz_model = "release"
    musicbrainz_mapping = {
        "mbid": {"musicbrainz_field_name": "id"},
        "position": {
            "musicbrainz_field_name": "release-list",
            "converter": lambda v: int(v[0]["medium-list"][0]["position"]),
        },
        "release_group_id": {
            "musicbrainz_field_name": "release-group",
            "converter": lambda v: v["id"],
        },
        "title": {"musicbrainz_field_name": "title"},
        "release_date": {"musicbrainz_field_name": "date", "converter": parse_date},
        "type": {"musicbrainz_field_name": "type", "converter": lambda v: v.lower()},
        "artist": {
            "musicbrainz_field_name": "artist-credit",
            "converter": import_artist,
        },
    }
    objects = AlbumQuerySet.as_manager()

    @property
    def cover(self):
        return self.attachment_cover

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return "/library/albums/{}".format(self.pk)

    def get_moderation_url(self):
        return "/manage/library/albums/{}".format(self.pk)

    @classmethod
    def get_or_create_from_title(cls, title, **kwargs):
        kwargs.update({"title": title})
        return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)


def import_tags(instance, cleaned_data, raw_data):
    MINIMUM_COUNT = 2
    tags_to_add = []
    for tag_data in raw_data.get("tag-list", []):
        try:
            if int(tag_data["count"]) < MINIMUM_COUNT:
                continue
        except ValueError:
            continue
        tags_to_add.append(tag_data["name"])

    tags_models.add_tags(instance, *tags_to_add)


def import_album(v):
    a = Album.get_or_create_from_api(mbid=v[0]["id"])[0]
    return a


class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
    def for_nested_serialization(self):
        return self.prefetch_related(
            "artist",
            Prefetch(
                "album",
                queryset=Album.objects.select_related(
                    "artist", "attachment_cover"
                ).annotate(_prefetched_tracks_count=Count("tracks")),
            ),
        )

    def annotate_playable_by_actor(self, actor):

        files = (
            Upload.objects.playable_by(actor)
            .filter(track=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(files)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):

        if settings.MUSIC_USE_DENORMALIZATION:
            if actor is not None:
                query = models.Q(actor=None) | models.Q(actor=actor)
            else:
                query = models.Q(actor=None, internal=False)
            if not include:
                query = ~query
            return self.filter(pk__in=TrackActor.objects.filter(query).values("track"))
        files = Upload.objects.playable_by(actor, include)
        matches = self.filter(uploads__in=files).values_list("pk")
        if include:
            return self.filter(pk__in=matches)
        else:
            return self.exclude(pk__in=matches)

    def with_playable_uploads(self, actor):
        uploads = Upload.objects.playable_by(actor)
        return self.prefetch_related(
            models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
        )

    def order_for_album(self):
        """
        Order by disc number then position
        """
        return self.order_by("disc_number", "position", "title")


def get_artist(release_list):
    return Artist.get_or_create_from_api(
        mbid=release_list[0]["artist-credits"][0]["artists"]["id"]
    )[0]


class Track(APIModelMixin):
    mbid = models.UUIDField(db_index=True, null=True, blank=True)
    title = models.CharField(max_length=MAX_LENGTHS["TRACK_TITLE"])
    artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
    disc_number = models.PositiveIntegerField(null=True, blank=True)
    position = models.PositiveIntegerField(null=True, blank=True)
    album = models.ForeignKey(
        Album, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
    )
    license = models.ForeignKey(
        License,
        null=True,
        blank=True,
        on_delete=models.DO_NOTHING,
        related_name="tracks",
    )
    # Music entities are attributed to actors, to validate that updates occur
    # from an authorized account. On top of that, we consider the instance actor
    # can update anything under it's own domain
    attributed_to = models.ForeignKey(
        "federation.Actor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="attributed_tracks",
    )
    copyright = models.CharField(
        max_length=MAX_LENGTHS["COPYRIGHT"], null=True, blank=True
    )
    description = models.ForeignKey(
        "common.Content", null=True, blank=True, on_delete=models.SET_NULL
    )
    attachment_cover = models.ForeignKey(
        "common.Attachment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="covered_track",
    )
    downloads_count = models.PositiveIntegerField(default=0)
    federation_namespace = "tracks"
    musicbrainz_model = "recording"
    api = musicbrainz.api.recordings
    api_includes = ["artist-credits", "releases", "media", "tags"]
    musicbrainz_mapping = {
        "mbid": {"musicbrainz_field_name": "id"},
        "title": {"musicbrainz_field_name": "title"},
        "artist": {
            "musicbrainz_field_name": "artist-credit",
            "converter": lambda v: Artist.get_or_create_from_api(
                mbid=v[0]["artist"]["id"]
            )[0],
        },
        "album": {"musicbrainz_field_name": "release-list", "converter": import_album},
    }
    import_hooks = [import_tags]
    objects = TrackQuerySet.as_manager()
    tagged_items = GenericRelation(tags_models.TaggedItem)
    fetches = GenericRelation(
        "federation.Fetch",
        content_type_field="object_content_type",
        object_id_field="object_id",
    )

    class Meta:
        ordering = ["album", "disc_number", "position"]
        indexes = [
            GinIndex(fields=["body_text"]),
        ]

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return "/library/tracks/{}".format(self.pk)

    def get_moderation_url(self):
        return "/manage/library/tracks/{}".format(self.pk)

    def save(self, **kwargs):
        try:
            self.artist
        except Artist.DoesNotExist:
            self.artist = self.album.artist
        super().save(**kwargs)

    @property
    def full_name(self):
        try:
            return "{} - {} - {}".format(self.artist.name, self.album.title, self.title)
        except AttributeError:
            return "{} - {}".format(self.artist.name, self.title)

    @property
    def cover(self):
        return self.attachment_cover

    def get_activity_url(self):
        if self.mbid:
            return "https://musicbrainz.org/recording/{}".format(self.mbid)
        return settings.FUNKWHALE_URL + "/tracks/{}".format(self.pk)

    @classmethod
    def get_or_create_from_title(cls, title, **kwargs):
        kwargs.update({"title": title})
        return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)

    @classmethod
    def get_or_create_from_release(cls, release_mbid, mbid):
        release_mbid = str(release_mbid)
        mbid = str(mbid)
        try:
            return cls.objects.get(mbid=mbid), False
        except cls.DoesNotExist:
            pass

        album = Album.get_or_create_from_api(release_mbid)[0]
        data = musicbrainz.client.api.releases.get(
            str(album.mbid), includes=Album.api_includes
        )
        tracks = [t for m in data["release"]["medium-list"] for t in m["track-list"]]
        track_data = None
        for track in tracks:
            if track["recording"]["id"] == str(mbid):
                track_data = track
                break
        if not track_data:
            raise ValueError("No track found matching this ID")

        track_artist_mbid = None
        for ac in track_data["recording"]["artist-credit"]:
            try:
                ac_mbid = ac["artist"]["id"]
            except TypeError:
                # it's probably a string, like "feat."
                continue

            if ac_mbid == str(album.artist.mbid):
                continue

            track_artist_mbid = ac_mbid
            break
        track_artist_mbid = track_artist_mbid or album.artist.mbid
        if track_artist_mbid == str(album.artist.mbid):
            track_artist = album.artist
        else:
            track_artist = Artist.get_or_create_from_api(track_artist_mbid)[0]
        return cls.objects.update_or_create(
            mbid=mbid,
            defaults={
                "position": int(track["position"]),
                "title": track["recording"]["title"],
                "album": album,
                "artist": track_artist,
            },
        )

    @property
    def listen_url(self) -> str:
        # Not using reverse because this is slow
        return "/api/v1/listen/{}/".format(self.uuid)

    @property
    def local_license(self):
        """
        Since license primary keys are strings, and we can get the data
        from our hardcoded licenses.LICENSES list, there is no need
        for extra SQL joins / queries.
        """
        from . import licenses

        return licenses.LICENSES_BY_ID.get(self.license_id)


class UploadQuerySet(common_models.NullsLastQuerySet):
    def in_place(self, include=True):
        query = models.Q(source__startswith="file://") & (
            models.Q(audio_file="") | models.Q(audio_file=None)
        )
        if not include:
            query = ~query
        return self.filter(query)

    def playable_by(self, actor, include=True):
        libraries = Library.objects.viewable_by(actor)

        if include:
            return self.filter(library__in=libraries, import_status="finished")
        return self.exclude(library__in=libraries, import_status="finished")

    def local(self, include=True):
        query = models.Q(library__actor__domain_id=settings.FEDERATION_HOSTNAME)
        if not include:
            query = ~query
        return self.filter(query)

    def for_federation(self):
        return self.filter(import_status="finished", mimetype__startswith="audio/")

    def with_file(self):
        return self.exclude(audio_file=None).exclude(audio_file="")


TRACK_FILE_IMPORT_STATUS_CHOICES = (
    ("draft", "Draft"),
    ("pending", "Pending"),
    ("finished", "Finished"),
    ("errored", "Errored"),
    ("skipped", "Skipped"),
)


def get_file_path(instance, filename):
    if isinstance(instance, UploadVersion):
        return common_utils.ChunkedPath("transcoded")(instance, filename)

    if instance.library.actor.get_user():
        return common_utils.ChunkedPath("tracks")(instance, filename)
    else:
        # we cache remote tracks in a different directory
        return common_utils.ChunkedPath("federation_cache/tracks")(instance, filename)


def get_import_reference():
    return str(uuid.uuid4())


class Upload(models.Model):
    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
    track = models.ForeignKey(
        Track, related_name="uploads", on_delete=models.CASCADE, null=True, blank=True
    )
    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
    source = models.CharField(
        # URL validators are not flexible enough for our file:// and upload:// schemes
        null=True,
        blank=True,
        max_length=500,
    )
    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
    modification_date = models.DateTimeField(default=timezone.now, null=True)
    accessed_date = models.DateTimeField(null=True, blank=True)
    duration = models.IntegerField(null=True, blank=True)
    size = models.IntegerField(null=True, blank=True)
    bitrate = models.IntegerField(null=True, blank=True)
    acoustid_track_id = models.UUIDField(null=True, blank=True)
    mimetype = models.CharField(null=True, blank=True, max_length=200)
    library = models.ForeignKey(
        "library",
        null=True,
        blank=True,
        related_name="uploads",
        on_delete=models.CASCADE,
    )

    # metadata from federation
    metadata = JSONField(
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
    )
    import_date = models.DateTimeField(null=True, blank=True)
    # optionnal metadata provided during import
    import_metadata = JSONField(
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
    )
    # status / error details for the import
    import_status = models.CharField(
        default="pending", choices=TRACK_FILE_IMPORT_STATUS_CHOICES, max_length=25
    )
    # a short reference provided by the client to group multiple files
    # in the same import
    import_reference = models.CharField(max_length=50, default=get_import_reference)

    # optionnal metadata about import results (error messages, etc.)
    import_details = JSONField(
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
    )
    from_activity = models.ForeignKey(
        "federation.Activity", null=True, on_delete=models.SET_NULL, blank=True
    )
    downloads_count = models.PositiveIntegerField(default=0)

    # stores checksums such as `sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
    checksum = models.CharField(max_length=100, db_index=True, null=True, blank=True)

    objects = UploadQuerySet.as_manager()

    @property
    def is_local(self) -> bool:
        return federation_utils.is_local(self.fid)

    @property
    def domain_name(self):
        if not self.fid:
            return

        parsed = urllib.parse.urlparse(self.fid)
        return parsed.hostname

    def download_audio_from_remote(self, actor):
        from funkwhale_api.federation import signing

        if actor:
            auth = signing.get_auth(actor.private_key, actor.private_key_id)
        else:
            auth = None

        remote_response = session.get_session().get(
            self.source,
            auth=auth,
            stream=True,
            timeout=20,
            headers={"Content-Type": "application/octet-stream"},
        )
        with remote_response as r:
            remote_response.raise_for_status()
            extension = utils.get_ext_from_type(self.mimetype)
            title_parts = []
            title_parts.append(self.track.title)
            if self.track.album:
                title_parts.append(self.track.album.title)
            title_parts.append(self.track.artist.name)

            title = " - ".join(title_parts)
            filename = "{}.{}".format(title, extension)
            tmp_file = tempfile.TemporaryFile()
            for chunk in r.iter_content(chunk_size=512):
                tmp_file.write(chunk)
            self.audio_file.save(filename, tmp_file, save=False)
            self.save(update_fields=["audio_file"])

    def get_federation_id(self):
        if self.fid:
            return self.fid

        return federation_utils.full_url(
            reverse("federation:music:uploads-detail", kwargs={"uuid": self.uuid})
        )

    @property
    def filename(self) -> str:
        return "{}.{}".format(self.track.full_name, self.extension)

    @property
    def extension(self):
        try:
            return utils.MIMETYPE_TO_EXTENSION[self.mimetype]
        except KeyError:
            pass
        if self.audio_file:
            return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
        if self.in_place_path:
            return os.path.splitext(self.in_place_path)[-1].replace(".", "", 1)

    def get_file_size(self):
        if self.audio_file:
            return self.audio_file.size

        if self.source.startswith("file://"):
            return os.path.getsize(self.source.replace("file://", "", 1))

    def get_audio_file(self):
        if self.audio_file:
            return self.audio_file.open()
        if self.source and self.source.startswith("file://"):
            return open(self.source.replace("file://", "", 1), "rb")

    def get_audio_data(self):
        audio_file = self.get_audio_file()
        if not audio_file:
            return
        audio_data = utils.get_audio_file_data(audio_file)
        if not audio_data:
            return
        return {
            "duration": int(audio_data["length"]),
            "bitrate": audio_data["bitrate"],
            "size": self.get_file_size(),
        }

    def get_audio_segment(self):
        input = self.get_audio_file()
        if not input:
            return

        audio = pydub.AudioSegment.from_file(input)
        return audio

    def save(self, **kwargs):
        if not self.mimetype:
            if self.audio_file:
                self.mimetype = utils.guess_mimetype(self.audio_file)
            elif self.source and self.source.startswith("file://"):
                self.mimetype = utils.guess_mimetype_from_name(self.source)
        if not self.size and self.audio_file:
            self.size = self.audio_file.size
        if not self.checksum:
            try:
                audio_file = self.get_audio_file()
            except FileNotFoundError:
                pass
            else:
                if audio_file:
                    self.checksum = common_utils.get_file_hash(audio_file)

        if not self.pk and not self.fid and self.library.actor.get_user():
            self.fid = self.get_federation_id()
        return super().save(**kwargs)

    def get_metadata(self):
        audio_file = self.get_audio_file()
        if not audio_file:
            return
        return metadata.Metadata(audio_file)

    @property
    def listen_url(self) -> str:
        return self.track.listen_url + "?upload={}".format(self.uuid)

    def get_listen_url(self, to=None, download=True) -> str:
        url = self.listen_url
        if to:
            url += "&to={}".format(to)
        if not download:
            url += "&download=false"

        return url

    @property
    def listen_url_no_download(self):
        # Not using reverse because this is slow
        return self.listen_url + "&download=false"

    def get_transcoded_version(self, format, max_bitrate=None):
        if format:
            mimetype = utils.EXTENSION_TO_MIMETYPE[format]
        else:
            mimetype = self.mimetype or "audio/mpeg"
            format = utils.MIMETYPE_TO_EXTENSION[mimetype]

        existing_versions = self.versions.filter(mimetype=mimetype)
        if max_bitrate is not None:
            # we don't want to transcode if a 320kbps version is available
            # and we're requestiong 300kbps
            acceptable_max_bitrate = max_bitrate * 1.2
            acceptable_min_bitrate = max_bitrate * 0.8
            existing_versions = existing_versions.filter(
                bitrate__gte=acceptable_min_bitrate, bitrate__lte=acceptable_max_bitrate
            ).order_by("-bitrate")
        if existing_versions:
            # we found an existing version, no need to transcode again
            return existing_versions[0]

        return self.create_transcoded_version(mimetype, format, bitrate=max_bitrate)

    @transaction.atomic
    def create_transcoded_version(self, mimetype, format, bitrate):
        # we create the version with an empty file, then
        # we'll write to it
        f = ContentFile(b"")
        bitrate = min(bitrate or 320000, self.bitrate or 320000)
        version = self.versions.create(mimetype=mimetype, bitrate=bitrate, size=0)
        # we keep the same name, but we update the extension
        new_name = os.path.splitext(os.path.basename(self.audio_file.name))[
            0
        ] + ".{}".format(format)
        version.audio_file.save(new_name, f)
        utils.transcode_audio(
            audio=self.get_audio_segment(),
            output=version.audio_file,
            output_format=utils.MIMETYPE_TO_EXTENSION[mimetype],
            bitrate=str(bitrate),
        )
        version.size = version.audio_file.size
        version.save(update_fields=["size"])

        return version

    @property
    def in_place_path(self):
        if not self.source or not self.source.startswith("file://"):
            return
        return self.source.lstrip("file://")

    @property
    def audio_file_path(self):
        if not self.audio_file:
            return None
        try:
            return self.audio_file.path
        except NotImplementedError:
            # external storage
            return self.audio_file.name

    def get_all_tagged_items(self):
        track_tags = self.track.tagged_items.all()
        album_tags = (
            self.track.album.tagged_items.all()
            if self.track.album
            else tags_models.TaggedItem.objects.none()
        )
        artist_tags = self.track.artist.tagged_items.all()

        items = (track_tags | album_tags | artist_tags).order_by("tag__name")
        return items


MIMETYPE_CHOICES = [(mt, ext) for ext, mt in utils.AUDIO_EXTENSIONS_AND_MIMETYPE]


class UploadVersion(models.Model):
    upload = models.ForeignKey(
        Upload, related_name="versions", on_delete=models.CASCADE
    )
    mimetype = models.CharField(max_length=50, choices=MIMETYPE_CHOICES)
    creation_date = models.DateTimeField(default=timezone.now)
    accessed_date = models.DateTimeField(null=True, blank=True)
    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
    bitrate = models.PositiveIntegerField()
    size = models.IntegerField()

    class Meta:
        unique_together = ("upload", "mimetype", "bitrate")

    @property
    def filename(self) -> str:
        try:
            return (
                self.upload.track.full_name
                + "."
                + utils.MIMETYPE_TO_EXTENSION[self.mimetype]
            )
        except KeyError:
            return self.upload.filename

    @property
    def audio_file_path(self):
        if not self.audio_file:
            return None
        try:
            return self.audio_file.path
        except NotImplementedError:
            # external storage
            return self.audio_file.name


IMPORT_STATUS_CHOICES = (
    ("pending", "Pending"),
    ("finished", "Finished"),
    ("errored", "Errored"),
    ("skipped", "Skipped"),
)


class ImportBatch(models.Model):
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
    IMPORT_BATCH_SOURCES = [
        ("api", "api"),
        ("shell", "shell"),
        ("federation", "federation"),
    ]
    source = models.CharField(
        max_length=30, default="api", choices=IMPORT_BATCH_SOURCES
    )
    creation_date = models.DateTimeField(default=timezone.now)
    submitted_by = models.ForeignKey(
        "users.User",
        related_name="imports",
        null=True,
        blank=True,
        on_delete=models.CASCADE,
    )
    status = models.CharField(
        choices=IMPORT_STATUS_CHOICES, default="pending", max_length=30
    )
    import_request = models.ForeignKey(
        "requests.ImportRequest",
        related_name="import_batches",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
    )
    library = models.ForeignKey(
        "Library",
        related_name="import_batches",
        null=True,
        blank=True,
        on_delete=models.CASCADE,
    )

    class Meta:
        ordering = ["-creation_date"]

    def __str__(self):
        return str(self.pk)

    def update_status(self):
        old_status = self.status
        self.status = utils.compute_status(self.jobs.all())
        if self.status == old_status:
            return
        self.save(update_fields=["status"])
        if self.status != old_status and self.status == "finished":
            from . import tasks

            tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)

    def get_federation_id(self):
        return federation_utils.full_url(
            "/federation/music/import/batch/{}".format(self.uuid)
        )


class ImportJob(models.Model):
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
    replace_if_duplicate = models.BooleanField(default=False)
    batch = models.ForeignKey(
        ImportBatch, related_name="jobs", on_delete=models.CASCADE
    )
    upload = models.ForeignKey(
        Upload, related_name="jobs", null=True, blank=True, on_delete=models.CASCADE
    )
    source = models.CharField(max_length=500)
    mbid = models.UUIDField(editable=False, null=True, blank=True)

    status = models.CharField(
        choices=IMPORT_STATUS_CHOICES, default="pending", max_length=30
    )
    audio_file = models.FileField(
        upload_to="imports/%Y/%m/%d", max_length=255, null=True, blank=True
    )

    library_track = models.ForeignKey(
        "federation.LibraryTrack",
        related_name="import_jobs",
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
    )
    audio_file_size = models.IntegerField(null=True, blank=True)

    class Meta:
        ordering = ("id",)

    def save(self, **kwargs):
        if self.audio_file and not self.audio_file_size:
            self.audio_file_size = self.audio_file.size
        return super().save(**kwargs)


LIBRARY_PRIVACY_LEVEL_CHOICES = [
    (k, l) for k, l in fields.PRIVACY_LEVEL_CHOICES if k != "followers"
]


class LibraryQuerySet(models.QuerySet):
    def local(self, include=True):
        query = models.Q(actor__domain_id=settings.FEDERATION_HOSTNAME)
        if not include:
            query = ~query
        return self.filter(query)

    def with_follows(self, actor):
        return self.prefetch_related(
            models.Prefetch(
                "received_follows",
                queryset=federation_models.LibraryFollow.objects.filter(actor=actor),
                to_attr="_follows",
            )
        )

    def viewable_by(self, actor):
        from funkwhale_api.federation.models import LibraryFollow, Follow

        if actor is None:
            return self.filter(privacy_level="everyone")

        me_query = models.Q(privacy_level="me", actor=actor)
        instance_query = models.Q(privacy_level="instance", actor__domain=actor.domain)
        followed_libraries = LibraryFollow.objects.filter(
            actor=actor, approved=True
        ).values_list("target", flat=True)
        followed_channels_libraries = (
            Follow.objects.exclude(target__channel=None)
            .filter(
                actor=actor,
                approved=True,
            )
            .values_list("target__channel__library", flat=True)
        )
        return self.filter(
            me_query
            | instance_query
            | models.Q(privacy_level="everyone")
            | models.Q(pk__in=followed_libraries)
            | models.Q(pk__in=followed_channels_libraries)
        )


class Library(federation_models.FederationMixin):
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
    actor = models.ForeignKey(
        "federation.Actor", related_name="libraries", on_delete=models.CASCADE
    )
    followers_url = models.URLField(max_length=500)
    creation_date = models.DateTimeField(default=timezone.now)
    name = models.CharField(max_length=100)
    description = models.TextField(max_length=5000, null=True, blank=True)
    privacy_level = models.CharField(
        choices=LIBRARY_PRIVACY_LEVEL_CHOICES, default="me", max_length=25
    )
    uploads_count = models.PositiveIntegerField(default=0)
    objects = LibraryQuerySet.as_manager()

    def __str__(self):
        return self.name

    def get_moderation_url(self) -> str:
        return "/manage/library/libraries/{}".format(self.uuid)

    def get_federation_id(self) -> str:
        return federation_utils.full_url(
            reverse("federation:music:libraries-detail", kwargs={"uuid": self.uuid})
        )

    def get_absolute_url(self) -> str:
        return "/library/{}".format(self.uuid)

    def save(self, **kwargs):
        if not self.pk and not self.fid and self.actor.is_local:
            self.fid = self.get_federation_id()
            self.followers_url = self.fid + "/followers"

        return super().save(**kwargs)

    def should_autoapprove_follow(self, actor) -> bool:
        if self.privacy_level == "everyone":
            return True
        if self.privacy_level == "instance" and actor.get_user():
            return True
        return False

    def schedule_scan(self, actor, force=False):
        latest_scan = (
            self.scans.exclude(status="errored").order_by("-creation_date").first()
        )
        delay_between_scans = datetime.timedelta(seconds=3600 * 24)
        now = timezone.now()
        if (
            not force
            and latest_scan
            and latest_scan.creation_date + delay_between_scans > now
        ):
            return

        scan = self.scans.create(total_files=self.uploads_count, actor=actor)
        from . import tasks

        common_utils.on_commit(tasks.start_library_scan.delay, library_scan_id=scan.pk)
        return scan

    def get_channel(self):
        try:
            return self.channel
        except ObjectDoesNotExist:
            return None

    def latest_scan(self):
        return self.scans.order_by("-creation_date").first()


SCAN_STATUS = [
    ("pending", "pending"),
    ("scanning", "scanning"),
    ("errored", "errored"),
    ("finished", "finished"),
]


class LibraryScan(models.Model):
    actor = models.ForeignKey(
        "federation.Actor", null=True, blank=True, on_delete=models.CASCADE
    )
    library = models.ForeignKey(Library, related_name="scans", on_delete=models.CASCADE)
    total_files = models.PositiveIntegerField(default=0)
    processed_files = models.PositiveIntegerField(default=0)
    errored_files = models.PositiveIntegerField(default=0)
    status = models.CharField(default="pending", max_length=25)
    creation_date = models.DateTimeField(default=timezone.now)
    modification_date = models.DateTimeField(null=True, blank=True)


class TrackActor(models.Model):
    """
    Denormalization table to store all playable tracks for a given user
    Empty user means the track is public or internal (cf internal flag too)
    """

    id = models.BigAutoField(primary_key=True)
    actor = models.ForeignKey(
        "federation.Actor",
        on_delete=models.CASCADE,
        related_name="track_actor_items",
        blank=True,
        null=True,
    )
    track = models.ForeignKey(
        Track, on_delete=models.CASCADE, related_name="track_actor_items"
    )
    upload = models.ForeignKey(
        Upload, on_delete=models.CASCADE, related_name="track_actor_items"
    )
    internal = models.BooleanField(default=False, db_index=True)

    class Meta:
        unique_together = ("track", "actor", "internal", "upload")

    @classmethod
    def get_objs(cls, library, actor_ids, upload_and_track_ids):
        upload_and_track_ids = upload_and_track_ids or library.uploads.filter(
            import_status="finished", track__isnull=False
        ).values_list("id", "track")
        objs = []
        if library.privacy_level == "me":
            if library.get_channel():
                follow_queryset = library.channel.actor.received_follows
            else:
                follow_queryset = library.received_follows
            follow_queryset = follow_queryset.filter(approved=True).exclude(
                actor__user__isnull=True
            )
            if actor_ids:
                follow_queryset = follow_queryset.filter(actor__pk__in=actor_ids)
            final_actor_ids = list(follow_queryset.values_list("actor", flat=True))

            owner = library.actor if library.actor.is_local else None
            if owner and (not actor_ids or owner in final_actor_ids):
                final_actor_ids.append(owner.pk)
            for actor_id in final_actor_ids:
                for upload_id, track_id in upload_and_track_ids:
                    objs.append(
                        cls(actor_id=actor_id, track_id=track_id, upload_id=upload_id)
                    )

        elif library.privacy_level == "instance":
            for upload_id, track_id in upload_and_track_ids:
                objs.append(
                    cls(
                        actor_id=None,
                        track_id=track_id,
                        upload_id=upload_id,
                        internal=True,
                    )
                )
        elif library.privacy_level == "everyone":
            for upload_id, track_id in upload_and_track_ids:
                objs.append(cls(actor_id=None, track_id=track_id, upload_id=upload_id))
        return objs

    @classmethod
    def create_entries(
        cls, library, delete_existing=True, actor_ids=None, upload_and_track_ids=None
    ):
        if not settings.MUSIC_USE_DENORMALIZATION:
            # skip
            return
        if delete_existing:
            to_delete = cls.objects.filter(upload__library=library)
            if actor_ids:
                to_delete = to_delete.filter(actor__pk__in=actor_ids)
            # we don't use .delete() here because we don't want signals to fire
            to_delete._raw_delete(to_delete.db)

        objs = cls.get_objs(
            library, actor_ids=actor_ids, upload_and_track_ids=upload_and_track_ids
        )
        return cls.objects.bulk_create(objs, ignore_conflicts=True, batch_size=5000)


@receiver(post_save, sender=ImportJob)
def update_batch_status(sender, instance, **kwargs):
    instance.batch.update_status()


@receiver(post_save, sender=Upload)
def update_denormalization_track_actor(sender, instance, created, **kwargs):
    if (
        created
        and settings.MUSIC_USE_DENORMALIZATION
        and instance.track_id
        and instance.import_status == "finished"
    ):
        TrackActor.create_entries(
            instance.library,
            delete_existing=False,
            upload_and_track_ids=[(instance.pk, instance.track_id)],
        )


@receiver(pre_save, sender=Library)
def set_privacy_level_updated(sender, instance, update_fields, **kwargs):
    if not instance.pk:
        return
    if update_fields is not None and "privacy_level" not in update_fields:
        return
    db_value = instance.__class__.objects.filter(pk=instance.pk).values_list(
        "privacy_level", flat=True
    )[0]
    if db_value != instance.privacy_level:
        # Needed to update denormalized permissions
        setattr(instance, "_privacy_level_updated", True)


@receiver(post_save, sender=Library)
def update_denormalization_track_user_library_privacy_level(
    sender, instance, created, **kwargs
):
    if created:
        return
    updated = getattr(instance, "_privacy_level_updated", False)
    if updated:
        TrackActor.create_entries(instance)


@receiver(post_save, sender=ImportBatch)
def update_request_status(sender, instance, created, **kwargs):
    update_fields = kwargs.get("update_fields", []) or []
    if not instance.import_request:
        return

    if not created and "status" not in update_fields:
        return

    r_status = instance.import_request.status
    status = instance.status

    if status == "pending" and r_status == "pending":
        # let's mark the request as accepted since we started an import
        instance.import_request.status = "accepted"
        return instance.import_request.save(update_fields=["status"])

    if status == "finished" and r_status == "accepted":
        # let's mark the request as imported since the import is over
        instance.import_request.status = "imported"
        return instance.import_request.save(update_fields=["status"])