Merge branch '836-playlist-import-export-backend' into 'develop'

Draft: Add backend logic to handle xspf file to import playlist (#836).

Closes #836

See merge request funkwhale/funkwhale!2317
merge-requests/2317/merge
petitminion 2024-04-16 12:28:06 +00:00
commit 08fcc5ec1a
16 zmienionych plików z 495 dodań i 10 usunięć

Wyświetl plik

@ -18,6 +18,6 @@ MEDIA_ROOT=/data/media
# FORCE_HTTPS_URLS=True
# Customize to your needs
POSTGRES_VERSION=11
POSTGRES_VERSION=12
DEBUG=true
TYPESENSE_API_KEY="apikey"

Wyświetl plik

@ -2,7 +2,8 @@ import logging.config
import sys
import warnings
from collections import OrderedDict
from urllib.parse import urlparse, urlsplit
from urllib.parse import urlsplit, urlparse
from . import testing
import environ
from celery.schedules import crontab
@ -1044,6 +1045,7 @@ REST_FRAMEWORK = {
"DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",),
"NUM_PROXIES": env.int("NUM_PROXIES", default=1),
}
REST_FRAMEWORK.update(testing.REST_FRAMEWORK)
THROTTLING_ENABLED = env.bool("THROTTLING_ENABLED", default=True)
"""
Whether to enable throttling (also known as rate-limiting).

Wyświetl plik

@ -7,3 +7,11 @@ from .common import * # noqa
DEBUG = True
SECRET_KEY = "a_super_secret_key!"
TYPESENSE_API_KEY = "apikey"
REST_FRAMEWORK = {
"TEST_REQUEST_RENDERER_CLASSES": [
"rest_framework.renderers.MultiPartRenderer",
"rest_framework.renderers.JSONRenderer",
"rest_framework.renderers.TemplateHTMLRenderer",
"funkwhale_api.playlists.renderers.PlaylistXspfRenderer",
],
}

Wyświetl plik

@ -15,6 +15,12 @@ v2_patterns += [
r"^radios/",
include(("funkwhale_api.radios.urls_v2", "radios"), namespace="radios"),
),
re_path(
r"^",
include(
("funkwhale_api.playlists.urls_v2", "playlists"), namespace="playlists"
),
),
]
urlpatterns = [re_path("", include((v2_patterns, "v2"), namespace="v2"))]

Wyświetl plik

@ -0,0 +1,57 @@
import xml.etree.ElementTree as ElementTree
from defusedxml import ElementTree as etree
from defusedxml.ElementTree import parse
from rest_framework.exceptions import ParseError
from rest_framework.parsers import BaseParser
from django.conf import settings
# stolen from https://github.com/jpadilla/django-rest-framework-xml/blob/master/rest_framework_xml/parsers.py
# class XSPFtoJSPF(BaseParser):
class XspfParser(BaseParser):
"""
Takes a xspf sting, validated it, and return an xspf json
"""
media_type = "application/octet-stream"
def parse(self, stream, media_type=None, parser_context=None):
playlist = {"tracks": []}
tree = parse(stream, forbid_dtd=True)
root = tree.getroot()
# Extract playlist information
playlist_info = root.find(".")
if playlist_info is not None:
playlist["title"] = playlist_info.findtext(
"{http://xspf.org/ns/0/}title", default=""
)
playlist["creator"] = playlist_info.findtext(
"{http://xspf.org/ns/0/}creator", default=""
)
playlist["creation_date"] = playlist_info.findtext(
"{http://xspf.org/ns/0/}date", default=""
)
playlist["version"] = playlist_info.attrib.get("version", "")
# Extract track information
for track in root.findall(".//{http://xspf.org/ns/0/}track"):
track_info = {
"location": track.findtext(
"{http://xspf.org/ns/0/}location", default=""
),
"title": track.findtext("{http://xspf.org/ns/0/}title", default=""),
"creator": track.findtext("{http://xspf.org/ns/0/}creator", default=""),
"album": track.findtext("{http://xspf.org/ns/0/}album", default=""),
"duration": track.findtext(
"{http://xspf.org/ns/0/}duration", default=""
),
}
playlist["tracks"].append(track_info)
return playlist

Wyświetl plik

@ -0,0 +1,80 @@
from rest_framework import renderers
from funkwhale_api.music.models import Album, Artist, Track
from funkwhale_api.playlists.models import Playlist
from xml.etree.ElementTree import Element, SubElement
import xml.etree.ElementTree as etree
from defusedxml import minidom
class PlaylistXspfRenderer(renderers.BaseRenderer):
media_type = "application/octet-stream"
format = "xspf"
def render(self, data, accepted_media_type=None, renderer_context=None):
if isinstance(data, bytes):
return data
fw_playlist = Playlist.objects.get(id=data["id"])
plt_tracks = fw_playlist.playlist_tracks.prefetch_related("track")
top = Element("playlist")
top.set("version", "1")
title_xspf = SubElement(top, "title")
title_xspf.text = fw_playlist.name
date_xspf = SubElement(top, "date")
date_xspf.text = str(fw_playlist.creation_date)
trackList_xspf = SubElement(top, "trackList")
for plt_track in plt_tracks:
track = plt_track.track
write_xspf_track_data(track, trackList_xspf)
return prettify(top)
def generate_xspf_from_playlist(playlist_id):
"""
This returns a string containing playlist data in xspf format
"""
fw_playlist = Playlist.objects.get(id=playlist_id)
plt_tracks = fw_playlist.playlist_tracks.prefetch_related("track")
top = Element("playlist")
top.set("version", "1")
title_xspf = SubElement(top, "title")
title_xspf.text = fw_playlist.name
date_xspf = SubElement(top, "date")
date_xspf.text = str(fw_playlist.creation_date)
trackList_xspf = SubElement(top, "trackList")
for plt_track in plt_tracks:
track = plt_track.track
write_xspf_track_data(track, trackList_xspf)
return prettify(top)
def write_xspf_track_data(track, trackList_xspf):
"""
Insert a track into the trackList subelement of a xspf file
"""
track_xspf = SubElement(trackList_xspf, "track")
location_xspf = SubElement(track_xspf, "location")
location_xspf.text = "https://" + track.domain_name + track.listen_url
title_xspf = SubElement(track_xspf, "title")
title_xspf.text = str(track.title)
creator_xspf = SubElement(track_xspf, "creator")
creator_xspf.text = str(track.artist)
if str(track.album) == "[non-album tracks]":
return
else:
album_xspf = SubElement(track_xspf, "album")
album_xspf.text = str(track.album)
def prettify(elem):
"""
Return a pretty-printed XML string for the Element.
"""
rough_string = etree.tostring(elem, "utf-8")
reparsed = minidom.parseString(rough_string)
return reparsed.toprettyxml(indent=" ")

Wyświetl plik

@ -1,14 +1,19 @@
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.music.models import Track
from funkwhale_api.music.models import Album, Artist, Track
from funkwhale_api.music.serializers import TrackSerializer
from funkwhale_api.users.serializers import UserBasicSerializer
from . import models
import logging
logger = logging.getLogger(__name__)
class PlaylistTrackSerializer(serializers.ModelSerializer):
# track = TrackSerializer()
@ -120,3 +125,57 @@ class PlaylistAddManySerializer(serializers.Serializer):
class Meta:
fields = "allow_duplicates"
class XspfTrackSerializer(serializers.Serializer):
location = serializers.CharField(allow_blank=True, required=False)
title = serializers.CharField()
creator = serializers.CharField()
album = serializers.CharField(allow_blank=True, required=False)
duration = serializers.CharField(allow_blank=True, required=False)
def validate(self, data):
artist = data["creator"]
title = data["title"]
album = data.get("album", None)
try:
artist_id = Artist.objects.get(name=artist)
except ObjectDoesNotExist:
raise ValidationError("Couldn't find artist in the database")
if album:
try:
album_id = Album.objects.get(title=album)
fw_track = Track.objects.get(
title=title, artist=artist_id.id, album=album_id
)
except ObjectDoesNotExist:
pass
try:
fw_track = Track.objects.get(title=title, artist=artist_id.id)
except ObjectDoesNotExist as e:
raise ValidationError(f"Couldn't find track in the database : {e!r}")
return fw_track
class XspfSerializer(serializers.Serializer):
title = serializers.CharField()
creator = serializers.CharField(allow_blank=True, required=False)
creation_date = serializers.DateTimeField(required=False)
version = serializers.IntegerField(required=False)
tracks = XspfTrackSerializer(many=True, required=False)
def create(self, validated_data):
pl = models.Playlist.objects.create(
name=validated_data["title"],
privacy_level="private",
user=validated_data["request"].user,
)
pl.insert_many(validated_data["tracks"])
return pl
def update(self, instance, validated_data):
instance.name = validated_data["title"]
instance.save()
return instance

Wyświetl plik

@ -0,0 +1,8 @@
from funkwhale_api.common import routers
from . import views
router = routers.OptionalSlashRouter()
router.register(r"playlists", views.PlaylistViewSet, "playlists")
urlpatterns = router.urls

Wyświetl plik

@ -0,0 +1,9 @@
from funkwhale_api.common import routers
from . import views
router = routers.OptionalSlashRouter()
router.register(r"playlists", views.PlaylistViewSet, "playlists")
urlpatterns = router.urls

Wyświetl plik

@ -1,15 +1,23 @@
from django.db import transaction
from django.db.models import Count
from django.http import HttpResponse, FileResponse
from drf_spectacular.utils import extend_schema
from rest_framework import exceptions, mixins, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.parsers import JSONParser, FormParser, MultiPartParser
from funkwhale_api.common import fields, permissions
from funkwhale_api.music import utils as music_utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import serializers as music_serializers
from funkwhale_api.users.oauth import permissions as oauth_permissions
from . import filters, models, serializers
from . import filters, models, serializers, renderers, parsers
class PlaylistViewSet(
@ -37,6 +45,65 @@ class PlaylistViewSet(
owner_checks = ["write"]
filterset_class = filters.PlaylistFilter
ordering_fields = ("id", "name", "creation_date", "modification_date")
parser_classes = [parsers.XspfParser, JSONParser, FormParser, MultiPartParser]
# https://docs.djangoproject.com/en/5.0/topics/class-based-views/generic-editing/#content-negotiation-example
# https://docs.djangoproject.com/en/5.0/ref/request-response/#django.http.HttpRequest.__iter__
def create(self, request, *args, **kwargs):
content_type = request.headers.get("Content-Type")
if content_type and "application/octet-stream" in content_type:
for track_data in request.data.get("tracks", []):
track_serializer = serializers.XspfTrackSerializer(data=track_data)
if not track_serializer.is_valid():
request.data["tracks"].remove(track_data)
serializer = serializers.XspfSerializer(data=request.data)
serializer.is_valid()
pl = serializer.save(request=request)
return Response(serializers.PlaylistSerializer(pl).data, status=201)
response = super().create(request, *args, **kwargs)
return response
def retrieve(self, request, pk, *args, **kwargs):
content_type = request.headers.get("Content-Type")
if content_type and "application/octet-stream" in content_type:
request.accepted_renderer = renderers.PlaylistXspfRenderer()
# https://docs.djangoproject.com/en/5.0/ref/request-response/#telling-the-browser-to-treat-the-response-as-a-file-attachment
# shoud we use https://docs.djangoproject.com/en/5.0/ref/request-response/#fileresponse-objects ?
# eg FileResponse(xspf, as_attachment=True, filename=f"{playlist.name}.xspf")
# return Response(playlist, content_type="xspf")
pl = self.get_object()
return Response(
serializers.PlaylistSerializer(pl).data,
headers={
"Content-Type": "application/octet-stream",
"Content-Disposition": f'attachment; filename="{self.get_object().name}.xspf"',
},
)
return super().retrieve(request, *args, **kwargs)
def update(self, request, *args, **kwargs):
playlist = self.get_object()
content_type = request.headers.get("Content-Type")
if content_type and "application/octet-stream" in content_type:
tracks = []
for track_data in request.data.get("tracks", []):
track_serializer = serializers.XspfTrackSerializer(data=track_data)
if track_serializer.is_valid():
tracks.append(track_serializer.validated_data)
else:
request.data["tracks"].remove(track_data)
playlist.playlist_tracks.all().delete()
playlist.insert_many(tracks)
serializer = serializers.XspfSerializer(playlist, data=request.data)
serializer.is_valid()
pl = serializer.save(request=request)
return Response(serializers.PlaylistSerializer(pl).data, status=201)
else:
return super().retrieve(request, *args, **kwargs)
@extend_schema(responses=serializers.PlaylistTrackSerializer(many=True))
@action(methods=["get"], detail=True)
@ -140,3 +207,35 @@ class PlaylistViewSet(
return Response(status=404)
playlist.insert(plt, to_index)
return Response(status=204)
@extend_schema(operation_id="get_playlist_releases")
@action(methods=["get"], detail=True)
@transaction.atomic
def releases(self, request, *args, **kwargs):
playlist = self.get_object()
try:
releases_pks = playlist.playlist_tracks.values_list(
"track__album__pk", flat=True
).distinct()
except models.PlaylistTrack.DoesNotExist:
return Response(status=404)
releases = music_models.Album.objects.filter(pk__in=releases_pks)
serializer = music_serializers.AlbumSerializer(data=releases, many=True)
serializer.is_valid()
return Response(serializer.data, status=200)
@extend_schema(operation_id="get_playlist_artits")
@action(methods=["get"], detail=True)
@transaction.atomic
def artists(self, request, *args, **kwargs):
playlist = self.get_object()
try:
artists_pks = playlist.playlist_tracks.values_list(
"track__artist__pk", flat=True
).distinct()
except models.PlaylistTrack.DoesNotExist:
return Response(status=404)
artists = music_models.Artist.objects.filter(pk__in=artists_pks)
serializer = music_serializers.SimpleArtistSerializer(data=artists, many=True)
serializer.is_valid()
return Response(serializer.data, status=200)

Wyświetl plik

@ -0,0 +1,19 @@
<?xml version="1.0" ?>
<playlist version="1" xmlns="http://xspf.org/ns/0/">
<title>Test</title>
<date>1312-01-08T17:10:47-05:00</date>
<trackList>
<track>
<location>https://maldonado-boyd.com/api/v1/listen/c5a0f3b9-1866-4258-98bc-1c6867d0b125/</location>
<title>Opinel 12</title>
<creator>Davinhor</creator>
<album>Racisme en pls</album>
</track>
<track>
<location>https://wilson.com/api/v1/listen/9fb8ee3f-caa3-4c7f-b7c1-2fd645f738f4/</location>
<title>lettre a la republique</title>
<creator>Kery James</creator>
<album>Racisme en pls</album>
</track>
</trackList>
</playlist>

Wyświetl plik

@ -0,0 +1,17 @@
from defusedxml import ElementTree as etree
from funkwhale_api.playlists import renderers, serializers
def test_generate_xspf_from_playlist(factories):
playlist_track = factories["playlists.PlaylistTrack"]()
playlist = playlist_track.playlist
xspf_test = renderers.PlaylistXspfRenderer().render(
serializers.PlaylistSerializer(playlist).data
)
tree = etree.fromstring(xspf_test)
# track1 = playlist_factory.playlist_tracks.get(id=1)
# track1_name = track1.track
track1_title = playlist_track.track.title
# assert playlist.name == tree.findtext("./title")
assert track1_title == tree.findtext("./trackList/track/title")

Wyświetl plik

@ -0,0 +1,120 @@
import json
from defusedxml import ElementTree as etree
from django.urls import reverse
from django.shortcuts import resolve_url
def test_can_get_playlists_list(factories, logged_in_api_client):
factories["playlists.Playlist"].create_batch(5)
url = reverse("api:v2:playlists:playlists-list")
headers = {"Content-Type": "application/json"}
response = logged_in_api_client.get(url, headers=headers)
data = json.loads(response.content)
assert response.status_code == 200
assert data["count"] == 5
def test_can_get_playlists_octet_stream(factories, logged_in_api_client):
pl = factories["playlists.Playlist"]()
factories["playlists.PlaylistTrack"](playlist=pl)
factories["playlists.PlaylistTrack"](playlist=pl)
factories["playlists.PlaylistTrack"](playlist=pl)
url = reverse("api:v2:playlists:playlists-detail", kwargs={"pk": pl.pk})
headers = {"Content-Type": "application/octet-stream"}
response = logged_in_api_client.get(url, headers=headers)
el = etree.fromstring(response.content)
assert response.status_code == 200
assert el.findtext("./title") == pl.name
def test_can_get_playlists_json(factories, logged_in_api_client):
pl = factories["playlists.Playlist"]()
url = reverse("api:v2:playlists:playlists-detail", kwargs={"pk": pl.pk})
response = logged_in_api_client.get(url, format="json")
assert response.status_code == 200
assert response.data["name"] == pl.name
def test_can_get_user_playlists_list(factories, logged_in_api_client):
user = factories["users.User"]()
factories["playlists.Playlist"](user=user)
url = reverse("api:v2:playlists:playlists-list")
url = resolve_url(url) + "?user=me"
response = logged_in_api_client.get(url)
data = json.loads(response.content.decode("utf-8"))
assert response.status_code == 200
assert data["count"] == 1
def test_can_post_user_playlists(factories, logged_in_api_client):
playlist = {"name": "Les chiennes de l'hexagone", "privacy_level": "me"}
url = reverse("api:v2:playlists:playlists-list")
response = logged_in_api_client.post(url, playlist, format="json")
data = json.loads(response.content.decode("utf-8"))
assert response.status_code == 201
assert data["name"] == "Les chiennes de l'hexagone"
assert data["privacy_level"] == "me"
def test_can_post_playlists_octet_stream(factories, logged_in_api_client):
artist = factories["music.Artist"](name="Davinhor")
album = factories["music.Album"](title="Racisme en pls", artist=artist)
factories["music.Track"](title="Opinel 12", artist=artist, album=album)
url = reverse("api:v2:playlists:playlists-list")
data = open("./tests/playlists/test.xspf", "rb").read()
response = logged_in_api_client.post(url, data=data, format="xspf")
data = json.loads(response.content)
assert response.status_code == 201
assert data["name"] == "Test"
def test_can_patch_playlists_octet_stream(factories, logged_in_api_client):
pl = factories["playlists.Playlist"](user=logged_in_api_client.user)
artist = factories["music.Artist"](name="Davinhor")
album = factories["music.Album"](title="Racisme en pls", artist=artist)
track = factories["music.Track"](title="Opinel 12", artist=artist, album=album)
url = reverse("api:v2:playlists:playlists-detail", kwargs={"pk": pl.pk})
data = open("./tests/playlists/test.xspf", "rb").read()
response = logged_in_api_client.patch(url, data=data, format="xspf")
pl.refresh_from_db()
assert response.status_code == 201
assert pl.name == "Test"
assert pl.playlist_tracks.all()[0].track.title == track.title
def test_can_get_playlists_track(factories, logged_in_api_client):
pl = factories["playlists.Playlist"]()
plt = factories["playlists.PlaylistTrack"](playlist=pl)
url = reverse("api:v2:playlists:playlists-tracks", kwargs={"pk": pl.pk})
response = logged_in_api_client.get(url)
data = json.loads(response.content.decode("utf-8"))
assert response.status_code == 200
assert data["count"] == 1
assert data["results"][0]["track"]["title"] == plt.track.title
def test_can_get_playlists_releases(factories, logged_in_api_client):
playlist = factories["playlists.Playlist"]()
plt = factories["playlists.PlaylistTrack"](playlist=playlist)
url = reverse("api:v2:playlists:playlists-releases", kwargs={"pk": playlist.pk})
response = logged_in_api_client.get(url)
data = json.loads(response.content)
assert response.status_code == 200
assert data[0]["title"] == plt.track.album.title
def test_can_get_playlists_artists(factories, logged_in_api_client):
playlist = factories["playlists.Playlist"]()
plt = factories["playlists.PlaylistTrack"](playlist=playlist)
url = reverse("api:v2:playlists:playlists-artists", kwargs={"pk": playlist.pk})
response = logged_in_api_client.get(url)
data = json.loads(response.content)
assert response.status_code == 200
assert data[0]["name"] == plt.track.artist.name

Wyświetl plik

@ -20,7 +20,7 @@ def test_serializer_includes_tracks_count(factories, logged_in_api_client):
factories["playlists.PlaylistTrack"](playlist=playlist)
url = reverse("api:v1:playlists-detail", kwargs={"pk": playlist.pk})
response = logged_in_api_client.get(url)
response = logged_in_api_client.get(url, content_type="application/json")
assert response.data["tracks_count"] == 1
@ -32,7 +32,7 @@ def test_serializer_includes_tracks_count_986(factories, logged_in_api_client):
3, track=plt.track, library__privacy_level="everyone", import_status="finished"
)
url = reverse("api:v1:playlists-detail", kwargs={"pk": playlist.pk})
response = logged_in_api_client.get(url)
response = logged_in_api_client.get(url, content_type="application/json")
assert response.data["tracks_count"] == 1
@ -42,7 +42,7 @@ def test_serializer_includes_is_playable(factories, logged_in_api_client):
factories["playlists.PlaylistTrack"](playlist=playlist)
url = reverse("api:v1:playlists-detail", kwargs={"pk": playlist.pk})
response = logged_in_api_client.get(url)
response = logged_in_api_client.get(url, content_type="application/json")
assert response.data["is_playable"] is False
@ -78,7 +78,7 @@ def test_only_can_add_track_on_own_playlist_via_api(factories, logged_in_api_cli
url = reverse("api:v1:playlists-add", kwargs={"pk": playlist.pk})
data = {"tracks": [track.pk]}
response = logged_in_api_client.post(url, data, format="json")
response = logged_in_api_client.post(url, data, content_type="application/json")
assert response.status_code == 404
assert playlist.playlist_tracks.count() == 0

Wyświetl plik

@ -0,0 +1 @@
Add backend logic to handle xspf file to import/export playlist (#836)

Wyświetl plik

@ -25,7 +25,7 @@ services:
env_file:
- .env.dev
- .env
image: postgres:${POSTGRES_VERSION-11}-alpine
image: postgres:${POSTGRES_VERSION-12}-alpine
environment:
- "POSTGRES_HOST_AUTH_METHOD=trust"
command: postgres ${POSTGRES_ARGS-}