From 641430289950d470b4e1b273217458b860d29b46 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 29 Nov 2023 21:12:23 +0100 Subject: [PATCH 01/16] implement listening and favorite sync with listenbrainz Part-of: --- api/config/plugins.py | 17 +++ api/config/settings/common.py | 1 + .../contrib/listenbrainz/funkwhale_ready.py | 102 +++++++++++++++++- .../contrib/listenbrainz/funkwhale_startup.py | 43 +++++++- .../contrib/listenbrainz/tasks.py | 90 ++++++++++++++++ .../0002_trackfavorite_from_listenbrainz.py | 18 ++++ api/funkwhale_api/favorites/models.py | 1 + api/funkwhale_api/favorites/views.py | 12 +++ .../0003_listening_from_listenbrainz.py | 18 ++++ api/funkwhale_api/history/models.py | 1 + api/funkwhale_api/typesense/utils.py | 1 - api/tests/contrib/__init__.py | 0 api/tests/contrib/listenbrainz/__init__.py | 0 .../contrib/listenbrainz/test_listenbrainz.py | 35 ++++++ api/tests/contrib/listenbrainz/test_tasks.py | 48 +++++++++ 15 files changed, 382 insertions(+), 5 deletions(-) create mode 100644 api/funkwhale_api/contrib/listenbrainz/tasks.py create mode 100644 api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py create mode 100644 api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py create mode 100644 api/tests/contrib/__init__.py create mode 100644 api/tests/contrib/listenbrainz/__init__.py create mode 100644 api/tests/contrib/listenbrainz/test_listenbrainz.py create mode 100644 api/tests/contrib/listenbrainz/test_tasks.py diff --git a/api/config/plugins.py b/api/config/plugins.py index ddd5afbb5..7df681d99 100644 --- a/api/config/plugins.py +++ b/api/config/plugins.py @@ -303,6 +303,23 @@ LISTENING_CREATED = "listening_created" """ Called when a track is being listened """ +LISTENING_SYNC = "listening_sync" +""" +Called by the task manager to trigger listening sync +""" +FAVORITE_CREATED = "favorite_created" +""" +Called when a track is being liked +""" +FAVORITE_DELETED = "favorite_deleted" +""" +Called when a favorite track is being unliked +""" +FAVORITE_SYNC = "favorite_sync" +""" +Called by the task manager to trigger favorite sync +""" + SCAN = "scan" """ diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 5865bf9a0..46ab61652 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -276,6 +276,7 @@ LOCAL_APPS = ( # Your stuff: custom apps go here "funkwhale_api.instance", "funkwhale_api.audio", + "funkwhale_api.contrib.listenbrainz", "funkwhale_api.music", "funkwhale_api.requests", "funkwhale_api.favorites", diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index 8671c129b..64b1b34ad 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -2,23 +2,40 @@ import liblistenbrainz from django.utils import timezone import funkwhale_api -from config import plugins +import pylistenbrainz +<<<<<<< HEAD +======= +from config import plugins +from django.utils import timezone + +from . import tasks +>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) from .funkwhale_startup import PLUGIN +from funkwhale_api.history import models as history_models +from funkwhale_api.favorites import models as favorites_models + @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN) def submit_listen(listening, conf, **kwargs): user_token = conf["user_token"] - if not user_token: + if not user_token and not conf["submit_listenings"]: return logger = PLUGIN["logger"] logger.info("Submitting listen to ListenBrainz") +<<<<<<< HEAD client = liblistenbrainz.ListenBrainz() client.set_auth_token(user_token) listen = get_listen(listening.track) +======= + + listen = get_listen(listening.track) + client = pylistenbrainz.ListenBrainz() + client.set_auth_token(user_token) +>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) client.submit_single_listen(listen) @@ -48,10 +65,91 @@ def get_listen(track): if upload: additional_info["duration"] = upload.duration +<<<<<<< HEAD return liblistenbrainz.Listen( +======= + return pylistenbrainz.Listen( +>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) track_name=track.title, artist_name=track.artist.name, listened_at=int(timezone.now()), release_name=release_name, additional_info=additional_info, ) +<<<<<<< HEAD +======= + + +@plugins.register_hook(plugins.FAVORITE_CREATED, PLUGIN) +def submit_favorite_creation(track_favorite, conf, **kwargs): + user_token = conf["user_token"] + if not user_token or not conf["submit_favorites"]: + return + logger = PLUGIN["logger"] + logger.info("Submitting favorite to ListenBrainz") + client = pylistenbrainz.ListenBrainz() + track = get_listen(track_favorite.track) + if not track.mbid: + logger.warning( + "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz" + ) + return + # client.feedback(track, 1) + + +@plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN) +def submit_favorite_deletion(track_favorite, conf, **kwargs): + user_token = conf["user_token"] + if not user_token or not conf["submit_favorites"]: + return + logger = PLUGIN["logger"] + logger.info("Submitting favorite deletion to ListenBrainz") + client = pylistenbrainz.ListenBrainz() + track = get_listen(track_favorite.track) + if not track.mbid: + logger.warning( + "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" + ) + return + # client.feedback(track, 0) + + +@plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN) +def sync_listenings_from_listenbrainz(user, conf): + user_name = conf["user_name"] + user_token = conf["user_token"] + + if not user_name or not conf["sync_listenings"]: + return + logger = PLUGIN["logger"] + logger.info("Getting listenings from ListenBrainz") + try: + last_ts = ( + history_models.Listening.objects.filter(user=user) + .filter(from_listenbrainz=True) + .latest("creation_date") + .values_list("creation_date", flat=True) + ) + except history_models.Listening.DoesNotExist: + tasks.import_listenbrainz_listenings(user, user_name, ts=0) + + tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts) + + +@plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN) +def sync_favorites_from_listenbrainz(user, conf): + user_name = conf["user_name"] + user_token = conf["user_token"] + + if not user_name or not conf["sync_favorites"]: + return + try: + last_ts = ( + favorites_models.TrackFavorite.objects.filter(user=user) + .filter(from_listenbrainz=True) + .latest("creation_date") + .values_list("creation_date", flat=True) + ) + except history_models.Listening.DoesNotExist: + tasks.import_listenbrainz_favorites(user, user_name, last_ts) +>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py index 16f58b3f4..b4ca600c3 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py @@ -3,7 +3,7 @@ from config import plugins PLUGIN = plugins.get_plugin_config( name="listenbrainz", label="ListenBrainz", - description="A plugin that allows you to submit your listens to ListenBrainz.", + description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.", homepage="https://docs.funkwhale.audio/users/builtinplugins.html#listenbrainz-plugin", # noqa version="0.3", user=True, @@ -13,6 +13,45 @@ PLUGIN = plugins.get_plugin_config( "type": "text", "label": "Your ListenBrainz user token", "help": "You can find your user token in your ListenBrainz profile at https://listenbrainz.org/profile/", - } + }, + { + "name": "user_name", + "type": "text", + "required": False, + "label": "Your ListenBrainz user name.", + "help": "It's needed for synchronisation with Listenbrainz (import listenings and favorites) \ + but not to send activities", + }, + { + "name": "submit_listenings", + "type": "boolean", + "default": True, + "label": "Enable listenings submission to Listenbrainz", + "help": "If enable, your listening from Funkwhale will be imported into ListenBrainz.", + }, + { + "name": "sync_listenings", + "type": "boolean", + "default": False, + "label": "Enable listenings sync", + "help": "If enable, your listening from Listenbrainz will be imported into Funkwhale. This means they \ + will be used has any other funkwhale listenings to filter out recently listened content or \ + generate recomendations", + }, + { + "name": "sync_facorites", + "type": "boolean", + "default": False, + "label": "Enable favorite sync", + "help": "If enable, your favorites from Listenbrainz will be imported into Funkwhale. This means they \ + will be used has any other funkwhale favorites (Ui display, federatipon activity)", + }, + { + "name": "submit_favorites", + "type": "boolean", + "default": False, + "label": "Enable favorite submition to Listenbrainz services", + "help": "If enable, your favorites from Funkwhale will be submit to Listenbrainz", + }, ], ) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py new file mode 100644 index 000000000..a498bb380 --- /dev/null +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -0,0 +1,90 @@ +import datetime +import pylistenbrainz + +from config import plugins +from django.utils import timezone + +from funkwhale_api.users import models +from funkwhale_api.taskapp import celery +from funkwhale_api.history import models as history_models +from funkwhale_api.music import models as music_models + + +@celery.app.task(name="listenbrainz.trigger_listening_sync_with_listenbrainz") +def trigger_listening_sync_with_listenbrainz(): + now = timezone.now() + active_month = now - datetime.timedelta(days=30) + users = ( + models.User.objects.filter(plugins__code="listenbrainz") + .filter(plugins__conf__sync_listenings=True) + .filter(last_activity__gte=active_month) + ) + for user in users: + plugins.trigger_hook( + plugins.LISTENING_SYNC, + user=user, + confs=plugins.get_confs(user), + ) + + +@celery.app.task(name="listenbrainz.trigger_favorite_sync_with_listenbrainz") +def trigger_favorite_sync_with_listenbrainz(): + now = timezone.now() + active_month = now - datetime.timedelta(days=30) + users = ( + models.User.objects.filter(plugins__code="listenbrainz") + .filter(plugins__conf__sync_listenings=True) + .filter(last_activity__gte=active_month) + ) + for user in users: + plugins.trigger_hook( + plugins.FAVORITE_SYNC, + user=user, + confs=plugins.get_confs(user), + ) + + +@celery.app.task(name="listenbrainz.import_listenbrainz_listenings") +def import_listenbrainz_listenings(user, user_name, ts): + client = pylistenbrainz.ListenBrainz() + listens = client.get_listens(username=user_name, min_ts=ts, count=100) + add_lb_listenings_to_db(listens, user) + new_ts = 13 + last_ts = 12 + while new_ts != last_ts: + last_ts = listens[0].listened_at + listens = client.get_listens(username=user_name, min_ts=new_ts, count=100) + new_ts = listens[0].listened_at + add_lb_listenings_to_db(listens, user) + + +def add_lb_listenings_to_db(listens, user): + fw_listens = [] + for listen in listens: + if ( + listen.additional_info.get("submission_client") + and listen.additional_info.get("submission_client") + == "Funkwhale ListenBrainz plugin" + ): + continue + try: + track = music_models.Track.objects.get(mbid=listen.recording_mbid) + except music_models.Track.DoesNotExist: + # to do : resolve non mbid listens ? + continue + + user = user + fw_listen = history_models.Listening( + creation_date=listen.listened_at, + track=track, + user=user, + from_listenbrainz=True, + ) + fw_listens.append(fw_listen) + + history_models.Listening.objects.bulk_create(fw_listens) + + +@celery.app.task(name="listenbrainz.import_listenbrainz_favorites") +def import_listenbrainz_favorites(): + return "to do" diff --git a/api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py new file mode 100644 index 000000000..f982f38d8 --- /dev/null +++ b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-11-29 15:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('favorites', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='trackfavorite', + name='from_listenbrainz', + field=models.BooleanField(default=None, null=True), + ), + ] diff --git a/api/funkwhale_api/favorites/models.py b/api/funkwhale_api/favorites/models.py index 5cba39f35..ca8051a31 100644 --- a/api/funkwhale_api/favorites/models.py +++ b/api/funkwhale_api/favorites/models.py @@ -12,6 +12,7 @@ class TrackFavorite(models.Model): track = models.ForeignKey( Track, related_name="track_favorites", on_delete=models.CASCADE ) + from_listenbrainz = models.BooleanField(default=None, null=True) class Meta: unique_together = ("track", "user") diff --git a/api/funkwhale_api/favorites/views.py b/api/funkwhale_api/favorites/views.py index 6a87d3b15..4edae860f 100644 --- a/api/funkwhale_api/favorites/views.py +++ b/api/funkwhale_api/favorites/views.py @@ -1,3 +1,5 @@ +from config import plugins + from django.db.models import Prefetch from drf_spectacular.utils import extend_schema from rest_framework import mixins, status, viewsets @@ -44,6 +46,11 @@ class TrackFavoriteViewSet( instance = self.perform_create(serializer) serializer = self.get_serializer(instance=instance) headers = self.get_success_headers(serializer.data) + plugins.trigger_hook( + plugins.FAVORITE_CREATED, + track_favorite=serializer.instance, + confs=plugins.get_confs(self.request.user), + ) record.send(instance) return Response( serializer.data, status=status.HTTP_201_CREATED, headers=headers @@ -76,6 +83,11 @@ class TrackFavoriteViewSet( except (AttributeError, ValueError, models.TrackFavorite.DoesNotExist): return Response({}, status=400) favorite.delete() + plugins.trigger_hook( + plugins.FAVORITE_DELETED, + track_favorite=favorite, + confs=plugins.get_confs(self.request.user), + ) return Response([], status=status.HTTP_204_NO_CONTENT) @extend_schema( diff --git a/api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py b/api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py new file mode 100644 index 000000000..3a9168c05 --- /dev/null +++ b/api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-11-29 15:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0002_auto_20180325_1433'), + ] + + operations = [ + migrations.AddField( + model_name='listening', + name='from_listenbrainz', + field=models.BooleanField(default=None, null=True), + ), + ] diff --git a/api/funkwhale_api/history/models.py b/api/funkwhale_api/history/models.py index 648c16cd8..7afcc0b2c 100644 --- a/api/funkwhale_api/history/models.py +++ b/api/funkwhale_api/history/models.py @@ -17,6 +17,7 @@ class Listening(models.Model): on_delete=models.CASCADE, ) session_key = models.CharField(max_length=100, null=True, blank=True) + from_listenbrainz = models.BooleanField(default=None, null=True) class Meta: ordering = ("-creation_date",) diff --git a/api/funkwhale_api/typesense/utils.py b/api/funkwhale_api/typesense/utils.py index 4e2d4b70d..324377bdf 100644 --- a/api/funkwhale_api/typesense/utils.py +++ b/api/funkwhale_api/typesense/utils.py @@ -36,7 +36,6 @@ def delete_non_alnum_characters(text): def resolve_recordings_to_fw_track(recordings): """ Tries to match a troi recording entity to a fw track using the typesense index. - It will save the results in the match_mbid attribute of the Track table. For test purposes : if multiple fw tracks are returned, we log the information but only keep the best result in db to avoid duplicates. """ diff --git a/api/tests/contrib/__init__.py b/api/tests/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/contrib/listenbrainz/__init__.py b/api/tests/contrib/listenbrainz/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py new file mode 100644 index 000000000..c47365c74 --- /dev/null +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -0,0 +1,35 @@ +import pytest +from django.urls import reverse +from config import plugins +from funkwhale_api.history import models as history_models + + +def test_listenbrainz_submit_listen(logged_in_client, mocker, factories): + plugin = plugins.get_plugin_config( + name="listenbrainz", + description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.", + conf=[], + source=False, + ) + handler = mocker.Mock() + plugins.register_hook(plugins.LISTENING_CREATED, plugin)(handler) + plugins.set_conf( + "listenbrainz", + { + "sync_listenings": True, + "sync_facorites": True, + "submit_favorites": True, + "sync_favorites": True, + "user_token": "blablabla", + }, + user=logged_in_client.user, + ) + plugins.enable_conf("listenbrainz", True, logged_in_client.user) + + track = factories["music.Track"]() + url = reverse("api:v1:history:listenings-list") + logged_in_client.post(url, {"track": track.pk}) + response = logged_in_client.get(url) + listening = history_models.Listening.objects.get(user=logged_in_client.user) + handler.assert_called_once_with(listening=listening, conf=None) + # why conf=none ? diff --git a/api/tests/contrib/listenbrainz/test_tasks.py b/api/tests/contrib/listenbrainz/test_tasks.py new file mode 100644 index 000000000..b53ee02a6 --- /dev/null +++ b/api/tests/contrib/listenbrainz/test_tasks.py @@ -0,0 +1,48 @@ +import datetime +import pytest + +import pylistenbrainz +from funkwhale_api.contrib.listenbrainz import tasks +from funkwhale_api.history import models as history_models + + +def test_import_listenbrainz_listenings(factories, mocker): + factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476") + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + + listens = [ + pylistenbrainz.utils.Listen( + track_name="test", + artist_name="artist_test", + recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", + additional_info={"submission_client": "not funkwhale"}, + listened_at=datetime.datetime.fromtimestamp(-3124224000), + ), + pylistenbrainz.utils.Listen( + track_name="test2", + artist_name="artist_test2", + recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", + additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, + listened_at=datetime.datetime.fromtimestamp(1871), + ), + pylistenbrainz.utils.Listen( + track_name="test3", + artist_name="artist_test3", + listened_at=0, + ), + ] + + mocker.patch.object( + tasks.pylistenbrainz.ListenBrainz, "get_listens", return_value=listens + ) + user = factories["users.User"]() + + tasks.import_listenbrainz_listenings(user, "user_name", ts=0) + + history_models.Listening.objects.filter( + track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476" + ).exists() + + assert not history_models.Listening.objects.filter( + track__mbid="54c60860-f43d-484e-b691-7ab7ec8de559" + ).exists() From 17c4a92f77c728c6b179a260f5ee13d941ce0c93 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Thu, 30 Nov 2023 19:38:25 +0100 Subject: [PATCH 02/16] lint Part-of: --- .../contrib/listenbrainz/funkwhale_ready.py | 21 ------------------- .../contrib/listenbrainz/tasks.py | 7 +++---- api/funkwhale_api/favorites/views.py | 3 +-- api/tests/conftest.py | 12 ++++++++++- api/tests/contrib/listenbrainz/test_tasks.py | 3 ++- changes/changelog.d/2079.enhancement | 1 + 6 files changed, 18 insertions(+), 29 deletions(-) create mode 100644 changes/changelog.d/2079.enhancement diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index 64b1b34ad..250019d00 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -4,13 +4,6 @@ from django.utils import timezone import funkwhale_api import pylistenbrainz -<<<<<<< HEAD -======= -from config import plugins -from django.utils import timezone - -from . import tasks ->>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) from .funkwhale_startup import PLUGIN from funkwhale_api.history import models as history_models @@ -25,17 +18,10 @@ def submit_listen(listening, conf, **kwargs): logger = PLUGIN["logger"] logger.info("Submitting listen to ListenBrainz") -<<<<<<< HEAD client = liblistenbrainz.ListenBrainz() client.set_auth_token(user_token) listen = get_listen(listening.track) -======= - - listen = get_listen(listening.track) - client = pylistenbrainz.ListenBrainz() - client.set_auth_token(user_token) ->>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) client.submit_single_listen(listen) @@ -65,19 +51,13 @@ def get_listen(track): if upload: additional_info["duration"] = upload.duration -<<<<<<< HEAD return liblistenbrainz.Listen( -======= - return pylistenbrainz.Listen( ->>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) track_name=track.title, artist_name=track.artist.name, listened_at=int(timezone.now()), release_name=release_name, additional_info=additional_info, ) -<<<<<<< HEAD -======= @plugins.register_hook(plugins.FAVORITE_CREATED, PLUGIN) @@ -152,4 +132,3 @@ def sync_favorites_from_listenbrainz(user, conf): ) except history_models.Listening.DoesNotExist: tasks.import_listenbrainz_favorites(user, user_name, last_ts) ->>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index a498bb380..a61a2c864 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -1,13 +1,12 @@ import datetime import pylistenbrainz -from config import plugins from django.utils import timezone - -from funkwhale_api.users import models -from funkwhale_api.taskapp import celery +from config import plugins from funkwhale_api.history import models as history_models from funkwhale_api.music import models as music_models +from funkwhale_api.taskapp import celery +from funkwhale_api.users import models @celery.app.task(name="listenbrainz.trigger_listening_sync_with_listenbrainz") diff --git a/api/funkwhale_api/favorites/views.py b/api/funkwhale_api/favorites/views.py index 4edae860f..1fecf3e5a 100644 --- a/api/funkwhale_api/favorites/views.py +++ b/api/funkwhale_api/favorites/views.py @@ -1,11 +1,10 @@ -from config import plugins - from django.db.models import Prefetch from drf_spectacular.utils import extend_schema from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.response import Response +from config import plugins from funkwhale_api.activity import record from funkwhale_api.common import fields, permissions from funkwhale_api.music import utils as music_utils diff --git a/api/tests/conftest.py b/api/tests/conftest.py index d241db5a7..0d5be2b79 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -27,7 +27,7 @@ from funkwhale_api.activity import record from funkwhale_api.federation import actors from funkwhale_api.moderation import mrf from funkwhale_api.music import licenses - +from funkwhale_api.contrib import listenbrainz from . import utils as test_utils pytest_plugins = "aiohttp.pytest_plugin" @@ -277,6 +277,16 @@ def disabled_musicbrainz(mocker): ) +# @pytest.fixture() +# def disabled_listenbrainz(mocker): +# # we ensure no listenbrainz requests gets out +# yield mocker.patch.object( +# listenbrainz.client.ListenBrainzClient, +# "_submit", +# return_value=None, +# ) + + @pytest.fixture(autouse=True) def r_mock(requests_mock): """ diff --git a/api/tests/contrib/listenbrainz/test_tasks.py b/api/tests/contrib/listenbrainz/test_tasks.py index b53ee02a6..7f9ee04df 100644 --- a/api/tests/contrib/listenbrainz/test_tasks.py +++ b/api/tests/contrib/listenbrainz/test_tasks.py @@ -1,7 +1,8 @@ import datetime -import pytest import pylistenbrainz +import pytest + from funkwhale_api.contrib.listenbrainz import tasks from funkwhale_api.history import models as history_models diff --git a/changes/changelog.d/2079.enhancement b/changes/changelog.d/2079.enhancement new file mode 100644 index 000000000..6c9cf7e31 --- /dev/null +++ b/changes/changelog.d/2079.enhancement @@ -0,0 +1 @@ +Add favorite and listening sync ith Listenbrainz (#2079) From f45fd1e465d37614bd1a8e7b424ffd59bb848116 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 13 Dec 2023 16:31:26 +0100 Subject: [PATCH 03/16] various reviews Part-of: --- .../contrib/listenbrainz/funkwhale_ready.py | 4 ++-- .../contrib/listenbrainz/tasks.py | 24 +++++++++++++++---- .../0002_trackfavorite_from_listenbrainz.py | 18 -------------- api/funkwhale_api/favorites/models.py | 2 +- .../0003_listening_from_listenbrainz.py | 18 -------------- api/funkwhale_api/history/models.py | 2 +- 6 files changed, 24 insertions(+), 44 deletions(-) delete mode 100644 api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py delete mode 100644 api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index 250019d00..cd164354c 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -106,7 +106,7 @@ def sync_listenings_from_listenbrainz(user, conf): try: last_ts = ( history_models.Listening.objects.filter(user=user) - .filter(from_listenbrainz=True) + .filter(source="Listenbrainz") .latest("creation_date") .values_list("creation_date", flat=True) ) @@ -126,7 +126,7 @@ def sync_favorites_from_listenbrainz(user, conf): try: last_ts = ( favorites_models.TrackFavorite.objects.filter(user=user) - .filter(from_listenbrainz=True) + .filter(source="Listenbrainz") .latest("creation_date") .values_list("creation_date", flat=True) ) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index a61a2c864..1c63100d6 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -8,6 +8,8 @@ from funkwhale_api.music import models as music_models from funkwhale_api.taskapp import celery from funkwhale_api.users import models +from .funkwhale_startup import PLUGIN + @celery.app.task(name="listenbrainz.trigger_listening_sync_with_listenbrainz") def trigger_listening_sync_with_listenbrainz(): @@ -64,20 +66,34 @@ def add_lb_listenings_to_db(listens, user): listen.additional_info.get("submission_client") and listen.additional_info.get("submission_client") == "Funkwhale ListenBrainz plugin" + and history_models.Listening.objects.filter( + creation_date=listen.listened_at + ).exists() ): continue + + mbid = ( + listen.mbid_mapping + if hasattr(listen, "mbid_mapping") + else listen.recording_mbid + ) + + if not mbid: + logger = PLUGIN["logger"] + logger.info("Received listening doesn't have a mbid. Skipping...") + try: - track = music_models.Track.objects.get(mbid=listen.recording_mbid) + track = music_models.Track.objects.get(mbid=mbid) except music_models.Track.DoesNotExist: - # to do : resolve non mbid listens ? + logger.info("Received listening doesn't exist in fw database. Skipping...") continue user = user fw_listen = history_models.Listening( - creation_date=listen.listened_at, + creation_date=timezone.make_aware(listen.listened_at), track=track, user=user, - from_listenbrainz=True, + source="Listenbrainz", ) fw_listens.append(fw_listen) diff --git a/api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py deleted file mode 100644 index f982f38d8..000000000 --- a/api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.20 on 2023-11-29 15:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('favorites', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='trackfavorite', - name='from_listenbrainz', - field=models.BooleanField(default=None, null=True), - ), - ] diff --git a/api/funkwhale_api/favorites/models.py b/api/funkwhale_api/favorites/models.py index ca8051a31..bd0f8be0b 100644 --- a/api/funkwhale_api/favorites/models.py +++ b/api/funkwhale_api/favorites/models.py @@ -12,7 +12,7 @@ class TrackFavorite(models.Model): track = models.ForeignKey( Track, related_name="track_favorites", on_delete=models.CASCADE ) - from_listenbrainz = models.BooleanField(default=None, null=True) + source = models.CharField(max_length=100, null=True, blank=True) class Meta: unique_together = ("track", "user") diff --git a/api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py b/api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py deleted file mode 100644 index 3a9168c05..000000000 --- a/api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.20 on 2023-11-29 15:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('history', '0002_auto_20180325_1433'), - ] - - operations = [ - migrations.AddField( - model_name='listening', - name='from_listenbrainz', - field=models.BooleanField(default=None, null=True), - ), - ] diff --git a/api/funkwhale_api/history/models.py b/api/funkwhale_api/history/models.py index 7afcc0b2c..046115d07 100644 --- a/api/funkwhale_api/history/models.py +++ b/api/funkwhale_api/history/models.py @@ -17,7 +17,7 @@ class Listening(models.Model): on_delete=models.CASCADE, ) session_key = models.CharField(max_length=100, null=True, blank=True) - from_listenbrainz = models.BooleanField(default=None, null=True) + source = models.CharField(max_length=100, null=True, blank=True) class Meta: ordering = ("-creation_date",) From 37acfa475dd777b65980dd5395495b946fc25938 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 20 Dec 2023 00:08:48 +0100 Subject: [PATCH 04/16] loads of things Part-of: --- .../contrib/listenbrainz/funkwhale_ready.py | 19 ++- .../contrib/listenbrainz/tasks.py | 73 +++++++++- .../migrations/0002_trackfavorite_source.py | 18 +++ .../migrations/0003_listening_source.py | 18 +++ .../contrib/listenbrainz/test_listenbrainz.py | 130 ++++++++++++++++++ api/tests/contrib/listenbrainz/test_tasks.py | 46 +------ 6 files changed, 250 insertions(+), 54 deletions(-) create mode 100644 api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py create mode 100644 api/funkwhale_api/history/migrations/0003_listening_source.py diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index cd164354c..a675b481b 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -25,7 +25,8 @@ def submit_listen(listening, conf, **kwargs): client.submit_single_listen(listen) -def get_listen(track): +def get_listen(listening): + track = listening.track additional_info = { "media_player": "Funkwhale", "media_player_version": funkwhale_api.__version__, @@ -54,7 +55,7 @@ def get_listen(track): return liblistenbrainz.Listen( track_name=track.title, artist_name=track.artist.name, - listened_at=int(timezone.now()), + listened_at=listening.creation_date.timestamp(), release_name=release_name, additional_info=additional_info, ) @@ -74,7 +75,7 @@ def submit_favorite_creation(track_favorite, conf, **kwargs): "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz" ) return - # client.feedback(track, 1) + client.submit_user_feedback(1, track.mbid) @plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN) @@ -91,13 +92,12 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs): "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" ) return - # client.feedback(track, 0) + client.submit_user_feedback(0, track.mbid) @plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN) def sync_listenings_from_listenbrainz(user, conf): user_name = conf["user_name"] - user_token = conf["user_token"] if not user_name or not conf["sync_listenings"]: return @@ -110,8 +110,10 @@ def sync_listenings_from_listenbrainz(user, conf): .latest("creation_date") .values_list("creation_date", flat=True) ) + last_ts.timestamp() except history_models.Listening.DoesNotExist: tasks.import_listenbrainz_listenings(user, user_name, ts=0) + return tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts) @@ -119,7 +121,6 @@ def sync_listenings_from_listenbrainz(user, conf): @plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN) def sync_favorites_from_listenbrainz(user, conf): user_name = conf["user_name"] - user_token = conf["user_token"] if not user_name or not conf["sync_favorites"]: return @@ -130,5 +131,9 @@ def sync_favorites_from_listenbrainz(user, conf): .latest("creation_date") .values_list("creation_date", flat=True) ) + last_ts.timestamp() except history_models.Listening.DoesNotExist: - tasks.import_listenbrainz_favorites(user, user_name, last_ts) + tasks.import_listenbrainz_favorites(user, user_name, ts=0) + return + + tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index 1c63100d6..2835a8484 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -3,6 +3,7 @@ import pylistenbrainz from django.utils import timezone from config import plugins +from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models from funkwhale_api.music import models as music_models from funkwhale_api.taskapp import celery @@ -53,13 +54,24 @@ def import_listenbrainz_listenings(user, user_name, ts): new_ts = 13 last_ts = 12 while new_ts != last_ts: - last_ts = listens[0].listened_at + last_ts = max( + listens, + key=lambda obj: datetime.datetime.fromtimestamp( + obj.listened_at, timezone.utc + ), + ) listens = client.get_listens(username=user_name, min_ts=new_ts, count=100) - new_ts = listens[0].listened_at + new_ts = max( + listens, + key=lambda obj: datetime.datetime.fromtimestamp( + obj.listened_at, timezone.utc + ), + ) add_lb_listenings_to_db(listens, user) def add_lb_listenings_to_db(listens, user): + logger = PLUGIN["logger"] fw_listens = [] for listen in listens: if ( @@ -67,9 +79,14 @@ def add_lb_listenings_to_db(listens, user): and listen.additional_info.get("submission_client") == "Funkwhale ListenBrainz plugin" and history_models.Listening.objects.filter( - creation_date=listen.listened_at + creation_date=datetime.datetime.fromtimestamp( + listen.listened_at, timezone.utc + ) ).exists() ): + logger.info( + f"Listen with ts {listen.listened_at} skipped because already in db" + ) continue mbid = ( @@ -79,7 +96,6 @@ def add_lb_listenings_to_db(listens, user): ) if not mbid: - logger = PLUGIN["logger"] logger.info("Received listening doesn't have a mbid. Skipping...") try: @@ -90,7 +106,9 @@ def add_lb_listenings_to_db(listens, user): user = user fw_listen = history_models.Listening( - creation_date=timezone.make_aware(listen.listened_at), + creation_date=datetime.datetime.fromtimestamp( + listen.listened_at, timezone.utc + ), track=track, user=user, source="Listenbrainz", @@ -101,5 +119,46 @@ def add_lb_listenings_to_db(listens, user): @celery.app.task(name="listenbrainz.import_listenbrainz_favorites") -def import_listenbrainz_favorites(): - return "to do" +def import_listenbrainz_favorites(user, user_name, last_sync): + client = pylistenbrainz.ListenBrainz() + last_ts = int(datetime.datetime.now(timezone.utc).timestamp()) + offset = 0 + while last_ts >= last_sync: + feedbacks = client.get_user_feedback(username=user_name, offset=offset) + add_lb_feedback_to_db(feedbacks, user) + offset = feedbacks.count + last_ts = max( + feedbacks.feedback, + key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc), + ) + # to do implement offset in pylb + + +def add_lb_feedback_to_db(feedbacks, user): + logger = PLUGIN["logger"] + fw_listens = [] + for feedback in feedbacks.feedback: + try: + track = music_models.Track.objects.get(mbid=feedback.recording_mbid) + except music_models.Track.DoesNotExist: + logger.info( + "Received feedback track doesn't exist in fw database. Skipping..." + ) + continue + + if feedback.score == 1: + favorites_models.TrackFavorite.objects.get_or_create( + user=user, + creation_date=datetime.datetime.fromtimestamp( + feedback.created, timezone.utc + ), + track=track, + source="Listenbrainz", + ) + elif feedback.score == 0: + try: + favorites_models.TrackFavorite.objects.delete(user=user, track=track) + except favorites_models.TrackFavorite.DoesNotExist: + continue + elif feedback.score == -1: + logger.info("Funkwhale doesn't support hate yet <3") diff --git a/api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py new file mode 100644 index 000000000..67a397aa0 --- /dev/null +++ b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-12-09 14:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('favorites', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='trackfavorite', + name='source', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/api/funkwhale_api/history/migrations/0003_listening_source.py b/api/funkwhale_api/history/migrations/0003_listening_source.py new file mode 100644 index 000000000..b50bc6220 --- /dev/null +++ b/api/funkwhale_api/history/migrations/0003_listening_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-12-09 14:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0002_auto_20180325_1433'), + ] + + operations = [ + migrations.AddField( + model_name='listening', + name='source', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index c47365c74..623c6b5b3 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -1,6 +1,15 @@ +import datetime +import logging + import pytest +import pylistenbrainz + from django.urls import reverse +from django.utils import timezone + from config import plugins +from funkwhale_api.contrib.listenbrainz import funkwhale_ready +from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models @@ -33,3 +42,124 @@ def test_listenbrainz_submit_listen(logged_in_client, mocker, factories): listening = history_models.Listening.objects.get(user=logged_in_client.user) handler.assert_called_once_with(listening=listening, conf=None) # why conf=none ? + + +def test_sync_listenings_from_listenbrainz(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476") + track = factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + factories["history.Listening"]( + creation_date=datetime.datetime.fromtimestamp(1871, timezone.utc), track=track + ) + + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "sync_listenings": True, + } + + listens = [ + pylistenbrainz.Listen( + track_name="test", + artist_name="artist_test", + recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", + additional_info={"submission_client": "not funkwhale"}, + listened_at=-3124224000, + ), + pylistenbrainz.Listen( + track_name="test2", + artist_name="artist_test2", + recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", + additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, + listened_at=1871, + ), + pylistenbrainz.Listen( + track_name="test3", + artist_name="artist_test3", + listened_at=0, + ), + ] + + mocker.patch.object( + funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + "get_listens", + return_value=listens, + ) + + funkwhale_ready.sync_listenings_from_listenbrainz(user, conf) + + assert history_models.Listening.objects.filter( + track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476" + ).exists() + + assert "Listen with ts 1871 skipped because already in db" in caplog.text + assert "Received listening doesn't have a mbid" in caplog.text + + +def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + + favorite = factories["favorites.TrackFavorite"](track=track) + + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "sync_favorites": True, + } + + feedbacks = { + "count": 5, + "feedback": [ + { + "created": 1701116226, + "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528", + "score": 1, + "user_id": user.username, + }, + { + "created": 1701116214, + "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986", + "score": 0, + "user_id": user.username, + }, + { + "created": 1690775094, + "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7", + "score": -1, + "user_id": user.username, + }, + { + "created": 1690775093, + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ", + "score": 1, + "user_id": user.username, + }, + ], + "offset": 0, + "total_count": 4, + } + mocker.patch.object( + funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + "get_user_feedback", + return_value=feedbacks, + ) + + funkwhale_ready.sync_favorites_from_listenbrainz(user, conf) + + assert favorites_models.TrackFavorite.objects.filter( + track__mbid="195565db-65f9-4d0d-b347-5f0c85509528" + ).exists() + with pytest.raises(deleted.DoesNotExist): + favorite.refresh_from_db() diff --git a/api/tests/contrib/listenbrainz/test_tasks.py b/api/tests/contrib/listenbrainz/test_tasks.py index 7f9ee04df..680662fe2 100644 --- a/api/tests/contrib/listenbrainz/test_tasks.py +++ b/api/tests/contrib/listenbrainz/test_tasks.py @@ -1,49 +1,15 @@ import datetime +import logging import pylistenbrainz import pytest +from django.utils import timezone + from funkwhale_api.contrib.listenbrainz import tasks from funkwhale_api.history import models as history_models -def test_import_listenbrainz_listenings(factories, mocker): - factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476") - factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") - - listens = [ - pylistenbrainz.utils.Listen( - track_name="test", - artist_name="artist_test", - recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", - additional_info={"submission_client": "not funkwhale"}, - listened_at=datetime.datetime.fromtimestamp(-3124224000), - ), - pylistenbrainz.utils.Listen( - track_name="test2", - artist_name="artist_test2", - recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", - additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, - listened_at=datetime.datetime.fromtimestamp(1871), - ), - pylistenbrainz.utils.Listen( - track_name="test3", - artist_name="artist_test3", - listened_at=0, - ), - ] - - mocker.patch.object( - tasks.pylistenbrainz.ListenBrainz, "get_listens", return_value=listens - ) - user = factories["users.User"]() - - tasks.import_listenbrainz_listenings(user, "user_name", ts=0) - - history_models.Listening.objects.filter( - track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476" - ).exists() - - assert not history_models.Listening.objects.filter( - track__mbid="54c60860-f43d-484e-b691-7ab7ec8de559" - ).exists() +def test_trigger_listening_sync_with_listenbrainz(): + # to do + pass From 5bc0171694cb32d55d4d2e55b61202e90e27a9ef Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 3 Jan 2024 16:46:12 +0100 Subject: [PATCH 05/16] delete test Part-of: --- api/tests/contrib/listenbrainz/test_tasks.py | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 api/tests/contrib/listenbrainz/test_tasks.py diff --git a/api/tests/contrib/listenbrainz/test_tasks.py b/api/tests/contrib/listenbrainz/test_tasks.py deleted file mode 100644 index 680662fe2..000000000 --- a/api/tests/contrib/listenbrainz/test_tasks.py +++ /dev/null @@ -1,15 +0,0 @@ -import datetime -import logging - -import pylistenbrainz -import pytest - -from django.utils import timezone - -from funkwhale_api.contrib.listenbrainz import tasks -from funkwhale_api.history import models as history_models - - -def test_trigger_listening_sync_with_listenbrainz(): - # to do - pass From 2a364d578547f73a4a1d85db0add1978fcb6fe43 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 5 Feb 2024 18:15:22 +0100 Subject: [PATCH 06/16] add favorite sync Part-of: --- api/config/settings/common.py | 10 + .../contrib/listenbrainz/funkwhale_ready.py | 43 ++-- .../contrib/listenbrainz/tasks.py | 70 +++--- api/funkwhale_api/favorites/factories.py | 2 + api/tests/conftest.py | 10 - .../contrib/listenbrainz/test_listenbrainz.py | 238 +++++++++++++++--- 6 files changed, 272 insertions(+), 101 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 46ab61652..4cbddfad1 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -950,6 +950,16 @@ CELERY_BEAT_SCHEDULE = { ), "options": {"expires": 60 * 60}, }, + "listenbrainz.trigger_listening_sync_with_listenbrainz": { + "task": "listenbrainz.trigger_listening_sync_with_listenbrainz", + "schedule": crontab(day_of_week="*", minute="0", hour="3"), + "options": {"expires": 60 * 60 * 24}, + }, + "listenbrainz.trigger_favorite_sync_with_listenbrainz": { + "task": "listenbrainz.trigger_favorite_sync_with_listenbrainz", + "schedule": crontab(day_of_week="*", minute="0", hour="3"), + "options": {"expires": 60 * 60 * 24}, + }, } if env.str("TYPESENSE_API_KEY", default=None): diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index a675b481b..5848ca746 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -1,14 +1,15 @@ -import liblistenbrainz -from django.utils import timezone - import funkwhale_api -import pylistenbrainz +import liblistenbrainz -from .funkwhale_startup import PLUGIN +from config import plugins +from django.utils import timezone from funkwhale_api.history import models as history_models from funkwhale_api.favorites import models as favorites_models +from .funkwhale_startup import PLUGIN +from . import tasks + @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN) def submit_listen(listening, conf, **kwargs): @@ -20,12 +21,12 @@ def submit_listen(listening, conf, **kwargs): logger.info("Submitting listen to ListenBrainz") client = liblistenbrainz.ListenBrainz() client.set_auth_token(user_token) - listen = get_listen(listening.track) + listen = get_lb_listen(listening) client.submit_single_listen(listen) -def get_listen(listening): +def get_lb_listen(listening): track = listening.track additional_info = { "media_player": "Funkwhale", @@ -68,11 +69,11 @@ def submit_favorite_creation(track_favorite, conf, **kwargs): return logger = PLUGIN["logger"] logger.info("Submitting favorite to ListenBrainz") - client = pylistenbrainz.ListenBrainz() - track = get_listen(track_favorite.track) + client = liblistenbrainz.ListenBrainz() + track = track_favorite.track if not track.mbid: logger.warning( - "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz" + "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" ) return client.submit_user_feedback(1, track.mbid) @@ -85,8 +86,8 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs): return logger = PLUGIN["logger"] logger.info("Submitting favorite deletion to ListenBrainz") - client = pylistenbrainz.ListenBrainz() - track = get_listen(track_favorite.track) + client = liblistenbrainz.ListenBrainz() + track = track_favorite.track if not track.mbid: logger.warning( "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" @@ -109,13 +110,12 @@ def sync_listenings_from_listenbrainz(user, conf): .filter(source="Listenbrainz") .latest("creation_date") .values_list("creation_date", flat=True) - ) - last_ts.timestamp() - except history_models.Listening.DoesNotExist: - tasks.import_listenbrainz_listenings(user, user_name, ts=0) + ).timestamp() + except funkwhale_api.history.models.Listening.DoesNotExist: + tasks.import_listenbrainz_listenings(user, user_name, 0) return - tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts) + tasks.import_listenbrainz_listenings(user, user_name, last_ts) @plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN) @@ -129,11 +129,10 @@ def sync_favorites_from_listenbrainz(user, conf): favorites_models.TrackFavorite.objects.filter(user=user) .filter(source="Listenbrainz") .latest("creation_date") - .values_list("creation_date", flat=True) + .creation_date.timestamp() ) - last_ts.timestamp() - except history_models.Listening.DoesNotExist: - tasks.import_listenbrainz_favorites(user, user_name, ts=0) + except favorites_models.TrackFavorite.DoesNotExist: + tasks.import_listenbrainz_favorites(user, user_name, 0) return - tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts) + tasks.import_listenbrainz_favorites(user, user_name, last_ts) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index 2835a8484..f23cd1cce 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -1,5 +1,5 @@ import datetime -import pylistenbrainz +import liblistenbrainz from django.utils import timezone from config import plugins @@ -47,27 +47,20 @@ def trigger_favorite_sync_with_listenbrainz(): @celery.app.task(name="listenbrainz.import_listenbrainz_listenings") -def import_listenbrainz_listenings(user, user_name, ts): - client = pylistenbrainz.ListenBrainz() - listens = client.get_listens(username=user_name, min_ts=ts, count=100) - add_lb_listenings_to_db(listens, user) - new_ts = 13 - last_ts = 12 - while new_ts != last_ts: - last_ts = max( - listens, - key=lambda obj: datetime.datetime.fromtimestamp( - obj.listened_at, timezone.utc - ), - ) - listens = client.get_listens(username=user_name, min_ts=new_ts, count=100) +def import_listenbrainz_listenings(user, user_name, since): + client = liblistenbrainz.ListenBrainz() + response = client.get_listens(username=user_name, min_ts=since, count=100) + listens = response["payload"]["listens"] + while listens: + add_lb_listenings_to_db(listens, user) new_ts = max( listens, key=lambda obj: datetime.datetime.fromtimestamp( obj.listened_at, timezone.utc ), ) - add_lb_listenings_to_db(listens, user) + response = client.get_listens(username=user_name, min_ts=new_ts, count=100) + listens = response["payload"]["listens"] def add_lb_listenings_to_db(listens, user): @@ -119,46 +112,51 @@ def add_lb_listenings_to_db(listens, user): @celery.app.task(name="listenbrainz.import_listenbrainz_favorites") -def import_listenbrainz_favorites(user, user_name, last_sync): - client = pylistenbrainz.ListenBrainz() - last_ts = int(datetime.datetime.now(timezone.utc).timestamp()) +def import_listenbrainz_favorites(user, user_name, since): + client = liblistenbrainz.ListenBrainz() + response = client.get_user_feedback(username=user_name) offset = 0 - while last_ts >= last_sync: - feedbacks = client.get_user_feedback(username=user_name, offset=offset) - add_lb_feedback_to_db(feedbacks, user) - offset = feedbacks.count - last_ts = max( - feedbacks.feedback, - key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc), - ) - # to do implement offset in pylb + while response["feedback"]: + count = response["count"] + offset = offset + count + last_sync = min( + response["feedback"], + key=lambda obj: datetime.datetime.fromtimestamp( + obj["created"], timezone.utc + ), + )["created"] + add_lb_feedback_to_db(response["feedback"], user) + if last_sync <= since or count == 0: + return + response = client.get_user_feedback(username=user_name, offset=offset) def add_lb_feedback_to_db(feedbacks, user): logger = PLUGIN["logger"] - fw_listens = [] - for feedback in feedbacks.feedback: + for feedback in feedbacks: try: - track = music_models.Track.objects.get(mbid=feedback.recording_mbid) + track = music_models.Track.objects.get(mbid=feedback["recording_mbid"]) except music_models.Track.DoesNotExist: logger.info( "Received feedback track doesn't exist in fw database. Skipping..." ) continue - if feedback.score == 1: + if feedback["score"] == 1: favorites_models.TrackFavorite.objects.get_or_create( user=user, creation_date=datetime.datetime.fromtimestamp( - feedback.created, timezone.utc + feedback["created"], timezone.utc ), track=track, source="Listenbrainz", ) - elif feedback.score == 0: + elif feedback["score"] == 0: try: - favorites_models.TrackFavorite.objects.delete(user=user, track=track) + favorites_models.TrackFavorite.objects.get( + user=user, track=track + ).delete() except favorites_models.TrackFavorite.DoesNotExist: continue - elif feedback.score == -1: + elif feedback["score"] == -1: logger.info("Funkwhale doesn't support hate yet <3") diff --git a/api/funkwhale_api/favorites/factories.py b/api/funkwhale_api/favorites/factories.py index df2f47335..c568a5a1d 100644 --- a/api/funkwhale_api/favorites/factories.py +++ b/api/funkwhale_api/favorites/factories.py @@ -1,4 +1,5 @@ import factory +from django.utils import timezone from funkwhale_api.factories import NoUpdateOnCreate, registry from funkwhale_api.music.factories import TrackFactory @@ -9,6 +10,7 @@ from funkwhale_api.users.factories import UserFactory class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory): track = factory.SubFactory(TrackFactory) user = factory.SubFactory(UserFactory) + creation_date = factory.Faker("date_time_this_decade", tzinfo=timezone.utc) class Meta: model = "favorites.TrackFavorite" diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 0d5be2b79..11496af7e 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -277,16 +277,6 @@ def disabled_musicbrainz(mocker): ) -# @pytest.fixture() -# def disabled_listenbrainz(mocker): -# # we ensure no listenbrainz requests gets out -# yield mocker.patch.object( -# listenbrainz.client.ListenBrainzClient, -# "_submit", -# return_value=None, -# ) - - @pytest.fixture(autouse=True) def r_mock(requests_mock): """ diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index 623c6b5b3..1efd6abd8 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -2,7 +2,7 @@ import datetime import logging import pytest -import pylistenbrainz +import liblistenbrainz from django.urls import reverse from django.utils import timezone @@ -62,32 +62,47 @@ def test_sync_listenings_from_listenbrainz(factories, mocker, caplog): "sync_listenings": True, } - listens = [ - pylistenbrainz.Listen( - track_name="test", - artist_name="artist_test", - recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", - additional_info={"submission_client": "not funkwhale"}, - listened_at=-3124224000, - ), - pylistenbrainz.Listen( - track_name="test2", - artist_name="artist_test2", - recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", - additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, - listened_at=1871, - ), - pylistenbrainz.Listen( - track_name="test3", - artist_name="artist_test3", - listened_at=0, - ), - ] + listens = { + "payload": { + "count": 25, + "user_id": "-- the MusicBrainz ID of the user --", + "listens": [ + liblistenbrainz.Listen( + track_name="test", + artist_name="artist_test", + recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", + additional_info={"submission_client": "not funkwhale"}, + listened_at=-3124224000, + ), + liblistenbrainz.Listen( + track_name="test2", + artist_name="artist_test2", + recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", + additional_info={ + "submission_client": "Funkwhale ListenBrainz plugin" + }, + listened_at=1871, + ), + liblistenbrainz.Listen( + track_name="test3", + artist_name="artist_test3", + listened_at=0, + ), + ], + } + } + no_more_listen = { + "payload": { + "count": 25, + "user_id": "Bilbo", + "listens": [], + } + } mocker.patch.object( - funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, "get_listens", - return_value=listens, + side_effect=[listens, no_more_listen], ) funkwhale_ready.sync_listenings_from_listenbrainz(user, conf) @@ -105,13 +120,20 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): caplog.set_level(logging.INFO) logger.addHandler(caplog.handler) user = factories["users.User"]() - + # track lb fav factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") - - factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + # random track + factories["music.Track"]() + # track lb neutral track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") - - favorite = factories["favorites.TrackFavorite"](track=track) + favorite = factories["favorites.TrackFavorite"](track=track, user=user) + # last_sync + track_last_sync = factories["music.Track"]( + mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7" + ) + favorite_last_sync = factories["favorites.TrackFavorite"]( + track=track_last_sync, source="Listenbrainz" + ) conf = { "user_name": user.username, @@ -135,6 +157,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): "user_id": user.username, }, { + # last sync "created": 1690775094, "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7", "score": -1, @@ -142,7 +165,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): }, { "created": 1690775093, - "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ", + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2", "score": 1, "user_id": user.username, }, @@ -150,10 +173,11 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): "offset": 0, "total_count": 4, } + empty_feedback = {"count": 0, "feedback": [], "offset": 0, "total_count": 0} mocker.patch.object( - funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, "get_user_feedback", - return_value=feedbacks, + side_effect=[feedbacks, empty_feedback], ) funkwhale_ready.sync_favorites_from_listenbrainz(user, conf) @@ -161,5 +185,153 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): assert favorites_models.TrackFavorite.objects.filter( track__mbid="195565db-65f9-4d0d-b347-5f0c85509528" ).exists() - with pytest.raises(deleted.DoesNotExist): + with pytest.raises(favorites_models.TrackFavorite.DoesNotExist): favorite.refresh_from_db() + + +def test_sync_favorites_from_listenbrainz_since(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + # track lb fav + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + # track lb neutral + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + favorite = factories["favorites.TrackFavorite"](track=track, user=user) + # track should be not synced + factories["music.Track"](mbid="1fd02cf2-7247-4715-8862-c378ec196000") + # last_sync + track_last_sync = factories["music.Track"]( + mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7" + ) + factories["favorites.TrackFavorite"]( + track=track_last_sync, + user=user, + source="Listenbrainz", + creation_date=datetime.datetime.fromtimestamp(1690775094), + ) + + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "sync_favorites": True, + } + + feedbacks = { + "count": 5, + "feedback": [ + { + "created": 1701116226, + "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528", + "score": 1, + "user_id": user.username, + }, + { + "created": 1701116214, + "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986", + "score": 0, + "user_id": user.username, + }, + { + # last sync + "created": 1690775094, + "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7", + "score": -1, + "user_id": user.username, + }, + { + "created": 1690775093, + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2", + "score": 1, + "user_id": user.username, + }, + ], + "offset": 0, + "total_count": 4, + } + second_feedback = { + "count": 0, + "feedback": [ + { + "created": 0, + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec196000", + "score": 1, + "user_id": user.username, + }, + ], + "offset": 0, + "total_count": 0, + } + mocker.patch.object( + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, + "get_user_feedback", + side_effect=[feedbacks, second_feedback], + ) + + funkwhale_ready.sync_favorites_from_listenbrainz(user, conf) + + assert favorites_models.TrackFavorite.objects.filter( + track__mbid="195565db-65f9-4d0d-b347-5f0c85509528" + ).exists() + assert not favorites_models.TrackFavorite.objects.filter( + track__mbid="1fd02cf2-7247-4715-8862-c378ec196000" + ).exists() + with pytest.raises(favorites_models.TrackFavorite.DoesNotExist): + favorite.refresh_from_db() + + +def test_submit_favorites_to_listenbrainz(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + + favorite = factories["favorites.TrackFavorite"](track=track) + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "submit_favorites": True, + } + + patch = mocker.patch.object( + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, + "submit_user_feedback", + return_value="Success", + ) + funkwhale_ready.submit_favorite_creation(favorite, conf) + + patch.assert_called_once_with(1, track.mbid) + + +def test_submit_favorites_deletion(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + + favorite = factories["favorites.TrackFavorite"](track=track) + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "submit_favorites": True, + } + + patch = mocker.patch.object( + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, + "submit_user_feedback", + return_value="Success", + ) + funkwhale_ready.submit_favorite_deletion(favorite, conf) + + patch.assert_called_once_with(0, track.mbid) From a03cc1db24fdf523bfb42b6b8f55ecb8985ccc9a Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 5 Feb 2024 18:30:44 +0100 Subject: [PATCH 07/16] lint Part-of: --- api/funkwhale_api/activity/utils.py | 2 ++ .../contrib/listenbrainz/funkwhale_ready.py | 12 ++++++------ api/funkwhale_api/contrib/listenbrainz/tasks.py | 2 ++ api/funkwhale_api/favorites/factories.py | 1 - api/tests/conftest.py | 1 - api/tests/contrib/listenbrainz/test_listenbrainz.py | 2 +- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/api/funkwhale_api/activity/utils.py b/api/funkwhale_api/activity/utils.py index 8d7cb3bd3..233f15a81 100644 --- a/api/funkwhale_api/activity/utils.py +++ b/api/funkwhale_api/activity/utils.py @@ -47,5 +47,7 @@ def get_activity(user, limit=20): "track", "user", "track__artist", "track__album__artist" ), ] + breakpoint() records = combined_recent(limit=limit, querysets=querysets) + return [r["object"] for r in records] diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index 5848ca746..d5d68e13e 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -1,14 +1,14 @@ -import funkwhale_api import liblistenbrainz -from config import plugins from django.utils import timezone -from funkwhale_api.history import models as history_models +import funkwhale_api +from config import plugins from funkwhale_api.favorites import models as favorites_models +from funkwhale_api.history import models as history_models -from .funkwhale_startup import PLUGIN from . import tasks +from .funkwhale_startup import PLUGIN @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN) @@ -73,7 +73,7 @@ def submit_favorite_creation(track_favorite, conf, **kwargs): track = track_favorite.track if not track.mbid: logger.warning( - "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" + "This tracks doesn't have a mbid. Feedback will not be submitted to Listenbrainz" ) return client.submit_user_feedback(1, track.mbid) @@ -90,7 +90,7 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs): track = track_favorite.track if not track.mbid: logger.warning( - "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" + "This tracks doesn't have a mbid. Feedback will not be submitted to Listenbrainz" ) return client.submit_user_feedback(0, track.mbid) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index f23cd1cce..5b60f74ba 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -1,7 +1,9 @@ import datetime + import liblistenbrainz from django.utils import timezone + from config import plugins from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models diff --git a/api/funkwhale_api/favorites/factories.py b/api/funkwhale_api/favorites/factories.py index c568a5a1d..871381eb4 100644 --- a/api/funkwhale_api/favorites/factories.py +++ b/api/funkwhale_api/favorites/factories.py @@ -10,7 +10,6 @@ from funkwhale_api.users.factories import UserFactory class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory): track = factory.SubFactory(TrackFactory) user = factory.SubFactory(UserFactory) - creation_date = factory.Faker("date_time_this_decade", tzinfo=timezone.utc) class Meta: model = "favorites.TrackFavorite" diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 11496af7e..41d9f63d7 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -27,7 +27,6 @@ from funkwhale_api.activity import record from funkwhale_api.federation import actors from funkwhale_api.moderation import mrf from funkwhale_api.music import licenses -from funkwhale_api.contrib import listenbrainz from . import utils as test_utils pytest_plugins = "aiohttp.pytest_plugin" diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index 1efd6abd8..a9802d509 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -1,9 +1,9 @@ import datetime import logging -import pytest import liblistenbrainz +import pytest from django.urls import reverse from django.utils import timezone From 05ec6f6d0fd2a7d2886b4827e2b3f86e621f61d4 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Thu, 15 Feb 2024 16:42:51 +0100 Subject: [PATCH 08/16] tests Part-of: --- api/funkwhale_api/activity/utils.py | 1 - .../contrib/listenbrainz/funkwhale_ready.py | 4 +--- .../contrib/listenbrainz/funkwhale_startup.py | 4 ++-- api/funkwhale_api/contrib/listenbrainz/tasks.py | 1 - api/funkwhale_api/favorites/factories.py | 1 - api/tests/conftest.py | 1 + api/tests/contrib/listenbrainz/test_listenbrainz.py | 12 ++++-------- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/api/funkwhale_api/activity/utils.py b/api/funkwhale_api/activity/utils.py index 233f15a81..28ddc2677 100644 --- a/api/funkwhale_api/activity/utils.py +++ b/api/funkwhale_api/activity/utils.py @@ -47,7 +47,6 @@ def get_activity(user, limit=20): "track", "user", "track__artist", "track__album__artist" ), ] - breakpoint() records = combined_recent(limit=limit, querysets=querysets) return [r["object"] for r in records] diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index d5d68e13e..02f698c8b 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -1,8 +1,6 @@ +import funkwhale_api import liblistenbrainz -from django.utils import timezone - -import funkwhale_api from config import plugins from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py index b4ca600c3..ce3878a56 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py @@ -36,7 +36,7 @@ PLUGIN = plugins.get_plugin_config( "label": "Enable listenings sync", "help": "If enable, your listening from Listenbrainz will be imported into Funkwhale. This means they \ will be used has any other funkwhale listenings to filter out recently listened content or \ - generate recomendations", + generate recommendations", }, { "name": "sync_facorites", @@ -50,7 +50,7 @@ PLUGIN = plugins.get_plugin_config( "name": "submit_favorites", "type": "boolean", "default": False, - "label": "Enable favorite submition to Listenbrainz services", + "label": "Enable favorite submission to Listenbrainz services", "help": "If enable, your favorites from Funkwhale will be submit to Listenbrainz", }, ], diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index 5b60f74ba..b3ba243ce 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -1,7 +1,6 @@ import datetime import liblistenbrainz - from django.utils import timezone from config import plugins diff --git a/api/funkwhale_api/favorites/factories.py b/api/funkwhale_api/favorites/factories.py index 871381eb4..df2f47335 100644 --- a/api/funkwhale_api/favorites/factories.py +++ b/api/funkwhale_api/favorites/factories.py @@ -1,5 +1,4 @@ import factory -from django.utils import timezone from funkwhale_api.factories import NoUpdateOnCreate, registry from funkwhale_api.music.factories import TrackFactory diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 41d9f63d7..d241db5a7 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -27,6 +27,7 @@ from funkwhale_api.activity import record from funkwhale_api.federation import actors from funkwhale_api.moderation import mrf from funkwhale_api.music import licenses + from . import utils as test_utils pytest_plugins = "aiohttp.pytest_plugin" diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index a9802d509..2af27da06 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -2,7 +2,6 @@ import datetime import logging import liblistenbrainz - import pytest from django.urls import reverse from django.utils import timezone @@ -14,14 +13,14 @@ from funkwhale_api.history import models as history_models def test_listenbrainz_submit_listen(logged_in_client, mocker, factories): - plugin = plugins.get_plugin_config( + config = plugins.get_plugin_config( name="listenbrainz", description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.", conf=[], source=False, ) handler = mocker.Mock() - plugins.register_hook(plugins.LISTENING_CREATED, plugin)(handler) + plugins.register_hook(plugins.LISTENING_CREATED, config)(handler) plugins.set_conf( "listenbrainz", { @@ -38,10 +37,9 @@ def test_listenbrainz_submit_listen(logged_in_client, mocker, factories): track = factories["music.Track"]() url = reverse("api:v1:history:listenings-list") logged_in_client.post(url, {"track": track.pk}) - response = logged_in_client.get(url) + logged_in_client.get(url) listening = history_models.Listening.objects.get(user=logged_in_client.user) handler.assert_called_once_with(listening=listening, conf=None) - # why conf=none ? def test_sync_listenings_from_listenbrainz(factories, mocker, caplog): @@ -131,9 +129,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): track_last_sync = factories["music.Track"]( mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7" ) - favorite_last_sync = factories["favorites.TrackFavorite"]( - track=track_last_sync, source="Listenbrainz" - ) + factories["favorites.TrackFavorite"](track=track_last_sync, source="Listenbrainz") conf = { "user_name": user.username, From 547bd6f37191c8cc61e66ba6f94a1ccb5a198274 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Thu, 15 Feb 2024 18:23:51 +0100 Subject: [PATCH 09/16] lint Part-of: --- api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index 02f698c8b..175c87904 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -1,6 +1,6 @@ -import funkwhale_api import liblistenbrainz +import funkwhale_api from config import plugins from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models From 67f74d40a662af83d7d95929a0af4ea6bf168108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20Ainsworth?= Date: Fri, 5 Apr 2024 11:13:25 +0000 Subject: [PATCH 10/16] Add ListenBrainz sync documentation Part-of: --- docs/user/plugins/listenbrainz.md | 49 +++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/user/plugins/listenbrainz.md b/docs/user/plugins/listenbrainz.md index d15cbfe7b..d2b828feb 100644 --- a/docs/user/plugins/listenbrainz.md +++ b/docs/user/plugins/listenbrainz.md @@ -19,8 +19,8 @@ The **ListenBrainz** plugin enables you to submit ({term}`scrobble`) ::: -:::{tab-item} Desktop -:sync: desktop +:::{tab-item} Mobile +:sync: mobile 1. Log in to your account. 2. Select the cog icon ({fa}`cog`) or your avatar to open the {guilabel}`Options` menu. @@ -36,3 +36,48 @@ The **ListenBrainz** plugin enables you to submit ({term}`scrobble`) :::: That's it! You've set up the **ListenBrainz** plugin. When you listen to tracks, the plugin sends the information to ListenBrainz. + +## Enable data synchronization + +The ListenBrainz plugin supports synchronizing listenings and track favorites with Funkwhale. To enable support for synchronization: + +::::{tab-set} + +:::{tab-item} Desktop +:sync: desktop + +1. Log in to your account. +2. Select the cog icon ({fa}`cog`) or your avatar to expand the user menu. +3. Select {guilabel}`Settings`. +4. Scroll down to the {guilabel}`Plugins` section. +5. Select {guilabel}`Manage plugins`. +6. Find the {guilabel}`ListenBrainz` plugin. +7. Enter {guilabel}`Your ListenBrainz user name`. You can find this on your ListenBrainz profile. +8. Select the data you want to synchronize. The following options are available: + - {guilabel}`Enable listenings submission to ListenBrainz`: submit your Funkwhale listens to ListenBrainz. + - {guilabel}`Enable listenings sync`: pull listening data from ListenBrainz into Funkwhale. + - {guilabel}`Enable favorite submission to ListenBrainz services`: submit your Funkwhale favorites activity to ListenBrainz. + - {guilabel}`Enable favorite sync`: pull favorites data from ListenBrainz into Funkwhale. +9. Select {guilabel}`Save`. + +::: + +:::{tab-item} Mobile +:sync: mobile + +1. Log in to your account. +2. Select the cog icon ({fa}`cog`) or your avatar to open the {guilabel}`Options` menu. +3. Select {guilabel}`Settings`. +4. Scroll down to the {guilabel}`Plugins` section. +5. Select {guilabel}`Manage plugins`. +6. Find the {guilabel}`ListenBrainz` plugin. +7. Enter {guilabel}`Your ListenBrainz user name`. You can find this on your ListenBrainz profile. +8. Select the data you want to synchronize. The following options are available: + - {guilabel}`Enable listenings submission to ListenBrainz`: submit your Funkwhale listens to ListenBrainz. + - {guilabel}`Enable listenings sync`: pull listening data from ListenBrainz into Funkwhale. + - {guilabel}`Enable favorite submission to ListenBrainz services`: submit your Funkwhale favorites activity to ListenBrainz. + - {guilabel}`Enable favorite sync`: pull favorites data from ListenBrainz into Funkwhale. +9. Select {guilabel}`Save`. + +::: +:::: From e028e8788baa528d44127bf350248b861ba8fa77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20Ainsworth?= Date: Mon, 8 Apr 2024 11:53:07 +0000 Subject: [PATCH 11/16] Apply 1 suggestion(s) to 1 file(s) Part-of: --- api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py index ce3878a56..a7184ff9a 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py @@ -19,7 +19,7 @@ PLUGIN = plugins.get_plugin_config( "type": "text", "required": False, "label": "Your ListenBrainz user name.", - "help": "It's needed for synchronisation with Listenbrainz (import listenings and favorites) \ + "help": "Required for importing listenings and favorites with ListenBrainz \ but not to send activities", }, { From b624fea2fab9e482c8fee3b7c9c6d401f94f8c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20Ainsworth?= Date: Mon, 8 Apr 2024 12:00:27 +0000 Subject: [PATCH 12/16] Apply 1 suggestion(s) to 1 file(s) Part-of: --- api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py index a7184ff9a..5eaee5f2b 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py @@ -26,7 +26,7 @@ PLUGIN = plugins.get_plugin_config( "name": "submit_listenings", "type": "boolean", "default": True, - "label": "Enable listenings submission to Listenbrainz", + "label": "Enable listening submission to ListenBrainz", "help": "If enable, your listening from Funkwhale will be imported into ListenBrainz.", }, { From 1b15fea1ab35dcb2477c2ac8523411d1bd4c2918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20Ainsworth?= Date: Mon, 8 Apr 2024 12:01:55 +0000 Subject: [PATCH 13/16] Apply 1 suggestion(s) to 1 file(s) Part-of: --- api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py index 5eaee5f2b..22b771e4c 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py @@ -27,7 +27,7 @@ PLUGIN = plugins.get_plugin_config( "type": "boolean", "default": True, "label": "Enable listening submission to ListenBrainz", - "help": "If enable, your listening from Funkwhale will be imported into ListenBrainz.", + "help": "If enabled, your listenings from Funkwhale will be imported into ListenBrainz.", }, { "name": "sync_listenings", From 97e24bcaa647baa869d36e2817556b768020631c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20Ainsworth?= Date: Mon, 8 Apr 2024 12:03:31 +0000 Subject: [PATCH 14/16] Apply 12 suggestion(s) to 4 file(s) Part-of: --- api/config/plugins.py | 4 ++-- .../contrib/listenbrainz/funkwhale_startup.py | 14 +++++++------- api/funkwhale_api/contrib/listenbrainz/tasks.py | 8 ++++---- .../contrib/listenbrainz/test_listenbrainz.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/config/plugins.py b/api/config/plugins.py index 7df681d99..cfdeabb3e 100644 --- a/api/config/plugins.py +++ b/api/config/plugins.py @@ -309,11 +309,11 @@ Called by the task manager to trigger listening sync """ FAVORITE_CREATED = "favorite_created" """ -Called when a track is being liked +Called when a track is being favorited """ FAVORITE_DELETED = "favorite_deleted" """ -Called when a favorite track is being unliked +Called when a favorited track is being unfavorited """ FAVORITE_SYNC = "favorite_sync" """ diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py index 22b771e4c..fef3235ed 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py @@ -34,24 +34,24 @@ PLUGIN = plugins.get_plugin_config( "type": "boolean", "default": False, "label": "Enable listenings sync", - "help": "If enable, your listening from Listenbrainz will be imported into Funkwhale. This means they \ - will be used has any other funkwhale listenings to filter out recently listened content or \ + "help": "If enabled, your listening from ListenBrainz will be imported into Funkwhale. This means they \ + will be used along with Funkwhale listenings to filter out recently listened content or \ generate recommendations", }, { - "name": "sync_facorites", + "name": "sync_favorites", "type": "boolean", "default": False, "label": "Enable favorite sync", - "help": "If enable, your favorites from Listenbrainz will be imported into Funkwhale. This means they \ - will be used has any other funkwhale favorites (Ui display, federatipon activity)", + "help": "If enabled, your favorites from ListenBrainz will be imported into Funkwhale. This means they \ + will be used along with Funkwhale favorites (UI display, federation activity)", }, { "name": "submit_favorites", "type": "boolean", "default": False, - "label": "Enable favorite submission to Listenbrainz services", - "help": "If enable, your favorites from Funkwhale will be submit to Listenbrainz", + "label": "Enable favorite submission to ListenBrainz services", + "help": "If enabled, your favorites from Funkwhale will be submitted to ListenBrainz", }, ], ) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index b3ba243ce..d43786678 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -90,12 +90,12 @@ def add_lb_listenings_to_db(listens, user): ) if not mbid: - logger.info("Received listening doesn't have a mbid. Skipping...") + logger.info("Received listening that doesn't have a mbid. Skipping...") try: track = music_models.Track.objects.get(mbid=mbid) except music_models.Track.DoesNotExist: - logger.info("Received listening doesn't exist in fw database. Skipping...") + logger.info("Received listening that doesn't exist in fw database. Skipping...") continue user = user @@ -139,7 +139,7 @@ def add_lb_feedback_to_db(feedbacks, user): track = music_models.Track.objects.get(mbid=feedback["recording_mbid"]) except music_models.Track.DoesNotExist: logger.info( - "Received feedback track doesn't exist in fw database. Skipping..." + "Received feedback track that doesn't exist in fw database. Skipping..." ) continue @@ -160,4 +160,4 @@ def add_lb_feedback_to_db(feedbacks, user): except favorites_models.TrackFavorite.DoesNotExist: continue elif feedback["score"] == -1: - logger.info("Funkwhale doesn't support hate yet <3") + logger.info("Funkwhale doesn't support disliked tracks") diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index 2af27da06..e917be7c9 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -25,7 +25,7 @@ def test_listenbrainz_submit_listen(logged_in_client, mocker, factories): "listenbrainz", { "sync_listenings": True, - "sync_facorites": True, + "sync_favorites": True, "submit_favorites": True, "sync_favorites": True, "user_token": "blablabla", From 4fc73c1430edd0d88cc3ccb4cd1d04862a1defda Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 8 Apr 2024 14:12:51 +0200 Subject: [PATCH 15/16] lint Part-of: --- api/funkwhale_api/contrib/listenbrainz/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index d43786678..85420c281 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -95,7 +95,9 @@ def add_lb_listenings_to_db(listens, user): try: track = music_models.Track.objects.get(mbid=mbid) except music_models.Track.DoesNotExist: - logger.info("Received listening that doesn't exist in fw database. Skipping...") + logger.info( + "Received listening that doesn't exist in fw database. Skipping..." + ) continue user = user From ba5b657b61246bae15c773aa60c906c1c9915f21 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 8 Apr 2024 14:36:12 +0200 Subject: [PATCH 16/16] lint Part-of: --- api/tests/contrib/listenbrainz/test_listenbrainz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index e917be7c9..7cf504707 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -110,7 +110,7 @@ def test_sync_listenings_from_listenbrainz(factories, mocker, caplog): ).exists() assert "Listen with ts 1871 skipped because already in db" in caplog.text - assert "Received listening doesn't have a mbid" in caplog.text + assert "Received listening that doesn't have a mbid. Skipping..." in caplog.text def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):