From 2a364d578547f73a4a1d85db0add1978fcb6fe43 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 5 Feb 2024 18:15:22 +0100 Subject: [PATCH] add favorite sync Part-of: --- api/config/settings/common.py | 10 + .../contrib/listenbrainz/funkwhale_ready.py | 43 ++-- .../contrib/listenbrainz/tasks.py | 70 +++--- api/funkwhale_api/favorites/factories.py | 2 + api/tests/conftest.py | 10 - .../contrib/listenbrainz/test_listenbrainz.py | 238 +++++++++++++++--- 6 files changed, 272 insertions(+), 101 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 46ab61652..4cbddfad1 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -950,6 +950,16 @@ CELERY_BEAT_SCHEDULE = { ), "options": {"expires": 60 * 60}, }, + "listenbrainz.trigger_listening_sync_with_listenbrainz": { + "task": "listenbrainz.trigger_listening_sync_with_listenbrainz", + "schedule": crontab(day_of_week="*", minute="0", hour="3"), + "options": {"expires": 60 * 60 * 24}, + }, + "listenbrainz.trigger_favorite_sync_with_listenbrainz": { + "task": "listenbrainz.trigger_favorite_sync_with_listenbrainz", + "schedule": crontab(day_of_week="*", minute="0", hour="3"), + "options": {"expires": 60 * 60 * 24}, + }, } if env.str("TYPESENSE_API_KEY", default=None): diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index a675b481b..5848ca746 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -1,14 +1,15 @@ -import liblistenbrainz -from django.utils import timezone - import funkwhale_api -import pylistenbrainz +import liblistenbrainz -from .funkwhale_startup import PLUGIN +from config import plugins +from django.utils import timezone from funkwhale_api.history import models as history_models from funkwhale_api.favorites import models as favorites_models +from .funkwhale_startup import PLUGIN +from . import tasks + @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN) def submit_listen(listening, conf, **kwargs): @@ -20,12 +21,12 @@ def submit_listen(listening, conf, **kwargs): logger.info("Submitting listen to ListenBrainz") client = liblistenbrainz.ListenBrainz() client.set_auth_token(user_token) - listen = get_listen(listening.track) + listen = get_lb_listen(listening) client.submit_single_listen(listen) -def get_listen(listening): +def get_lb_listen(listening): track = listening.track additional_info = { "media_player": "Funkwhale", @@ -68,11 +69,11 @@ def submit_favorite_creation(track_favorite, conf, **kwargs): return logger = PLUGIN["logger"] logger.info("Submitting favorite to ListenBrainz") - client = pylistenbrainz.ListenBrainz() - track = get_listen(track_favorite.track) + client = liblistenbrainz.ListenBrainz() + track = track_favorite.track if not track.mbid: logger.warning( - "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz" + "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" ) return client.submit_user_feedback(1, track.mbid) @@ -85,8 +86,8 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs): return logger = PLUGIN["logger"] logger.info("Submitting favorite deletion to ListenBrainz") - client = pylistenbrainz.ListenBrainz() - track = get_listen(track_favorite.track) + client = liblistenbrainz.ListenBrainz() + track = track_favorite.track if not track.mbid: logger.warning( "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" @@ -109,13 +110,12 @@ def sync_listenings_from_listenbrainz(user, conf): .filter(source="Listenbrainz") .latest("creation_date") .values_list("creation_date", flat=True) - ) - last_ts.timestamp() - except history_models.Listening.DoesNotExist: - tasks.import_listenbrainz_listenings(user, user_name, ts=0) + ).timestamp() + except funkwhale_api.history.models.Listening.DoesNotExist: + tasks.import_listenbrainz_listenings(user, user_name, 0) return - tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts) + tasks.import_listenbrainz_listenings(user, user_name, last_ts) @plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN) @@ -129,11 +129,10 @@ def sync_favorites_from_listenbrainz(user, conf): favorites_models.TrackFavorite.objects.filter(user=user) .filter(source="Listenbrainz") .latest("creation_date") - .values_list("creation_date", flat=True) + .creation_date.timestamp() ) - last_ts.timestamp() - except history_models.Listening.DoesNotExist: - tasks.import_listenbrainz_favorites(user, user_name, ts=0) + except favorites_models.TrackFavorite.DoesNotExist: + tasks.import_listenbrainz_favorites(user, user_name, 0) return - tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts) + tasks.import_listenbrainz_favorites(user, user_name, last_ts) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index 2835a8484..f23cd1cce 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -1,5 +1,5 @@ import datetime -import pylistenbrainz +import liblistenbrainz from django.utils import timezone from config import plugins @@ -47,27 +47,20 @@ def trigger_favorite_sync_with_listenbrainz(): @celery.app.task(name="listenbrainz.import_listenbrainz_listenings") -def import_listenbrainz_listenings(user, user_name, ts): - client = pylistenbrainz.ListenBrainz() - listens = client.get_listens(username=user_name, min_ts=ts, count=100) - add_lb_listenings_to_db(listens, user) - new_ts = 13 - last_ts = 12 - while new_ts != last_ts: - last_ts = max( - listens, - key=lambda obj: datetime.datetime.fromtimestamp( - obj.listened_at, timezone.utc - ), - ) - listens = client.get_listens(username=user_name, min_ts=new_ts, count=100) +def import_listenbrainz_listenings(user, user_name, since): + client = liblistenbrainz.ListenBrainz() + response = client.get_listens(username=user_name, min_ts=since, count=100) + listens = response["payload"]["listens"] + while listens: + add_lb_listenings_to_db(listens, user) new_ts = max( listens, key=lambda obj: datetime.datetime.fromtimestamp( obj.listened_at, timezone.utc ), ) - add_lb_listenings_to_db(listens, user) + response = client.get_listens(username=user_name, min_ts=new_ts, count=100) + listens = response["payload"]["listens"] def add_lb_listenings_to_db(listens, user): @@ -119,46 +112,51 @@ def add_lb_listenings_to_db(listens, user): @celery.app.task(name="listenbrainz.import_listenbrainz_favorites") -def import_listenbrainz_favorites(user, user_name, last_sync): - client = pylistenbrainz.ListenBrainz() - last_ts = int(datetime.datetime.now(timezone.utc).timestamp()) +def import_listenbrainz_favorites(user, user_name, since): + client = liblistenbrainz.ListenBrainz() + response = client.get_user_feedback(username=user_name) offset = 0 - while last_ts >= last_sync: - feedbacks = client.get_user_feedback(username=user_name, offset=offset) - add_lb_feedback_to_db(feedbacks, user) - offset = feedbacks.count - last_ts = max( - feedbacks.feedback, - key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc), - ) - # to do implement offset in pylb + while response["feedback"]: + count = response["count"] + offset = offset + count + last_sync = min( + response["feedback"], + key=lambda obj: datetime.datetime.fromtimestamp( + obj["created"], timezone.utc + ), + )["created"] + add_lb_feedback_to_db(response["feedback"], user) + if last_sync <= since or count == 0: + return + response = client.get_user_feedback(username=user_name, offset=offset) def add_lb_feedback_to_db(feedbacks, user): logger = PLUGIN["logger"] - fw_listens = [] - for feedback in feedbacks.feedback: + for feedback in feedbacks: try: - track = music_models.Track.objects.get(mbid=feedback.recording_mbid) + track = music_models.Track.objects.get(mbid=feedback["recording_mbid"]) except music_models.Track.DoesNotExist: logger.info( "Received feedback track doesn't exist in fw database. Skipping..." ) continue - if feedback.score == 1: + if feedback["score"] == 1: favorites_models.TrackFavorite.objects.get_or_create( user=user, creation_date=datetime.datetime.fromtimestamp( - feedback.created, timezone.utc + feedback["created"], timezone.utc ), track=track, source="Listenbrainz", ) - elif feedback.score == 0: + elif feedback["score"] == 0: try: - favorites_models.TrackFavorite.objects.delete(user=user, track=track) + favorites_models.TrackFavorite.objects.get( + user=user, track=track + ).delete() except favorites_models.TrackFavorite.DoesNotExist: continue - elif feedback.score == -1: + elif feedback["score"] == -1: logger.info("Funkwhale doesn't support hate yet <3") diff --git a/api/funkwhale_api/favorites/factories.py b/api/funkwhale_api/favorites/factories.py index df2f47335..c568a5a1d 100644 --- a/api/funkwhale_api/favorites/factories.py +++ b/api/funkwhale_api/favorites/factories.py @@ -1,4 +1,5 @@ import factory +from django.utils import timezone from funkwhale_api.factories import NoUpdateOnCreate, registry from funkwhale_api.music.factories import TrackFactory @@ -9,6 +10,7 @@ from funkwhale_api.users.factories import UserFactory class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory): track = factory.SubFactory(TrackFactory) user = factory.SubFactory(UserFactory) + creation_date = factory.Faker("date_time_this_decade", tzinfo=timezone.utc) class Meta: model = "favorites.TrackFavorite" diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 0d5be2b79..11496af7e 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -277,16 +277,6 @@ def disabled_musicbrainz(mocker): ) -# @pytest.fixture() -# def disabled_listenbrainz(mocker): -# # we ensure no listenbrainz requests gets out -# yield mocker.patch.object( -# listenbrainz.client.ListenBrainzClient, -# "_submit", -# return_value=None, -# ) - - @pytest.fixture(autouse=True) def r_mock(requests_mock): """ diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index 623c6b5b3..1efd6abd8 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -2,7 +2,7 @@ import datetime import logging import pytest -import pylistenbrainz +import liblistenbrainz from django.urls import reverse from django.utils import timezone @@ -62,32 +62,47 @@ def test_sync_listenings_from_listenbrainz(factories, mocker, caplog): "sync_listenings": True, } - listens = [ - pylistenbrainz.Listen( - track_name="test", - artist_name="artist_test", - recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", - additional_info={"submission_client": "not funkwhale"}, - listened_at=-3124224000, - ), - pylistenbrainz.Listen( - track_name="test2", - artist_name="artist_test2", - recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", - additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, - listened_at=1871, - ), - pylistenbrainz.Listen( - track_name="test3", - artist_name="artist_test3", - listened_at=0, - ), - ] + listens = { + "payload": { + "count": 25, + "user_id": "-- the MusicBrainz ID of the user --", + "listens": [ + liblistenbrainz.Listen( + track_name="test", + artist_name="artist_test", + recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", + additional_info={"submission_client": "not funkwhale"}, + listened_at=-3124224000, + ), + liblistenbrainz.Listen( + track_name="test2", + artist_name="artist_test2", + recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", + additional_info={ + "submission_client": "Funkwhale ListenBrainz plugin" + }, + listened_at=1871, + ), + liblistenbrainz.Listen( + track_name="test3", + artist_name="artist_test3", + listened_at=0, + ), + ], + } + } + no_more_listen = { + "payload": { + "count": 25, + "user_id": "Bilbo", + "listens": [], + } + } mocker.patch.object( - funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, "get_listens", - return_value=listens, + side_effect=[listens, no_more_listen], ) funkwhale_ready.sync_listenings_from_listenbrainz(user, conf) @@ -105,13 +120,20 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): caplog.set_level(logging.INFO) logger.addHandler(caplog.handler) user = factories["users.User"]() - + # track lb fav factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") - - factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + # random track + factories["music.Track"]() + # track lb neutral track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") - - favorite = factories["favorites.TrackFavorite"](track=track) + favorite = factories["favorites.TrackFavorite"](track=track, user=user) + # last_sync + track_last_sync = factories["music.Track"]( + mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7" + ) + favorite_last_sync = factories["favorites.TrackFavorite"]( + track=track_last_sync, source="Listenbrainz" + ) conf = { "user_name": user.username, @@ -135,6 +157,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): "user_id": user.username, }, { + # last sync "created": 1690775094, "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7", "score": -1, @@ -142,7 +165,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): }, { "created": 1690775093, - "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ", + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2", "score": 1, "user_id": user.username, }, @@ -150,10 +173,11 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): "offset": 0, "total_count": 4, } + empty_feedback = {"count": 0, "feedback": [], "offset": 0, "total_count": 0} mocker.patch.object( - funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, "get_user_feedback", - return_value=feedbacks, + side_effect=[feedbacks, empty_feedback], ) funkwhale_ready.sync_favorites_from_listenbrainz(user, conf) @@ -161,5 +185,153 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): assert favorites_models.TrackFavorite.objects.filter( track__mbid="195565db-65f9-4d0d-b347-5f0c85509528" ).exists() - with pytest.raises(deleted.DoesNotExist): + with pytest.raises(favorites_models.TrackFavorite.DoesNotExist): favorite.refresh_from_db() + + +def test_sync_favorites_from_listenbrainz_since(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + # track lb fav + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + # track lb neutral + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + favorite = factories["favorites.TrackFavorite"](track=track, user=user) + # track should be not synced + factories["music.Track"](mbid="1fd02cf2-7247-4715-8862-c378ec196000") + # last_sync + track_last_sync = factories["music.Track"]( + mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7" + ) + factories["favorites.TrackFavorite"]( + track=track_last_sync, + user=user, + source="Listenbrainz", + creation_date=datetime.datetime.fromtimestamp(1690775094), + ) + + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "sync_favorites": True, + } + + feedbacks = { + "count": 5, + "feedback": [ + { + "created": 1701116226, + "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528", + "score": 1, + "user_id": user.username, + }, + { + "created": 1701116214, + "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986", + "score": 0, + "user_id": user.username, + }, + { + # last sync + "created": 1690775094, + "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7", + "score": -1, + "user_id": user.username, + }, + { + "created": 1690775093, + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2", + "score": 1, + "user_id": user.username, + }, + ], + "offset": 0, + "total_count": 4, + } + second_feedback = { + "count": 0, + "feedback": [ + { + "created": 0, + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec196000", + "score": 1, + "user_id": user.username, + }, + ], + "offset": 0, + "total_count": 0, + } + mocker.patch.object( + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, + "get_user_feedback", + side_effect=[feedbacks, second_feedback], + ) + + funkwhale_ready.sync_favorites_from_listenbrainz(user, conf) + + assert favorites_models.TrackFavorite.objects.filter( + track__mbid="195565db-65f9-4d0d-b347-5f0c85509528" + ).exists() + assert not favorites_models.TrackFavorite.objects.filter( + track__mbid="1fd02cf2-7247-4715-8862-c378ec196000" + ).exists() + with pytest.raises(favorites_models.TrackFavorite.DoesNotExist): + favorite.refresh_from_db() + + +def test_submit_favorites_to_listenbrainz(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + + favorite = factories["favorites.TrackFavorite"](track=track) + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "submit_favorites": True, + } + + patch = mocker.patch.object( + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, + "submit_user_feedback", + return_value="Success", + ) + funkwhale_ready.submit_favorite_creation(favorite, conf) + + patch.assert_called_once_with(1, track.mbid) + + +def test_submit_favorites_deletion(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + + favorite = factories["favorites.TrackFavorite"](track=track) + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "submit_favorites": True, + } + + patch = mocker.patch.object( + funkwhale_ready.tasks.liblistenbrainz.ListenBrainz, + "submit_user_feedback", + return_value="Success", + ) + funkwhale_ready.submit_favorite_deletion(favorite, conf) + + patch.assert_called_once_with(0, track.mbid)