From 37acfa475dd777b65980dd5395495b946fc25938 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 20 Dec 2023 00:08:48 +0100 Subject: [PATCH] loads of things Part-of: --- .../contrib/listenbrainz/funkwhale_ready.py | 19 ++- .../contrib/listenbrainz/tasks.py | 73 +++++++++- .../migrations/0002_trackfavorite_source.py | 18 +++ .../migrations/0003_listening_source.py | 18 +++ .../contrib/listenbrainz/test_listenbrainz.py | 130 ++++++++++++++++++ api/tests/contrib/listenbrainz/test_tasks.py | 46 +------ 6 files changed, 250 insertions(+), 54 deletions(-) create mode 100644 api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py create mode 100644 api/funkwhale_api/history/migrations/0003_listening_source.py diff --git a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py index cd164354c..a675b481b 100644 --- a/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py @@ -25,7 +25,8 @@ def submit_listen(listening, conf, **kwargs): client.submit_single_listen(listen) -def get_listen(track): +def get_listen(listening): + track = listening.track additional_info = { "media_player": "Funkwhale", "media_player_version": funkwhale_api.__version__, @@ -54,7 +55,7 @@ def get_listen(track): return liblistenbrainz.Listen( track_name=track.title, artist_name=track.artist.name, - listened_at=int(timezone.now()), + listened_at=listening.creation_date.timestamp(), release_name=release_name, additional_info=additional_info, ) @@ -74,7 +75,7 @@ def submit_favorite_creation(track_favorite, conf, **kwargs): "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz" ) return - # client.feedback(track, 1) + client.submit_user_feedback(1, track.mbid) @plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN) @@ -91,13 +92,12 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs): "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz" ) return - # client.feedback(track, 0) + client.submit_user_feedback(0, track.mbid) @plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN) def sync_listenings_from_listenbrainz(user, conf): user_name = conf["user_name"] - user_token = conf["user_token"] if not user_name or not conf["sync_listenings"]: return @@ -110,8 +110,10 @@ def sync_listenings_from_listenbrainz(user, conf): .latest("creation_date") .values_list("creation_date", flat=True) ) + last_ts.timestamp() except history_models.Listening.DoesNotExist: tasks.import_listenbrainz_listenings(user, user_name, ts=0) + return tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts) @@ -119,7 +121,6 @@ def sync_listenings_from_listenbrainz(user, conf): @plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN) def sync_favorites_from_listenbrainz(user, conf): user_name = conf["user_name"] - user_token = conf["user_token"] if not user_name or not conf["sync_favorites"]: return @@ -130,5 +131,9 @@ def sync_favorites_from_listenbrainz(user, conf): .latest("creation_date") .values_list("creation_date", flat=True) ) + last_ts.timestamp() except history_models.Listening.DoesNotExist: - tasks.import_listenbrainz_favorites(user, user_name, last_ts) + tasks.import_listenbrainz_favorites(user, user_name, ts=0) + return + + tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts) diff --git a/api/funkwhale_api/contrib/listenbrainz/tasks.py b/api/funkwhale_api/contrib/listenbrainz/tasks.py index 1c63100d6..2835a8484 100644 --- a/api/funkwhale_api/contrib/listenbrainz/tasks.py +++ b/api/funkwhale_api/contrib/listenbrainz/tasks.py @@ -3,6 +3,7 @@ import pylistenbrainz from django.utils import timezone from config import plugins +from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models from funkwhale_api.music import models as music_models from funkwhale_api.taskapp import celery @@ -53,13 +54,24 @@ def import_listenbrainz_listenings(user, user_name, ts): new_ts = 13 last_ts = 12 while new_ts != last_ts: - last_ts = listens[0].listened_at + last_ts = max( + listens, + key=lambda obj: datetime.datetime.fromtimestamp( + obj.listened_at, timezone.utc + ), + ) listens = client.get_listens(username=user_name, min_ts=new_ts, count=100) - new_ts = listens[0].listened_at + new_ts = max( + listens, + key=lambda obj: datetime.datetime.fromtimestamp( + obj.listened_at, timezone.utc + ), + ) add_lb_listenings_to_db(listens, user) def add_lb_listenings_to_db(listens, user): + logger = PLUGIN["logger"] fw_listens = [] for listen in listens: if ( @@ -67,9 +79,14 @@ def add_lb_listenings_to_db(listens, user): and listen.additional_info.get("submission_client") == "Funkwhale ListenBrainz plugin" and history_models.Listening.objects.filter( - creation_date=listen.listened_at + creation_date=datetime.datetime.fromtimestamp( + listen.listened_at, timezone.utc + ) ).exists() ): + logger.info( + f"Listen with ts {listen.listened_at} skipped because already in db" + ) continue mbid = ( @@ -79,7 +96,6 @@ def add_lb_listenings_to_db(listens, user): ) if not mbid: - logger = PLUGIN["logger"] logger.info("Received listening doesn't have a mbid. Skipping...") try: @@ -90,7 +106,9 @@ def add_lb_listenings_to_db(listens, user): user = user fw_listen = history_models.Listening( - creation_date=timezone.make_aware(listen.listened_at), + creation_date=datetime.datetime.fromtimestamp( + listen.listened_at, timezone.utc + ), track=track, user=user, source="Listenbrainz", @@ -101,5 +119,46 @@ def add_lb_listenings_to_db(listens, user): @celery.app.task(name="listenbrainz.import_listenbrainz_favorites") -def import_listenbrainz_favorites(): - return "to do" +def import_listenbrainz_favorites(user, user_name, last_sync): + client = pylistenbrainz.ListenBrainz() + last_ts = int(datetime.datetime.now(timezone.utc).timestamp()) + offset = 0 + while last_ts >= last_sync: + feedbacks = client.get_user_feedback(username=user_name, offset=offset) + add_lb_feedback_to_db(feedbacks, user) + offset = feedbacks.count + last_ts = max( + feedbacks.feedback, + key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc), + ) + # to do implement offset in pylb + + +def add_lb_feedback_to_db(feedbacks, user): + logger = PLUGIN["logger"] + fw_listens = [] + for feedback in feedbacks.feedback: + try: + track = music_models.Track.objects.get(mbid=feedback.recording_mbid) + except music_models.Track.DoesNotExist: + logger.info( + "Received feedback track doesn't exist in fw database. Skipping..." + ) + continue + + if feedback.score == 1: + favorites_models.TrackFavorite.objects.get_or_create( + user=user, + creation_date=datetime.datetime.fromtimestamp( + feedback.created, timezone.utc + ), + track=track, + source="Listenbrainz", + ) + elif feedback.score == 0: + try: + favorites_models.TrackFavorite.objects.delete(user=user, track=track) + except favorites_models.TrackFavorite.DoesNotExist: + continue + elif feedback.score == -1: + logger.info("Funkwhale doesn't support hate yet <3") diff --git a/api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py new file mode 100644 index 000000000..67a397aa0 --- /dev/null +++ b/api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-12-09 14:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('favorites', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='trackfavorite', + name='source', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/api/funkwhale_api/history/migrations/0003_listening_source.py b/api/funkwhale_api/history/migrations/0003_listening_source.py new file mode 100644 index 000000000..b50bc6220 --- /dev/null +++ b/api/funkwhale_api/history/migrations/0003_listening_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-12-09 14:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0002_auto_20180325_1433'), + ] + + operations = [ + migrations.AddField( + model_name='listening', + name='source', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/api/tests/contrib/listenbrainz/test_listenbrainz.py b/api/tests/contrib/listenbrainz/test_listenbrainz.py index c47365c74..623c6b5b3 100644 --- a/api/tests/contrib/listenbrainz/test_listenbrainz.py +++ b/api/tests/contrib/listenbrainz/test_listenbrainz.py @@ -1,6 +1,15 @@ +import datetime +import logging + import pytest +import pylistenbrainz + from django.urls import reverse +from django.utils import timezone + from config import plugins +from funkwhale_api.contrib.listenbrainz import funkwhale_ready +from funkwhale_api.favorites import models as favorites_models from funkwhale_api.history import models as history_models @@ -33,3 +42,124 @@ def test_listenbrainz_submit_listen(logged_in_client, mocker, factories): listening = history_models.Listening.objects.get(user=logged_in_client.user) handler.assert_called_once_with(listening=listening, conf=None) # why conf=none ? + + +def test_sync_listenings_from_listenbrainz(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476") + track = factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + factories["history.Listening"]( + creation_date=datetime.datetime.fromtimestamp(1871, timezone.utc), track=track + ) + + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "sync_listenings": True, + } + + listens = [ + pylistenbrainz.Listen( + track_name="test", + artist_name="artist_test", + recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", + additional_info={"submission_client": "not funkwhale"}, + listened_at=-3124224000, + ), + pylistenbrainz.Listen( + track_name="test2", + artist_name="artist_test2", + recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", + additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, + listened_at=1871, + ), + pylistenbrainz.Listen( + track_name="test3", + artist_name="artist_test3", + listened_at=0, + ), + ] + + mocker.patch.object( + funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + "get_listens", + return_value=listens, + ) + + funkwhale_ready.sync_listenings_from_listenbrainz(user, conf) + + assert history_models.Listening.objects.filter( + track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476" + ).exists() + + assert "Listen with ts 1871 skipped because already in db" in caplog.text + assert "Received listening doesn't have a mbid" in caplog.text + + +def test_sync_favorites_from_listenbrainz(factories, mocker, caplog): + logger = logging.getLogger("plugins") + caplog.set_level(logging.INFO) + logger.addHandler(caplog.handler) + user = factories["users.User"]() + + factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528") + + factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") + track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986") + + favorite = factories["favorites.TrackFavorite"](track=track) + + conf = { + "user_name": user.username, + "user_token": "user_tolkien", + "sync_favorites": True, + } + + feedbacks = { + "count": 5, + "feedback": [ + { + "created": 1701116226, + "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528", + "score": 1, + "user_id": user.username, + }, + { + "created": 1701116214, + "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986", + "score": 0, + "user_id": user.username, + }, + { + "created": 1690775094, + "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7", + "score": -1, + "user_id": user.username, + }, + { + "created": 1690775093, + "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ", + "score": 1, + "user_id": user.username, + }, + ], + "offset": 0, + "total_count": 4, + } + mocker.patch.object( + funkwhale_ready.tasks.pylistenbrainz.ListenBrainz, + "get_user_feedback", + return_value=feedbacks, + ) + + funkwhale_ready.sync_favorites_from_listenbrainz(user, conf) + + assert favorites_models.TrackFavorite.objects.filter( + track__mbid="195565db-65f9-4d0d-b347-5f0c85509528" + ).exists() + with pytest.raises(deleted.DoesNotExist): + favorite.refresh_from_db() diff --git a/api/tests/contrib/listenbrainz/test_tasks.py b/api/tests/contrib/listenbrainz/test_tasks.py index 7f9ee04df..680662fe2 100644 --- a/api/tests/contrib/listenbrainz/test_tasks.py +++ b/api/tests/contrib/listenbrainz/test_tasks.py @@ -1,49 +1,15 @@ import datetime +import logging import pylistenbrainz import pytest +from django.utils import timezone + from funkwhale_api.contrib.listenbrainz import tasks from funkwhale_api.history import models as history_models -def test_import_listenbrainz_listenings(factories, mocker): - factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476") - factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559") - - listens = [ - pylistenbrainz.utils.Listen( - track_name="test", - artist_name="artist_test", - recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476", - additional_info={"submission_client": "not funkwhale"}, - listened_at=datetime.datetime.fromtimestamp(-3124224000), - ), - pylistenbrainz.utils.Listen( - track_name="test2", - artist_name="artist_test2", - recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559", - additional_info={"submission_client": "Funkwhale ListenBrainz plugin"}, - listened_at=datetime.datetime.fromtimestamp(1871), - ), - pylistenbrainz.utils.Listen( - track_name="test3", - artist_name="artist_test3", - listened_at=0, - ), - ] - - mocker.patch.object( - tasks.pylistenbrainz.ListenBrainz, "get_listens", return_value=listens - ) - user = factories["users.User"]() - - tasks.import_listenbrainz_listenings(user, "user_name", ts=0) - - history_models.Listening.objects.filter( - track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476" - ).exists() - - assert not history_models.Listening.objects.filter( - track__mbid="54c60860-f43d-484e-b691-7ab7ec8de559" - ).exists() +def test_trigger_listening_sync_with_listenbrainz(): + # to do + pass