loads of things

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2658>

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

@@ -25,7 +25,8 @@ def submit_listen(listening, conf, **kwargs):
     client.submit_single_listen(listen)
 
 
-def get_listen(track):
+def get_listen(listening):
+    track = listening.track
     additional_info = {
         "media_player": "Funkwhale",
         "media_player_version": funkwhale_api.__version__,
@@ -54,7 +55,7 @@ def get_listen(track):
     return liblistenbrainz.Listen(
         track_name=track.title,
         artist_name=track.artist.name,
-        listened_at=int(timezone.now()),
+        listened_at=listening.creation_date.timestamp(),
         release_name=release_name,
         additional_info=additional_info,
     )
@@ -74,7 +75,7 @@ def submit_favorite_creation(track_favorite, conf, **kwargs):
             "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz"
         )
         return
-    # client.feedback(track, 1)
+    client.submit_user_feedback(1, track.mbid)
 
 
 @plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN)
@@ -91,13 +92,12 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs):
             "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz"
         )
         return
-    # client.feedback(track, 0)
+    client.submit_user_feedback(0, track.mbid)
 
 
 @plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN)
 def sync_listenings_from_listenbrainz(user, conf):
     user_name = conf["user_name"]
-    user_token = conf["user_token"]
 
     if not user_name or not conf["sync_listenings"]:
         return
@@ -110,8 +110,10 @@ def sync_listenings_from_listenbrainz(user, conf):
             .latest("creation_date")
             .values_list("creation_date", flat=True)
         )
+        last_ts.timestamp()
     except history_models.Listening.DoesNotExist:
         tasks.import_listenbrainz_listenings(user, user_name, ts=0)
+        return
 
     tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts)
 
@@ -119,7 +121,6 @@ def sync_listenings_from_listenbrainz(user, conf):
 @plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN)
 def sync_favorites_from_listenbrainz(user, conf):
     user_name = conf["user_name"]
-    user_token = conf["user_token"]
 
     if not user_name or not conf["sync_favorites"]:
         return
@@ -130,5 +131,9 @@ def sync_favorites_from_listenbrainz(user, conf):
             .latest("creation_date")
             .values_list("creation_date", flat=True)
         )
+        last_ts.timestamp()
     except history_models.Listening.DoesNotExist:
-        tasks.import_listenbrainz_favorites(user, user_name, last_ts)
+        tasks.import_listenbrainz_favorites(user, user_name, ts=0)
+        return
+
+    tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts)

api/funkwhale_api/contrib/listenbrainz/tasks.py

@@ -3,6 +3,7 @@ import pylistenbrainz
 
 from django.utils import timezone
 from config import plugins
+from funkwhale_api.favorites import models as favorites_models
 from funkwhale_api.history import models as history_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.taskapp import celery
@@ -53,13 +54,24 @@ def import_listenbrainz_listenings(user, user_name, ts):
     new_ts = 13
     last_ts = 12
     while new_ts != last_ts:
-        last_ts = listens[0].listened_at
+        last_ts = max(
+            listens,
+            key=lambda obj: datetime.datetime.fromtimestamp(
+                obj.listened_at, timezone.utc
+            ),
+        )
         listens = client.get_listens(username=user_name, min_ts=new_ts, count=100)
-        new_ts = listens[0].listened_at
+        new_ts = max(
+            listens,
+            key=lambda obj: datetime.datetime.fromtimestamp(
+                obj.listened_at, timezone.utc
+            ),
+        )
         add_lb_listenings_to_db(listens, user)
 
 
 def add_lb_listenings_to_db(listens, user):
+    logger = PLUGIN["logger"]
     fw_listens = []
     for listen in listens:
         if (
@@ -67,9 +79,14 @@ def add_lb_listenings_to_db(listens, user):
             and listen.additional_info.get("submission_client")
             == "Funkwhale ListenBrainz plugin"
             and history_models.Listening.objects.filter(
-                creation_date=listen.listened_at
+                creation_date=datetime.datetime.fromtimestamp(
+                    listen.listened_at, timezone.utc
+                )
             ).exists()
         ):
+            logger.info(
+                f"Listen with ts {listen.listened_at} skipped because already in db"
+            )
             continue
 
         mbid = (
@@ -79,7 +96,6 @@ def add_lb_listenings_to_db(listens, user):
         )
 
         if not mbid:
-            logger = PLUGIN["logger"]
             logger.info("Received listening doesn't have a mbid. Skipping...")
 
         try:
@@ -90,7 +106,9 @@ def add_lb_listenings_to_db(listens, user):
 
         user = user
         fw_listen = history_models.Listening(
-            creation_date=timezone.make_aware(listen.listened_at),
+            creation_date=datetime.datetime.fromtimestamp(
+                listen.listened_at, timezone.utc
+            ),
             track=track,
             user=user,
             source="Listenbrainz",
@@ -101,5 +119,46 @@ def add_lb_listenings_to_db(listens, user):
 
 
 @celery.app.task(name="listenbrainz.import_listenbrainz_favorites")
-def import_listenbrainz_favorites():
-    return "to do"
+def import_listenbrainz_favorites(user, user_name, last_sync):
+    client = pylistenbrainz.ListenBrainz()
+    last_ts = int(datetime.datetime.now(timezone.utc).timestamp())
+    offset = 0
+    while last_ts >= last_sync:
+        feedbacks = client.get_user_feedback(username=user_name, offset=offset)
+        add_lb_feedback_to_db(feedbacks, user)
+        offset = feedbacks.count
+        last_ts = max(
+            feedbacks.feedback,
+            key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc),
+        )
+        # to do implement offset in pylb
+
+
+def add_lb_feedback_to_db(feedbacks, user):
+    logger = PLUGIN["logger"]
+    fw_listens = []
+    for feedback in feedbacks.feedback:
+        try:
+            track = music_models.Track.objects.get(mbid=feedback.recording_mbid)
+        except music_models.Track.DoesNotExist:
+            logger.info(
+                "Received feedback track doesn't exist in fw database. Skipping..."
+            )
+            continue
+
+        if feedback.score == 1:
+            favorites_models.TrackFavorite.objects.get_or_create(
+                user=user,
+                creation_date=datetime.datetime.fromtimestamp(
+                    feedback.created, timezone.utc
+                ),
+                track=track,
+                source="Listenbrainz",
+            )
+        elif feedback.score == 0:
+            try:
+                favorites_models.TrackFavorite.objects.delete(user=user, track=track)
+            except favorites_models.TrackFavorite.DoesNotExist:
+                continue
+        elif feedback.score == -1:
+            logger.info("Funkwhale doesn't support hate yet <3")

api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-12-09 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('favorites', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfavorite',
+            name='source',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

api/funkwhale_api/history/migrations/0003_listening_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-12-09 14:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('history', '0002_auto_20180325_1433'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='listening',
+            name='source',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

api/tests/contrib/listenbrainz/test_listenbrainz.py

@@ -1,6 +1,15 @@
+import datetime
+import logging
+
 import pytest
+import pylistenbrainz
+
 from django.urls import reverse
+from django.utils import timezone
+
 from config import plugins
+from funkwhale_api.contrib.listenbrainz import funkwhale_ready
+from funkwhale_api.favorites import models as favorites_models
 from funkwhale_api.history import models as history_models
 
 
@@ -33,3 +42,124 @@ def test_listenbrainz_submit_listen(logged_in_client, mocker, factories):
     listening = history_models.Listening.objects.get(user=logged_in_client.user)
     handler.assert_called_once_with(listening=listening, conf=None)
     # why conf=none ?
+
+
+def test_sync_listenings_from_listenbrainz(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476")
+    track = factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    factories["history.Listening"](
+        creation_date=datetime.datetime.fromtimestamp(1871, timezone.utc), track=track
+    )
+
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "sync_listenings": True,
+    }
+
+    listens = [
+        pylistenbrainz.Listen(
+            track_name="test",
+            artist_name="artist_test",
+            recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
+            additional_info={"submission_client": "not funkwhale"},
+            listened_at=-3124224000,
+        ),
+        pylistenbrainz.Listen(
+            track_name="test2",
+            artist_name="artist_test2",
+            recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
+            additional_info={"submission_client": "Funkwhale ListenBrainz plugin"},
+            listened_at=1871,
+        ),
+        pylistenbrainz.Listen(
+            track_name="test3",
+            artist_name="artist_test3",
+            listened_at=0,
+        ),
+    ]
+
+    mocker.patch.object(
+        funkwhale_ready.tasks.pylistenbrainz.ListenBrainz,
+        "get_listens",
+        return_value=listens,
+    )
+
+    funkwhale_ready.sync_listenings_from_listenbrainz(user, conf)
+
+    assert history_models.Listening.objects.filter(
+        track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476"
+    ).exists()
+
+    assert "Listen with ts 1871 skipped because already in db" in caplog.text
+    assert "Received listening doesn't have a mbid" in caplog.text
+
+
+def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+
+    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+
+    favorite = factories["favorites.TrackFavorite"](track=track)
+
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "sync_favorites": True,
+    }
+
+    feedbacks = {
+        "count": 5,
+        "feedback": [
+            {
+                "created": 1701116226,
+                "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528",
+                "score": 1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1701116214,
+                "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986",
+                "score": 0,
+                "user_id": user.username,
+            },
+            {
+                "created": 1690775094,
+                "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
+                "score": -1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1690775093,
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ",
+                "score": 1,
+                "user_id": user.username,
+            },
+        ],
+        "offset": 0,
+        "total_count": 4,
+    }
+    mocker.patch.object(
+        funkwhale_ready.tasks.pylistenbrainz.ListenBrainz,
+        "get_user_feedback",
+        return_value=feedbacks,
+    )
+
+    funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
+
+    assert favorites_models.TrackFavorite.objects.filter(
+        track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
+    ).exists()
+    with pytest.raises(deleted.DoesNotExist):
+        favorite.refresh_from_db()

api/tests/contrib/listenbrainz/test_tasks.py

@@ -1,49 +1,15 @@
 import datetime
+import logging
 
 import pylistenbrainz
 import pytest
 
+from django.utils import timezone
+
 from funkwhale_api.contrib.listenbrainz import tasks
 from funkwhale_api.history import models as history_models
 
 
-def test_import_listenbrainz_listenings(factories, mocker):
-    factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476")
-    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
-
-    listens = [
-        pylistenbrainz.utils.Listen(
-            track_name="test",
-            artist_name="artist_test",
-            recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
-            additional_info={"submission_client": "not funkwhale"},
-            listened_at=datetime.datetime.fromtimestamp(-3124224000),
-        ),
-        pylistenbrainz.utils.Listen(
-            track_name="test2",
-            artist_name="artist_test2",
-            recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
-            additional_info={"submission_client": "Funkwhale ListenBrainz plugin"},
-            listened_at=datetime.datetime.fromtimestamp(1871),
-        ),
-        pylistenbrainz.utils.Listen(
-            track_name="test3",
-            artist_name="artist_test3",
-            listened_at=0,
-        ),
-    ]
-
-    mocker.patch.object(
-        tasks.pylistenbrainz.ListenBrainz, "get_listens", return_value=listens
-    )
-    user = factories["users.User"]()
-
-    tasks.import_listenbrainz_listenings(user, "user_name", ts=0)
-
-    history_models.Listening.objects.filter(
-        track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476"
-    ).exists()
-
-    assert not history_models.Listening.objects.filter(
-        track__mbid="54c60860-f43d-484e-b691-7ab7ec8de559"
-    ).exists()
+def test_trigger_listening_sync_with_listenbrainz():
+    # to do
+    pass