chore(front): update dependency universal-cookie to v7

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2771>

.env.dev

@@ -18,6 +18,6 @@ MEDIA_ROOT=/data/media
 # FORCE_HTTPS_URLS=True
 
 # Customize to your needs
-POSTGRES_VERSION=11
+POSTGRES_VERSION=15
 DEBUG=true
 TYPESENSE_API_KEY="apikey"

api/Dockerfile

@@ -39,7 +39,7 @@ RUN set -eux; \
   zlib-dev \
   py3-cryptography=41.0.7-r0 \
   py3-lxml=4.9.3-r1 \
-  py3-pillow=10.2.0-r0 \
+  py3-pillow=10.3.0-r0 \
   py3-psycopg2=2.9.9-r0 \
   py3-watchfiles=0.19.0-r1 \
   python3-dev
@@ -99,7 +99,7 @@ RUN set -eux; \
   libxslt \
   py3-cryptography=41.0.7-r0 \
   py3-lxml=4.9.3-r1 \
-  py3-pillow=10.2.0-r0 \
+  py3-pillow=10.3.0-r0 \
   py3-psycopg2=2.9.9-r0 \
   py3-watchfiles=0.19.0-r1 \
   python3 \

api/config/plugins.py

@@ -303,6 +303,23 @@ LISTENING_CREATED = "listening_created"
 """
 Called when a track is being listened
 """
+LISTENING_SYNC = "listening_sync"
+"""
+Called by the task manager to trigger listening sync
+"""
+FAVORITE_CREATED = "favorite_created"
+"""
+Called when a track is being favorited
+"""
+FAVORITE_DELETED = "favorite_deleted"
+"""
+Called when a favorited track is being unfavorited
+"""
+FAVORITE_SYNC = "favorite_sync"
+"""
+Called by the task manager to trigger favorite sync
+"""
+
 SCAN = "scan"
 """
 

api/config/settings/common.py

@@ -276,6 +276,7 @@ LOCAL_APPS = (
     # Your stuff: custom apps go here
     "funkwhale_api.instance",
     "funkwhale_api.audio",
+    "funkwhale_api.contrib.listenbrainz",
     "funkwhale_api.music",
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
@@ -949,6 +950,16 @@ CELERY_BEAT_SCHEDULE = {
         ),
         "options": {"expires": 60 * 60},
     },
+    "listenbrainz.trigger_listening_sync_with_listenbrainz": {
+        "task": "listenbrainz.trigger_listening_sync_with_listenbrainz",
+        "schedule": crontab(day_of_week="*", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    },
+    "listenbrainz.trigger_favorite_sync_with_listenbrainz": {
+        "task": "listenbrainz.trigger_favorite_sync_with_listenbrainz",
+        "schedule": crontab(day_of_week="*", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    },
 }
 
 if env.str("TYPESENSE_API_KEY", default=None):

api/funkwhale_api/activity/utils.py

@@ -48,4 +48,5 @@ def get_activity(user, limit=20):
         ),
     ]
     records = combined_recent(limit=limit, querysets=querysets)
+
     return [r["object"] for r in records]

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

@@ -1,28 +1,31 @@
 import liblistenbrainz
-from django.utils import timezone
 
 import funkwhale_api
 from config import plugins
+from funkwhale_api.favorites import models as favorites_models
+from funkwhale_api.history import models as history_models
 
+from . import tasks
 from .funkwhale_startup import PLUGIN
 
 
 @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
 def submit_listen(listening, conf, **kwargs):
     user_token = conf["user_token"]
-    if not user_token:
+    if not user_token and not conf["submit_listenings"]:
         return
 
     logger = PLUGIN["logger"]
     logger.info("Submitting listen to ListenBrainz")
     client = liblistenbrainz.ListenBrainz()
     client.set_auth_token(user_token)
-    listen = get_listen(listening.track)
+    listen = get_lb_listen(listening)
 
     client.submit_single_listen(listen)
 
 
-def get_listen(track):
+def get_lb_listen(listening):
+    track = listening.track
     additional_info = {
         "media_player": "Funkwhale",
         "media_player_version": funkwhale_api.__version__,
@@ -51,7 +54,83 @@ def get_listen(track):
     return liblistenbrainz.Listen(
         track_name=track.title,
         artist_name=track.artist.name,
-        listened_at=int(timezone.now()),
+        listened_at=listening.creation_date.timestamp(),
         release_name=release_name,
         additional_info=additional_info,
     )
+
+
+@plugins.register_hook(plugins.FAVORITE_CREATED, PLUGIN)
+def submit_favorite_creation(track_favorite, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token or not conf["submit_favorites"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Submitting favorite to ListenBrainz")
+    client = liblistenbrainz.ListenBrainz()
+    track = track_favorite.track
+    if not track.mbid:
+        logger.warning(
+            "This tracks doesn't have a mbid. Feedback will not be submitted to Listenbrainz"
+        )
+        return
+    client.submit_user_feedback(1, track.mbid)
+
+
+@plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN)
+def submit_favorite_deletion(track_favorite, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token or not conf["submit_favorites"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Submitting favorite deletion to ListenBrainz")
+    client = liblistenbrainz.ListenBrainz()
+    track = track_favorite.track
+    if not track.mbid:
+        logger.warning(
+            "This tracks doesn't have a mbid. Feedback will not be submitted to Listenbrainz"
+        )
+        return
+    client.submit_user_feedback(0, track.mbid)
+
+
+@plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN)
+def sync_listenings_from_listenbrainz(user, conf):
+    user_name = conf["user_name"]
+
+    if not user_name or not conf["sync_listenings"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Getting listenings from ListenBrainz")
+    try:
+        last_ts = (
+            history_models.Listening.objects.filter(user=user)
+            .filter(source="Listenbrainz")
+            .latest("creation_date")
+            .values_list("creation_date", flat=True)
+        ).timestamp()
+    except funkwhale_api.history.models.Listening.DoesNotExist:
+        tasks.import_listenbrainz_listenings(user, user_name, 0)
+        return
+
+    tasks.import_listenbrainz_listenings(user, user_name, last_ts)
+
+
+@plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN)
+def sync_favorites_from_listenbrainz(user, conf):
+    user_name = conf["user_name"]
+
+    if not user_name or not conf["sync_favorites"]:
+        return
+    try:
+        last_ts = (
+            favorites_models.TrackFavorite.objects.filter(user=user)
+            .filter(source="Listenbrainz")
+            .latest("creation_date")
+            .creation_date.timestamp()
+        )
+    except favorites_models.TrackFavorite.DoesNotExist:
+        tasks.import_listenbrainz_favorites(user, user_name, 0)
+        return
+
+    tasks.import_listenbrainz_favorites(user, user_name, last_ts)

api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py

@@ -3,7 +3,7 @@ from config import plugins
 PLUGIN = plugins.get_plugin_config(
     name="listenbrainz",
     label="ListenBrainz",
-    description="A plugin that allows you to submit your listens to ListenBrainz.",
+    description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.",
     homepage="https://docs.funkwhale.audio/users/builtinplugins.html#listenbrainz-plugin",  # noqa
     version="0.3",
     user=True,
@@ -13,6 +13,45 @@ PLUGIN = plugins.get_plugin_config(
             "type": "text",
             "label": "Your ListenBrainz user token",
             "help": "You can find your user token in your ListenBrainz profile at https://listenbrainz.org/profile/",
-        }
+        },
+        {
+            "name": "user_name",
+            "type": "text",
+            "required": False,
+            "label": "Your ListenBrainz user name.",
+            "help": "Required for importing listenings and favorites with ListenBrainz \
+                but not to send activities",
+        },
+        {
+            "name": "submit_listenings",
+            "type": "boolean",
+            "default": True,
+            "label": "Enable listening submission to ListenBrainz",
+            "help": "If enabled, your listenings from Funkwhale will be imported into ListenBrainz.",
+        },
+        {
+            "name": "sync_listenings",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable listenings sync",
+            "help": "If enabled, your listening from ListenBrainz will be imported into Funkwhale. This means they \
+                will be used along with Funkwhale listenings to filter out recently listened content or \
+                generate recommendations",
+        },
+        {
+            "name": "sync_favorites",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable favorite sync",
+            "help": "If enabled, your favorites from ListenBrainz will be imported into Funkwhale. This means they \
+                will be used along with Funkwhale favorites (UI display, federation activity)",
+        },
+        {
+            "name": "submit_favorites",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable favorite submission to ListenBrainz services",
+            "help": "If enabled, your favorites from Funkwhale will be submitted to ListenBrainz",
+        },
     ],
 )

api/funkwhale_api/contrib/listenbrainz/tasks.py

@@ -0,0 +1,165 @@
+import datetime
+
+import liblistenbrainz
+from django.utils import timezone
+
+from config import plugins
+from funkwhale_api.favorites import models as favorites_models
+from funkwhale_api.history import models as history_models
+from funkwhale_api.music import models as music_models
+from funkwhale_api.taskapp import celery
+from funkwhale_api.users import models
+
+from .funkwhale_startup import PLUGIN
+
+
+@celery.app.task(name="listenbrainz.trigger_listening_sync_with_listenbrainz")
+def trigger_listening_sync_with_listenbrainz():
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    users = (
+        models.User.objects.filter(plugins__code="listenbrainz")
+        .filter(plugins__conf__sync_listenings=True)
+        .filter(last_activity__gte=active_month)
+    )
+    for user in users:
+        plugins.trigger_hook(
+            plugins.LISTENING_SYNC,
+            user=user,
+            confs=plugins.get_confs(user),
+        )
+
+
+@celery.app.task(name="listenbrainz.trigger_favorite_sync_with_listenbrainz")
+def trigger_favorite_sync_with_listenbrainz():
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    users = (
+        models.User.objects.filter(plugins__code="listenbrainz")
+        .filter(plugins__conf__sync_listenings=True)
+        .filter(last_activity__gte=active_month)
+    )
+    for user in users:
+        plugins.trigger_hook(
+            plugins.FAVORITE_SYNC,
+            user=user,
+            confs=plugins.get_confs(user),
+        )
+
+
+@celery.app.task(name="listenbrainz.import_listenbrainz_listenings")
+def import_listenbrainz_listenings(user, user_name, since):
+    client = liblistenbrainz.ListenBrainz()
+    response = client.get_listens(username=user_name, min_ts=since, count=100)
+    listens = response["payload"]["listens"]
+    while listens:
+        add_lb_listenings_to_db(listens, user)
+        new_ts = max(
+            listens,
+            key=lambda obj: datetime.datetime.fromtimestamp(
+                obj.listened_at, timezone.utc
+            ),
+        )
+        response = client.get_listens(username=user_name, min_ts=new_ts, count=100)
+        listens = response["payload"]["listens"]
+
+
+def add_lb_listenings_to_db(listens, user):
+    logger = PLUGIN["logger"]
+    fw_listens = []
+    for listen in listens:
+        if (
+            listen.additional_info.get("submission_client")
+            and listen.additional_info.get("submission_client")
+            == "Funkwhale ListenBrainz plugin"
+            and history_models.Listening.objects.filter(
+                creation_date=datetime.datetime.fromtimestamp(
+                    listen.listened_at, timezone.utc
+                )
+            ).exists()
+        ):
+            logger.info(
+                f"Listen with ts {listen.listened_at} skipped because already in db"
+            )
+            continue
+
+        mbid = (
+            listen.mbid_mapping
+            if hasattr(listen, "mbid_mapping")
+            else listen.recording_mbid
+        )
+
+        if not mbid:
+            logger.info("Received listening that doesn't have a mbid. Skipping...")
+
+        try:
+            track = music_models.Track.objects.get(mbid=mbid)
+        except music_models.Track.DoesNotExist:
+            logger.info(
+                "Received listening that doesn't exist in fw database. Skipping..."
+            )
+            continue
+
+        user = user
+        fw_listen = history_models.Listening(
+            creation_date=datetime.datetime.fromtimestamp(
+                listen.listened_at, timezone.utc
+            ),
+            track=track,
+            user=user,
+            source="Listenbrainz",
+        )
+        fw_listens.append(fw_listen)
+
+    history_models.Listening.objects.bulk_create(fw_listens)
+
+
+@celery.app.task(name="listenbrainz.import_listenbrainz_favorites")
+def import_listenbrainz_favorites(user, user_name, since):
+    client = liblistenbrainz.ListenBrainz()
+    response = client.get_user_feedback(username=user_name)
+    offset = 0
+    while response["feedback"]:
+        count = response["count"]
+        offset = offset + count
+        last_sync = min(
+            response["feedback"],
+            key=lambda obj: datetime.datetime.fromtimestamp(
+                obj["created"], timezone.utc
+            ),
+        )["created"]
+        add_lb_feedback_to_db(response["feedback"], user)
+        if last_sync <= since or count == 0:
+            return
+        response = client.get_user_feedback(username=user_name, offset=offset)
+
+
+def add_lb_feedback_to_db(feedbacks, user):
+    logger = PLUGIN["logger"]
+    for feedback in feedbacks:
+        try:
+            track = music_models.Track.objects.get(mbid=feedback["recording_mbid"])
+        except music_models.Track.DoesNotExist:
+            logger.info(
+                "Received feedback track that doesn't exist in fw database. Skipping..."
+            )
+            continue
+
+        if feedback["score"] == 1:
+            favorites_models.TrackFavorite.objects.get_or_create(
+                user=user,
+                creation_date=datetime.datetime.fromtimestamp(
+                    feedback["created"], timezone.utc
+                ),
+                track=track,
+                source="Listenbrainz",
+            )
+        elif feedback["score"] == 0:
+            try:
+                favorites_models.TrackFavorite.objects.get(
+                    user=user, track=track
+                ).delete()
+            except favorites_models.TrackFavorite.DoesNotExist:
+                continue
+        elif feedback["score"] == -1:
+            logger.info("Funkwhale doesn't support disliked tracks")

api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-12-09 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('favorites', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfavorite',
+            name='source',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

api/funkwhale_api/favorites/models.py

@@ -12,6 +12,7 @@ class TrackFavorite(models.Model):
     track = models.ForeignKey(
         Track, related_name="track_favorites", on_delete=models.CASCADE
     )
+    source = models.CharField(max_length=100, null=True, blank=True)
 
     class Meta:
         unique_together = ("track", "user")

api/funkwhale_api/favorites/views.py

@@ -4,6 +4,7 @@ from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
+from config import plugins
 from funkwhale_api.activity import record
 from funkwhale_api.common import fields, permissions
 from funkwhale_api.music import utils as music_utils
@@ -44,6 +45,11 @@ class TrackFavoriteViewSet(
         instance = self.perform_create(serializer)
         serializer = self.get_serializer(instance=instance)
         headers = self.get_success_headers(serializer.data)
+        plugins.trigger_hook(
+            plugins.FAVORITE_CREATED,
+            track_favorite=serializer.instance,
+            confs=plugins.get_confs(self.request.user),
+        )
         record.send(instance)
         return Response(
             serializer.data, status=status.HTTP_201_CREATED, headers=headers
@@ -76,6 +82,11 @@ class TrackFavoriteViewSet(
         except (AttributeError, ValueError, models.TrackFavorite.DoesNotExist):
             return Response({}, status=400)
         favorite.delete()
+        plugins.trigger_hook(
+            plugins.FAVORITE_DELETED,
+            track_favorite=favorite,
+            confs=plugins.get_confs(self.request.user),
+        )
         return Response([], status=status.HTTP_204_NO_CONTENT)
 
     @extend_schema(

api/funkwhale_api/history/migrations/0003_listening_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-12-09 14:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('history', '0002_auto_20180325_1433'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='listening',
+            name='source',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

api/funkwhale_api/history/models.py

@@ -17,6 +17,7 @@ class Listening(models.Model):
         on_delete=models.CASCADE,
     )
     session_key = models.CharField(max_length=100, null=True, blank=True)
+    source = models.CharField(max_length=100, null=True, blank=True)
 
     class Meta:
         ordering = ("-creation_date",)

api/funkwhale_api/typesense/utils.py

@@ -36,7 +36,6 @@ def delete_non_alnum_characters(text):
 def resolve_recordings_to_fw_track(recordings):
     """
     Tries to match a troi recording entity to a fw track using the typesense index.
-    It will save the results in the match_mbid attribute of the Track table.
     For test purposes : if multiple fw tracks are returned, we log the information
     but only keep the best result in db to avoid duplicates.
     """

api/tests/contrib/listenbrainz/test_listenbrainz.py

@@ -0,0 +1,333 @@
+import datetime
+import logging
+
+import liblistenbrainz
+import pytest
+from django.urls import reverse
+from django.utils import timezone
+
+from config import plugins
+from funkwhale_api.contrib.listenbrainz import funkwhale_ready
+from funkwhale_api.favorites import models as favorites_models
+from funkwhale_api.history import models as history_models
+
+
+def test_listenbrainz_submit_listen(logged_in_client, mocker, factories):
+    config = plugins.get_plugin_config(
+        name="listenbrainz",
+        description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.",
+        conf=[],
+        source=False,
+    )
+    handler = mocker.Mock()
+    plugins.register_hook(plugins.LISTENING_CREATED, config)(handler)
+    plugins.set_conf(
+        "listenbrainz",
+        {
+            "sync_listenings": True,
+            "sync_favorites": True,
+            "submit_favorites": True,
+            "sync_favorites": True,
+            "user_token": "blablabla",
+        },
+        user=logged_in_client.user,
+    )
+    plugins.enable_conf("listenbrainz", True, logged_in_client.user)
+
+    track = factories["music.Track"]()
+    url = reverse("api:v1:history:listenings-list")
+    logged_in_client.post(url, {"track": track.pk})
+    logged_in_client.get(url)
+    listening = history_models.Listening.objects.get(user=logged_in_client.user)
+    handler.assert_called_once_with(listening=listening, conf=None)
+
+
+def test_sync_listenings_from_listenbrainz(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476")
+    track = factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    factories["history.Listening"](
+        creation_date=datetime.datetime.fromtimestamp(1871, timezone.utc), track=track
+    )
+
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "sync_listenings": True,
+    }
+
+    listens = {
+        "payload": {
+            "count": 25,
+            "user_id": "-- the MusicBrainz ID of the user --",
+            "listens": [
+                liblistenbrainz.Listen(
+                    track_name="test",
+                    artist_name="artist_test",
+                    recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
+                    additional_info={"submission_client": "not funkwhale"},
+                    listened_at=-3124224000,
+                ),
+                liblistenbrainz.Listen(
+                    track_name="test2",
+                    artist_name="artist_test2",
+                    recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
+                    additional_info={
+                        "submission_client": "Funkwhale ListenBrainz plugin"
+                    },
+                    listened_at=1871,
+                ),
+                liblistenbrainz.Listen(
+                    track_name="test3",
+                    artist_name="artist_test3",
+                    listened_at=0,
+                ),
+            ],
+        }
+    }
+    no_more_listen = {
+        "payload": {
+            "count": 25,
+            "user_id": "Bilbo",
+            "listens": [],
+        }
+    }
+
+    mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "get_listens",
+        side_effect=[listens, no_more_listen],
+    )
+
+    funkwhale_ready.sync_listenings_from_listenbrainz(user, conf)
+
+    assert history_models.Listening.objects.filter(
+        track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476"
+    ).exists()
+
+    assert "Listen with ts 1871 skipped because already in db" in caplog.text
+    assert "Received listening that doesn't have a mbid. Skipping..." in caplog.text
+
+
+def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+    # track lb fav
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+    # random track
+    factories["music.Track"]()
+    # track lb neutral
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+    favorite = factories["favorites.TrackFavorite"](track=track, user=user)
+    # last_sync
+    track_last_sync = factories["music.Track"](
+        mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7"
+    )
+    factories["favorites.TrackFavorite"](track=track_last_sync, source="Listenbrainz")
+
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "sync_favorites": True,
+    }
+
+    feedbacks = {
+        "count": 5,
+        "feedback": [
+            {
+                "created": 1701116226,
+                "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528",
+                "score": 1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1701116214,
+                "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986",
+                "score": 0,
+                "user_id": user.username,
+            },
+            {
+                # last sync
+                "created": 1690775094,
+                "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
+                "score": -1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1690775093,
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2",
+                "score": 1,
+                "user_id": user.username,
+            },
+        ],
+        "offset": 0,
+        "total_count": 4,
+    }
+    empty_feedback = {"count": 0, "feedback": [], "offset": 0, "total_count": 0}
+    mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "get_user_feedback",
+        side_effect=[feedbacks, empty_feedback],
+    )
+
+    funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
+
+    assert favorites_models.TrackFavorite.objects.filter(
+        track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
+    ).exists()
+    with pytest.raises(favorites_models.TrackFavorite.DoesNotExist):
+        favorite.refresh_from_db()
+
+
+def test_sync_favorites_from_listenbrainz_since(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+    # track lb fav
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+    # track lb neutral
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+    favorite = factories["favorites.TrackFavorite"](track=track, user=user)
+    # track should be not synced
+    factories["music.Track"](mbid="1fd02cf2-7247-4715-8862-c378ec196000")
+    # last_sync
+    track_last_sync = factories["music.Track"](
+        mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7"
+    )
+    factories["favorites.TrackFavorite"](
+        track=track_last_sync,
+        user=user,
+        source="Listenbrainz",
+        creation_date=datetime.datetime.fromtimestamp(1690775094),
+    )
+
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "sync_favorites": True,
+    }
+
+    feedbacks = {
+        "count": 5,
+        "feedback": [
+            {
+                "created": 1701116226,
+                "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528",
+                "score": 1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1701116214,
+                "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986",
+                "score": 0,
+                "user_id": user.username,
+            },
+            {
+                # last sync
+                "created": 1690775094,
+                "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
+                "score": -1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1690775093,
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2",
+                "score": 1,
+                "user_id": user.username,
+            },
+        ],
+        "offset": 0,
+        "total_count": 4,
+    }
+    second_feedback = {
+        "count": 0,
+        "feedback": [
+            {
+                "created": 0,
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec196000",
+                "score": 1,
+                "user_id": user.username,
+            },
+        ],
+        "offset": 0,
+        "total_count": 0,
+    }
+    mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "get_user_feedback",
+        side_effect=[feedbacks, second_feedback],
+    )
+
+    funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
+
+    assert favorites_models.TrackFavorite.objects.filter(
+        track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
+    ).exists()
+    assert not favorites_models.TrackFavorite.objects.filter(
+        track__mbid="1fd02cf2-7247-4715-8862-c378ec196000"
+    ).exists()
+    with pytest.raises(favorites_models.TrackFavorite.DoesNotExist):
+        favorite.refresh_from_db()
+
+
+def test_submit_favorites_to_listenbrainz(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+
+    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+
+    favorite = factories["favorites.TrackFavorite"](track=track)
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "submit_favorites": True,
+    }
+
+    patch = mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "submit_user_feedback",
+        return_value="Success",
+    )
+    funkwhale_ready.submit_favorite_creation(favorite, conf)
+
+    patch.assert_called_once_with(1, track.mbid)
+
+
+def test_submit_favorites_deletion(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+
+    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+
+    favorite = factories["favorites.TrackFavorite"](track=track)
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "submit_favorites": True,
+    }
+
+    patch = mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "submit_user_feedback",
+        return_value="Success",
+    )
+    funkwhale_ready.submit_favorite_deletion(favorite, conf)
+
+    patch.assert_called_once_with(0, track.mbid)

changes/changelog.d/2079.enhancement

@@ -0,0 +1 @@
+Add favorite and listening sync ith Listenbrainz (#2079)

changes/changelog.d/genre-tags.doc

@@ -0,0 +1 @@
+Add genre tags spec.

dev.yml

@@ -25,7 +25,7 @@ services:
     env_file:
       - .env.dev
       - .env
-    image: postgres:${POSTGRES_VERSION-11}-alpine
+    image: postgres:${POSTGRES_VERSION-15}-alpine
     environment:
       - "POSTGRES_HOST_AUTH_METHOD=trust"
     command: postgres ${POSTGRES_ARGS-}

docs/index.md

@@ -109,6 +109,7 @@ specs/multi-artist/index
 specs/user-follow/index
 specs/user-deletion/index
 specs/upload-process/index
+specs/genre-tags/index
 
 ```
 

docs/specs/genre-tags/index.md

@@ -0,0 +1,239 @@
+# Genre tags
+
+## The issue
+
+Funkwhale offers users a facility to assign genre tags to items such as tracks, albums, and artists. The `tags_tag` table is populated automatically when new tags are found in uploaded content, and users can also enter custom tags. By default, the table is empty. This means that a user on a new pod won't see any results when attempting to tag items in the frontend.
+
+## The solution
+
+To provide the best experience for new Funkwhale users, we should pre-populate this table with [genre tags from Musicbrainz](https://musicbrainz.org/genres). Doing this enables users to easily search for and select the tags they want to assign to their content without needing to create custom tags or upload tagged content.
+
+Having these tags easily available also facilitates better tagging within Funkwhale in future, reducing the reliance on external tools such as Picard.
+
+## Feature behavior
+
+### Backend behavior
+
+The `tags_tag` table contains the following fields:
+
+| Field            | Data type                | Description                                                                                                             | Relations                            | Constraints    |
+| ---------------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | -------------- |
+| `id`             | Integer                  | The randomly generated table ID                                                                                         | `tags_taggeditem.tag_id` foreign key | None           |
+| `musicbrainz_id` | UUID                     | The Musicbrainz genre tag `id`. Used to identify the tag in Musicbrainz fetches                                         | None                                 | None           |
+| `name`           | String                   | The name of the tag. Assigned by Funkwhale during creation for use in URLs. Uses Pascal casing for consistency          | None                                 | Must be unique |
+| `display_name`   | String                   | The name of the tag as the user entered it or as it was originally written by Musicbrainz. Lowercase, normalizes spaces | None                                 | None           |
+| `creation_date`  | Timestamp with time zone | The date on which the tag was created                                                                                   | None                                 | None           |
+
+#### Musicbrainz fetch task
+
+To keep Funkwhale's database up-to-date with Musicbrainz's genre tags, we must fetch information from Musicbrainz periodically. We can use the following endpoint to fetch the information:
+
+```text
+https://musicbrainz.org/ws/2/genre/all
+```
+
+This endpoint accepts the `application/json` header for a JSON response. See the [Musicbrainz API documentation](https://musicbrainz.org/doc/MusicBrainz_API) for more information. The pagination can be controlled by passing the following options:
+
+- `limit`: the number of results to return
+- `offset`: the starting point of the page
+
+The fetch task should fetch **all** pages, using the response `genre-count` to determine how many offset positions to pass.
+
+```json
+{
+  "genre-count": 1913,
+  "genre-offset": 24,
+  "genres": [
+    {
+      "disambiguation": "",
+      "id": "243975aa-1250-4429-8bd3-97080af44cf7",
+      "name": "afro trap"
+    },
+    {
+      "name": "afro-cuban jazz",
+      "id": "cdb11433-1ff1-4c88-be16-717567e1342f",
+      "disambiguation": ""
+    },
+    {
+      "name": "afro-funk",
+      "disambiguation": "",
+      "id": "fc00175b-2be9-4d73-ba91-27b3ca827223"
+    },
+    {
+      "name": "afro-jazz",
+      "disambiguation": "",
+      "id": "6f33d775-b4e2-473c-a7db-e34c525cc52d"
+    },
+    {
+      "disambiguation": "",
+      "id": "a7e0229c-6e53-45f1-a6f2-a791e78b159e",
+      "name": "afro-zouk"
+    },
+    {
+      "disambiguation": "funk/soul + West African sounds",
+      "id": "fcc58a18-9326-4c92-8b29-c294d44379c3",
+      "name": "afrobeat"
+    },
+    {
+      "id": "b8793fdb-bbc8-4418-a6f8-05eafbbe07ef",
+      "disambiguation": "West African urban/pop music",
+      "name": "afrobeats"
+    },
+    {
+      "name": "afropiano",
+      "id": "d42b567f-0952-424b-959d-bee6e5961cc0",
+      "disambiguation": ""
+    },
+    {
+      "disambiguation": "",
+      "id": "52349b68-9cad-496e-8785-00d53f410246",
+      "name": "afroswing"
+    },
+    {
+      "name": "agbadza",
+      "disambiguation": "",
+      "id": "c6d1e78b-ac82-4bb8-89d5-21e3226dc906"
+    },
+    {
+      "disambiguation": "",
+      "id": "b8ae0a3c-5826-4104-9663-fe8f828effa9",
+      "name": "agbekor"
+    },
+    {
+      "name": "aggrotech",
+      "disambiguation": "",
+      "id": "c844c144-90a8-4288-981e-e38275592688"
+    },
+    {
+      "name": "ahwash",
+      "id": "4802e6e4-f403-41d1-8e58-76e5cf4df81d",
+      "disambiguation": ""
+    },
+    {
+      "id": "50cc5641-b4f9-40b7-bf7a-6d903ac6c1c5",
+      "disambiguation": "",
+      "name": "aita"
+    },
+    {
+      "id": "aebbce35-0e8b-40e9-b04c-bebbbda124d0",
+      "disambiguation": "",
+      "name": "akishibu-kei"
+    },
+    {
+      "name": "al jeel",
+      "disambiguation": "",
+      "id": "0f8d3ff4-8cda-42c4-b462-10352cd01606"
+    },
+    {
+      "name": "algerian chaabi",
+      "id": "998efb76-2f98-41c8-8c5f-74c32e405e9f",
+      "disambiguation": ""
+    },
+    {
+      "name": "algorave",
+      "id": "e0a9d0d1-b86f-4344-82a9-022a84627087",
+      "disambiguation": ""
+    },
+    {
+      "name": "alloukou",
+      "disambiguation": "",
+      "id": "e367c884-d94d-4fba-abc4-8ac51d167ccf"
+    },
+    {
+      "disambiguation": "",
+      "id": "ef1d11cc-e70f-4885-ad6c-103f060d33b2",
+      "name": "alpenrock"
+    },
+    {
+      "disambiguation": "",
+      "id": "5f9cba3d-1a9f-46cd-8c49-7ed78d1f3354",
+      "name": "alternative country"
+    },
+    {
+      "name": "alternative dance",
+      "id": "8301f73c-9166-4108-bfeb-4fd22dc19083",
+      "disambiguation": ""
+    },
+    {
+      "name": "alternative folk",
+      "id": "0b48a36c-630f-4ee7-8cf3-480e3dd8be65",
+      "disambiguation": ""
+    },
+    {
+      "name": "alternative hip hop",
+      "disambiguation": "",
+      "id": "924943cd-73c8-45c0-96eb-74f2a15e5d6e"
+    },
+    {
+      "disambiguation": "",
+      "id": "7c4d0994-4c49-4c74-8763-df27fc0084cc",
+      "name": "alté"
+    }
+  ]
+}
+```
+
+The fetch task should run _initially upon first startup_ and then _monthly_ thereafter. The pod admin must be able to disable this job or run it manually at their discretion.
+
+The task should use the following logic:
+
+1. Call the Musicbrainz API to fetch new data
+2. Verify the listed entries against the Funkwhale tag table. The `id` field in the response should be checked against the `musicbrainz_id` field
+3. Any entries that do not currently exist in Funkwhale should be added with the following mapping:
+
+| Musicbrainz response field | Tags table column | Notes                                                                             |
+| -------------------------- | ----------------- | --------------------------------------------------------------------------------- |
+| `id`                       | `musicbrainz_id`  |                                                                                   |
+| `name`                     | `display_name`    | Funkwhale should automatically generate a Pascal cased `name` based on this entry |
+
+4. If the `display_name` of a tag **exactly matches** a `name` in the Musicbrainz response but the tag has no `musicbrainz_id`, the `musicbrainz_id` should be populated
+
+### Frontend behavior
+
+#### Tagged uploads
+
+When a user uploads new content with genre tags, the tagged item should be linked to any existing tags and new ones should be created if they're not found.
+
+#### In-app tagging
+
+When a user uploads new content with _no_ genre tags, they should be able to select tags from a dropdown menu. This menu is populated with the tags from the database with the `display_name` shown in the list. When a tag is selected, the item is linked to the associated tag.
+
+If a user inserts a new tag, Funkwhale should:
+
+1. Store the entered string as the tag's `display_name`
+2. Generate a Pascal cased `name` for the tag
+3. Associate the targeted object with the new tag
+
+#### Search results
+
+Users should be able to search for tags using Funkwhale's in-app search. In search autocomplete and search results page, the `display_name` should be used. The `name` of the tag should be used to populate the search URL.
+
+#### Cards
+
+The `display_name` of the tag should be shown in pills against cards.
+
+### Admin options
+
+If the admin of a server wants to **disable** MusicBrainz tagging, they should be able to toggle this in their instance settings. If the setting is **disabled**:
+
+- The sync task should stop running
+- Any tags with an `musicbrainz_id` should be excluded from API queries.
+
+## Availability
+
+- [x] Admin panel
+- [x] App frontend
+- [x] CLI
+
+## Responsible parties
+
+- Backend group:
+  - Update the tracks table to support the new information
+  - Update the API to support the new information, or create a new v2 endpoint
+  - Create the new fetch task
+  - Add admin controls for the new task
+- Frontend group:
+  - Update views to use `display_name` instead of `name` for tag results
+  - Update API calls to use the new API structure created by the backend group
+- Documentation group:
+  - Document the new task and settings for admins

docs/user/plugins/listenbrainz.md

@@ -19,8 +19,8 @@ The **ListenBrainz** plugin enables you to submit ({term}`scrobble<Scrobbling>`)
 
 :::
 
-:::{tab-item} Desktop
-:sync: desktop
+:::{tab-item} Mobile
+:sync: mobile
 
 1. Log in to your account.
 2. Select the cog icon ({fa}`cog`) or your avatar to open the {guilabel}`Options` menu.
@@ -36,3 +36,48 @@ The **ListenBrainz** plugin enables you to submit ({term}`scrobble<Scrobbling>`)
 ::::
 
 That's it! You've set up the **ListenBrainz** plugin. When you listen to tracks, the plugin sends the information to ListenBrainz.
+
+## Enable data synchronization
+
+The ListenBrainz plugin supports synchronizing listenings and track favorites with Funkwhale. To enable support for synchronization:
+
+::::{tab-set}
+
+:::{tab-item} Desktop
+:sync: desktop
+
+1. Log in to your account.
+2. Select the cog icon ({fa}`cog`) or your avatar to expand the user menu.
+3. Select {guilabel}`Settings`.
+4. Scroll down to the {guilabel}`Plugins` section.
+5. Select {guilabel}`Manage plugins`.
+6. Find the {guilabel}`ListenBrainz` plugin.
+7. Enter {guilabel}`Your ListenBrainz user name`. You can find this on your ListenBrainz profile.
+8. Select the data you want to synchronize. The following options are available:
+   - {guilabel}`Enable listenings submission to ListenBrainz`: submit your Funkwhale listens to ListenBrainz.
+   - {guilabel}`Enable listenings sync`: pull listening data from ListenBrainz into Funkwhale.
+   - {guilabel}`Enable favorite submission to ListenBrainz services`: submit your Funkwhale favorites activity to ListenBrainz.
+   - {guilabel}`Enable favorite sync`: pull favorites data from ListenBrainz into Funkwhale.
+9. Select {guilabel}`Save`.
+
+:::
+
+:::{tab-item} Mobile
+:sync: mobile
+
+1. Log in to your account.
+2. Select the cog icon ({fa}`cog`) or your avatar to open the {guilabel}`Options` menu.
+3. Select {guilabel}`Settings`.
+4. Scroll down to the {guilabel}`Plugins` section.
+5. Select {guilabel}`Manage plugins`.
+6. Find the {guilabel}`ListenBrainz` plugin.
+7. Enter {guilabel}`Your ListenBrainz user name`. You can find this on your ListenBrainz profile.
+8. Select the data you want to synchronize. The following options are available:
+   - {guilabel}`Enable listenings submission to ListenBrainz`: submit your Funkwhale listens to ListenBrainz.
+   - {guilabel}`Enable listenings sync`: pull listening data from ListenBrainz into Funkwhale.
+   - {guilabel}`Enable favorite submission to ListenBrainz services`: submit your Funkwhale favorites activity to ListenBrainz.
+   - {guilabel}`Enable favorite sync`: pull favorites data from ListenBrainz into Funkwhale.
+9. Select {guilabel}`Save`.
+
+:::
+::::

front/package.json

@@ -44,7 +44,7 @@
     "standardized-audio-context": "25.3.60",
     "text-clipper": "2.2.0",
     "transliteration": "2.3.5",
-    "universal-cookie": "4.0.4",
+    "universal-cookie": "7.1.4",
     "vite-plugin-pwa": "0.14.4",
     "vue": "3.3.11",
     "vue-gettext": "2.1.12",

front/yarn.lock

@@ -1808,11 +1808,6 @@
     "@tauri-apps/cli-win32-ia32-msvc" "2.0.0-beta.2"
     "@tauri-apps/cli-win32-x64-msvc" "2.0.0-beta.2"
 
-"@types/cookie@^0.3.3":
-  version "0.3.3"
-  resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.3.3.tgz#85bc74ba782fb7aa3a514d11767832b0e3bc6803"
-  integrity sha512-LKVP3cgXBT9RYj+t+9FDKwS5tdI+rPBXaNSkma7hvqy35lc7mAokC2zsqWJH0LaqIt3B962nuYI77hsJoT1gow==
-
 "@types/cookie@^0.6.0":
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.6.0.tgz#eac397f28bf1d6ae0ae081363eca2f425bedf0d5"
@@ -3671,16 +3666,16 @@ convert-source-map@^2.0.0:
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a"
   integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==
 
-cookie@^0.4.0:
-  version "0.4.2"
-  resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432"
-  integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==
-
 cookie@^0.5.0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b"
   integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==
 
+cookie@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051"
+  integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==
+
 core-js-compat@^3.31.0, core-js-compat@^3.34.0:
   version "3.36.0"
   resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.36.0.tgz#087679119bc2fdbdefad0d45d8e5d307d45ba190"
@@ -7518,13 +7513,13 @@ unique-string@^2.0.0:
   dependencies:
     crypto-random-string "^2.0.0"
 
-universal-cookie@4.0.4:
-  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/universal-cookie/-/universal-cookie-4.0.4.tgz#06e8b3625bf9af049569ef97109b4bb226ad798d"
-  integrity sha512-lbRVHoOMtItjWbM7TwDLdl8wug7izB0tq3/YVKhT/ahB4VDvWMyvnADfnJI8y6fSvsjh51Ix7lTGC6Tn4rMPhw==
+universal-cookie@7.1.4:
+  version "7.1.4"
+  resolved "https://registry.yarnpkg.com/universal-cookie/-/universal-cookie-7.1.4.tgz#d11bb95e405639c0ff0b467a64a5ccc5ce97dfc6"
+  integrity sha512-Q+DVJsdykStWRMtXr2Pdj3EF98qZHUH/fXv/gwFz/unyToy1Ek1w5GsWt53Pf38tT8Gbcy5QNsj61Xe9TggP4g==
   dependencies:
-    "@types/cookie" "^0.3.3"
-    cookie "^0.4.0"
+    "@types/cookie" "^0.6.0"
+    cookie "^0.6.0"
 
 universalify@^0.2.0:
   version "0.2.0"