System actor

api/config/settings/common.py

@@ -94,6 +94,9 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
 )
 # XXX: deprecated, see #186
 FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
+FEDERATION_SERVICE_ACTOR_USERNAME = env(
+    "FEDERATION_SERVICE_ACTOR_USERNAME", default="service"
+)
 ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
 
 # APP CONFIGURATION

api/funkwhale_api/common/utils.py

@@ -147,3 +147,24 @@ def order_for_search(qs, field):
     this function will order the given qs based on the length of the given field
     """
     return qs.annotate(__size=models.functions.Length(field)).order_by("__size")
+
+
+def recursive_getattr(obj, key, permissive=False):
+    """
+    Given a dictionary such as {'user': {'name': 'Bob'}} and
+    a dotted string such as user.name, returns 'Bob'.
+
+    If the value is not present, returns None
+    """
+    v = obj
+    for k in key.split("."):
+        try:
+            v = v.get(k)
+        except (TypeError, AttributeError):
+            if not permissive:
+                raise
+            return
+        if v is None:
+            return
+
+    return v

api/funkwhale_api/federation/activity.py

@@ -9,6 +9,8 @@ from django.db.models import Q
 from funkwhale_api.common import channels
 from funkwhale_api.common import utils as funkwhale_utils
 
+recursive_getattr = funkwhale_utils.recursive_getattr
+
 
 logger = logging.getLogger(__name__)
 PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
@@ -89,9 +91,9 @@ def should_reject(id, actor_id=None, payload={}):
 
     media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"]
     relevant_values = [
-        recursive_gettattr(payload, "type", permissive=True),
-        recursive_gettattr(payload, "object.type", permissive=True),
-        recursive_gettattr(payload, "target.type", permissive=True),
+        recursive_getattr(payload, "type", permissive=True),
+        recursive_getattr(payload, "object.type", permissive=True),
+        recursive_getattr(payload, "target.type", permissive=True),
     ]
     # if one of the payload types match our internal media types, then
     # we apply policies that reject media
@@ -343,7 +345,7 @@ class OutboxRouter(Router):
             return activities
 
 
-def recursive_gettattr(obj, key, permissive=False):
+def recursive_getattr(obj, key, permissive=False):
     """
     Given a dictionary such as {'user': {'name': 'Bob'}} and
     a dotted string such as user.name, returns 'Bob'.
@@ -366,7 +368,7 @@ def recursive_gettattr(obj, key, permissive=False):
 
 def match_route(route, payload):
     for key, value in route.items():
-        payload_value = recursive_gettattr(payload, key)
+        payload_value = recursive_getattr(payload, key)
         if payload_value != value:
             return False
 

api/funkwhale_api/federation/actors.py

@@ -5,8 +5,9 @@ from django.conf import settings
 from django.utils import timezone
 
 from funkwhale_api.common import preferences, session
+from funkwhale_api.users import models as users_models
 
-from . import models, serializers
+from . import keys, models, serializers
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +29,7 @@ def get_actor_data(actor_url):
 def get_actor(fid, skip_cache=False):
     if not skip_cache:
         try:
-            actor = models.Actor.objects.get(fid=fid)
+            actor = models.Actor.objects.select_related().get(fid=fid)
         except models.Actor.DoesNotExist:
             actor = None
         fetch_delta = datetime.timedelta(
@@ -42,3 +43,23 @@ def get_actor(fid, skip_cache=False):
     serializer.is_valid(raise_exception=True)
 
     return serializer.save(last_fetch_date=timezone.now())
+
+
+def get_service_actor():
+    name, domain = (
+        settings.FEDERATION_SERVICE_ACTOR_USERNAME,
+        settings.FEDERATION_HOSTNAME,
+    )
+    try:
+        return models.Actor.objects.select_related().get(
+            preferred_username=name, domain__name=domain
+        )
+    except models.Actor.DoesNotExist:
+        pass
+
+    args = users_models.get_actor_data(name)
+    private, public = keys.get_key_pair()
+    args["private_key"] = private.decode("utf-8")
+    args["public_key"] = public.decode("utf-8")
+    args["type"] = "Service"
+    return models.Actor.objects.create(**args)

api/funkwhale_api/federation/authentication.py

@@ -1,11 +1,13 @@
 import cryptography
 import logging
+import datetime
 
 from django.contrib.auth.models import AnonymousUser
-from rest_framework import authentication, exceptions as rest_exceptions
+from django.utils import timezone
 
+from rest_framework import authentication, exceptions as rest_exceptions
 from funkwhale_api.moderation import models as moderation_models
-from . import actors, exceptions, keys, signing, utils
+from . import actors, exceptions, keys, signing, tasks, utils
 
 
 logger = logging.getLogger(__name__)
@@ -57,6 +59,15 @@ class SignatureAuthentication(authentication.BaseAuthentication):
             actor = actors.get_actor(actor_url, skip_cache=True)
             signing.verify_django(request, actor.public_key.encode("utf-8"))
 
+        # we trigger a nodeinfo update on the actor's domain, if needed
+        fetch_delay = 24 * 3600
+        now = timezone.now()
+        last_fetch = actor.domain.nodeinfo_fetch_date
+        if not last_fetch or (
+            last_fetch < (now - datetime.timedelta(seconds=fetch_delay))
+        ):
+            tasks.update_domain_nodeinfo(domain_name=actor.domain.name)
+            actor.domain.refresh_from_db()
         return actor
 
     def authenticate(self, request):

api/funkwhale_api/federation/factories.py

@@ -69,6 +69,7 @@ def create_user(actor):
 @registry.register
 class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("domain_name")
+    nodeinfo_fetch_date = factory.LazyFunction(lambda: timezone.now())
 
     class Meta:
         model = "federation.Domain"

api/funkwhale_api/federation/migrations/0017_auto_20190130_0926.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.1.5 on 2019-01-30 09:26
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import funkwhale_api.common.validators
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0016_auto_20181227_1605'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='actor',
+            name='old_domain',
+        ),
+        migrations.AddField(
+            model_name='domain',
+            name='service_actor',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='managed_domains', to='federation.Actor'),
+        ),
+        migrations.AlterField(
+            model_name='domain',
+            name='name',
+            field=models.CharField(max_length=255, primary_key=True, serialize=False, validators=[funkwhale_api.common.validators.DomainValidator()]),
+        ),
+        migrations.AlterField(
+            model_name='domain',
+            name='nodeinfo',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, max_length=50000),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -46,7 +46,9 @@ class FederationMixin(models.Model):
 
 class ActorQuerySet(models.QuerySet):
     def local(self, include=True):
-        return self.exclude(user__isnull=include)
+        if include:
+            return self.filter(domain__name=settings.FEDERATION_HOSTNAME)
+        return self.exclude(domain__name=settings.FEDERATION_HOSTNAME)
 
     def with_current_usage(self):
         qs = self
@@ -92,7 +94,13 @@ class Domain(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
     nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
-
+    service_actor = models.ForeignKey(
+        "Actor",
+        related_name="managed_domains",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
     objects = DomainQuerySet.as_manager()
 
     def __str__(self):

api/funkwhale_api/federation/tasks.py

@@ -11,6 +11,7 @@ from requests.exceptions import RequestException
 
 from funkwhale_api.common import preferences
 from funkwhale_api.common import session
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.music import models as music_models
 from funkwhale_api.taskapp import celery
 
@@ -18,6 +19,7 @@ from . import keys
 from . import models, signing
 from . import serializers
 from . import routes
+from . import utils
 
 logger = logging.getLogger(__name__)
 
@@ -184,9 +186,27 @@ def update_domain_nodeinfo(domain):
         nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
     except (requests.RequestException, serializers.serializers.ValidationError) as e:
         nodeinfo = {"status": "error", "error": str(e)}
+
+    service_actor_id = common_utils.recursive_getattr(
+        nodeinfo, "payload.metadata.actorId", permissive=True
+    )
+    try:
+        domain.service_actor = (
+            utils.retrieve_ap_object(
+                service_actor_id,
+                queryset=models.Actor,
+                serializer_class=serializers.ActorSerializer,
+            )
+            if service_actor_id
+            else None
+        )
+    except (serializers.serializers.ValidationError, RequestException) as e:
+        logger.warning(
+            "Cannot fetch system actor for domain %s: %s", domain.name, str(e)
+        )
     domain.nodeinfo_fetch_date = now
     domain.nodeinfo = nodeinfo
-    domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])
+    domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
 
 
 def delete_qs(qs):

api/funkwhale_api/instance/nodeinfo.py

@@ -2,6 +2,7 @@ import memoize.djangocache
 
 import funkwhale_api
 from funkwhale_api.common import preferences
+from funkwhale_api.federation import actors
 
 from . import stats
 
@@ -19,6 +20,7 @@ def get():
         "openRegistrations": preferences.get("users__registration_enabled"),
         "usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
         "metadata": {
+            "actorId": actors.get_service_actor().fid,
             "private": preferences.get("instance__nodeinfo_private"),
             "shortDescription": preferences.get("instance__short_description"),
             "longDescription": preferences.get("instance__long_description"),

api/funkwhale_api/users/models.py

@@ -245,41 +245,52 @@ class Invitation(models.Model):
         return super().save(**kwargs)
 
 
-def get_actor_data(user):
-    username = federation_utils.slugify_username(user.username)
+def get_actor_data(username):
+    slugified_username = federation_utils.slugify_username(username)
     return {
-        "preferred_username": username,
+        "preferred_username": slugified_username,
         "domain": federation_models.Domain.objects.get_or_create(
             name=settings.FEDERATION_HOSTNAME
         )[0],
         "type": "Person",
-        "name": user.username,
+        "name": username,
         "manually_approves_followers": False,
         "fid": federation_utils.full_url(
-            reverse("federation:actors-detail", kwargs={"preferred_username": username})
+            reverse(
+                "federation:actors-detail",
+                kwargs={"preferred_username": slugified_username},
+            )
         ),
         "shared_inbox_url": federation_models.get_shared_inbox_url(),
         "inbox_url": federation_utils.full_url(
-            reverse("federation:actors-inbox", kwargs={"preferred_username": username})
+            reverse(
+                "federation:actors-inbox",
+                kwargs={"preferred_username": slugified_username},
+            )
         ),
         "outbox_url": federation_utils.full_url(
-            reverse("federation:actors-outbox", kwargs={"preferred_username": username})
+            reverse(
+                "federation:actors-outbox",
+                kwargs={"preferred_username": slugified_username},
+            )
         ),
         "followers_url": federation_utils.full_url(
             reverse(
-                "federation:actors-followers", kwargs={"preferred_username": username}
+                "federation:actors-followers",
+                kwargs={"preferred_username": slugified_username},
             )
         ),
         "following_url": federation_utils.full_url(
             reverse(
-                "federation:actors-following", kwargs={"preferred_username": username}
+                "federation:actors-following",
+                kwargs={"preferred_username": slugified_username},
             )
         ),
     }
 
 
 def create_actor(user):
-    args = get_actor_data(user)
+    args = get_actor_data(user.username)
     private, public = keys.get_key_pair()
     args["private_key"] = private.decode("utf-8")
     args["public_key"] = public.decode("utf-8")

api/tests/federation/test_actors.py

@@ -46,3 +46,15 @@ def test_get_actor_refresh(factories, preferences, mocker):
     assert new_actor == actor
     assert new_actor.last_fetch_date > actor.last_fetch_date
     assert new_actor.preferred_username == "New me"
+
+
+def test_get_service_actor(db, settings):
+    settings.FEDERATION_HOSTNAME = "test.hello"
+    settings.FEDERATION_SERVICE_ACTOR_USERNAME = "bob"
+    actor = actors.get_service_actor()
+
+    assert actor.preferred_username == "bob"
+    assert actor.domain.name == "test.hello"
+    assert actor.private_key is not None
+    assert actor.type == "Service"
+    assert actor.public_key is not None

api/tests/federation/test_authentication.py

@@ -5,6 +5,7 @@ from funkwhale_api.federation import authentication, exceptions, keys
 
 def test_authenticate(factories, mocker, api_request):
     private, public = keys.get_key_pair()
+    factories["federation.Domain"](name="test.federation", nodeinfo_fetch_date=None)
     actor_url = "https://test.federation/actor"
     mocker.patch(
         "funkwhale_api.federation.actors.get_actor_data",
@@ -22,6 +23,10 @@ def test_authenticate(factories, mocker, api_request):
             },
         },
     )
+    update_domain_nodeinfo = mocker.patch(
+        "funkwhale_api.federation.tasks.update_domain_nodeinfo"
+    )
+
     signed_request = factories["federation.SignedRequest"](
         auth__key=private, auth__key_id=actor_url + "#main-key", auth__headers=["date"]
     )
@@ -40,6 +45,7 @@ def test_authenticate(factories, mocker, api_request):
     assert user.is_anonymous is True
     assert actor.public_key == public.decode("utf-8")
     assert actor.fid == actor_url
+    update_domain_nodeinfo.assert_called_once_with(domain_name="test.federation")
 
 
 def test_authenticate_skips_blocked_domain(factories, api_request):

api/tests/federation/test_tasks.py

@@ -161,22 +161,32 @@ def test_fetch_nodeinfo(factories, r_mock, now):
 
 
 def test_update_domain_nodeinfo(factories, mocker, now):
-    domain = factories["federation.Domain"]()
-    mocker.patch.object(tasks, "fetch_nodeinfo", return_value={"hello": "world"})
+    domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
+    actor = factories["federation.Actor"](fid="https://actor.id")
+    mocker.patch.object(
+        tasks,
+        "fetch_nodeinfo",
+        return_value={"hello": "world", "metadata": {"actorId": "https://actor.id"}},
+    )
 
     assert domain.nodeinfo == {}
     assert domain.nodeinfo_fetch_date is None
+    assert domain.service_actor is None
 
     tasks.update_domain_nodeinfo(domain_name=domain.name)
 
     domain.refresh_from_db()
 
     assert domain.nodeinfo_fetch_date == now
-    assert domain.nodeinfo == {"status": "ok", "payload": {"hello": "world"}}
+    assert domain.nodeinfo == {
+        "status": "ok",
+        "payload": {"hello": "world", "metadata": {"actorId": "https://actor.id"}},
+    }
+    assert domain.service_actor == actor
 
 
 def test_update_domain_nodeinfo_error(factories, r_mock, now):
-    domain = factories["federation.Domain"]()
+    domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
     wellknown_url = "https://{}/.well-known/nodeinfo".format(domain.name)
 
     r_mock.get(wellknown_url, status_code=500)

api/tests/federation/test_views.py

@@ -2,7 +2,7 @@ import pytest
 from django.core.paginator import Paginator
 from django.urls import reverse
 
-from funkwhale_api.federation import serializers, webfinger
+from funkwhale_api.federation import actors, serializers, webfinger
 
 
 def test_wellknown_webfinger_validates_resource(db, api_client, settings, mocker):
@@ -54,6 +54,19 @@ def test_local_actor_detail(factories, api_client):
     assert response.data == serializer.data
 
 
+def test_service_actor_detail(factories, api_client):
+    actor = actors.get_service_actor()
+    url = reverse(
+        "federation:actors-detail",
+        kwargs={"preferred_username": actor.preferred_username},
+    )
+    serializer = serializers.ActorSerializer(actor)
+    response = api_client.get(url)
+
+    assert response.status_code == 200
+    assert response.data == serializer.data
+
+
 def test_local_actor_inbox_post_requires_auth(factories, api_client):
     user = factories["users.User"](with_actor=True)
     url = reverse(

api/tests/instance/test_nodeinfo.py

@@ -1,5 +1,6 @@
 import funkwhale_api
 from funkwhale_api.instance import nodeinfo
+from funkwhale_api.federation import actors
 
 
 def test_nodeinfo_dump(preferences, mocker):
@@ -23,6 +24,7 @@ def test_nodeinfo_dump(preferences, mocker):
         "openRegistrations": preferences["users__registration_enabled"],
         "usage": {"users": {"total": 1, "activeHalfyear": 12, "activeMonth": 13}},
         "metadata": {
+            "actorId": actors.get_service_actor().fid,
             "private": preferences["instance__nodeinfo_private"],
             "shortDescription": preferences["instance__short_description"],
             "longDescription": preferences["instance__long_description"],
@@ -60,6 +62,7 @@ def test_nodeinfo_dump_stats_disabled(preferences, mocker):
         "openRegistrations": preferences["users__registration_enabled"],
         "usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
         "metadata": {
+            "actorId": actors.get_service_actor().fid,
             "private": preferences["instance__nodeinfo_private"],
             "shortDescription": preferences["instance__short_description"],
             "longDescription": preferences["instance__long_description"],

api/tests/manage/test_serializers.py

@@ -40,7 +40,7 @@ def test_user_update_permission(factories):
 
 
 def test_manage_domain_serializer(factories, now):
-    domain = factories["federation.Domain"]()
+    domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
     setattr(domain, "actors_count", 42)
     setattr(domain, "outbox_activities_count", 23)
     expected = {

changes/changelog.d/system-actor.enhancement

@@ -0,0 +1 @@
+Expose an instance-level actor (service@domain) in nodeinfo endpoint (#689)