Add remote profile fetcher

High level method to fetch a remote profile. Currently falls back to Diaspora protocol as no others are supported.

Returns a Profile entity.

Closes #15

CHANGELOG.md

@@ -11,6 +11,7 @@
 - `Profile` base entity and Diaspora counterpart `DiasporaProfile`. Represents a user profile.
 - `federation.utils.network.fetch_document` utility function to fetch a remote document. Returns document, status code and possible exception. Takes either `url` or a `host` + `path` combination. With `host`, https is first tried and optionally fall back to http.
 - Utility methods to retrieve Diaspora user discovery related documents. These include the host-meta, webfinger and hCard documents. The utility methods are in `federation.utils.diaspora`.
+- Utility to fetch remote profile, `federation.fetchers.retrieve_remote_profile`. Currently always uses Diaspora protocol. Returns a `Profile` entity.
 
 ## Changed
 - Unlock most of the direct dependencies to a certain version range. Unlock all of test requirements to any version.

federation/fetchers.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+import importlib
+import warnings
+
+
+def retrieve_remote_profile(handle, protocol=None):
+    """High level retrieve profile method.
+
+    Retrieve the profile from a remote location, using either the given protocol or by checking each
+    protocol until a user can be constructed from the remote documents.
+
+    Currently, due to no other protocols supported, always use the Diaspora protocol.
+
+    Args:
+        handle (str) - The profile handle in format username@domain.tld
+    """
+    if protocol:
+        warnings.warn("Currently retrieve_remote_profile doesn't use the protocol argument. Diaspora protocol"
+                      "will always be used.")
+    protocol_name = "diaspora"
+    utils = importlib.import_module("federation.utils.%s" % protocol_name)
+    return utils.retrieve_and_parse_profile(handle)

federation/tests/test_fetchers.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from unittest.mock import patch, Mock
+
+from federation.fetchers import retrieve_remote_profile
+
+
+class TestRetrieveRemoteProfile(object):
+    @patch("federation.fetchers.importlib.import_module")
+    def test_calls_diaspora_retrieve_and_parse_profile(self, mock_import):
+        class MockRetrieve(Mock):
+            def retrieve_and_parse_profile(self, handle):
+                return "called with %s" % handle
+
+        mock_retrieve = MockRetrieve()
+        mock_import.return_value = mock_retrieve
+        assert retrieve_remote_profile("foo@bar") == "called with foo@bar"

federation/tests/utils/test_diaspora.py

@@ -2,8 +2,11 @@
 from unittest.mock import patch
 from urllib.parse import quote
 
-from federation.hostmeta.generators import DiasporaWebFinger, DiasporaHostMeta
-from federation.utils.diaspora import retrieve_diaspora_hcard, retrieve_diaspora_webfinger, retrieve_diaspora_host_meta
+from lxml import html
+
+from federation.hostmeta.generators import DiasporaWebFinger, DiasporaHostMeta, DiasporaHCard, generate_hcard
+from federation.utils.diaspora import retrieve_diaspora_hcard, retrieve_diaspora_webfinger, retrieve_diaspora_host_meta, \
+    _get_element_text_or_none, _get_element_attr_or_none, parse_profile_from_hcard, retrieve_and_parse_profile
 
 
 class TestRetrieveDiasporaHCard(object):
@@ -82,3 +85,81 @@ class TestRetrieveDiasporaHostMeta(object):
         document = retrieve_diaspora_host_meta("localhost")
         mock_fetch.assert_called_with(host="localhost", path="/.well-known/host-meta")
         assert document == None
+
+
+class TestGetElementTextOrNone(object):
+    doc = html.fromstring("<foo>bar</foo>")
+
+    def test_text_returned_on_element(self):
+        assert _get_element_text_or_none(self.doc, "foo") == "bar"
+
+    def test_none_returned_on_no_element(self):
+        assert _get_element_text_or_none(self.doc, "bar") == None
+
+
+class TestGetElementAttrOrNone(object):
+    doc = html.fromstring("<foo src='baz'>bar</foo>")
+
+    def test_attr_returned_on_attr(self):
+        assert _get_element_attr_or_none(self.doc, "foo", "src") == "baz"
+
+    def test_none_returned_on_attr(self):
+        assert _get_element_attr_or_none(self.doc, "foo", "href") == None
+
+    def test_none_returned_on_no_element(self):
+        assert _get_element_attr_or_none(self.doc, "bar", "href") == None
+
+
+class TestParseProfileFromHCard(object):
+    def test_profile_is_parsed(self):
+        hcard = generate_hcard(
+            "diaspora",
+            hostname="https://hostname",
+            fullname="fullname",
+            firstname="firstname",
+            lastname="lastname",
+            photo300="photo300",
+            photo100="photo100",
+            photo50="photo50",
+            searchable="true",
+            guid="guid",
+            public_key="public_key",
+            username="username",
+        )
+        profile = parse_profile_from_hcard(hcard)
+        assert profile.name == "fullname"
+        assert profile.image_urls == {
+            "small": "photo50", "medium": "photo100", "large": "photo300"
+        }
+        assert profile.public == True
+        assert profile.handle == "username@hostname"
+        assert profile.guid == "guid"
+        assert profile.public_key == "public_key\n"
+
+
+class TestRetrieveAndParseProfile(object):
+    @patch("federation.utils.diaspora.retrieve_diaspora_hcard", return_value=None)
+    def test_retrieve_diaspora_hcard_is_called(self, mock_retrieve):
+        retrieve_and_parse_profile("foo@bar")
+        mock_retrieve.assert_called_with("foo@bar")
+
+    @patch("federation.utils.diaspora.parse_profile_from_hcard")
+    @patch("federation.utils.diaspora.retrieve_diaspora_hcard")
+    def test_parse_profile_from_hcard_called(self, mock_retrieve, mock_parse):
+        hcard = generate_hcard(
+            "diaspora",
+            hostname="https://hostname",
+            fullname="fullname",
+            firstname="firstname",
+            lastname="lastname",
+            photo300="photo300",
+            photo100="photo100",
+            photo50="photo50",
+            searchable="true",
+            guid="guid",
+            public_key="public_key",
+            username="username",
+        )
+        mock_retrieve.return_value = hcard
+        retrieve_and_parse_profile("foo@bar")
+        mock_parse.assert_called_with(hcard)

federation/utils/diaspora.py

@@ -1,13 +1,16 @@
 # -*- coding: utf-8 -*-
-from urllib.parse import quote
+from urllib.parse import quote, urlparse
 
+from lxml import html
 from xrd import XRD
 
+from federation.entities.base import Profile
 from federation.utils.network import fetch_document
 
 
 def retrieve_diaspora_hcard(handle):
-    """Retrieve a remote Diaspora hCard document.
+    """
+    Retrieve a remote Diaspora hCard document.
 
     Args:
         handle (str) - Remote handle to retrieve
@@ -26,7 +29,8 @@ def retrieve_diaspora_hcard(handle):
 
 
 def retrieve_diaspora_webfinger(handle):
-    """Retrieve a remote Diaspora webfinger document.
+    """
+    Retrieve a remote Diaspora webfinger document.
 
     Args:
         handle (str) - Remote handle to retrieve
@@ -46,7 +50,8 @@ def retrieve_diaspora_webfinger(handle):
 
 
 def retrieve_diaspora_host_meta(host):
-    """Retrieve a remote Diaspora host-meta document.
+    """
+    Retrieve a remote Diaspora host-meta document.
 
     Args:
         host (str) - Host to retrieve from
@@ -59,3 +64,72 @@ def retrieve_diaspora_host_meta(host):
         return None
     xrd = XRD.parse_xrd(document)
     return xrd
+
+
+def _get_element_text_or_none(document, selector):
+    """
+    Using a CSS selector, get the element and return the text, or None if no element.
+
+    Args:
+        document (HTMLElement) - HTMLElement document
+        selector (str) - CSS selector
+    """
+    element = document.cssselect(selector)
+    if element:
+        return element[0].text
+    return None
+
+
+def _get_element_attr_or_none(document, selector, attribute):
+    """
+    Using a CSS selector, get the element and return the given attribute value, or None if no element.
+
+    Args:
+        document (HTMLElement) - HTMLElement document
+        selector (str) - CSS selector
+        attribute (str) - The attribute to get from the element
+    """
+    element = document.cssselect(selector)
+    if element:
+        return element[0].get(attribute)
+    return None
+
+
+def parse_profile_from_hcard(hcard):
+    """
+    Parse all the fields we can from a hCard document to get a Profile.
+
+    Args:
+         hcard (str) - HTML hcard document
+    """
+    doc = html.fromstring(hcard)
+    domain = urlparse(_get_element_attr_or_none(doc, "a#pod_location", "href")).netloc
+    profile = Profile(
+        name=_get_element_text_or_none(doc, "dl.entity_full_name span.fn"),
+        image_urls={
+            "small": _get_element_attr_or_none(doc, "dl.entity_photo_small img.photo", "src"),
+            "medium": _get_element_attr_or_none(doc, "dl.entity_photo_medium img.photo", "src"),
+            "large": _get_element_attr_or_none(doc, "dl.entity_photo img.photo", "src"),
+        },
+        public=True if _get_element_text_or_none(doc, "dl.entity_searchable span.searchable") == "true" else False,
+        handle="%s@%s" % (_get_element_text_or_none(doc, "dl.entity_nickname span.nickname"), domain),
+        guid=_get_element_text_or_none(doc, "dl.entity_uid span.uid"),
+        public_key=_get_element_text_or_none(doc, "dl.entity_key pre.key"),
+    )
+    return profile
+
+
+def retrieve_and_parse_profile(handle):
+    """
+    Retrieve the remote user and return a Profile object.
+
+    Args:
+        handle (str) - User handle in username@domain.tld format
+
+    Returns:
+        Profile
+    """
+    hcard = retrieve_diaspora_hcard(handle)
+    if not hcard:
+        return None
+    return parse_profile_from_hcard(hcard)

setup.py

@@ -21,6 +21,7 @@ setup(
     packages=find_packages(),
     license="BSD 3-clause",
     install_requires=[
+        "cssselect>=0.9.2",
         "dirty-validators>=0.3.0, <0.4.0",
         "lxml>=3.4.0, <4.0.0",
         "jsonschema>=2.0.0, <3.0.0",