From 945c9495f56742334ddd464e14f9c7a4e3160438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20=C5=A0korpil?= Date: Sat, 8 Jan 2022 02:16:16 +0100 Subject: [PATCH] Added pleroma --- .../migration.sql | 167 ++++++++++++++++++ .../migration.sql | 4 + application/prisma/schema.prisma | 4 +- .../src/Fediverse/Providers/Pleroma/index.ts | 19 ++ .../Pleroma/retrieveLocalPublicUsersPage.ts | 100 +++++++++++ .../Providers/Pleroma/retrievePeers.ts | 23 +++ application/src/Fediverse/Providers/index.ts | 2 + 7 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 application/prisma/migrations/20220108001026_subscription_counts_made_optional/migration.sql create mode 100644 application/prisma/migrations/20220108001027_plan_pleroma_refresh/migration.sql create mode 100644 application/src/Fediverse/Providers/Pleroma/index.ts create mode 100644 application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts create mode 100644 application/src/Fediverse/Providers/Pleroma/retrievePeers.ts diff --git a/application/prisma/migrations/20220108001026_subscription_counts_made_optional/migration.sql b/application/prisma/migrations/20220108001026_subscription_counts_made_optional/migration.sql new file mode 100644 index 0000000..5586970 --- /dev/null +++ b/application/prisma/migrations/20220108001026_subscription_counts_made_optional/migration.sql @@ -0,0 +1,167 @@ +/* + Warnings: + + - A unique constraint covering the columns `[name,nodeId]` on the table `Feed` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[domain]` on the table `Node` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[name]` on the table `Tag` will be added. If there are existing duplicate values, this will fail. + +*/ +-- DropIndex +DROP INDEX "Email_address_idx"; + +-- DropIndex +DROP INDEX "Feed_bot_idx"; + +-- DropIndex +DROP INDEX "Feed_createdAt_idx"; + +-- DropIndex +DROP INDEX "Feed_description_idx"; + +-- DropIndex +DROP INDEX "Feed_displayName_idx"; + +-- DropIndex +DROP INDEX "Feed_fulltext_idx"; + +-- DropIndex +DROP INDEX "Feed_lastStatusAt_idx"; + +-- DropIndex +DROP INDEX "Feed_locked_idx"; + +-- DropIndex +DROP INDEX "Feed_name_nodeId_key"; + +-- DropIndex +DROP INDEX "Feed_parentFeedName_parentFeedDomain_idx"; + +-- DropIndex +DROP INDEX "Feed_refreshedAt_idx"; + +-- DropIndex +DROP INDEX "Feed_type_idx"; + +-- DropIndex +DROP INDEX "Field_name_idx"; + +-- DropIndex +DROP INDEX "Field_value_idx"; + +-- DropIndex +DROP INDEX "Node_domain_key"; + +-- DropIndex +DROP INDEX "Node_foundAt_idx"; + +-- DropIndex +DROP INDEX "Node_halfYearActiveUserCount_idx"; + +-- DropIndex +DROP INDEX "Node_monthActiveUserCount_idx"; + +-- DropIndex +DROP INDEX "Node_openRegistrations_idx"; + +-- DropIndex +DROP INDEX "Node_refreshAttemptedAt_idx"; + +-- DropIndex +DROP INDEX "Node_refreshedAt_idx"; + +-- DropIndex +DROP INDEX "Node_softwareName_idx"; + +-- DropIndex +DROP INDEX "Node_softwareVersion_idx"; + +-- DropIndex +DROP INDEX "Node_statusesCount_idx"; + +-- DropIndex +DROP INDEX "Node_totalUserCount_idx"; + +-- DropIndex +DROP INDEX "Tag_name_key"; + +-- AlterTable +ALTER TABLE "Feed" ALTER COLUMN "followersCount" DROP NOT NULL, +ALTER COLUMN "followingCount" DROP NOT NULL; + +-- CreateIndex +CREATE INDEX "Email_address_idx" ON "Email"("address"); + +-- CreateIndex +CREATE INDEX "Feed_displayName_idx" ON "Feed"("displayName"); + +-- CreateIndex +CREATE INDEX "Feed_description_idx" ON "Feed"("description"); + +-- CreateIndex +CREATE INDEX "Feed_bot_idx" ON "Feed"("bot"); + +-- CreateIndex +CREATE INDEX "Feed_locked_idx" ON "Feed"("locked"); + +-- CreateIndex +CREATE INDEX "Feed_lastStatusAt_idx" ON "Feed"("lastStatusAt"); + +-- CreateIndex +CREATE INDEX "Feed_createdAt_idx" ON "Feed"("createdAt"); + +-- CreateIndex +CREATE INDEX "Feed_refreshedAt_idx" ON "Feed"("refreshedAt"); + +-- CreateIndex +CREATE INDEX "Feed_parentFeedName_parentFeedDomain_idx" ON "Feed"("parentFeedName", "parentFeedDomain"); + +-- CreateIndex +CREATE INDEX "Feed_type_idx" ON "Feed"("type"); + +-- CreateIndex +CREATE INDEX "Feed_fulltext_idx" ON "Feed"("fulltext"); + +-- CreateIndex +CREATE UNIQUE INDEX "Feed_name_nodeId_key" ON "Feed"("name", "nodeId"); + +-- CreateIndex +CREATE INDEX "Field_name_idx" ON "Field"("name"); + +-- CreateIndex +CREATE INDEX "Field_value_idx" ON "Field"("value"); + +-- CreateIndex +CREATE UNIQUE INDEX "Node_domain_key" ON "Node"("domain"); + +-- CreateIndex +CREATE INDEX "Node_softwareName_idx" ON "Node"("softwareName"); + +-- CreateIndex +CREATE INDEX "Node_softwareVersion_idx" ON "Node"("softwareVersion"); + +-- CreateIndex +CREATE INDEX "Node_totalUserCount_idx" ON "Node"("totalUserCount"); + +-- CreateIndex +CREATE INDEX "Node_monthActiveUserCount_idx" ON "Node"("monthActiveUserCount"); + +-- CreateIndex +CREATE INDEX "Node_halfYearActiveUserCount_idx" ON "Node"("halfYearActiveUserCount"); + +-- CreateIndex +CREATE INDEX "Node_statusesCount_idx" ON "Node"("statusesCount"); + +-- CreateIndex +CREATE INDEX "Node_openRegistrations_idx" ON "Node"("openRegistrations"); + +-- CreateIndex +CREATE INDEX "Node_refreshedAt_idx" ON "Node"("refreshedAt"); + +-- CreateIndex +CREATE INDEX "Node_refreshAttemptedAt_idx" ON "Node"("refreshAttemptedAt"); + +-- CreateIndex +CREATE INDEX "Node_foundAt_idx" ON "Node"("foundAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "Tag_name_key" ON "Tag"("name"); diff --git a/application/prisma/migrations/20220108001027_plan_pleroma_refresh/migration.sql b/application/prisma/migrations/20220108001027_plan_pleroma_refresh/migration.sql new file mode 100644 index 0000000..e29d20b --- /dev/null +++ b/application/prisma/migrations/20220108001027_plan_pleroma_refresh/migration.sql @@ -0,0 +1,4 @@ +update "Node" +set "refreshedAt"=NULL, + "refreshAttemptedAt"=NULL +where "Node"."softwareName" like 'pleroma'; diff --git a/application/prisma/schema.prisma b/application/prisma/schema.prisma index 67a1633..833b9a6 100644 --- a/application/prisma/schema.prisma +++ b/application/prisma/schema.prisma @@ -60,8 +60,8 @@ model Feed { feedToTags FeedToTag[] fields Field[] emails Email[] - followersCount Int - followingCount Int + followersCount Int? + followingCount Int? statusesCount Int? bot Boolean? url String diff --git a/application/src/Fediverse/Providers/Pleroma/index.ts b/application/src/Fediverse/Providers/Pleroma/index.ts new file mode 100644 index 0000000..341ec9e --- /dev/null +++ b/application/src/Fediverse/Providers/Pleroma/index.ts @@ -0,0 +1,19 @@ +import { Provider } from '../Provider' +import { retrievePeers } from './retrievePeers' +import { retrieveLocalPublicUsersPage } from './retrieveLocalPublicUsersPage' +import { NodeProvider } from '../NodeProvider' +import { FeedProvider } from '../FeedProvider' + +const PleromaProvider: Provider = { + getKey: () => 'pleroma', + getNodeProviders: ():NodeProvider[] => [{ + getKey: () => 'peers', + retrieveNodes: retrievePeers + }], + getFeedProviders: ():FeedProvider[] => [{ + getKey: () => 'users', + retrieveFeeds: retrieveLocalPublicUsersPage + }] +} + +export default PleromaProvider diff --git a/application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts b/application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts new file mode 100644 index 0000000..b5c4a81 --- /dev/null +++ b/application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts @@ -0,0 +1,100 @@ +import axios from 'axios' +import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' +import { FeedData } from '../FeedData' +import { z } from 'zod' +import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' + +const limit = 500 + +const emojiSchema = z.object({ + shortcode: z.string(), + url: z.string() +}) + +const schema = z.array( + z.object({ + id: z.string(), + username: z.string(), + display_name: z.string(), + locked: z.boolean(), + bot: z.boolean(), + created_at: z.string(), + note: z.string(), + url: z.string(), + avatar: z.string(), + followers_count: z.number(), + following_count: z.number(), + statuses_count: z.number(), + last_status_at: z.nullable(z.string()), + emojis: z.array(emojiSchema), + fields: z.array( + z.object({ + name: z.string(), + value: z.string() + }) + ), + pleroma: z.object({ + hide_followers_count: z.boolean(), + hide_follows_count: z.boolean() + }) + }) +) + +type Emoji = z.infer + +const replaceEmojis = (text: string, emojis: Emoji[]): string => { + emojis.forEach(emoji => { + text = text.replace( + RegExp(`:${emoji.shortcode}:`, 'gi'), + `${emoji.shortcode}` + ) + }) + return text +} + +export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise => { + try { + const response = await axios.get('https://' + domain + '/api/v1/directory', { + params: { + limit: limit, + offset: page * limit, + local: true + }, + timeout: getDefaultTimeoutMilliseconds() + }) + assertSuccessJsonResponse(response) + const responseData = schema.parse(response.data) + if (responseData.length === 0) { + throw new Error('No more users') + } + return responseData.map( + item => { + return { + name: item.username, + displayName: replaceEmojis(item.display_name, item.emojis), + description: replaceEmojis(item.note, item.emojis), + followersCount: item.pleroma.hide_followers_count ? null : item.followers_count, + followingCount: item.pleroma.hide_follows_count ? null : item.following_count, + statusesCount: item.statuses_count, + bot: item.bot, + url: item.url, + avatar: item.avatar, + locked: item.locked, + lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null, + createdAt: new Date(item.created_at), + fields: item.fields.map(field => { + return { + name: replaceEmojis(field.name, item.emojis), + value: replaceEmojis(field.value, item.emojis), + verifiedAt: null + } + }), + type: 'account', + parentFeed: null + } + } + ) + } catch (error) { + throw new Error('Invalid response: ' + error) + } +} diff --git a/application/src/Fediverse/Providers/Pleroma/retrievePeers.ts b/application/src/Fediverse/Providers/Pleroma/retrievePeers.ts new file mode 100644 index 0000000..b8389cc --- /dev/null +++ b/application/src/Fediverse/Providers/Pleroma/retrievePeers.ts @@ -0,0 +1,23 @@ +import axios from 'axios' +import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' +import { z } from 'zod' +import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' + +const schema = z.array( + z.string() +) + +export const retrievePeers = async (domain:string, page:number):Promise => { + if (page !== 0) { + throw new Error('No more peer pages') + } + try { + const response = await axios.get('https://' + domain + '/api/v1/instance/peers', { + timeout: getDefaultTimeoutMilliseconds() + }) + assertSuccessJsonResponse(response) + return schema.parse(response.data) + } catch (error) { + throw new Error('Invalid response') + } +} diff --git a/application/src/Fediverse/Providers/index.ts b/application/src/Fediverse/Providers/index.ts index 3506f78..ea337c6 100644 --- a/application/src/Fediverse/Providers/index.ts +++ b/application/src/Fediverse/Providers/index.ts @@ -1,8 +1,10 @@ import { providerRegistry } from './ProviderRegistry' import MastodonProvider from './Mastodon' import PeertubeProvider from './Peertube' +import PleromaProvider from './Pleroma' providerRegistry.registerProvider(MastodonProvider) providerRegistry.registerProvider(PeertubeProvider) +providerRegistry.registerProvider(PleromaProvider) export default providerRegistry