diff --git a/application/package-lock.json b/application/package-lock.json index 6bced4c..b6e2ef8 100644 --- a/application/package-lock.json +++ b/application/package-lock.json @@ -3469,9 +3469,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.14.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz", - "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==", + "version": "1.14.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==", "funding": [ { "type": "individual", @@ -6344,9 +6344,10 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "dev": true, - "license": "MIT" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "dev": true }, "node_modules/ms": { "version": "2.1.2", @@ -11163,9 +11164,9 @@ "dev": true }, "follow-redirects": { - "version": "1.14.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz", - "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==" + "version": "1.14.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" }, "form-data": { "version": "3.0.1", @@ -13249,7 +13250,9 @@ } }, "minimist": { - "version": "1.2.5", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "ms": { diff --git a/application/package.json b/application/package.json index a95aff4..133b711 100644 --- a/application/package.json +++ b/application/package.json @@ -26,16 +26,16 @@ "migrate:resolve": "npx prisma migrate resolve", "prisma:generate": "npx prisma generate", "prisma:studio": "npx prisma studio", - "start:deploy": "npm run migrate:deploy && npm run start" + "start:deploy": "npm run migrate:deploy && npm run start" }, "dependencies": { "@prisma/client": "^3.6.0", "axios": "^0.21.1", "npmlog": "^6.0.0", - "typescript-collections": "^1.3.3", - "zod": "^3.11.6", "rimraf": "^3.0.2", - "striptags": "^3.2.0" + "striptags": "^3.2.0", + "typescript-collections": "^1.3.3", + "zod": "^3.11.6" }, "devDependencies": { "@types/jest": "^27.0.2", @@ -49,11 +49,11 @@ "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^5.1.1", "eslint-plugin-react": "^7.27.1", + "jest": "^27.3.0", "prisma": "^3.6.0", "standard": "*", - "typescript": "^4.3.5", "ts-jest": "^27.0.7", - "jest": "^27.3.0" + "typescript": "^4.3.5" }, "jest": { "moduleFileExtensions": [ diff --git a/application/prisma/migrations/20220417112556_add_delete_cascade/migration.sql b/application/prisma/migrations/20220417112556_add_delete_cascade/migration.sql new file mode 100644 index 0000000..cb778f2 --- /dev/null +++ b/application/prisma/migrations/20220417112556_add_delete_cascade/migration.sql @@ -0,0 +1,193 @@ +/* + Warnings: + + - A unique constraint covering the columns `[name,nodeId]` on the table `Feed` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[domain]` on the table `Node` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[name]` on the table `Tag` will be added. If there are existing duplicate values, this will fail. + +*/ +-- DropForeignKey +ALTER TABLE "Email" DROP CONSTRAINT "Email_feedId_fkey"; + +-- DropForeignKey +ALTER TABLE "Feed" DROP CONSTRAINT "Feed_nodeId_fkey"; + +-- DropForeignKey +ALTER TABLE "FeedToTag" DROP CONSTRAINT "FeedToTag_feedId_fkey"; + +-- DropForeignKey +ALTER TABLE "FeedToTag" DROP CONSTRAINT "FeedToTag_tagId_fkey"; + +-- DropForeignKey +ALTER TABLE "Field" DROP CONSTRAINT "Field_feedId_fkey"; + +-- DropIndex +DROP INDEX "Email_address_idx"; + +-- DropIndex +DROP INDEX "Feed_bot_idx"; + +-- DropIndex +DROP INDEX "Feed_createdAt_idx"; + +-- DropIndex +DROP INDEX "Feed_description_idx"; + +-- DropIndex +DROP INDEX "Feed_displayName_idx"; + +-- DropIndex +DROP INDEX "Feed_fulltext_idx"; + +-- DropIndex +DROP INDEX "Feed_lastStatusAt_idx"; + +-- DropIndex +DROP INDEX "Feed_locked_idx"; + +-- DropIndex +DROP INDEX "Feed_name_nodeId_key"; + +-- DropIndex +DROP INDEX "Feed_parentFeedName_parentFeedDomain_idx"; + +-- DropIndex +DROP INDEX "Feed_refreshedAt_idx"; + +-- DropIndex +DROP INDEX "Feed_type_idx"; + +-- DropIndex +DROP INDEX "Field_name_idx"; + +-- DropIndex +DROP INDEX "Field_value_idx"; + +-- DropIndex +DROP INDEX "Node_domain_key"; + +-- DropIndex +DROP INDEX "Node_foundAt_idx"; + +-- DropIndex +DROP INDEX "Node_halfYearActiveUserCount_idx"; + +-- DropIndex +DROP INDEX "Node_monthActiveUserCount_idx"; + +-- DropIndex +DROP INDEX "Node_openRegistrations_idx"; + +-- DropIndex +DROP INDEX "Node_refreshAttemptedAt_idx"; + +-- DropIndex +DROP INDEX "Node_refreshedAt_idx"; + +-- DropIndex +DROP INDEX "Node_softwareName_idx"; + +-- DropIndex +DROP INDEX "Node_softwareVersion_idx"; + +-- DropIndex +DROP INDEX "Node_statusesCount_idx"; + +-- DropIndex +DROP INDEX "Node_totalUserCount_idx"; + +-- DropIndex +DROP INDEX "Tag_name_key"; + +-- CreateIndex +CREATE INDEX "Email_address_idx" ON "Email"("address"); + +-- CreateIndex +CREATE INDEX "Feed_displayName_idx" ON "Feed"("displayName"); + +-- CreateIndex +CREATE INDEX "Feed_description_idx" ON "Feed"("description"); + +-- CreateIndex +CREATE INDEX "Feed_bot_idx" ON "Feed"("bot"); + +-- CreateIndex +CREATE INDEX "Feed_locked_idx" ON "Feed"("locked"); + +-- CreateIndex +CREATE INDEX "Feed_lastStatusAt_idx" ON "Feed"("lastStatusAt"); + +-- CreateIndex +CREATE INDEX "Feed_createdAt_idx" ON "Feed"("createdAt"); + +-- CreateIndex +CREATE INDEX "Feed_refreshedAt_idx" ON "Feed"("refreshedAt"); + +-- CreateIndex +CREATE INDEX "Feed_parentFeedName_parentFeedDomain_idx" ON "Feed"("parentFeedName", "parentFeedDomain"); + +-- CreateIndex +CREATE INDEX "Feed_type_idx" ON "Feed"("type"); + +-- CreateIndex +CREATE INDEX "Feed_fulltext_idx" ON "Feed"("fulltext"); + +-- CreateIndex +CREATE UNIQUE INDEX "Feed_name_nodeId_key" ON "Feed"("name", "nodeId"); + +-- CreateIndex +CREATE INDEX "Field_name_idx" ON "Field"("name"); + +-- CreateIndex +CREATE INDEX "Field_value_idx" ON "Field"("value"); + +-- CreateIndex +CREATE UNIQUE INDEX "Node_domain_key" ON "Node"("domain"); + +-- CreateIndex +CREATE INDEX "Node_softwareName_idx" ON "Node"("softwareName"); + +-- CreateIndex +CREATE INDEX "Node_softwareVersion_idx" ON "Node"("softwareVersion"); + +-- CreateIndex +CREATE INDEX "Node_totalUserCount_idx" ON "Node"("totalUserCount"); + +-- CreateIndex +CREATE INDEX "Node_monthActiveUserCount_idx" ON "Node"("monthActiveUserCount"); + +-- CreateIndex +CREATE INDEX "Node_halfYearActiveUserCount_idx" ON "Node"("halfYearActiveUserCount"); + +-- CreateIndex +CREATE INDEX "Node_statusesCount_idx" ON "Node"("statusesCount"); + +-- CreateIndex +CREATE INDEX "Node_openRegistrations_idx" ON "Node"("openRegistrations"); + +-- CreateIndex +CREATE INDEX "Node_refreshedAt_idx" ON "Node"("refreshedAt"); + +-- CreateIndex +CREATE INDEX "Node_refreshAttemptedAt_idx" ON "Node"("refreshAttemptedAt"); + +-- CreateIndex +CREATE INDEX "Node_foundAt_idx" ON "Node"("foundAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "Tag_name_key" ON "Tag"("name"); + +-- AddForeignKey +ALTER TABLE "Email" ADD CONSTRAINT "Email_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "FeedToTag" ADD CONSTRAINT "FeedToTag_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "FeedToTag" ADD CONSTRAINT "FeedToTag_tagId_fkey" FOREIGN KEY ("tagId") REFERENCES "Tag"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Field" ADD CONSTRAINT "Field_feedId_fkey" FOREIGN KEY ("feedId") REFERENCES "Feed"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Feed" ADD CONSTRAINT "Feed_nodeId_fkey" FOREIGN KEY ("nodeId") REFERENCES "Node"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/application/prisma/schema.prisma b/application/prisma/schema.prisma index 833b9a6..f398f05 100644 --- a/application/prisma/schema.prisma +++ b/application/prisma/schema.prisma @@ -5,7 +5,7 @@ datasource db { generator client { provider = "prisma-client-js" - previewFeatures = ["extendedIndexes","fullTextSearch"] + previewFeatures = ["extendedIndexes","fullTextSearch","referentialActions"] } model Tag { @@ -17,16 +17,16 @@ model Tag { model Email { id String @id @default(uuid()) @db.Uuid address String - feed Feed @relation(fields: [feedId], references: [id]) + feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade) feedId String @db.Uuid @@index([address]) } model FeedToTag { - feed Feed @relation(fields: [feedId], references: [id]) + feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade) feedId String @db.Uuid - tag Tag @relation(fields: [tagId], references: [id]) + tag Tag @relation(fields: [tagId], references: [id], onDelete: Cascade) tagId String @db.Uuid @@id([feedId, tagId]) @@ -36,7 +36,7 @@ model Field { id String @id @default(uuid()) @db.Uuid name String value String - feed Feed @relation(fields: [feedId], references: [id]) + feed Feed @relation(fields: [feedId], references: [id], onDelete: Cascade) feedId String @db.Uuid @@index([name]) @@ -50,7 +50,7 @@ enum FeedType{ model Feed { id String @id @default(uuid()) @db.Uuid - node Node @relation(fields: [nodeId], references: [id]) + node Node @relation(fields: [nodeId], references: [id], onDelete: Cascade) nodeId String @db.Uuid foundAt DateTime @default(now()) refreshedAt DateTime @updatedAt diff --git a/application/src/Fediverse/NodeInfo/NoSupportedLinkError.ts b/application/src/Fediverse/NodeInfo/NoSupportedLinkError.ts new file mode 100644 index 0000000..aaf702a --- /dev/null +++ b/application/src/Fediverse/NodeInfo/NoSupportedLinkError.ts @@ -0,0 +1,5 @@ +export class NoSupportedLinkError extends Error { + public constructor (domain:string) { + super(`No supported link node info link for ${domain}`) + } +} diff --git a/application/src/Fediverse/NodeInfo/retrieveDomainNodeInfo.ts b/application/src/Fediverse/NodeInfo/retrieveDomainNodeInfo.ts index 0911ec1..c519e41 100644 --- a/application/src/Fediverse/NodeInfo/retrieveDomainNodeInfo.ts +++ b/application/src/Fediverse/NodeInfo/retrieveDomainNodeInfo.ts @@ -1,11 +1,12 @@ import { retrieveWellKnown } from './retrieveWellKnown' import { retrieveNodeInfo, NodeInfo } from './retrieveNodeInfo' +import { NoSupportedLinkError } from './NoSupportedLinkError' export const retrieveDomainNodeInfo = async (domain:string):Promise => { const wellKnown = await retrieveWellKnown(domain) const link = wellKnown.links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0') if (typeof link === 'undefined') { - throw new Error(`No supported link node info link for ${domain}`) + throw new NoSupportedLinkError(domain) } return await retrieveNodeInfo(link.href) } diff --git a/application/src/Fediverse/Providers/Ecko/index.ts b/application/src/Fediverse/Providers/Ecko/index.ts index bdb80e5..6418163 100644 --- a/application/src/Fediverse/Providers/Ecko/index.ts +++ b/application/src/Fediverse/Providers/Ecko/index.ts @@ -1,6 +1,9 @@ import { Provider } from '../Provider' import MastodonProvider from '../Mastodon' +/** + * Ecko is Mastodon's fork + */ const EckoProvider: Provider = { getKey: () => 'ecko', getNodeProviders: MastodonProvider.getNodeProviders, diff --git a/application/src/Fediverse/Providers/FeedProvider.ts b/application/src/Fediverse/Providers/FeedProvider.ts index cb847b9..8442d12 100644 --- a/application/src/Fediverse/Providers/FeedProvider.ts +++ b/application/src/Fediverse/Providers/FeedProvider.ts @@ -1,6 +1,6 @@ -import { FeedData } from './FeedData' +import { FeedProviderMethod } from './FeedProviderMethod' export interface FeedProvider { getKey: ()=>string - retrieveFeeds: (domain:string, page:number)=> Promise + retrieveFeeds: FeedProviderMethod } diff --git a/application/src/Fediverse/Providers/FeedProviderMethod.ts b/application/src/Fediverse/Providers/FeedProviderMethod.ts new file mode 100644 index 0000000..eb41a99 --- /dev/null +++ b/application/src/Fediverse/Providers/FeedProviderMethod.ts @@ -0,0 +1,3 @@ +import { FeedData } from './FeedData' + +export type FeedProviderMethod = (domain: string, page: number) => Promise diff --git a/application/src/Fediverse/Providers/Hometown/index.ts b/application/src/Fediverse/Providers/Hometown/index.ts index e0ce5f8..87a8261 100644 --- a/application/src/Fediverse/Providers/Hometown/index.ts +++ b/application/src/Fediverse/Providers/Hometown/index.ts @@ -1,6 +1,9 @@ import { Provider } from '../Provider' import MastodonProvider from '../Mastodon' +/** + * Hometown is Mastodon's fork + */ const HometownProvider: Provider = { getKey: () => 'hometown', getNodeProviders: MastodonProvider.getNodeProviders, diff --git a/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts b/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts index 511c215..e3af23f 100644 --- a/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts +++ b/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts @@ -1,8 +1,9 @@ import axios from 'axios' import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' -import { FeedData } from '../FeedData' import { z } from 'zod' import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' +import { FeedProviderMethod } from '../FeedProviderMethod' +import { NoMoreFeedsError } from '../NoMoreFeedsError' const limit = 500 @@ -49,49 +50,45 @@ const replaceEmojis = (text: string, emojis: Emoji[]): string => { return text } -export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise => { - try { - const response = await axios.get('https://' + domain + '/api/v1/directory', { - params: { - limit: limit, - offset: page * limit, - local: true - }, - timeout: getDefaultTimeoutMilliseconds() - }) - assertSuccessJsonResponse(response) - const responseData = schema.parse(response.data) - if (responseData.length === 0) { - throw new Error('No more users') - } - return responseData.map( - item => { - return { - name: item.username, - displayName: replaceEmojis(item.display_name, item.emojis), - description: replaceEmojis(item.note, item.emojis), - followersCount: item.followers_count, - followingCount: item.following_count, - statusesCount: item.statuses_count, - bot: item.bot, - url: item.url, - avatar: item.avatar, - locked: item.locked, - lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null, - createdAt: new Date(item.created_at), - fields: item.fields.map(field => { - return { - name: replaceEmojis(field.name, item.emojis), - value: replaceEmojis(field.value, item.emojis), - verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null - } - }), - type: 'account', - parentFeed: null - } - } - ) - } catch (error) { - throw new Error('Invalid response: ' + error) +export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (domain, page) => { + const response = await axios.get('https://' + domain + '/api/v1/directory', { + params: { + limit: limit, + offset: page * limit, + local: true + }, + timeout: getDefaultTimeoutMilliseconds() + }) + assertSuccessJsonResponse(response) + const responseData = schema.parse(response.data) + if (responseData.length === 0) { + throw new NoMoreFeedsError('user') } + return responseData.map( + item => { + return { + name: item.username, + displayName: replaceEmojis(item.display_name, item.emojis), + description: replaceEmojis(item.note, item.emojis), + followersCount: item.followers_count, + followingCount: item.following_count, + statusesCount: item.statuses_count, + bot: item.bot, + url: item.url, + avatar: item.avatar, + locked: item.locked, + lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null, + createdAt: new Date(item.created_at), + fields: item.fields.map(field => { + return { + name: replaceEmojis(field.name, item.emojis), + value: replaceEmojis(field.value, item.emojis), + verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null + } + }), + type: 'account', + parentFeed: null + } + } + ) } diff --git a/application/src/Fediverse/Providers/Mastodon/retrievePeers.ts b/application/src/Fediverse/Providers/Mastodon/retrievePeers.ts index b8389cc..1c4ff5f 100644 --- a/application/src/Fediverse/Providers/Mastodon/retrievePeers.ts +++ b/application/src/Fediverse/Providers/Mastodon/retrievePeers.ts @@ -2,22 +2,20 @@ import axios from 'axios' import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' import { z } from 'zod' import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' +import { NodeProviderMethod } from '../NodeProviderMethod' +import { NoMoreNodesError } from '../NoMoreNodesError' const schema = z.array( z.string() ) -export const retrievePeers = async (domain:string, page:number):Promise => { +export const retrievePeers:NodeProviderMethod = async (domain, page) => { if (page !== 0) { - throw new Error('No more peer pages') - } - try { - const response = await axios.get('https://' + domain + '/api/v1/instance/peers', { - timeout: getDefaultTimeoutMilliseconds() - }) - assertSuccessJsonResponse(response) - return schema.parse(response.data) - } catch (error) { - throw new Error('Invalid response') + throw new NoMoreNodesError('peer') } + const response = await axios.get('https://' + domain + '/api/v1/instance/peers', { + timeout: getDefaultTimeoutMilliseconds() + }) + assertSuccessJsonResponse(response) + return schema.parse(response.data) } diff --git a/application/src/Fediverse/Providers/Misskey/retrieveInstancesPage.ts b/application/src/Fediverse/Providers/Misskey/retrieveInstancesPage.ts index 0b652c7..44723a6 100644 --- a/application/src/Fediverse/Providers/Misskey/retrieveInstancesPage.ts +++ b/application/src/Fediverse/Providers/Misskey/retrieveInstancesPage.ts @@ -2,6 +2,8 @@ import axios from 'axios' import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' import { z } from 'zod' import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' +import { NodeProviderMethod } from '../NodeProviderMethod' +import { NoMoreNodesError } from '../NoMoreNodesError' const limit = 100 @@ -11,33 +13,29 @@ const schema = z.array( }) ) -export const retrieveInstancesPage = async (domain: string, page: number): Promise => { - try { - const response = await axios.post('https://' + domain + '/api/federation/instances', { - host: null, - blocked: null, - notResponding: null, - suspended: null, - federating: null, - subscribing: null, - publishing: null, - limit: limit, - offset: page * limit, - sort: '+id' - }, { - timeout: getDefaultTimeoutMilliseconds() - }) - assertSuccessJsonResponse(response) - const responseData = schema.parse(response.data) - if (responseData.length === 0) { - throw new Error('No more instances') - } - return responseData.map( - item => { - return item.host - } - ) - } catch (error) { - throw new Error('Invalid response: ' + error) +export const retrieveInstancesPage:NodeProviderMethod = async (domain, page) => { + const response = await axios.post('https://' + domain + '/api/federation/instances', { + host: null, + blocked: null, + notResponding: null, + suspended: null, + federating: null, + subscribing: null, + publishing: null, + limit: limit, + offset: page * limit, + sort: '+id' + }, { + timeout: getDefaultTimeoutMilliseconds() + }) + assertSuccessJsonResponse(response) + const responseData = schema.parse(response.data) + if (responseData.length === 0) { + throw new NoMoreNodesError('instance') } + return responseData.map( + item => { + return item.host + } + ) } diff --git a/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts b/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts index 3b43e4b..7d85e62 100644 --- a/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts +++ b/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts @@ -3,6 +3,9 @@ import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' import { FeedData } from '../FeedData' import { z } from 'zod' import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' +import { NodeProviderMethod } from '../NodeProviderMethod' +import { NoMoreFeedsError } from '../NoMoreFeedsError' +import { FeedProviderMethod } from '../FeedProviderMethod' const limit = 100 @@ -60,57 +63,53 @@ const parseDescription = (description:string|null):string => { }).join('\n') } -export const retrieveUsersPage = async (domain: string, page: number): Promise => { - try { - const response = await axios.post('https://' + domain + '/api/users', { - state: 'all', - origin: 'local', - sort: '+createdAt', - limit: limit, - offset: limit * page - }, { - timeout: getDefaultTimeoutMilliseconds() - }) - assertSuccessJsonResponse(response) - const responseData = schema.parse(response.data) - if (responseData.length === 0) { - throw new Error('No more users') - } - return responseData.map( - item => { - return { - name: item.username, - displayName: replaceEmojis(item.name ?? item.username, item.emojis), - description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis), - followersCount: item.followersCount, - followingCount: item.followingCount, - statusesCount: item.notesCount, - bot: item.isBot, - url: `https://${domain}/@${item.username}`, - avatar: item.avatarUrl, - locked: item.isLocked, - lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : null, - createdAt: new Date(item.createdAt), - fields: [ - ...item.fields.map(field => { - return { - name: replaceEmojis(field.name, item.emojis), - value: replaceEmojis(field.value, item.emojis), - verifiedAt: null - } - }), - ...[ - { name: 'Location', value: item.location, verifiedAt: null }, - { name: 'Birthday', value: item.birthday, verifiedAt: null }, - { name: 'Language', value: item.lang, verifiedAt: null } - ].filter(field => field.value !== null) - ], - type: 'account', - parentFeed: null - } - } - ) - } catch (error) { - throw new Error('Invalid response: ' + error) +export const retrieveUsersPage:FeedProviderMethod = async (domain, page) => { + const response = await axios.post('https://' + domain + '/api/users', { + state: 'all', + origin: 'local', + sort: '+createdAt', + limit: limit, + offset: limit * page + }, { + timeout: getDefaultTimeoutMilliseconds() + }) + assertSuccessJsonResponse(response) + const responseData = schema.parse(response.data) + if (responseData.length === 0) { + throw new NoMoreFeedsError('user') } + return responseData.map( + item => { + return { + name: item.username, + displayName: replaceEmojis(item.name ?? item.username, item.emojis), + description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis), + followersCount: item.followersCount, + followingCount: item.followingCount, + statusesCount: item.notesCount, + bot: item.isBot, + url: `https://${domain}/@${item.username}`, + avatar: item.avatarUrl, + locked: item.isLocked, + lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : null, + createdAt: new Date(item.createdAt), + fields: [ + ...item.fields.map(field => { + return { + name: replaceEmojis(field.name, item.emojis), + value: replaceEmojis(field.value, item.emojis), + verifiedAt: null + } + }), + ...[ + { name: 'Location', value: item.location, verifiedAt: null }, + { name: 'Birthday', value: item.birthday, verifiedAt: null }, + { name: 'Language', value: item.lang, verifiedAt: null } + ].filter(field => field.value !== null) + ], + type: 'account', + parentFeed: null + } + } + ) } diff --git a/application/src/Fediverse/Providers/NoMoreFeedsError.ts b/application/src/Fediverse/Providers/NoMoreFeedsError.ts new file mode 100644 index 0000000..f79b0b0 --- /dev/null +++ b/application/src/Fediverse/Providers/NoMoreFeedsError.ts @@ -0,0 +1,5 @@ +export class NoMoreFeedsError extends Error { + public constructor (feedType:string) { + super(`No more feeds of type ${feedType}`) + } +} diff --git a/application/src/Fediverse/Providers/NoMoreNodesError.ts b/application/src/Fediverse/Providers/NoMoreNodesError.ts new file mode 100644 index 0000000..9c49e5d --- /dev/null +++ b/application/src/Fediverse/Providers/NoMoreNodesError.ts @@ -0,0 +1,5 @@ +export class NoMoreNodesError extends Error { + public constructor (nodeType:string) { + super(`No more nodes of type ${nodeType}`) + } +} diff --git a/application/src/Fediverse/Providers/NodeProvider.ts b/application/src/Fediverse/Providers/NodeProvider.ts index b6d2fc3..29bd5b6 100644 --- a/application/src/Fediverse/Providers/NodeProvider.ts +++ b/application/src/Fediverse/Providers/NodeProvider.ts @@ -1,4 +1,6 @@ +import { NodeProviderMethod } from './NodeProviderMethod' + export interface NodeProvider { getKey:()=>string, - retrieveNodes: (domain: string, page:number)=> Promise + retrieveNodes: NodeProviderMethod } diff --git a/application/src/Fediverse/Providers/NodeProviderMethod.ts b/application/src/Fediverse/Providers/NodeProviderMethod.ts new file mode 100644 index 0000000..305ee7d --- /dev/null +++ b/application/src/Fediverse/Providers/NodeProviderMethod.ts @@ -0,0 +1,2 @@ + +export type NodeProviderMethod = (domain: string, page:number)=> Promise diff --git a/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts b/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts index 20add9b..e691d31 100644 --- a/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts +++ b/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts @@ -6,6 +6,8 @@ import { avatarSchema } from './Avatar' import { parseAvatarUrl } from './parseAvatarUrl' import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' import { parseDescription } from './parseDescription' +import { NoMoreFeedsError } from '../NoMoreFeedsError' +import { FeedProviderMethod } from '../FeedProviderMethod' const limit = 100 @@ -27,7 +29,7 @@ const schema = z.object({ ) }) -export const retrieveAccounts = async (domain: string, page: number): Promise => { +export const retrieveAccounts:FeedProviderMethod = async (domain, page) => { const response = await axios.get(`https://${domain}/api/v1/accounts`, { params: { count: limit, @@ -39,7 +41,7 @@ export const retrieveAccounts = async (domain: string, page: number): Promise item.host === domain) diff --git a/application/src/Fediverse/Providers/Peertube/retrieveFollowers.ts b/application/src/Fediverse/Providers/Peertube/retrieveFollowers.ts index bc0392c..49ab100 100644 --- a/application/src/Fediverse/Providers/Peertube/retrieveFollowers.ts +++ b/application/src/Fediverse/Providers/Peertube/retrieveFollowers.ts @@ -2,6 +2,8 @@ import axios from 'axios' import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' import { z } from 'zod' import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' +import { NodeProviderMethod } from '../NodeProviderMethod' +import { NoMoreNodesError } from '../NoMoreNodesError' const limit = 100 @@ -19,7 +21,7 @@ const schema = z.object({ ) }) -export const retrieveFollowers = async (domain: string, page: number): Promise => { +export const retrieveFollowers:NodeProviderMethod = async (domain, page) => { const response = await axios.get(`https://${domain}/api/v1/server/followers`, { params: { count: limit, @@ -36,7 +38,7 @@ export const retrieveFollowers = async (domain: string, page: number): Promise => { +export const retrieveVideoChannels:FeedProviderMethod = async (domain, page) => { const response = await axios.get(`https://${domain}/api/v1/video-channels`, { params: { count: limit, @@ -46,7 +48,7 @@ export const retrieveVideoChannels = async (domain: string, page: number): Promi assertSuccessJsonResponse(response) const responseData = schema.parse(response.data) if (responseData.data.length === 0) { - throw new Error('No more channels') + throw new NoMoreFeedsError('channel') } return responseData.data .filter(item => item.host === domain) diff --git a/application/src/Fediverse/Providers/Pleroma/index.ts b/application/src/Fediverse/Providers/Pleroma/index.ts index 341ec9e..bad572b 100644 --- a/application/src/Fediverse/Providers/Pleroma/index.ts +++ b/application/src/Fediverse/Providers/Pleroma/index.ts @@ -1,19 +1,13 @@ import { Provider } from '../Provider' -import { retrievePeers } from './retrievePeers' -import { retrieveLocalPublicUsersPage } from './retrieveLocalPublicUsersPage' -import { NodeProvider } from '../NodeProvider' -import { FeedProvider } from '../FeedProvider' +import MastodonProvider from '../Mastodon' +/** + * Pleroma implements Mastodon's api + */ const PleromaProvider: Provider = { getKey: () => 'pleroma', - getNodeProviders: ():NodeProvider[] => [{ - getKey: () => 'peers', - retrieveNodes: retrievePeers - }], - getFeedProviders: ():FeedProvider[] => [{ - getKey: () => 'users', - retrieveFeeds: retrieveLocalPublicUsersPage - }] + getNodeProviders: MastodonProvider.getNodeProviders, + getFeedProviders: MastodonProvider.getFeedProviders } export default PleromaProvider diff --git a/application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts b/application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts deleted file mode 100644 index b5c4a81..0000000 --- a/application/src/Fediverse/Providers/Pleroma/retrieveLocalPublicUsersPage.ts +++ /dev/null @@ -1,100 +0,0 @@ -import axios from 'axios' -import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' -import { FeedData } from '../FeedData' -import { z } from 'zod' -import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' - -const limit = 500 - -const emojiSchema = z.object({ - shortcode: z.string(), - url: z.string() -}) - -const schema = z.array( - z.object({ - id: z.string(), - username: z.string(), - display_name: z.string(), - locked: z.boolean(), - bot: z.boolean(), - created_at: z.string(), - note: z.string(), - url: z.string(), - avatar: z.string(), - followers_count: z.number(), - following_count: z.number(), - statuses_count: z.number(), - last_status_at: z.nullable(z.string()), - emojis: z.array(emojiSchema), - fields: z.array( - z.object({ - name: z.string(), - value: z.string() - }) - ), - pleroma: z.object({ - hide_followers_count: z.boolean(), - hide_follows_count: z.boolean() - }) - }) -) - -type Emoji = z.infer - -const replaceEmojis = (text: string, emojis: Emoji[]): string => { - emojis.forEach(emoji => { - text = text.replace( - RegExp(`:${emoji.shortcode}:`, 'gi'), - `${emoji.shortcode}` - ) - }) - return text -} - -export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise => { - try { - const response = await axios.get('https://' + domain + '/api/v1/directory', { - params: { - limit: limit, - offset: page * limit, - local: true - }, - timeout: getDefaultTimeoutMilliseconds() - }) - assertSuccessJsonResponse(response) - const responseData = schema.parse(response.data) - if (responseData.length === 0) { - throw new Error('No more users') - } - return responseData.map( - item => { - return { - name: item.username, - displayName: replaceEmojis(item.display_name, item.emojis), - description: replaceEmojis(item.note, item.emojis), - followersCount: item.pleroma.hide_followers_count ? null : item.followers_count, - followingCount: item.pleroma.hide_follows_count ? null : item.following_count, - statusesCount: item.statuses_count, - bot: item.bot, - url: item.url, - avatar: item.avatar, - locked: item.locked, - lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null, - createdAt: new Date(item.created_at), - fields: item.fields.map(field => { - return { - name: replaceEmojis(field.name, item.emojis), - value: replaceEmojis(field.value, item.emojis), - verifiedAt: null - } - }), - type: 'account', - parentFeed: null - } - } - ) - } catch (error) { - throw new Error('Invalid response: ' + error) - } -} diff --git a/application/src/Fediverse/Providers/Pleroma/retrievePeers.ts b/application/src/Fediverse/Providers/Pleroma/retrievePeers.ts deleted file mode 100644 index b8389cc..0000000 --- a/application/src/Fediverse/Providers/Pleroma/retrievePeers.ts +++ /dev/null @@ -1,23 +0,0 @@ -import axios from 'axios' -import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse' -import { z } from 'zod' -import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds' - -const schema = z.array( - z.string() -) - -export const retrievePeers = async (domain:string, page:number):Promise => { - if (page !== 0) { - throw new Error('No more peer pages') - } - try { - const response = await axios.get('https://' + domain + '/api/v1/instance/peers', { - timeout: getDefaultTimeoutMilliseconds() - }) - assertSuccessJsonResponse(response) - return schema.parse(response.data) - } catch (error) { - throw new Error('Invalid response') - } -} diff --git a/application/src/Fediverse/Providers/ProviderKeyAlreadyRegisteredError.ts b/application/src/Fediverse/Providers/ProviderKeyAlreadyRegisteredError.ts new file mode 100644 index 0000000..29ac7d0 --- /dev/null +++ b/application/src/Fediverse/Providers/ProviderKeyAlreadyRegisteredError.ts @@ -0,0 +1,12 @@ +export class ProviderKeyAlreadyRegisteredError extends Error { + private readonly _key:string + + public constructor (key:string) { + super(`Provider with the key ${key} is already registered`) + this._key = key + } + + public get key (): string { + return this._key + } +} diff --git a/application/src/Fediverse/Providers/ProviderRegistry.ts b/application/src/Fediverse/Providers/ProviderRegistry.ts index cde94bf..5405c5c 100644 --- a/application/src/Fediverse/Providers/ProviderRegistry.ts +++ b/application/src/Fediverse/Providers/ProviderRegistry.ts @@ -1,5 +1,6 @@ import { Provider } from './Provider' import { Dictionary } from 'typescript-collections' +import { ProviderKeyAlreadyRegisteredError } from './ProviderKeyAlreadyRegisteredError' export interface ProviderCallback { (key: string, provider: Provider): void @@ -10,7 +11,7 @@ const providers: Dictionary = new Dictionary const registerProvider = (provider: Provider): void => { const key = provider.getKey() if (providers.containsKey(key)) { - throw new Error(`Provider with the key ${key} is already registered`) + throw new ProviderKeyAlreadyRegisteredError(key) } providers.setValue(key, provider) console.info('Added provider to registry', { key: key }) diff --git a/application/src/Fediverse/UnexpectedContentTypeError.ts b/application/src/Fediverse/UnexpectedContentTypeError.ts new file mode 100644 index 0000000..8e20a3d --- /dev/null +++ b/application/src/Fediverse/UnexpectedContentTypeError.ts @@ -0,0 +1,20 @@ +import { UnexpectedResponseError } from './UnexpectedResponseError' + +export class UnexpectedContentTypeError extends UnexpectedResponseError { + private readonly _expectedContentType: string + private readonly _actualContentType: string + + public constructor (actualContentType: string, expectedContentType:string) { + super(`Expected content type '${expectedContentType}' but got '${actualContentType}'`) + this._expectedContentType = expectedContentType + this._actualContentType = actualContentType + } + + get expectedContentType (): string { + return this._expectedContentType + } + + get actualContentType (): string { + return this._actualContentType + } +} diff --git a/application/src/Fediverse/UnexpectedResponseError.ts b/application/src/Fediverse/UnexpectedResponseError.ts new file mode 100644 index 0000000..dd4cc48 --- /dev/null +++ b/application/src/Fediverse/UnexpectedResponseError.ts @@ -0,0 +1,3 @@ +export class UnexpectedResponseError extends Error { + +} diff --git a/application/src/Fediverse/UnexpectedResponseStatusError.ts b/application/src/Fediverse/UnexpectedResponseStatusError.ts new file mode 100644 index 0000000..235d7f2 --- /dev/null +++ b/application/src/Fediverse/UnexpectedResponseStatusError.ts @@ -0,0 +1,20 @@ +import { UnexpectedResponseError } from './UnexpectedResponseError' + +export class UnexpectedResponseStatusError extends UnexpectedResponseError { + private readonly _expectedStatusCode: number + private readonly _actualStatusCode: number + + public constructor (expectedStatusCode:number, actualStatusCode:number) { + super(`Expected response code ${expectedStatusCode} but got ${actualStatusCode}`) + this._actualStatusCode = actualStatusCode + this._expectedStatusCode = expectedStatusCode + } + + get expectedStatusCode (): number { + return this._expectedStatusCode + } + + get actualStatusCode (): number { + return this._actualStatusCode + } +} diff --git a/application/src/Fediverse/assertSuccessJsonResponse.ts b/application/src/Fediverse/assertSuccessJsonResponse.ts index b549fef..a1e3e12 100644 --- a/application/src/Fediverse/assertSuccessJsonResponse.ts +++ b/application/src/Fediverse/assertSuccessJsonResponse.ts @@ -1,10 +1,16 @@ import { AxiosResponse } from 'axios' +import { UnexpectedResponseStatusError } from './UnexpectedResponseStatusError' +import { UnexpectedContentTypeError } from './UnexpectedContentTypeError' -export const assertSuccessJsonResponse = (response:AxiosResponse):void => { - if (response.status !== 200) { - throw new Error('Unexpected response ' + response.status) +export const assertSuccessJsonResponse = (response: AxiosResponse): void => { + const expectedStatus = 200 + const actualStatus = response.status + if (actualStatus !== expectedStatus) { + throw new UnexpectedResponseStatusError(expectedStatus, actualStatus) } - if (!response.headers['content-type'].startsWith('application/json')) { - throw new Error('Unexpected content-type ' + response.headers['content-type']) + const expectedContentType = 'application/json' + const actualContentType = response.headers['content-type'] + if (!actualContentType.startsWith(expectedContentType)) { + throw new UnexpectedContentTypeError(expectedContentType, actualContentType) } } diff --git a/application/src/Jobs/processNextNode.ts b/application/src/Jobs/processNextNode.ts index 403ceb6..6c5657c 100644 --- a/application/src/Jobs/processNextNode.ts +++ b/application/src/Jobs/processNextNode.ts @@ -8,6 +8,7 @@ import { findNewNodes } from './Nodes/findNewNodes' import { NodeProvider } from '../Fediverse/Providers/NodeProvider' import { FeedProvider } from '../Fediverse/Providers/FeedProvider' import { refreshFeeds } from './Feeds/refreshFeeds' +import { deleteOldFeeds } from '../Storage/Feeds/deleteOldFeeds' export const processNextNode = async (prisma:PrismaClient, providerRegistry:ProviderRegistry):Promise => { console.info('#############################################') @@ -18,6 +19,7 @@ export const processNextNode = async (prisma:PrismaClient, providerRegistry:Prov if (!providerRegistry.containsKey(node.softwareName)) { console.warn('Unknown software', { domain: node.domain, software: node.softwareName }) + await deleteOldFeeds(prisma, node) await setNodeRefreshed(prisma, node) return } @@ -37,5 +39,7 @@ export const processNextNode = async (prisma:PrismaClient, providerRegistry:Prov }) ) + await deleteOldFeeds(prisma, node) + await setNodeRefreshed(prisma, node) } diff --git a/application/src/Storage/Feeds/deleteOldFeeds.ts b/application/src/Storage/Feeds/deleteOldFeeds.ts new file mode 100644 index 0000000..55db6e6 --- /dev/null +++ b/application/src/Storage/Feeds/deleteOldFeeds.ts @@ -0,0 +1,18 @@ +import { Node, PrismaClient } from '@prisma/client' + +export const deleteOldFeeds = async (prisma: PrismaClient, node: Node): Promise => { + const result = await prisma.feed.deleteMany({ + where: { + nodeId: { + equals: node.id + }, + refreshedAt: { + lt: node.refreshAttemptedAt + } + } + }) + console.info('Deleted old feeds', { + count: result.count, olderThen: node.refreshAttemptedAt, nodeDomain: node.domain + }) + return result.count +} diff --git a/application/src/Storage/Nodes/NoNodeFoundError.ts b/application/src/Storage/Nodes/NoNodeFoundError.ts new file mode 100644 index 0000000..61dcce8 --- /dev/null +++ b/application/src/Storage/Nodes/NoNodeFoundError.ts @@ -0,0 +1,5 @@ +export class NoNodeFoundError extends Error { + public constructor () { + super('No node found') + } +} diff --git a/application/src/Storage/Nodes/fetchNodeToProcess.ts b/application/src/Storage/Nodes/fetchNodeToProcess.ts index e55d1e9..1959c68 100644 --- a/application/src/Storage/Nodes/fetchNodeToProcess.ts +++ b/application/src/Storage/Nodes/fetchNodeToProcess.ts @@ -1,4 +1,5 @@ import { Node, PrismaClient } from '@prisma/client' +import { NoNodeFoundError } from './NoNodeFoundError' export const fetchNodeToProcess = async (prisma: PrismaClient): Promise => { const currentTimestamp = Date.now() @@ -56,10 +57,10 @@ export const fetchNodeToProcess = async (prisma: PrismaClient): Promise => ] } }) - if (node) { - console.log('Found oldest node', { domain: node.domain }) - } else { - throw new Error('No node found') + if (!node) { + throw new NoNodeFoundError() } + + console.log('Found oldest node', { domain: node.domain }) return node }