kopia lustrzana https://github.com/Stopka/fedicrawl
Added pleroma
rodzic
de5e989bbb
commit
945c9495f5
|
@ -0,0 +1,167 @@
|
|||
/*
|
||||
Warnings:
|
||||
|
||||
- A unique constraint covering the columns `[name,nodeId]` on the table `Feed` will be added. If there are existing duplicate values, this will fail.
|
||||
- A unique constraint covering the columns `[domain]` on the table `Node` will be added. If there are existing duplicate values, this will fail.
|
||||
- A unique constraint covering the columns `[name]` on the table `Tag` will be added. If there are existing duplicate values, this will fail.
|
||||
|
||||
*/
|
||||
-- DropIndex
|
||||
DROP INDEX "Email_address_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_bot_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_createdAt_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_description_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_displayName_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_fulltext_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_lastStatusAt_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_locked_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_name_nodeId_key";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_parentFeedName_parentFeedDomain_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_refreshedAt_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Feed_type_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Field_name_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Field_value_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_domain_key";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_foundAt_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_halfYearActiveUserCount_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_monthActiveUserCount_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_openRegistrations_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_refreshAttemptedAt_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_refreshedAt_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_softwareName_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_softwareVersion_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_statusesCount_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Node_totalUserCount_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "Tag_name_key";
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "Feed" ALTER COLUMN "followersCount" DROP NOT NULL,
|
||||
ALTER COLUMN "followingCount" DROP NOT NULL;
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Email_address_idx" ON "Email"("address");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_displayName_idx" ON "Feed"("displayName");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_description_idx" ON "Feed"("description");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_bot_idx" ON "Feed"("bot");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_locked_idx" ON "Feed"("locked");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_lastStatusAt_idx" ON "Feed"("lastStatusAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_createdAt_idx" ON "Feed"("createdAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_refreshedAt_idx" ON "Feed"("refreshedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_parentFeedName_parentFeedDomain_idx" ON "Feed"("parentFeedName", "parentFeedDomain");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_type_idx" ON "Feed"("type");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Feed_fulltext_idx" ON "Feed"("fulltext");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "Feed_name_nodeId_key" ON "Feed"("name", "nodeId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Field_name_idx" ON "Field"("name");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Field_value_idx" ON "Field"("value");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "Node_domain_key" ON "Node"("domain");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_softwareName_idx" ON "Node"("softwareName");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_softwareVersion_idx" ON "Node"("softwareVersion");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_totalUserCount_idx" ON "Node"("totalUserCount");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_monthActiveUserCount_idx" ON "Node"("monthActiveUserCount");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_halfYearActiveUserCount_idx" ON "Node"("halfYearActiveUserCount");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_statusesCount_idx" ON "Node"("statusesCount");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_openRegistrations_idx" ON "Node"("openRegistrations");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_refreshedAt_idx" ON "Node"("refreshedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_refreshAttemptedAt_idx" ON "Node"("refreshAttemptedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Node_foundAt_idx" ON "Node"("foundAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "Tag_name_key" ON "Tag"("name");
|
|
@ -0,0 +1,4 @@
|
|||
update "Node"
|
||||
set "refreshedAt"=NULL,
|
||||
"refreshAttemptedAt"=NULL
|
||||
where "Node"."softwareName" like 'pleroma';
|
|
@ -60,8 +60,8 @@ model Feed {
|
|||
feedToTags FeedToTag[]
|
||||
fields Field[]
|
||||
emails Email[]
|
||||
followersCount Int
|
||||
followingCount Int
|
||||
followersCount Int?
|
||||
followingCount Int?
|
||||
statusesCount Int?
|
||||
bot Boolean?
|
||||
url String
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
import { Provider } from '../Provider'
|
||||
import { retrievePeers } from './retrievePeers'
|
||||
import { retrieveLocalPublicUsersPage } from './retrieveLocalPublicUsersPage'
|
||||
import { NodeProvider } from '../NodeProvider'
|
||||
import { FeedProvider } from '../FeedProvider'
|
||||
|
||||
const PleromaProvider: Provider = {
|
||||
getKey: () => 'pleroma',
|
||||
getNodeProviders: ():NodeProvider[] => [{
|
||||
getKey: () => 'peers',
|
||||
retrieveNodes: retrievePeers
|
||||
}],
|
||||
getFeedProviders: ():FeedProvider[] => [{
|
||||
getKey: () => 'users',
|
||||
retrieveFeeds: retrieveLocalPublicUsersPage
|
||||
}]
|
||||
}
|
||||
|
||||
export default PleromaProvider
|
|
@ -0,0 +1,100 @@
|
|||
import axios from 'axios'
|
||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||
import { FeedData } from '../FeedData'
|
||||
import { z } from 'zod'
|
||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||
|
||||
const limit = 500
|
||||
|
||||
const emojiSchema = z.object({
|
||||
shortcode: z.string(),
|
||||
url: z.string()
|
||||
})
|
||||
|
||||
const schema = z.array(
|
||||
z.object({
|
||||
id: z.string(),
|
||||
username: z.string(),
|
||||
display_name: z.string(),
|
||||
locked: z.boolean(),
|
||||
bot: z.boolean(),
|
||||
created_at: z.string(),
|
||||
note: z.string(),
|
||||
url: z.string(),
|
||||
avatar: z.string(),
|
||||
followers_count: z.number(),
|
||||
following_count: z.number(),
|
||||
statuses_count: z.number(),
|
||||
last_status_at: z.nullable(z.string()),
|
||||
emojis: z.array(emojiSchema),
|
||||
fields: z.array(
|
||||
z.object({
|
||||
name: z.string(),
|
||||
value: z.string()
|
||||
})
|
||||
),
|
||||
pleroma: z.object({
|
||||
hide_followers_count: z.boolean(),
|
||||
hide_follows_count: z.boolean()
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
type Emoji = z.infer<typeof emojiSchema>
|
||||
|
||||
const replaceEmojis = (text: string, emojis: Emoji[]): string => {
|
||||
emojis.forEach(emoji => {
|
||||
text = text.replace(
|
||||
RegExp(`:${emoji.shortcode}:`, 'gi'),
|
||||
`<img draggable="false" class="emoji" title="${emoji.shortcode}" alt="${emoji.shortcode}" src="${emoji.url}" />`
|
||||
)
|
||||
})
|
||||
return text
|
||||
}
|
||||
|
||||
export const retrieveLocalPublicUsersPage = async (domain: string, page: number): Promise<FeedData[]> => {
|
||||
try {
|
||||
const response = await axios.get('https://' + domain + '/api/v1/directory', {
|
||||
params: {
|
||||
limit: limit,
|
||||
offset: page * limit,
|
||||
local: true
|
||||
},
|
||||
timeout: getDefaultTimeoutMilliseconds()
|
||||
})
|
||||
assertSuccessJsonResponse(response)
|
||||
const responseData = schema.parse(response.data)
|
||||
if (responseData.length === 0) {
|
||||
throw new Error('No more users')
|
||||
}
|
||||
return responseData.map(
|
||||
item => {
|
||||
return {
|
||||
name: item.username,
|
||||
displayName: replaceEmojis(item.display_name, item.emojis),
|
||||
description: replaceEmojis(item.note, item.emojis),
|
||||
followersCount: item.pleroma.hide_followers_count ? null : item.followers_count,
|
||||
followingCount: item.pleroma.hide_follows_count ? null : item.following_count,
|
||||
statusesCount: item.statuses_count,
|
||||
bot: item.bot,
|
||||
url: item.url,
|
||||
avatar: item.avatar,
|
||||
locked: item.locked,
|
||||
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
|
||||
createdAt: new Date(item.created_at),
|
||||
fields: item.fields.map(field => {
|
||||
return {
|
||||
name: replaceEmojis(field.name, item.emojis),
|
||||
value: replaceEmojis(field.value, item.emojis),
|
||||
verifiedAt: null
|
||||
}
|
||||
}),
|
||||
type: 'account',
|
||||
parentFeed: null
|
||||
}
|
||||
}
|
||||
)
|
||||
} catch (error) {
|
||||
throw new Error('Invalid response: ' + error)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
import axios from 'axios'
|
||||
import { assertSuccessJsonResponse } from '../../assertSuccessJsonResponse'
|
||||
import { z } from 'zod'
|
||||
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
|
||||
|
||||
const schema = z.array(
|
||||
z.string()
|
||||
)
|
||||
|
||||
export const retrievePeers = async (domain:string, page:number):Promise<string[]> => {
|
||||
if (page !== 0) {
|
||||
throw new Error('No more peer pages')
|
||||
}
|
||||
try {
|
||||
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
|
||||
timeout: getDefaultTimeoutMilliseconds()
|
||||
})
|
||||
assertSuccessJsonResponse(response)
|
||||
return schema.parse(response.data)
|
||||
} catch (error) {
|
||||
throw new Error('Invalid response')
|
||||
}
|
||||
}
|
|
@ -1,8 +1,10 @@
|
|||
import { providerRegistry } from './ProviderRegistry'
|
||||
import MastodonProvider from './Mastodon'
|
||||
import PeertubeProvider from './Peertube'
|
||||
import PleromaProvider from './Pleroma'
|
||||
|
||||
providerRegistry.registerProvider(MastodonProvider)
|
||||
providerRegistry.registerProvider(PeertubeProvider)
|
||||
providerRegistry.registerProvider(PleromaProvider)
|
||||
|
||||
export default providerRegistry
|
||||
|
|
Ładowanie…
Reference in New Issue