mirror
/
fedicrawl
kopia lustrzana https://github.com/Stopka/fedicrawl
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Properly applied code quality tools

main
Štěpán Škorpil 2022-09-18 13:32:25 +02:00
rodzic 405cf2f181
commit 29acce3906
88 zmienionych plików z 4426 dodań i 13190 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
Dockerfile
Wyświetl plik

@ -12,9 +12,9 @@ ENV ELASTIC_URL='http://elastic:9200' \
FROM prebuild AS build
WORKDIR /srv
COPY application/package*.json ./
RUN npm install
RUN yarn
COPY application/. .
RUN npm run build
RUN yarn build
FROM build AS dev
CMD npx tsc --watch

24
application/.eslintrc.json
Wyświetl plik

@ -1,24 +0,0 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"plugin:react/recommended",
"standard"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaFeatures": {
"jsx": true
},
"ecmaVersion": 12,
"sourceType": "module"
},
"plugins": [
"react",
"@typescript-eslint"
],
"rules": {
}
}

4
application/.prettierrc
Wyświetl plik

@ -1,4 +0,0 @@
{
"printWidth": 100,
"singleQuote": true
}

12635
application/package-lock.json wygenerowano
Wyświetl plik

Plik diff jest za duży Load Diff

63
application/package.json
Wyświetl plik

@ -8,46 +8,43 @@
"url": "skorpil.cz",
"license": "MIT",
"scripts": {
"dev": "npx tsc --watch",
"clean": "npx rimraf dist",
"build": "npx tsc",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"dev": "tsc --watch",
"clean": "rimraf dist",
"build": "tsc",
"start": "node dist/app",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"lint": "eslint \"{src,test}/**/*.{ts,js}\" --fix",
"test": "jest",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json",
"start:deploy": "npm run start"
"start:deploy": "yarn start"
},
"dependencies": {
"@elastic/elasticsearch": "^8.2.1",
"axios": "^0.21.1",
"@elastic/elasticsearch": "^8.4.0",
"axios": "^0.27.2",
"geoip-lite": "^1.4.6",
"npmlog": "^6.0.0",
"rimraf": "^3.0.2",
"striptags": "^3.2.0",
"typescript-collections": "^1.3.3",
"zod": "^3.11.6"
"zod": "^3.19.1"
},
"devDependencies": {
"@types/geoip-lite": "^1.4.1",
"@types/jest": "^27.0.2",
"@types/jest": "^29.0.3",
"@types/node": "^18.7.18",
"@types/npmlog": "^4.1.3",
"@typescript-eslint/eslint-plugin": "^5.4.0",
"@typescript-eslint/parser": "^5.4.0",
"eslint": "^7.32.0",
"eslint-config-standard": "^16.0.3",
"eslint-plugin-import": "^2.25.3",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^5.1.1",
"eslint-plugin-react": "^7.27.1",
"jest": "^27.3.0",
"standard": "*",
"ts-jest": "^27.0.7",
"typescript": "^4.3.5"
"@types/npmlog": "^4.1.4",
"@typescript-eslint/eslint-plugin": "^5.0.0",
"@typescript-eslint/parser": "^5.0.0",
"eslint": "^8.23.1",
"eslint-config-standard-with-typescript": "^23.0.0",
"eslint-plugin-import": "^2.25.2",
"eslint-plugin-n": "^15.0.0",
"eslint-plugin-promise": "^6.0.0",
"jest": "^29.0.3",
"ts-jest": "^29.0.1",
"typescript": "^4.3.0"
},
"jest": {
"moduleFileExtensions": [
@ -62,5 +59,23 @@
},
"coverageDirectory": "../coverage",
"testEnvironment": "node"
}
},
"eslintConfig": {
"env": {
"es2021": true
},
"extends": [
"standard-with-typescript"
],
"parserOptions": {
"project": [
"tsconfig.json"
]
}
},
"eslintIgnore": [
"dist",
"node_modules"
],
"prettier": "prettier-config-standard"
}

2
application/src/Fediverse/NodeInfo/NoSupportedLinkError.ts
Wyświetl plik

@ -1,5 +1,5 @@
export class NoSupportedLinkError extends Error {
public constructor (domain:string) {
public constructor (domain: string) {
super(`No supported link node info link for ${domain}`)
}
}

8
application/src/Fediverse/NodeInfo/retrieveDomainNodeInfo.ts
Wyświetl plik

@ -2,9 +2,13 @@ import { retrieveWellKnown } from './retrieveWellKnown'
import { retrieveNodeInfo, NodeInfo } from './retrieveNodeInfo'
import { NoSupportedLinkError } from './NoSupportedLinkError'
export const retrieveDomainNodeInfo = async (domain:string):Promise<NodeInfo> => {
export const retrieveDomainNodeInfo = async (
domain: string
): Promise<NodeInfo> => {
const wellKnown = await retrieveWellKnown(domain)
const link = wellKnown.links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0')
const link = wellKnown.links.find(
(link) => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0'
)
if (typeof link === 'undefined') {
throw new NoSupportedLinkError(domain)
}

24
application/src/Fediverse/NodeInfo/retrieveNodeInfo.ts
Wyświetl plik

@ -9,24 +9,26 @@ const schema = z.object({
name: z.string(),
version: z.string()
}),
protocols: z.array(
z.string()
protocols: z.array(z.string()),
usage: z.optional(
z.object({
users: z.optional(
z.object({
total: z.optional(z.number()),
activeMonth: z.optional(z.number()),
activeHalfyear: z.optional(z.number())
})
),
localPosts: z.optional(z.number())
})
),
usage: z.optional(z.object({
users: z.optional(z.object({
total: z.optional(z.number()),
activeMonth: z.optional(z.number()),
activeHalfyear: z.optional(z.number())
})),
localPosts: z.optional(z.number())
})),
openRegistrations: z.optional(z.boolean())
})
export type NodeInfo = z.infer<typeof schema>
export const retrieveNodeInfo = async (url: string): Promise<NodeInfo> => {
console.info('Retrieving node info', { url: url })
console.info('Retrieving node info', { url })
const nodeInfoResponse = await axios.get(url, {
timeout: getDefaultTimeoutMilliseconds()
})

2
application/src/Fediverse/NodeInfo/retrieveWellKnown.ts
Wyświetl plik

@ -15,7 +15,7 @@ const wellKnownSchema = z.object({
export type WellKnown = z.infer<typeof wellKnownSchema>
export const retrieveWellKnown = async (domain: string): Promise<WellKnown> => {
console.info('Retrieving well known', { domain: domain })
console.info('Retrieving well known', { domain })
const wellKnownUrl = `https://${domain}/.well-known/nodeinfo`
const wellKnownResponse = await axios.get(wellKnownUrl, {
timeout: getDefaultTimeoutMilliseconds(),

36
application/src/Fediverse/Providers/FeedData.ts
Wyświetl plik

@ -1,22 +1,22 @@
import { FieldData } from './FieldData'
export interface FeedData {
name:string,
displayName:string,
description:string,
followersCount: number,
followingCount:number,
statusesCount?:number,
bot?:boolean,
url: string,
avatar?:string,
locked:boolean,
lastStatusAt?:Date,
createdAt:Date
fields: FieldData[],
type: 'account'|'channel'
parentFeed?: {
name:string
hostDomain:string
}
name: string
displayName: string
description: string
followersCount: number
followingCount: number
statusesCount?: number
bot?: boolean
url: string
avatar?: string
locked: boolean
lastStatusAt?: Date
createdAt: Date
fields: FieldData[]
type: 'account' | 'channel'
parentFeed?: {
name: string
hostDomain: string
}
}

4
application/src/Fediverse/Providers/FeedProvider.ts
Wyświetl plik

@ -1,6 +1,6 @@
import { FeedProviderMethod } from './FeedProviderMethod'
export interface FeedProvider {
getKey: ()=>string
retrieveFeeds: FeedProviderMethod
getKey: () => string
retrieveFeeds: FeedProviderMethod
}

5
application/src/Fediverse/Providers/FeedProviderMethod.ts
Wyświetl plik

@ -1,3 +1,6 @@
import { FeedData } from './FeedData'
export type FeedProviderMethod = (domain: string, page: number) => Promise<FeedData[]>
export type FeedProviderMethod = (
domain: string,
page: number
) => Promise<FeedData[]>

8
application/src/Fediverse/Providers/FieldData.ts
Wyświetl plik

@ -1,5 +1,5 @@
export interface FieldData{
name: string,
value: string,
verifiedAt: Date|undefined
export interface FieldData {
name: string
value: string
verifiedAt: Date | undefined
}

20
application/src/Fediverse/Providers/Mastodon/index.ts
Wyświetl plik

@ -6,14 +6,18 @@ import { FeedProvider } from '../FeedProvider'
const MastodonProvider: Provider = {
getKey: () => 'mastodon',
getNodeProviders: ():NodeProvider[] => [{
getKey: () => 'peers',
retrieveNodes: retrievePeers
}],
getFeedProviders: ():FeedProvider[] => [{
getKey: () => 'users',
retrieveFeeds: retrieveLocalPublicUsersPage
}]
getNodeProviders: (): NodeProvider[] => [
{
getKey: () => 'peers',
retrieveNodes: retrievePeers
}
],
getFeedProviders: (): FeedProvider[] => [
{
getKey: () => 'users',
retrieveFeeds: retrieveLocalPublicUsersPage
}
]
}
export default MastodonProvider

66
application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts
Wyświetl plik

@ -4,6 +4,7 @@ import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { FeedProviderMethod } from '../FeedProviderMethod'
import { NoMoreFeedsError } from '../NoMoreFeedsError'
import { FeedData } from '../FeedData'
const limit = 500
@ -41,19 +42,22 @@ const schema = z.array(
type Emoji = z.infer<typeof emojiSchema>
const replaceEmojis = (text: string, emojis: Emoji[]): string => {
emojis.forEach(emoji => {
emojis.forEach((emoji) => {
text = text.replace(
RegExp(`:${emoji.shortcode}:`, 'gi'),
`<img draggable="false" class="emoji" title="${emoji.shortcode}" alt="${emoji.shortcode}" src="${emoji.url}" />`
`<img draggable="false" class="emoji" title="${emoji.shortcode}" alt="${emoji.shortcode}" src="${emoji.url}" />`
)
})
return text
}
export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (domain, page) => {
export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (
domain,
page
): Promise<FeedData[]> => {
const response = await axios.get('https://' + domain + '/api/v1/directory', {
params: {
limit: limit,
limit,
offset: page * limit,
local: true
},
@ -64,31 +68,33 @@ export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (domain, p
if (responseData.length === 0) {
throw new NoMoreFeedsError('user')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.display_name, item.emojis),
description: replaceEmojis(item.note, item.emojis),
followersCount: item.followers_count,
followingCount: item.following_count,
statusesCount: item.statuses_count,
bot: item.bot,
url: item.url,
avatar: item.avatar,
locked: item.locked,
lastStatusAt: item.last_status_at !== null ? new Date(item.last_status_at) : null,
createdAt: new Date(item.created_at),
fields: item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: field.verified_at !== null ? new Date(field.verified_at) : null
}
}),
type: 'account',
parentFeed: null
}
return responseData.map((item) => {
return {
name: item.username,
displayName: replaceEmojis(item.display_name, item.emojis),
description: replaceEmojis(item.note, item.emojis),
followersCount: item.followers_count,
followingCount: item.following_count,
statusesCount: item.statuses_count,
bot: item.bot,
url: item.url,
avatar: item.avatar,
locked: item.locked,
lastStatusAt:
item.last_status_at !== null
? new Date(item.last_status_at)
: undefined,
createdAt: new Date(item.created_at),
fields: item.fields.map((field) => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt:
field.verified_at !== null ? new Date(field.verified_at) : undefined
}
}),
type: 'account',
parentFeed: undefined
}
)
})
}

15
application/src/Fediverse/Providers/Mastodon/retrievePeers.ts
Wyświetl plik

@ -5,17 +5,18 @@ import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMillisecon
import { NodeProviderMethod } from '../NodeProviderMethod'
import { NoMoreNodesError } from '../NoMoreNodesError'
const schema = z.array(
z.string()
)
const schema = z.array(z.string())
export const retrievePeers:NodeProviderMethod = async (domain, page) => {
export const retrievePeers: NodeProviderMethod = async (domain, page) => {
if (page !== 0) {
throw new NoMoreNodesError('peer')
}
const response = await axios.get('https://' + domain + '/api/v1/instance/peers', {
timeout: getDefaultTimeoutMilliseconds()
})
const response = await axios.get(
'https://' + domain + '/api/v1/instance/peers',
{
timeout: getDefaultTimeoutMilliseconds()
}
)
assertSuccessJsonResponse(response)
return schema.parse(response.data)
}

20
application/src/Fediverse/Providers/Misskey/index.ts
Wyświetl plik

@ -6,14 +6,18 @@ import { retrieveUsersPage } from './retrieveUsersPage'
const MisskeyProvider: Provider = {
getKey: () => 'misskey',
getNodeProviders: ():NodeProvider[] => [{
getKey: () => 'federation-instances',
retrieveNodes: retrieveInstancesPage
}],
getFeedProviders: ():FeedProvider[] => [{
getKey: () => 'users',
retrieveFeeds: retrieveUsersPage
}]
getNodeProviders: (): NodeProvider[] => [
{
getKey: () => 'federation-instances',
retrieveNodes: retrieveInstancesPage
}
],
getFeedProviders: (): FeedProvider[] => [
{
getKey: () => 'users',
retrieveFeeds: retrieveUsersPage
}
]
}
export default MisskeyProvider

45
application/src/Fediverse/Providers/Misskey/retrieveInstancesPage.ts
Wyświetl plik

@ -13,29 +13,34 @@ const schema = z.array(
})
)
export const retrieveInstancesPage:NodeProviderMethod = async (domain, page) => {
const response = await axios.post('https://' + domain + '/api/federation/instances', {
host: null,
blocked: null,
notResponding: null,
suspended: null,
federating: null,
subscribing: null,
publishing: null,
limit: limit,
offset: page * limit,
sort: '+id'
}, {
timeout: getDefaultTimeoutMilliseconds()
})
export const retrieveInstancesPage: NodeProviderMethod = async (
domain,
page
) => {
const response = await axios.post(
'https://' + domain + '/api/federation/instances',
{
host: null,
blocked: null,
notResponding: null,
suspended: null,
federating: null,
subscribing: null,
publishing: null,
limit,
offset: page * limit,
sort: '+id'
},
{
timeout: getDefaultTimeoutMilliseconds()
}
)
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new NoMoreNodesError('instance')
}
return responseData.map(
item => {
return item.host
}
)
return responseData.map((item) => {
return item.host
})
}

114
application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts
Wyświetl plik

@ -4,6 +4,8 @@ import { z } from 'zod'
import { getDefaultTimeoutMilliseconds } from '../../getDefaultTimeoutMilliseconds'
import { NoMoreFeedsError } from '../NoMoreFeedsError'
import { FeedProviderMethod } from '../FeedProviderMethod'
import { FeedData } from '../FeedData'
import { FieldData } from '../FieldData'
const limit = 100
@ -42,72 +44,84 @@ const schema = z.array(
type Emoji = z.infer<typeof emojiSchema>
const replaceEmojis = (text: string, emojis: Emoji[]): string => {
emojis.forEach(emoji => {
emojis.forEach((emoji) => {
text = text.replace(
RegExp(`:${emoji.name}:`, 'gi'),
`<img draggable="false" class="emoji" title="${emoji.name}" alt="${emoji.name}" src="${emoji.url}" />`
`<img draggable="false" class="emoji" title="${emoji.name}" alt="${emoji.name}" src="${emoji.url}" />`
)
})
return text
}
const parseDescription = (description:string|null):string => {
const parseDescription = (description: string | null): string => {
if (typeof description !== 'string') {
return ''
}
return description.split('\n\n').map(paragraph => {
paragraph = paragraph.replace('\n', '</br>\n')
return `<p>${paragraph}</p>`
}).join('\n')
return description
.split('\n\n')
.map((paragraph) => {
paragraph = paragraph.replace('\n', '</br>\n')
return `<p>${paragraph}</p>`
})
.join('\n')
}
export const retrieveUsersPage:FeedProviderMethod = async (domain, page) => {
const response = await axios.post('https://' + domain + '/api/users', {
state: 'all',
origin: 'local',
sort: '+createdAt',
limit: limit,
offset: limit * page
}, {
timeout: getDefaultTimeoutMilliseconds()
})
export const retrieveUsersPage: FeedProviderMethod = async (
domain,
page
): Promise<FeedData[]> => {
const response = await axios.post(
'https://' + domain + '/api/users',
{
state: 'all',
origin: 'local',
sort: '+createdAt',
limit,
offset: limit * page
},
{
timeout: getDefaultTimeoutMilliseconds()
}
)
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
if (responseData.length === 0) {
throw new NoMoreFeedsError('user')
}
return responseData.map(
item => {
return {
name: item.username,
displayName: replaceEmojis(item.name ?? item.username, item.emojis),
description: replaceEmojis(parseDescription(item.description ?? ''), item.emojis),
followersCount: item.followersCount,
followingCount: item.followingCount,
statusesCount: item.notesCount,
bot: item.isBot,
url: `https://${domain}/@${item.username}`,
avatar: item.avatarUrl,
locked: item.isLocked,
lastStatusAt: item.updatedAt !== null ? new Date(item.updatedAt) : undefined,
createdAt: new Date(item.createdAt),
fields: [
...item.fields.map(field => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: undefined
}
}),
...[
{ name: 'Location', value: item.location, verifiedAt: undefined },
{ name: 'Birthday', value: item.birthday, verifiedAt: undefined },
{ name: 'Language', value: item.lang, verifiedAt: undefined }
].filter(field => field.value !== null)
],
type: 'account',
parentFeed: undefined
}
return responseData.map((item) => {
return {
name: item.username,
displayName: replaceEmojis(item.name ?? item.username, item.emojis),
description: replaceEmojis(
parseDescription(item.description ?? ''),
item.emojis
),
followersCount: item.followersCount,
followingCount: item.followingCount,
statusesCount: item.notesCount,
bot: item.isBot,
url: `https://${domain}/@${item.username}`,
avatar: item.avatarUrl ?? undefined,
locked: item.isLocked,
lastStatusAt:
item.updatedAt !== null ? new Date(item.updatedAt) : undefined,
createdAt: new Date(item.createdAt),
fields: [
...item.fields.map((field) => {
return {
name: replaceEmojis(field.name, item.emojis),
value: replaceEmojis(field.value, item.emojis),
verifiedAt: undefined
}
}),
...([
{ name: 'Location', value: item.location, verifiedAt: undefined },
{ name: 'Birthday', value: item.birthday, verifiedAt: undefined },
{ name: 'Language', value: item.lang, verifiedAt: undefined }
].filter((field) => field.value !== null) as FieldData[])
],
type: 'account',
parentFeed: undefined
}
)
})
}

2
application/src/Fediverse/Providers/NoMoreFeedsError.ts
Wyświetl plik

@ -1,5 +1,5 @@
export class NoMoreFeedsError extends Error {
public constructor (feedType:string) {
public constructor (feedType: string) {
super(`No more feeds of type ${feedType}`)
}
}

2
application/src/Fediverse/Providers/NoMoreNodesError.ts
Wyświetl plik

@ -1,5 +1,5 @@
export class NoMoreNodesError extends Error {
public constructor (nodeType:string) {
public constructor (nodeType: string) {
super(`No more nodes of type ${nodeType}`)
}
}

4
application/src/Fediverse/Providers/NodeProvider.ts
Wyświetl plik

@ -1,6 +1,6 @@
import { NodeProviderMethod } from './NodeProviderMethod'
export interface NodeProvider {
getKey:()=>string,
retrieveNodes: NodeProviderMethod
getKey: () => string
retrieveNodes: NodeProviderMethod
}

6
application/src/Fediverse/Providers/NodeProviderMethod.ts
Wyświetl plik

@ -1,2 +1,4 @@
export type NodeProviderMethod = (domain: string, page:number)=> Promise<string[]>
export type NodeProviderMethod = (
domain: string,
page: number
) => Promise<string[]>

10
application/src/Fediverse/Providers/Peertube/Avatar.ts
Wyświetl plik

@ -1,7 +1,11 @@
import { z } from 'zod'
export const avatarSchema = z.optional(z.nullable(z.object({
path: z.string()
})))
export const avatarSchema = z.optional(
z.nullable(
z.object({
path: z.string()
})
)
)
export type Avatar = z.infer<typeof avatarSchema>

27
application/src/Fediverse/Providers/Peertube/index.ts
Wyświetl plik

@ -7,16 +7,21 @@ import { retrieveFollowers } from './retrieveFollowers'
const PeertubeProvider: Provider = {
getKey: () => 'peertube',
getNodeProviders: ():NodeProvider[] => [{
getKey: () => 'followers',
retrieveNodes: retrieveFollowers
}],
getFeedProviders: ():FeedProvider[] => [{
getKey: () => 'accounts',
retrieveFeeds: retrieveAccounts
}, {
getKey: () => 'video-channels',
retrieveFeeds: retrieveVideoChannels
}]
getNodeProviders: (): NodeProvider[] => [
{
getKey: () => 'followers',
retrieveNodes: retrieveFollowers
}
],
getFeedProviders: (): FeedProvider[] => [
{
getKey: () => 'accounts',
retrieveFeeds: retrieveAccounts
},
{
getKey: () => 'video-channels',
retrieveFeeds: retrieveVideoChannels
}
]
}
export default PeertubeProvider

7
application/src/Fediverse/Providers/Peertube/parseAvatarUrl.ts
Wyświetl plik

@ -1,7 +1,10 @@
import { Avatar } from './Avatar'
export const parseAvatarUrl = (data:Avatar, domain:string):string|undefined => {
if (data === null) {
export const parseAvatarUrl = (
data: Avatar,
domain: string
): string | undefined => {
if (data === null || data === undefined) {
return undefined
}
return `https://${domain}${data.path}`

13
application/src/Fediverse/Providers/Peertube/parseDescription.ts
Wyświetl plik

@ -1,9 +1,12 @@
export const parseDescription = (description:string|null):string => {
export const parseDescription = (description: string | null): string => {
if (typeof description !== 'string') {
return ''
}
return description.split('\n\n').map(paragraph => {
paragraph = paragraph.replace('\n', '</br>\n')
return `<p>${paragraph}</p>`
}).join('\n')
return description
.split('\n\n')
.map((paragraph) => {
paragraph = paragraph.replace('\n', '</br>\n')
return `<p>${paragraph}</p>`
})
.join('\n')
}

6
application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts
Wyświetl plik

@ -29,7 +29,7 @@ const schema = z.object({
)
})
export const retrieveAccounts:FeedProviderMethod = async (domain, page) => {
export const retrieveAccounts: FeedProviderMethod = async (domain, page) => {
const response = await axios.get(`https://${domain}/api/v1/accounts`, {
params: {
count: limit,
@ -44,8 +44,8 @@ export const retrieveAccounts:FeedProviderMethod = async (domain, page) => {
throw new NoMoreFeedsError('account')
}
return responseData.data
.filter(item => item.host === domain)
.map((item):FeedData => {
.filter((item) => item.host === domain)
.map((item): FeedData => {
return {
name: item.name,
url: item.url ?? `https://${domain}/accounts/${item.name}/`,

23
application/src/Fediverse/Providers/Peertube/retrieveFollowers.ts
Wyświetl plik

@ -21,19 +21,22 @@ const schema = z.object({
)
})
export const retrieveFollowers:NodeProviderMethod = async (domain, page) => {
const response = await axios.get(`https://${domain}/api/v1/server/followers`, {
params: {
count: limit,
sort: 'createdAt',
start: page * limit
},
timeout: getDefaultTimeoutMilliseconds()
})
export const retrieveFollowers: NodeProviderMethod = async (domain, page) => {
const response = await axios.get(
`https://${domain}/api/v1/server/followers`,
{
params: {
count: limit,
sort: 'createdAt',
start: page * limit
},
timeout: getDefaultTimeoutMilliseconds()
}
)
assertSuccessJsonResponse(response)
const responseData = schema.parse(response.data)
const hosts = new Set<string>()
responseData.data.forEach(item => {
responseData.data.forEach((item) => {
hosts.add(item.follower.host)
hosts.add(item.following.host)
})

18
application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts
Wyświetl plik

@ -36,7 +36,10 @@ const schema = z.object({
)
})
export const retrieveVideoChannels:FeedProviderMethod = async (domain, page) => {
export const retrieveVideoChannels: FeedProviderMethod = async (
domain,
page
) => {
const response = await axios.get(`https://${domain}/api/v1/video-channels`, {
params: {
count: limit,
@ -51,17 +54,18 @@ export const retrieveVideoChannels:FeedProviderMethod = async (domain, page) =>
throw new NoMoreFeedsError('channel')
}
return responseData.data
.filter(item => item.host === domain)
.map((item):FeedData => {
const fields:FieldData[] = item.support
? [{ name: 'support', value: item.support, verifiedAt: undefined }]
: []
.filter((item) => item.host === domain)
.map((item): FeedData => {
const fields: FieldData[] =
item.support !== null
? [{ name: 'support', value: item.support, verifiedAt: undefined }]
: []
return {
name: item.name,
url: item.url ?? `https://${domain}/video-channels/${item.name}/`,
avatar: parseAvatarUrl(item.avatar, domain),
locked: false,
fields: fields,
fields,
description: parseDescription(item.description),
displayName: item.displayName,
followersCount: item.followersCount,

6
application/src/Fediverse/Providers/Provider.ts
Wyświetl plik

@ -2,9 +2,9 @@ import { NodeProvider } from './NodeProvider'
import { FeedProvider } from './FeedProvider'
export interface Provider {
getKey(): string
getKey: () => string
getNodeProviders(): NodeProvider[]
getNodeProviders: () => NodeProvider[]
getFeedProviders(): FeedProvider[]
getFeedProviders: () => FeedProvider[]
}

4
application/src/Fediverse/Providers/ProviderKeyAlreadyRegisteredError.ts
Wyświetl plik

@ -1,7 +1,7 @@
export class ProviderKeyAlreadyRegisteredError extends Error {
private readonly _key:string
private readonly _key: string
public constructor (key:string) {
public constructor (key: string) {
super(`Provider with the key ${key} is already registered`)
this._key = key
}

12
application/src/Fediverse/Providers/ProviderKeyNotFoundError.ts 100644
Wyświetl plik

@ -0,0 +1,12 @@
export class ProviderKeyNotFoundError extends Error {
private readonly _key: string
public constructor (key: string) {
super(`Provider with the key ${key} is not registered`)
this._key = key
}
public get key (): string {
return this._key
}
}

30
application/src/Fediverse/Providers/ProviderRegistry.ts
Wyświetl plik

@ -1,10 +1,8 @@
import { Provider } from './Provider'
import { Dictionary } from 'typescript-collections'
import { ProviderKeyAlreadyRegisteredError } from './ProviderKeyAlreadyRegisteredError'
export interface ProviderCallback {
(key: string, provider: Provider): void
}
import { ProviderKeyNotFoundError } from './ProviderKeyNotFoundError'
export type ProviderCallback = (key: string, provider: Provider) => void
const providers: Dictionary<string, Provider> = new Dictionary<string, Provider>()
@ -14,14 +12,18 @@ const registerProvider = (provider: Provider): void => {
throw new ProviderKeyAlreadyRegisteredError(key)
}
providers.setValue(key, provider)
console.info('Added provider to registry', { key: key })
console.info('Added provider to registry', { key })
}
const getProviderByKey = (key: string): Provider => {
return providers.getValue(key)
const provider = providers.getValue(key)
if (provider === undefined) {
throw new ProviderKeyNotFoundError(key)
}
return provider
}
const getKeys = ():string[] => {
const getKeys = (): string[] => {
return providers.keys()
}
@ -29,19 +31,19 @@ const forEachProvider = (callback: ProviderCallback): void => {
return providers.forEach(callback)
}
const containsKey = (key:string):boolean => {
const containsKey = (key: string): boolean => {
return providers.containsKey(key)
}
export interface ProviderRegistry {
registerProvider: (provider: Provider)=> void,
getProviderByKey:(key: string)=> Provider
forEachProvider:(callback: ProviderCallback)=> void
getKeys:()=>string[]
containsKey:(key:string)=>boolean
registerProvider: (provider: Provider) => void
getProviderByKey: (key: string) => Provider
forEachProvider: (callback: ProviderCallback) => void
getKeys: () => string[]
containsKey: (key: string) => boolean
}
export const providerRegistry:ProviderRegistry = {
export const providerRegistry: ProviderRegistry = {
registerProvider,
getProviderByKey,
forEachProvider,

28
application/src/Fediverse/UnexpectedContentTypeError.ts
Wyświetl plik

@ -1,20 +1,22 @@
import { UnexpectedResponseError } from './UnexpectedResponseError'
export class UnexpectedContentTypeError extends UnexpectedResponseError {
private readonly _expectedContentType: string
private readonly _actualContentType: string
private readonly _expectedContentType: string
private readonly _actualContentType: string
public constructor (actualContentType: string, expectedContentType:string) {
super(`Expected content type '${expectedContentType}' but got '${actualContentType}'`)
this._expectedContentType = expectedContentType
this._actualContentType = actualContentType
}
public constructor (actualContentType: string, expectedContentType: string) {
super(
`Expected content type '${expectedContentType}' but got '${actualContentType}'`
)
this._expectedContentType = expectedContentType
this._actualContentType = actualContentType
}
get expectedContentType (): string {
return this._expectedContentType
}
get expectedContentType (): string {
return this._expectedContentType
}
get actualContentType (): string {
return this._actualContentType
}
get actualContentType (): string {
return this._actualContentType
}
}

4
application/src/Fediverse/UnexpectedResponseError.ts
Wyświetl plik

@ -1,3 +1 @@
export class UnexpectedResponseError extends Error {
}
export class UnexpectedResponseError extends Error {}

28
application/src/Fediverse/UnexpectedResponseStatusError.ts
Wyświetl plik

@ -1,20 +1,22 @@
import { UnexpectedResponseError } from './UnexpectedResponseError'
export class UnexpectedResponseStatusError extends UnexpectedResponseError {
private readonly _expectedStatusCode: number
private readonly _actualStatusCode: number
private readonly _expectedStatusCode: number
private readonly _actualStatusCode: number
public constructor (expectedStatusCode:number, actualStatusCode:number) {
super(`Expected response code ${expectedStatusCode} but got ${actualStatusCode}`)
this._actualStatusCode = actualStatusCode
this._expectedStatusCode = expectedStatusCode
}
public constructor (expectedStatusCode: number, actualStatusCode: number) {
super(
`Expected response code ${expectedStatusCode} but got ${actualStatusCode}`
)
this._actualStatusCode = actualStatusCode
this._expectedStatusCode = expectedStatusCode
}
get expectedStatusCode (): number {
return this._expectedStatusCode
}
get expectedStatusCode (): number {
return this._expectedStatusCode
}
get actualStatusCode (): number {
return this._actualStatusCode
}
get actualStatusCode (): number {
return this._actualStatusCode
}
}

4
application/src/Fediverse/assertSuccessJsonResponse.ts
Wyświetl plik

@ -2,7 +2,9 @@ import { AxiosResponse } from 'axios'
import { UnexpectedResponseStatusError } from './UnexpectedResponseStatusError'
import { UnexpectedContentTypeError } from './UnexpectedContentTypeError'
export const assertSuccessJsonResponse = (response: AxiosResponse<unknown>): void => {
export const assertSuccessJsonResponse = (
response: AxiosResponse<unknown>
): void => {
const expectedStatus = 200
const actualStatus = response.status
if (actualStatus !== expectedStatus) {

2
application/src/Fediverse/getDefaultTimeoutMilliseconds.ts
Wyświetl plik

@ -1,3 +1,3 @@
export const getDefaultTimeoutMilliseconds = () :number => {
export const getDefaultTimeoutMilliseconds = (): number => {
return parseInt(process.env.DEFAULT_TIMEOUT_MILLISECONDS ?? '10000')
}

11
application/src/Jobs/Dns/refreshNodeIps.ts
Wyświetl plik

@ -3,13 +3,20 @@ import { updateNodeIps } from '../../Storage/Nodes/updateNodeIps'
import { ElasticClient } from '../../Storage/ElasticClient'
import Node from '../../Storage/Definitions/Node'
const refreshNodeIps = async (elastic: ElasticClient, node:Node):Promise<Node> => {
const refreshNodeIps = async (
elastic: ElasticClient,
node: Node
): Promise<Node> => {
console.info('Looking up node ip addresses', {
nodeDomain: node.domain
})
try {
const addresses = await lookup(node.domain, { all: true })
return updateNodeIps(elastic, node, addresses.map(resolution => resolution.address))
return await updateNodeIps(
elastic,
node,
addresses.map((resolution) => resolution.address)
)
} catch (error) {
console.warn('Could not lookup the domain', { node, error })
return node

12
application/src/Jobs/Feeds/addFeed.ts
Wyświetl plik

@ -7,9 +7,17 @@ import Feed from '../../Storage/Definitions/Feed'
import Node from '../../Storage/Definitions/Node'
import { ElasticClient } from '../../Storage/ElasticClient'
export const addFeed = async (elastic: ElasticClient, node: Node, feedData: FeedData): Promise<Feed> => {
export const addFeed = async (
elastic: ElasticClient,
node: Node,
feedData: FeedData
): Promise<Feed> => {
const fulltext = prepareFulltext(feedData, node)
const extractedTags = extractTags(fulltext)
const extractedEmails = extractEmails(fulltext)
return await createFeed(elastic, { ...feedData, extractedTags, extractedEmails }, node)
return await createFeed(
elastic,
{ ...feedData, extractedTags, extractedEmails },
node
)
}

20
application/src/Jobs/Feeds/prepareFulltext.ts
Wyświetl plik

@ -2,13 +2,21 @@ import { FeedData } from '../../Fediverse/Providers/FeedData'
import striptags from 'striptags'
import Node from '../../Storage/Definitions/Node'
export default function (feedData: FeedData, node: Node):string {
export default function (feedData: FeedData, node: Node): string {
return striptags(
feedData.displayName +
' ' + feedData.description +
' ' + feedData.fields.map(field => field.name).join(' ') +
' ' + feedData.fields.map(field => field.value).join(' ') +
' ' + feedData.name + '@' + node.domain +
(feedData.parentFeed ? (' ' + feedData.parentFeed.name + '@' + feedData.parentFeed.hostDomain) : '')
' ' +
feedData.description +
' ' +
feedData.fields.map((field) => field.name).join(' ') +
' ' +
feedData.fields.map((field) => field.value).join(' ') +
' ' +
feedData.name +
'@' +
node.domain +
(feedData.parentFeed != null
? ' ' + feedData.parentFeed.name + '@' + feedData.parentFeed.hostDomain
: '')
)
}

13
application/src/Jobs/Feeds/refreshFeed.ts
Wyświetl plik

@ -7,11 +7,20 @@ import Node from '../../Storage/Definitions/Node'
import prepareFulltext from './prepareFulltext'
import { ElasticClient } from '../../Storage/ElasticClient'
export const refreshFeed = async (elastic: ElasticClient, feed:Feed, feedData: FeedData, node: Node): Promise<Feed> => {
export const refreshFeed = async (
elastic: ElasticClient,
feed: Feed,
feedData: FeedData,
node: Node
): Promise<Feed> => {
const fulltext = prepareFulltext(feedData, node)
const extractedTags = extractTags(fulltext)
const extractedEmails = extractEmails(fulltext)
return await updateFeed(elastic, feed, { ...feedData, extractedTags, extractedEmails })
return await updateFeed(elastic, feed, {
...feedData,
extractedTags,
extractedEmails
})
}

21
application/src/Jobs/Feeds/refreshFeeds.ts
Wyświetl plik

@ -3,13 +3,26 @@ import { FeedProvider } from '../../Fediverse/Providers/FeedProvider'
import Node from '../../Storage/Definitions/Node'
import { ElasticClient } from '../../Storage/ElasticClient'
export const refreshFeeds = async (elastic: ElasticClient, provider:FeedProvider, node:Node):Promise<void> => {
export const refreshFeeds = async (
elastic: ElasticClient,
provider: FeedProvider,
node: Node
): Promise<void> => {
try {
// noinspection InfiniteLoopJS
for (let page = 0; true; page++) {
console.info('Retrieve feeds page', { nodeDomain: node.domain, provider: provider.getKey(), page: page })
console.info('Retrieve feeds page', {
nodeDomain: node.domain,
provider: provider.getKey(),
page
})
await refreshFeedsOnPage(elastic, provider, node, page)
}
} catch (e) {
console.info('Feed search finished: ' + e, { nodeDomain: node.domain, provider: provider.getKey() })
} catch (error) {
console.info('Feed search finished', {
error,
nodeDomain: node.domain,
provider: provider.getKey()
})
}
}

23
application/src/Jobs/Feeds/refreshFeedsOnPage.ts
Wyświetl plik

@ -4,10 +4,23 @@ import Node from '../../Storage/Definitions/Node'
import Feed from '../../Storage/Definitions/Feed'
import { ElasticClient } from '../../Storage/ElasticClient'
export const refreshFeedsOnPage = async (elastic: ElasticClient, provider:FeedProvider, node:Node, page:number):Promise<Feed[]> => {
export const refreshFeedsOnPage = async (
elastic: ElasticClient,
provider: FeedProvider,
node: Node,
page: number
): Promise<Feed[]> => {
const feedData = await provider.retrieveFeeds(node.domain, page)
console.info('Retrieved feeds', { count: feedData.length, domain: node.domain, provider: provider.getKey(), page: page })
return Promise.all(feedData.map(
feedDataItem => refreshOrAddFeed(elastic, node, feedDataItem)
))
console.info('Retrieved feeds', {
count: feedData.length,
domain: node.domain,
provider: provider.getKey(),
page
})
return await Promise.all(
feedData.map(
async (feedDataItem) =>
await refreshOrAddFeed(elastic, node, feedDataItem)
)
)
}

26
application/src/Jobs/Feeds/refreshOrAddFeed.ts
Wyświetl plik

@ -6,17 +6,27 @@ import Node from '../../Storage/Definitions/Node'
import { ElasticClient } from '../../Storage/ElasticClient'
import getFeed from '../../Storage/Feeds/getFeed'
export const refreshOrAddFeed = async (elastic: ElasticClient, node:Node, feedData:FeedData):Promise<Feed> => {
let feed:Feed|null = null
export const refreshOrAddFeed = async (
elastic: ElasticClient,
node: Node,
feedData: FeedData
): Promise<Feed> => {
let feed: Feed | null | undefined
try {
feed = await getFeed(elastic, `${feedData.name}@${node.domain}`)
} catch (e) {
}
if (feed) {
console.info('Refreshing feed', { nodeDomain: node.domain, feedName: feedData.name, feedType: feedData.type })
} catch (e) {}
if (feed !== null && feed !== undefined) {
console.info('Refreshing feed', {
nodeDomain: node.domain,
feedName: feedData.name,
feedType: feedData.type
})
return await refreshFeed(elastic, feed, feedData, node)
}
console.info('Adding feed', { nodeDomain: node.domain, feedName: feedData.name, feedType: feedData.type })
console.info('Adding feed', {
nodeDomain: node.domain,
feedName: feedData.name,
feedType: feedData.type
})
return await addFeed(elastic, node, feedData)
}

7
application/src/Jobs/NodeInfo/refreshNodeInfo.ts
Wyświetl plik

@ -3,13 +3,16 @@ import { updateNodeInfo } from '../../Storage/Nodes/updateNodeInfo'
import Node from '../../Storage/Definitions/Node'
import { ElasticClient } from '../../Storage/ElasticClient'
export const refreshNodeInfo = async (elastic: ElasticClient, node:Node):Promise<Node> => {
export const refreshNodeInfo = async (
elastic: ElasticClient,
node: Node
): Promise<Node> => {
console.info('Updating info of node', { nodeDomain: node.domain })
try {
const nodeInfo = await retrieveDomainNodeInfo(node.domain)
return await updateNodeInfo(elastic, node, nodeInfo)
} catch (error) {
console.warn('Failed to update node info: ' + error)
console.warn('Failed to update node info', error)
return node
}
}

20
application/src/Jobs/Nodes/findNewNodes.ts
Wyświetl plik

@ -3,13 +3,25 @@ import { findNewNodesOnPage } from './findNewNodesOnPage'
import Node from '../../Storage/Definitions/Node'
import { ElasticClient } from '../../Storage/ElasticClient'
export const findNewNodes = async (elastic: ElasticClient, provider:NodeProvider, node:Node):Promise<void> => {
export const findNewNodes = async (
elastic: ElasticClient,
provider: NodeProvider,
node: Node
): Promise<void> => {
try {
// noinspection InfiniteLoopJS
for (let page = 0; true; page++) {
console.info('Retrieve node page', { domain: node.domain, provider: provider.getKey() })
console.info('Retrieve node page', {
domain: node.domain,
provider: provider.getKey()
})
await findNewNodesOnPage(elastic, provider, node, page)
}
} catch (e) {
console.info('Node search finished: ' + e, { domain: node.domain, provider: provider.getKey() })
} catch (error) {
console.info('Node search finished', {
error,
domain: node.domain,
provider: provider.getKey()
})
}
}

14
application/src/Jobs/Nodes/findNewNodesOnPage.ts
Wyświetl plik

@ -5,10 +5,18 @@ import { ElasticClient } from '../../Storage/ElasticClient'
import isDomainNotBanned from '../../Storage/Nodes/isDomainNotBanned'
export const findNewNodesOnPage = async (
elastic: ElasticClient, provider: NodeProvider, node:Node, page:number
):Promise<number> => {
elastic: ElasticClient,
provider: NodeProvider,
node: Node,
page: number
): Promise<number> => {
let domains = await provider.retrieveNodes(node.domain, page)
domains = domains.filter(isDomainNotBanned)
console.log('Found nodes', { count: domains.length, domain: node.domain, provider: provider.getKey(), page: page })
console.log('Found nodes', {
count: domains.length,
domain: node.domain,
provider: provider.getKey(),
page
})
return await createMissingNodes(elastic, domains, node.domain)
}

11
application/src/Jobs/Nodes/updateNodeFeedStats.ts
Wyświetl plik

@ -3,12 +3,15 @@ import { setNodeStats } from '../../Storage/Nodes/setNodeStats'
import { ElasticClient } from '../../Storage/ElasticClient'
import Node from '../../Storage/Definitions/Node'
export type NodeStats ={
account: number,
channel: number,
export interface NodeStats {
account: number
channel: number
}
export default async function updateNodeFeedStats (elasticClient: ElasticClient, node: Node) {
export default async function updateNodeFeedStats (
elasticClient: ElasticClient,
node: Node
): Promise<void> {
await setNodeStats(
elasticClient,
node,

7
application/src/Jobs/Seed/addNodeSeed.ts
Wyświetl plik

@ -1,8 +1,11 @@
import { createMissingNodes } from '../../Storage/Nodes/createMissingNodes'
import { ElasticClient } from '../../Storage/ElasticClient'
export const addNodeSeed = async (elastic: ElasticClient, domains:string[]):Promise<boolean> => {
console.info('Trying to add seed domain nodes', { domains: domains })
export const addNodeSeed = async (
elastic: ElasticClient,
domains: string[]
): Promise<boolean> => {
console.info('Trying to add seed domain nodes', { domains })
const result = await createMissingNodes(elastic, domains, undefined)
return result > 0
}

11
application/src/Jobs/Seed/deleteBannedNodes.ts
Wyświetl plik

@ -2,10 +2,13 @@ import { deleteDomainFeeds } from '../../Storage/Feeds/deleteDomainFeeds'
import { deleteDomainNodes } from '../../Storage/Nodes/deleteDomainNodes'
import { ElasticClient } from '../../Storage/ElasticClient'
export default async function deleteDomains (elastic: ElasticClient, domains:string[]):Promise<number> {
if (domains === []) {
return
export default async function deleteDomains (
elastic: ElasticClient,
domains: string[]
): Promise<number> {
if (domains.length === 0) {
return 0
}
await deleteDomainFeeds(elastic, domains)
return deleteDomainNodes(elastic, domains)
return await deleteDomainNodes(elastic, domains)
}

4
application/src/Jobs/Seed/getBannedDomains.ts
Wyświetl plik

@ -1,7 +1,7 @@
export default function getBannedDomains ():string[] {
export default function getBannedDomains (): string[] {
const domains = process.env.BANNED_DOMAINS ?? ''
if (domains === '') {
return []
}
return domains.split(',').map(domain => domain.toLowerCase())
return domains.split(',').map((domain) => domain.toLowerCase())
}

33
application/src/Jobs/processNextNode.ts
Wyświetl plik

@ -12,7 +12,10 @@ import refreshNodeIps from './Dns/refreshNodeIps'
import { ElasticClient } from '../Storage/ElasticClient'
import updateNodeFeedStats from './Nodes/updateNodeFeedStats'
export const processNextNode = async (elastic: ElasticClient, providerRegistry:ProviderRegistry):Promise<void> => {
export const processNextNode = async (
elastic: ElasticClient,
providerRegistry: ProviderRegistry
): Promise<void> => {
console.info('#############################################')
let node = await fetchNodeToProcess(elastic)
node = await setNodeRefreshAttempted(elastic, node)
@ -20,25 +23,35 @@ export const processNextNode = async (elastic: ElasticClient, providerRegistry:P
node = await refreshNodeIps(elastic, node)
node = await refreshNodeInfo(elastic, node)
if (!providerRegistry.containsKey(node.softwareName)) {
console.warn('Unknown software', { domain: node.domain, software: node.softwareName })
const softwareName = node.softwareName ?? ''
if (!providerRegistry.containsKey(softwareName)) {
console.warn('Unknown software', {
domain: node.domain,
software: node.softwareName
})
await deleteOldFeeds(elastic, node)
await setNodeRefreshed(elastic, node)
return
}
const provider = providerRegistry.getProviderByKey(node.softwareName)
const provider = providerRegistry.getProviderByKey(softwareName)
await Promise.all(
provider.getNodeProviders().map((nodeProvider:NodeProvider) => {
console.info('Searching for nodes', { domain: node.domain, provider: nodeProvider.getKey() })
return findNewNodes(elastic, nodeProvider, node)
provider.getNodeProviders().map(async (nodeProvider: NodeProvider) => {
console.info('Searching for nodes', {
domain: node.domain,
provider: nodeProvider.getKey()
})
return await findNewNodes(elastic, nodeProvider, node)
})
)
await Promise.all(
provider.getFeedProviders().map((feedProvider:FeedProvider) => {
console.info('Searching for feeds', { domain: node.domain, provider: feedProvider.getKey() })
return refreshFeeds(elastic, feedProvider, node)
provider.getFeedProviders().map(async (feedProvider: FeedProvider) => {
console.info('Searching for feeds', {
domain: node.domain,
provider: feedProvider.getKey()
})
return await refreshFeeds(elastic, feedProvider, node)
})
)

46
application/src/Storage/Definitions/Feed.ts
Wyświetl plik

@ -1,29 +1,29 @@
import Field from './Field'
interface Feed {
domain: string
foundAt: number,
refreshedAt?: number,
name: string,
fullName: string,
displayName: string,
description: string,
strippedDescription?: string,
followersCount?: number,
followingCount?: number,
statusesCount?: number,
lastStatusAt?: number,
createdAt?: number,
bot?: boolean,
locked: boolean,
url: string,
avatar?: string,
type: 'account' | 'channel',
parentFeedName?: string,
parentFeedDomain?: string
fields: Field[],
extractedEmails: string[],
extractedTags: string[]
domain: string
foundAt: number
refreshedAt?: number
name: string
fullName: string
displayName: string
description: string
strippedDescription?: string
followersCount?: number
followingCount?: number
statusesCount?: number
lastStatusAt?: number
createdAt?: number
bot?: boolean
locked: boolean
url: string
avatar?: string
type: 'account' | 'channel'
parentFeedName?: string
parentFeedDomain?: string
fields: Field[]
extractedEmails: string[]
extractedTags: string[]
}
export default Feed

8
application/src/Storage/Definitions/Field.ts
Wyświetl plik

@ -1,8 +1,8 @@
interface Field {
name: string,
value: string
strippedName?: string
strippedValue?: string
name: string
value: string
strippedName?: string
strippedValue?: string
}
export default Field

18
application/src/Storage/Definitions/Geo.ts
Wyświetl plik

@ -1,13 +1,13 @@
interface Geo {
cityName?: string,
continentName?: string,
countryIsoCode?: string,
countryName?: string,
latitude: number
longitude: number
location?: string,
regionIsoCode?: string,
regionName?: string
cityName?: string
continentName?: string
countryIsoCode?: string
countryName?: string
latitude: number
longitude: number
location?: string
regionIsoCode?: string
regionName?: string
}
export default Geo

38
application/src/Storage/Definitions/Node.ts
Wyświetl plik

@ -1,25 +1,25 @@
import Geo from './Geo'
interface Node {
name?:string,
strippedName?:string,
foundAt: number,
refreshAttemptedAt?: number
refreshedAt?: number
openRegistrations?: boolean
domain: string,
serverIps?: string[],
geoip?: Geo[],
softwareName?: string;
softwareVersion?: string
standardizedSoftwareVersion?: string
halfYearActiveUserCount?: number,
monthActiveUserCount?: number,
statusesCount?: number,
totalUserCount?: number,
discoveredByDomain?:string,
accountFeedCount?: number,
channelFeedCount?: number,
name?: string
strippedName?: string
foundAt: number
refreshAttemptedAt?: number
refreshedAt?: number
openRegistrations?: boolean
domain: string
serverIps?: string[]
geoip?: Geo[]
softwareName?: string
softwareVersion?: string
standardizedSoftwareVersion?: string
halfYearActiveUserCount?: number
monthActiveUserCount?: number
statusesCount?: number
totalUserCount?: number
discoveredByDomain?: string
accountFeedCount?: number
channelFeedCount?: number
}
export default Node

2
application/src/Storage/ElasticClient.ts
Wyświetl plik

@ -6,7 +6,7 @@ const elasticClient = new Client({
},
auth: {
username: process.env.ELASTIC_USER ?? 'elastic',
password: process.env.ELASTIC_PASSWORD
password: process.env.ELASTIC_PASSWORD ?? ''
}
})

6
application/src/Storage/Feeds/StorageFeedData.ts
Wyświetl plik

@ -1,6 +1,6 @@
import { FeedData } from '../../Fediverse/Providers/FeedData'
export default interface StorageFeedData extends FeedData{
extractedTags:string[],
extractedEmails:string[],
export default interface StorageFeedData extends FeedData {
extractedTags: string[]
extractedEmails: string[]
}

10
application/src/Storage/Feeds/assertFeedIndex.ts
Wyświetl plik

@ -9,7 +9,7 @@ const assertFeedIndex = async (elastic: ElasticClient): Promise<void> => {
await elastic.ingest.putPipeline({
id: 'feed',
description: 'Default feed pipeline',
processors: processors
processors
})
console.info('Checking feed index')
const exists = await elastic.indices.exists({
@ -31,12 +31,8 @@ const assertFeedIndex = async (elastic: ElasticClient): Promise<void> => {
html: {
type: 'custom',
tokenizer: 'standard',
filter: [
'lowercase'
],
char_filter: [
'html_strip'
]
filter: ['lowercase'],
char_filter: ['html_strip']
}
}
}

23
application/src/Storage/Feeds/countNodeFeeds.ts
Wyświetl plik

@ -3,15 +3,18 @@ import Feed from '../Definitions/Feed'
import feedIndex from '../Definitions/feedIndex'
import { NodeStats } from '../../Jobs/Nodes/updateNodeFeedStats'
type Aggregation = {
buckets:{
key:'account'|'channel',
// eslint-disable-next-line camelcase
doc_count:number
}[]
interface Aggregation {
buckets: Array<{
key: 'account' | 'channel'
// eslint-disable-next-line camelcase
doc_count: number
}>
}
export default async function countNodeFeeds (elastic: ElasticClient, domain:string):Promise<NodeStats> {
export default async function countNodeFeeds (
elastic: ElasticClient,
domain: string
): Promise<NodeStats> {
await elastic.indices.refresh({ index: feedIndex })
const response = await elastic.search<Feed>({
index: feedIndex,
@ -27,12 +30,12 @@ export default async function countNodeFeeds (elastic: ElasticClient, domain:str
}
}
})
const types = response.aggregations.types as Aggregation
const result:NodeStats = {
const types = response?.aggregations?.types as Aggregation
const result: NodeStats = {
channel: 0,
account: 0
}
types.buckets.forEach(item => {
types.buckets.forEach((item) => {
result[item.key] += item.doc_count
})
return result

21
application/src/Storage/Feeds/createFeed.ts
Wyświetl plik

@ -4,8 +4,13 @@ import feedIndex from '../Definitions/feedIndex'
import getFeed from './getFeed'
import Feed from '../Definitions/Feed'
import Node from '../Definitions/Node'
import assertDefined from '../assertDefined'
export const createFeed = async (elastic: ElasticClient, feedData: StorageFeedData, node: Node): Promise<Feed> => {
export const createFeed = async (
elastic: ElasticClient,
feedData: StorageFeedData,
node: Node
): Promise<Feed> => {
const fullName = `${feedData.name}@${node.domain}`
await elastic.create<Feed>({
index: feedIndex,
@ -25,8 +30,8 @@ export const createFeed = async (elastic: ElasticClient, feedData: StorageFeedDa
displayName: feedData.displayName,
locked: feedData.locked,
createdAt: feedData.createdAt.getTime(),
foundAt: (new Date()).getTime(),
fields: feedData.fields.map(field => {
foundAt: new Date().getTime(),
fields: feedData.fields.map((field) => {
return { name: field.name, value: field.value }
}),
extractedEmails: feedData.extractedEmails,
@ -36,6 +41,12 @@ export const createFeed = async (elastic: ElasticClient, feedData: StorageFeedDa
type: feedData.type
}
})
console.info('Created new feed', { feedName: feedData.name, nodeDomain: node.domain })
return getFeed(elastic, fullName)
console.info('Created new feed', {
feedName: feedData.name,
nodeDomain: node.domain
})
return assertDefined(
await getFeed(elastic, fullName),
'Missing feed after creating it'
)
}

12
application/src/Storage/Feeds/deleteDomainFeeds.ts
Wyświetl plik

@ -1,13 +1,16 @@
import { ElasticClient } from '../ElasticClient'
import feedIndex from '../Definitions/feedIndex'
export const deleteDomainFeeds = async (elastic: ElasticClient, domains:string[]): Promise<number> => {
export const deleteDomainFeeds = async (
elastic: ElasticClient,
domains: string[]
): Promise<number> => {
await elastic.indices.refresh({ index: feedIndex })
const result = await elastic.deleteByQuery({
index: feedIndex,
query: {
bool: {
should: domains.map(domain => {
should: domains.map((domain) => {
return {
regexp: {
domain: {
@ -22,7 +25,8 @@ export const deleteDomainFeeds = async (elastic: ElasticClient, domains:string[]
}
})
console.info('Deleted domain feeds', {
count: result.deleted, domains
count: result.deleted ?? 0,
domains
})
return result.deleted
return result.deleted ?? 0
}

11
application/src/Storage/Feeds/deleteOldFeeds.ts
Wyświetl plik

@ -2,7 +2,10 @@ import { ElasticClient } from '../ElasticClient'
import Node from '../Definitions/Node'
import feedIndex from '../Definitions/feedIndex'
export const deleteOldFeeds = async (elastic: ElasticClient, node: Node): Promise<number> => {
export const deleteOldFeeds = async (
elastic: ElasticClient,
node: Node
): Promise<number> => {
await elastic.indices.refresh({ index: feedIndex })
const result = await elastic.deleteByQuery({
index: feedIndex,
@ -16,7 +19,9 @@ export const deleteOldFeeds = async (elastic: ElasticClient, node: Node): Promis
}
})
console.info('Deleted old feeds', {
count: result.deleted, olderThen: node.refreshAttemptedAt, nodeDomain: node.domain
count: result.deleted ?? 0,
olderThen: node.refreshAttemptedAt,
nodeDomain: node.domain
})
return result.deleted
return result.deleted ?? 0
}

5
application/src/Storage/Feeds/getFeed.ts
Wyświetl plik

@ -2,7 +2,10 @@ import Feed from '../Definitions/Feed'
import { ElasticClient } from '../ElasticClient'
import feedIndex from '../Definitions/feedIndex'
const getFeed = async (elastic: ElasticClient, feedFullName:string):Promise<Feed> => {
const getFeed = async (
elastic: ElasticClient,
feedFullName: string
): Promise<Feed | undefined> => {
const result = await elastic.get<Feed>({
index: feedIndex,
id: feedFullName

16
application/src/Storage/Feeds/updateFeed.ts
Wyświetl plik

@ -3,8 +3,13 @@ import Feed from '../Definitions/Feed'
import { ElasticClient } from '../ElasticClient'
import feedIndex from '../Definitions/feedIndex'
import getFeed from './getFeed'
import assertDefined from '../assertDefined'
export const updateFeed = async (elastic: ElasticClient, feed:Feed, feedData:StorageFeedData):Promise<Feed> => {
export const updateFeed = async (
elastic: ElasticClient,
feed: Feed,
feedData: StorageFeedData
): Promise<Feed> => {
await elastic.update<Feed>({
index: feedIndex,
id: feed.fullName,
@ -20,9 +25,9 @@ export const updateFeed = async (elastic: ElasticClient, feed:Feed, feedData:Sto
displayName: feedData.displayName,
locked: feedData.locked,
createdAt: feedData.createdAt,
refreshedAt: (new Date()).getTime(),
refreshedAt: new Date().getTime(),
type: feedData.type,
fields: feedData.fields.map(field => {
fields: feedData.fields.map((field) => {
return { name: field.name, value: field.value }
}),
extractedEmails: feedData.extractedEmails,
@ -30,5 +35,8 @@ export const updateFeed = async (elastic: ElasticClient, feed:Feed, feedData:Sto
}
})
console.info('Updated feed', { feedName: feed.name, nodeDomain: feed.domain })
return getFeed(elastic, feed.fullName)
return assertDefined(
await getFeed(elastic, feed.fullName),
'Missing updated feed'
)
}

4
application/src/Storage/Nodes/assertNodeIndex.ts
Wyświetl plik

@ -2,14 +2,14 @@ import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import dateProperty from '../Properties/dateProperty'
const assertNodeIndex = async (elastic: ElasticClient):Promise<void> => {
const assertNodeIndex = async (elastic: ElasticClient): Promise<void> => {
console.info('Setting node pipeline')
await elastic.ingest.putPipeline({
id: 'node',
description: 'Default node pipeline',
processors: [
{
// @ts-ignore
// @ts-expect-error
geoip: {
ignore_missing: true,
field: 'serverIps',

22
application/src/Storage/Nodes/createMissingNodes.ts
Wyświetl plik

@ -1,24 +1,32 @@
import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
export const createMissingNodes = async (elastic: ElasticClient, domains:string[], discoveredByDomain:string|undefined):Promise<number> => {
export const createMissingNodes = async (
elastic: ElasticClient,
domains: string[],
discoveredByDomain: string | undefined
): Promise<number> => {
const response = await elastic.bulk({
index: nodeIndex,
body: domains.flatMap(domain => [
body: domains.flatMap((domain) => [
{
create: { _id: domain }
},
{
domain: domain,
domain,
discoveredByDomain,
foundAt: (new Date()).getTime()
foundAt: new Date().getTime()
}
])
})
const createdCount = response.items.filter(item => item.create.status === 201).length
const createdCount = response.items.filter(
(item) => item.create?.status === 201
).length
console.warn('Created new nodes', {
requestedCount: domains.length,
createdCount: createdCount,
errors: response.items.filter(item => item.create.status !== 201).map(item => item.create.error.reason)
createdCount,
errors: response.items
.filter((item) => item.create?.status !== 201)
.map((item) => item.create?.error?.reason)
})
return createdCount
}

12
application/src/Storage/Nodes/deleteDomainNodes.ts
Wyświetl plik

@ -1,13 +1,16 @@
import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
export const deleteDomainNodes = async (elastic: ElasticClient, domains:string[]): Promise<number> => {
export const deleteDomainNodes = async (
elastic: ElasticClient,
domains: string[]
): Promise<number> => {
await elastic.indices.refresh({ index: nodeIndex })
const result = await elastic.deleteByQuery({
index: nodeIndex,
query: {
bool: {
should: domains.map(domain => {
should: domains.map((domain) => {
return {
regexp: {
domain: {
@ -22,7 +25,8 @@ export const deleteDomainNodes = async (elastic: ElasticClient, domains:string[]
}
})
console.info('Deleted domain nodes', {
count: result.deleted, domains
count: result.deleted ?? 0,
domains
})
return result.deleted
return result.deleted ?? 0
}

4
application/src/Storage/Nodes/fetchNodeToProcess.ts
Wyświetl plik

@ -5,7 +5,9 @@ import Node from '../Definitions/Node'
import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
export const fetchNodeToProcess = async (elastic: ElasticClient): Promise<Node> => {
export const fetchNodeToProcess = async (
elastic: ElasticClient
): Promise<Node> => {
await elastic.indices.refresh({ index: nodeIndex })
let node = await findNotProcessedNodeWithAttemptLimit(elastic)
if (node !== null) {

47
application/src/Storage/Nodes/findNodeWithOldestRefreshWithLimits.ts
Wyświetl plik

@ -2,42 +2,55 @@ import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import Node from '../Definitions/Node'
const findNodeWithOldestRefreshWithLimits = async (elastic: ElasticClient): Promise<Node | null> => {
const findNodeWithOldestRefreshWithLimits = async (
elastic: ElasticClient
): Promise<Node | null> => {
const currentTimestamp = Date.now()
const attemptLimitMilliseconds = parseInt(process.env.REATTEMPT_MINUTES ?? '60') * 60 * 1000
const attemptLimitMilliseconds =
parseInt(process.env.REATTEMPT_MINUTES ?? '60') * 60 * 1000
const attemptLimitDate = new Date(currentTimestamp - attemptLimitMilliseconds)
const refreshLimitMilliseconds = parseInt(process.env.REFRESH_HOURS ?? '168') * 60 * 60 * 1000
const refreshLimitMilliseconds =
parseInt(process.env.REFRESH_HOURS ?? '168') * 60 * 60 * 1000
const refreshLimitDate = new Date(currentTimestamp - refreshLimitMilliseconds)
console.log('Searching instance not refreshed for longest time and before refreshLimit and attemptLimit', {
refreshLimitMilliseconds,
refreshLimitDate,
attemptLimitDate,
attemptLimitMilliseconds
})
console.log(
'Searching instance not refreshed for longest time and before refreshLimit and attemptLimit',
{
refreshLimitMilliseconds,
refreshLimitDate,
attemptLimitDate,
attemptLimitMilliseconds
}
)
const result = await elastic.search<Node>({
index: nodeIndex,
body: {
size: 1,
sort: [{
refreshedAt: { order: 'asc' }
}],
sort: [
{
refreshedAt: { order: 'asc' }
}
],
query: {
bool: {
must: [
{ range: { refreshedAt: { lt: refreshLimitDate.getTime() } } }
],
should: [
{ range: { refreshAttemptedAt: { lt: attemptLimitDate.getTime() } } },
{ bool: { must_not: [{ exists: { field: 'refreshAttemptedAt' } }] } }
{
range: { refreshAttemptedAt: { lt: attemptLimitDate.getTime() } }
},
{
bool: { must_not: [{ exists: { field: 'refreshAttemptedAt' } }] }
}
],
minimum_should_match: 1
}
}
}
})
if (result.hits.hits.length > 0) {
const node = result.hits.hits[0]._source
console.log('Found oldest node', { node })
const node = result.hits.hits.pop()?._source
if (node !== undefined) {
console.info('Found oldest node', { node })
return node
}
return null

35
application/src/Storage/Nodes/findNotProcessedNodeWithAttemptLimit.ts
Wyświetl plik

@ -2,33 +2,44 @@ import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import Node from '../Definitions/Node'
const findNotProcessedNodeWithAttemptLimit = async (elastic: ElasticClient): Promise<Node|null> => {
const findNotProcessedNodeWithAttemptLimit = async (
elastic: ElasticClient
): Promise<Node | null> => {
const currentTimestamp = Date.now()
const attemptLimitMilliseconds = parseInt(process.env.REATTEMPT_MINUTES ?? '60') * 60 * 1000
const attemptLimitMilliseconds =
parseInt(process.env.REATTEMPT_MINUTES ?? '60') * 60 * 1000
const attemptLimitDate = new Date(currentTimestamp - attemptLimitMilliseconds)
console.log('Searching for not yet processed node not attempted before attemptLimit', { attemptLimitDate, attemptLimitMilliseconds })
console.log(
'Searching for not yet processed node not attempted before attemptLimit',
{ attemptLimitDate, attemptLimitMilliseconds }
)
const result = await elastic.search<Node>({
index: nodeIndex,
body: {
size: 1,
sort: [{
foundAt: { order: 'asc' }
}],
sort: [
{
foundAt: { order: 'asc' }
}
],
query: {
bool: {
must_not: [
{ exists: { field: 'refreshedAt' } }],
must_not: [{ exists: { field: 'refreshedAt' } }],
should: [
{ bool: { must_not: [{ exists: { field: 'refreshAttemptedAt' } }] } },
{ range: { refreshAttemptedAt: { lt: attemptLimitDate.getTime() } } }
{
bool: { must_not: [{ exists: { field: 'refreshAttemptedAt' } }] }
},
{
range: { refreshAttemptedAt: { lt: attemptLimitDate.getTime() } }
}
],
minimum_should_match: 1
}
}
}
})
if (result.hits.hits.length > 0) {
const node = result.hits.hits[0]._source
const node = result.hits.hits.pop()?._source
if (node !== undefined) {
console.log('Found not yet processed node', { node })
return node
}

5
application/src/Storage/Nodes/getNode.ts
Wyświetl plik

@ -2,7 +2,10 @@ import { ElasticClient } from '../ElasticClient'
import Node from '../Definitions/Node'
import nodeIndex from '../Definitions/nodeIndex'
const getNode = async (elastic:ElasticClient, domain:string):Promise<Node> => {
const getNode = async (
elastic: ElasticClient,
domain: string
): Promise<Node | undefined> => {
const result = await elastic.get<Node>({
index: nodeIndex,
id: domain

10
application/src/Storage/Nodes/isDomainNotBanned.ts
Wyświetl plik

@ -1,7 +1,9 @@
import getBannedDomains from '../../Jobs/Seed/getBannedDomains'
export default function isDomainNotBanned (domain):boolean {
return getBannedDomains().filter(
banned => domain.match(new RegExp('(.*\\.)?' + banned, 'gi')) !== null
).length === 0
export default function isDomainNotBanned (domain): boolean {
return (
getBannedDomains().filter(
(banned) => domain.match(new RegExp('(.*\\.)?' + banned, 'gi')) !== null
).length === 0
)
}

13
application/src/Storage/Nodes/setNodeRefreshAttempted.ts
Wyświetl plik

@ -2,10 +2,14 @@ import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import Node from '../Definitions/Node'
import getNode from './getNode'
import assertDefined from '../assertDefined'
export const setNodeRefreshAttempted = async (elastic: ElasticClient, node:Node):Promise<Node> => {
export const setNodeRefreshAttempted = async (
elastic: ElasticClient,
node: Node
): Promise<Node> => {
const date = new Date()
console.info('Setting node refresh attempt', { domain: node.domain, date: date })
console.info('Setting node refresh attempt', { domain: node.domain, date })
await elastic.update<Node>({
index: nodeIndex,
id: node.domain,
@ -13,5 +17,8 @@ export const setNodeRefreshAttempted = async (elastic: ElasticClient, node:Node)
refreshAttemptedAt: date.getTime()
}
})
return getNode(elastic, node.domain)
return assertDefined(
await getNode(elastic, node.domain),
'Missing node after updating it'
)
}

13
application/src/Storage/Nodes/setNodeRefreshed.ts
Wyświetl plik

@ -2,10 +2,14 @@ import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import Node from '../Definitions/Node'
import getNode from './getNode'
import assertDefined from '../assertDefined'
export const setNodeRefreshed = async (elastic: ElasticClient, node:Node):Promise<Node> => {
export const setNodeRefreshed = async (
elastic: ElasticClient,
node: Node
): Promise<Node> => {
const date = new Date()
console.info('Setting node refreshed', { domain: node.domain, date: date })
console.info('Setting node refreshed', { domain: node.domain, date })
await elastic.update<Node>({
index: nodeIndex,
id: node.domain,
@ -13,5 +17,8 @@ export const setNodeRefreshed = async (elastic: ElasticClient, node:Node):Promis
refreshedAt: date.getTime()
}
})
return getNode(elastic, node.domain)
return assertDefined(
await getNode(elastic, node.domain),
'Missing node after updating it'
)
}

12
application/src/Storage/Nodes/setNodeStats.ts
Wyświetl plik

@ -3,8 +3,13 @@ import nodeIndex from '../Definitions/nodeIndex'
import Node from '../Definitions/Node'
import getNode from './getNode'
import { NodeStats } from '../../Jobs/Nodes/updateNodeFeedStats'
import assertDefined from '../assertDefined'
export const setNodeStats = async (elastic: ElasticClient, node:Node, stats: NodeStats):Promise<Node> => {
export const setNodeStats = async (
elastic: ElasticClient,
node: Node,
stats: NodeStats
): Promise<Node> => {
console.info('Setting node stats', { domain: node.domain, stats })
await elastic.update<Node>({
index: nodeIndex,
@ -14,5 +19,8 @@ export const setNodeStats = async (elastic: ElasticClient, node:Node, stats: Nod
channelFeedCount: stats.channel
}
})
return getNode(elastic, node.domain)
return assertDefined(
await getNode(elastic, node.domain),
'Missing node after updating it'
)
}

22
application/src/Storage/Nodes/updateNodeInfo.ts
Wyświetl plik

@ -3,15 +3,20 @@ import Node from '../Definitions/Node'
import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import getNode from './getNode'
import assertDefined from '../assertDefined'
const assertPositiveInt = (number:number|undefined):number|undefined => {
const assertPositiveInt = (number: number | undefined): number | undefined => {
if (number === undefined) {
return undefined
}
return Math.max(0, Math.round(number))
}
export const updateNodeInfo = async (elastic: ElasticClient, node: Node, nodeInfo:NodeInfo):Promise<Node> => {
export const updateNodeInfo = async (
elastic: ElasticClient,
node: Node,
nodeInfo: NodeInfo
): Promise<Node> => {
await elastic.update<Node>({
index: nodeIndex,
id: node.domain,
@ -20,14 +25,21 @@ export const updateNodeInfo = async (elastic: ElasticClient, node: Node, nodeInf
openRegistrations: nodeInfo?.openRegistrations,
softwareName: nodeInfo?.software?.name?.toLocaleLowerCase(),
softwareVersion: nodeInfo?.software?.version,
halfYearActiveUserCount: assertPositiveInt(nodeInfo?.usage?.users?.activeHalfyear),
monthActiveUserCount: assertPositiveInt(nodeInfo?.usage?.users.activeMonth),
halfYearActiveUserCount: assertPositiveInt(
nodeInfo?.usage?.users?.activeHalfyear
),
monthActiveUserCount: assertPositiveInt(
nodeInfo?.usage?.users?.activeMonth
),
statusesCount: assertPositiveInt(nodeInfo?.usage?.localPosts),
totalUserCount: assertPositiveInt(nodeInfo?.usage?.users?.total)
}
})
const resultNode = await getNode(elastic, node.domain)
const resultNode = assertDefined(
await getNode(elastic, node.domain),
'Missing node after updating it'
)
console.info('Updated node info', { node })
return resultNode
}

12
application/src/Storage/Nodes/updateNodeIps.ts
Wyświetl plik

@ -2,8 +2,13 @@ import Node from '../Definitions/Node'
import { ElasticClient } from '../ElasticClient'
import nodeIndex from '../Definitions/nodeIndex'
import getNode from './getNode'
import assertDefined from '../assertDefined'
export const updateNodeIps = async (elastic:ElasticClient, node: Node, ips:string[]):Promise<Node> => {
export const updateNodeIps = async (
elastic: ElasticClient,
node: Node,
ips: string[]
): Promise<Node> => {
await elastic.update<Node>({
index: nodeIndex,
id: node.domain,
@ -11,7 +16,10 @@ export const updateNodeIps = async (elastic:ElasticClient, node: Node, ips:strin
serverIps: ips
}
})
const resultNode = await getNode(elastic, node.domain)
const resultNode = assertDefined(
await getNode(elastic, node.domain),
'Missing node after updating it'
)
console.info('Updated node ips', { resultNode })
return resultNode
}

2
application/src/Storage/Properties/dateProperty.ts
Wyświetl plik

@ -1,5 +1,5 @@
import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'
const dateProperty:MappingProperty = { type: 'date', format: 'epoch_millis' }
const dateProperty: MappingProperty = { type: 'date', format: 'epoch_millis' }
export default dateProperty

9
application/src/Storage/assertDefined.ts 100644
Wyświetl plik

@ -0,0 +1,9 @@
export default function assertDefined<T> (
value: T | undefined | null,
errorMessage: string
): T {
if (value === null || value === undefined) {
throw new Error(errorMessage)
}
return value
}

11
application/src/StringTools/extractEmails.ts
Wyświetl plik

@ -1,4 +1,9 @@
export const extractEmails = (text:string):string[] => {
return (text.match(/([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)/gi) || [])
.map(email => email.toLowerCase()) || []
export const extractEmails = (text: string): string[] => {
const matches = text.match(
/([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)/gi
)
if (matches === null) {
return []
}
return matches.map((email) => email.toLowerCase())
}

9
application/src/StringTools/extractTags.ts
Wyświetl plik

@ -1,4 +1,7 @@
export const extractTags = (text:string):string[] => {
return (text.match(/#[a-z0-9_]+/gi) || [])
.map(hashtag => hashtag.substring(1).toLowerCase()) || []
export const extractTags = (text: string): string[] => {
const matches = text.match(/#[a-z0-9_]+/gi)
if (matches === null) {
return []
}
return matches.map((hashtag) => hashtag.substring(1).toLowerCase())
}

25
application/src/app.ts
Wyświetl plik

@ -7,16 +7,25 @@ import elasticClient from './Storage/ElasticClient'
import deleteDomains from './Jobs/Seed/deleteBannedNodes'
import getBannedDomains from './Jobs/Seed/getBannedDomains'
const timeout = async (ms: number): Promise<void> => {
return await new Promise((resolve) => setTimeout(resolve, ms))
}
const loop = async (): Promise<void> => {
// noinspection InfiniteLoopJS
while (true) {
try {
await processNextNode(elasticClient, providerRegistry)
} catch (err) {
console.warn(err)
const waitForJobMilliseconds = parseInt(process.env.WAIT_FOR_JOB_MINUTES ?? '60') * 60 * 1000
console.info('Delaying next node process', { timeoutMilliseconds: waitForJobMilliseconds, timeoutDate: new Date(Date.now() + waitForJobMilliseconds), now: new Date() })
setTimeout(loop, waitForJobMilliseconds)
return
const waitForJobMilliseconds =
parseInt(process.env.WAIT_FOR_JOB_MINUTES ?? '60') * 60 * 1000
console.info('Delaying next node process', {
timeoutMilliseconds: waitForJobMilliseconds,
timeoutDate: new Date(Date.now() + waitForJobMilliseconds),
now: new Date()
})
await timeout(waitForJobMilliseconds)
}
}
}
@ -25,9 +34,13 @@ const app = async (): Promise<void> => {
await assertNodeIndex(elasticClient)
await assertFeedIndex(elasticClient)
await deleteDomains(elasticClient, getBannedDomains())
const seedDomains = (process.env.SEED_NODE_DOMAIN ?? 'mastodon.social').split(',')
const seedDomains = (process.env.SEED_NODE_DOMAIN ?? 'mastodon.social').split(
','
)
await addNodeSeed(elasticClient, seedDomains)
setTimeout(loop)
await loop()
}
app()
.then(() => console.info('App finished'))
.catch((error) => console.error('App was interrupted', { error }))

3
application/tsconfig.json
Wyświetl plik

@ -7,7 +7,8 @@
"outDir": "./dist/",
"sourceMap": true,
"alwaysStrict": true,
"allowSyntheticDefaultImports": true
"allowSyntheticDefaultImports": true,
"strictNullChecks": true,
},
"files": ["./src/app.ts"]
}

11
application/tslint.json
Wyświetl plik

@ -1,11 +0,0 @@
{
"defaultSeverity": "error",
"extends": [
"tslint:recommended"
],
"jsRules": {},
"rules": {
"no-console": false
},
"rulesDirectory": []
}

3551
application/yarn.lock 100644
Wyświetl plik

Plik diff jest za duży Load Diff