kopia lustrzana https://github.com/Stopka/fedicrawl
It respects noindex option in mastodon
rodzic
704c7c066e
commit
4113d78a17
|
@ -19,4 +19,5 @@ export interface FeedData {
|
|||
name: string
|
||||
hostDomain: string
|
||||
}
|
||||
indexable: boolean
|
||||
}
|
||||
|
|
|
@ -34,7 +34,8 @@ const schema = z.array(
|
|||
value: z.string(),
|
||||
verified_at: z.nullable(z.string())
|
||||
})
|
||||
)
|
||||
),
|
||||
noindex: z.boolean().optional().nullable()
|
||||
})
|
||||
)
|
||||
|
||||
|
@ -94,7 +95,8 @@ export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (
|
|||
}
|
||||
}),
|
||||
type: 'account',
|
||||
parentFeed: undefined
|
||||
parentFeed: undefined,
|
||||
indexable: !(item.noindex ?? false)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
@ -121,7 +121,8 @@ export const retrieveUsersPage: FeedProviderMethod = async (
|
|||
].filter((field) => field.value !== null) as FieldData[])
|
||||
],
|
||||
type: 'account',
|
||||
parentFeed: undefined
|
||||
parentFeed: undefined,
|
||||
indexable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
@ -60,7 +60,8 @@ export const retrieveAccounts: FeedProviderMethod = async (domain, page, robotsT
|
|||
lastStatusAt: undefined,
|
||||
statusesCount: undefined,
|
||||
type: 'account',
|
||||
parentFeed: undefined
|
||||
parentFeed: undefined,
|
||||
indexable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
@ -78,7 +78,8 @@ export const retrieveVideoChannels: FeedProviderMethod = async (
|
|||
parentFeed: {
|
||||
name: item.ownerAccount.name,
|
||||
hostDomain: item.ownerAccount.host
|
||||
}
|
||||
},
|
||||
indexable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
@ -13,14 +13,16 @@ export const refreshFeedsOnPage = async (
|
|||
robotsTxt: RobotsTxt
|
||||
): Promise<Feed[]> => {
|
||||
const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
|
||||
const indexableFeedData = feedData.filter(item => item.indexable)
|
||||
console.info('Retrieved feeds', {
|
||||
count: feedData.length,
|
||||
indexableCount: indexableFeedData.length,
|
||||
domain: node.domain,
|
||||
provider: provider.getKey(),
|
||||
page
|
||||
})
|
||||
return await Promise.all(
|
||||
feedData.map(
|
||||
indexableFeedData.map(
|
||||
async (feedDataItem) =>
|
||||
await refreshOrAddFeed(elastic, node, feedDataItem)
|
||||
)
|
||||
|
|
Ładowanie…
Reference in New Issue