kopia lustrzana https://github.com/Stopka/fedicrawl
It respects noindex option in mastodon
rodzic
4113d78a17
commit
c0b10f2e7a
|
@ -13,7 +13,7 @@ export const refreshFeedsOnPage = async (
|
|||
robotsTxt: RobotsTxt
|
||||
): Promise<Feed[]> => {
|
||||
const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
|
||||
const indexableFeedData = feedData.filter(item => item.indexable)
|
||||
const indexableFeedData = feedData.filter(item => item.indexable && !item.description.includes('#noindex'))
|
||||
console.info('Retrieved feeds', {
|
||||
count: feedData.length,
|
||||
indexableCount: indexableFeedData.length,
|
||||
|
|
Ładowanie…
Reference in New Issue