kopia lustrzana https://github.com/Stopka/fedicrawl
Fixed robots.txt fetching
rodzic
4064db521f
commit
0723c2508d
|
@ -15,7 +15,7 @@ export default async function fetchRobotsTxt (domain: string): Promise<RobotsTxt
|
|||
headers: { 'User-Agent': userAgent },
|
||||
timeout: getDefaultTimeoutMilliseconds()
|
||||
})
|
||||
content = robotsTxt.data
|
||||
content = String(robotsTxt.data)
|
||||
} catch (error) {
|
||||
console.info('Robots.txt not found', { error, url })
|
||||
}
|
||||
|
|
|
@ -11,8 +11,8 @@ export const assertSuccessJsonResponse = (
|
|||
throw new UnexpectedResponseStatusError(expectedStatus, actualStatus)
|
||||
}
|
||||
const expectedContentType = 'application/json'
|
||||
const actualContentType = response.headers['content-type']
|
||||
const actualContentType = String(response.headers['content-type'])
|
||||
if (!actualContentType.startsWith(expectedContentType)) {
|
||||
throw new UnexpectedContentTypeError(expectedContentType, actualContentType)
|
||||
throw new UnexpectedContentTypeError(actualContentType, expectedContentType)
|
||||
}
|
||||
}
|
||||
|
|
Ładowanie…
Reference in New Issue