diff --git a/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts b/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts index a2b3743..13cb1a8 100644 --- a/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts +++ b/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts @@ -1,5 +1,6 @@ import axios, { AxiosRequestConfig, AxiosResponse } from 'axios' import robotsParser from 'robots-parser' +import { getDefaultTimeoutMilliseconds } from '../getDefaultTimeoutMilliseconds.js' import RobotsTxt from './RobotsTxt.js' import { RobotsTxtError } from './RobotsTxtError.js' @@ -10,7 +11,10 @@ export default async function fetchRobotsTxt (domain: string): Promise