kopia lustrzana https://github.com/tsileo/little-boxes
Add caching to urlutils.check_url
rodzic
ebcfc6e0a5
commit
714461d1d0
|
@ -15,11 +15,10 @@ from .backend import Backend
|
|||
from .collection import parse_collection
|
||||
from .errors import BadActivityError
|
||||
from .errors import Error
|
||||
from .errors import RemoteActivityGoneError
|
||||
from .errors import NotFromOutboxError
|
||||
from .errors import RemoteActivityGoneError
|
||||
from .errors import UnexpectedActivityTypeError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UninitializedBackendError = Error("a backend must be initialized")
|
||||
|
|
|
@ -2,6 +2,7 @@ import ipaddress
|
|||
import logging
|
||||
import os
|
||||
import socket
|
||||
from typing import Dict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import strtobool
|
||||
|
@ -10,6 +11,9 @@ from .errors import Error
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_CACHE: Dict[str, bool] = {}
|
||||
|
||||
|
||||
class InvalidURLError(Error):
|
||||
pass
|
||||
|
||||
|
@ -27,6 +31,9 @@ def is_url_valid(url: str) -> bool:
|
|||
if parsed.hostname in ["localhost"]:
|
||||
return False
|
||||
|
||||
if _CACHE.get(parsed.hostname, False):
|
||||
return True
|
||||
|
||||
try:
|
||||
ip_address = ipaddress.ip_address(parsed.hostname)
|
||||
except ValueError:
|
||||
|
@ -35,14 +42,17 @@ def is_url_valid(url: str) -> bool:
|
|||
logger.debug(f"dns lookup: {parsed.hostname} -> {ip_address}")
|
||||
except socket.gaierror:
|
||||
logger.exception(f"failed to lookup url {url}")
|
||||
_CACHE[parsed.hostname] = False
|
||||
return False
|
||||
|
||||
logger.debug(f"{ip_address}")
|
||||
|
||||
if ipaddress.ip_address(ip_address).is_private:
|
||||
logger.info(f"rejecting private URL {url}")
|
||||
_CACHE[parsed.hostname] = False
|
||||
return False
|
||||
|
||||
_CACHE[parsed.hostname] = True
|
||||
return True
|
||||
|
||||
|
||||
|
|
Ładowanie…
Reference in New Issue