mirrors/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-05-19 20:08:04 +00:00
Code Issues Projects Releases Wiki Activity

Merge f49d1a9b90 into 11fe88bb40

Browse source
This commit is contained in:
Alexandre Flament 2024-05-02 12:22:07 +02:00 committed by GitHub
parent 11fe88bb40 f49d1a9b90
commit 8a1fb7cf25
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
searx/search/checker/impl.py
View file

@ -2,11 +2,11 @@
# pylint: disable=missing-module-docstring, invalid-name
import gc
import time
import typing
import types
import functools
import itertools
from time import time
from timeit import default_timer
from urllib.parse import urlparse
@ -66,15 +66,15 @@ def _download_and_check_if_image(image_url: str) -> bool:
This function should not be called directly: use _is_url_image
otherwise the cache of functools.lru_cache contains data: URL which might be huge.
"""
retry = 2
retry = 3
while retry > 0:
a = time()
a = default_timer()
try:
# use "image_proxy" (avoid HTTP/2)
network.set_timeout_for_thread(2)
network.set_context_network_name('image_proxy')
r, stream = network.stream(
'GET',
r = network.get(
image_url,
timeout=10.0,
allow_redirects=True,
@ -90,19 +90,20 @@ def _download_and_check_if_image(image_url: str) -> bool:
'Cache-Control': 'max-age=0',
},
)
r.close()
if r.status_code == 200:
is_image = r.headers.get('content-type', '').startswith('image/')
else:
is_image = False
del r
del stream
return is_image
except httpx.TimeoutException:
logger.error('Timeout for %s: %i', image_url, int(time() - a))
logger.error('Timeout for %s: %i', image_url, int(default_timer() - a))
time.sleep(1)
retry -= 1
except httpx.HTTPError:
logger.exception('Exception for %s', image_url)
except httpx.HTTPStatusError as e:
logger.error('Exception for %s: HTTP status=%i', image_url, e.response.status_code)
except httpx.HTTPError as e:
logger.error('Exception for %s: %s, %s', image_url, e.__class__.__name__, ",".join(e.args))
return False
return False