From e92755d358df5b34b0181f48f8ba02c7f2939e8f Mon Sep 17 00:00:00 2001 From: Alexandre FLAMENT Date: Fri, 14 Oct 2022 13:27:07 +0000 Subject: [PATCH] Initialize Redis in searx/webapp.py settings.yml: * The default URL was unix:///usr/local/searxng-redis/run/redis.sock?db=0 * The default URL is now "false" The default URL makes the log difficult to deal with: if the admin didn't install a Redis instance, the logs record a false error. It worked before because SearXNG initialized the Redis connection when the limiter started. In this commit, SearXNG initializes Redis in searx/webapp.py so various components can use Redis without taking care of the initialization step. --- docs/admin/engines/settings.rst | 17 +++++++++-------- searx/search/checker/scheduler.lua | 6 +++--- searx/settings.yml | 2 +- searx/settings_defaults.py | 2 +- searx/shared/redisdb.py | 27 +++++++++++++++++++-------- utils/searxng.sh | 4 ++-- utils/searxng_check.py | 2 +- 7 files changed, 36 insertions(+), 24 deletions(-) diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 086b3ccbe..3ac992d3e 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -314,20 +314,21 @@ developer) account needs to be added to the *searxng-redis* group. .. admonition:: Tip for developers - To set up a local redis instance using sockets simply use:: - - $ ./manage redis.build - $ sudo -H ./manage redis.install - $ sudo -H ./manage redis.addgrp "${USER}" - # don't forget to logout & login to get member of group - - The YAML setting for such a redis instance is: + To set up a local redis instance, first set the socket path of the Redis DB + in your YAML setting: .. code:: yaml redis: url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + Then use the following commands to install the redis instance :: + + $ ./manage redis.build + $ sudo -H ./manage redis.install + $ sudo -H ./manage redis.addgrp "${USER}" + # don't forget to logout & login to get member of group + .. _settings outgoing: diff --git a/searx/search/checker/scheduler.lua b/searx/search/checker/scheduler.lua index b3c6023fe..c5ebb05bc 100644 --- a/searx/search/checker/scheduler.lua +++ b/searx/search/checker/scheduler.lua @@ -18,8 +18,8 @@ if (next_call_ts == false or next_call_ts == nil) then -- the scheduler has never run on this Redis instance, so: -- 1/ the scheduler does not run now -- 2/ the next call is a random time between start_after_from and start_after_to - local delay = start_after_from + math.random(start_after_to - start_after_from) - redis.call('SET', redis_key, now + delay) + local initial_delay = math.random(start_after_from, start_after_to) + redis.call('SET', redis_key, now + initial_delay) return { false, delay } end @@ -30,7 +30,7 @@ local call_now = next_call_ts <= now if call_now then -- the checker runs now, define the timestamp of the next call: -- this is a random delay between every_from and every_to - local periodic_delay = every_from + math.random(every_to - every_from) + local periodic_delay = math.random(every_from, every_to) next_call_ts = redis.call('INCRBY', redis_key, periodic_delay) end return { call_now, next_call_ts - now } diff --git a/searx/settings.yml b/searx/settings.yml index 2304c6fe7..36ed90b22 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -78,7 +78,7 @@ server: redis: # https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url - url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + url: false ui: # Custom static path - leave it blank if you didn't change diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index a4f6ce607..470290f92 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -174,7 +174,7 @@ SCHEMA = { 'default_http_headers': SettingsValue(dict, {}), }, 'redis': { - 'url': SettingsValue(str, 'unix:///usr/local/searxng-redis/run/redis.sock?db=0'), + 'url': SettingsValue((None, False, str), False), }, 'ui': { 'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')), diff --git a/searx/shared/redisdb.py b/searx/shared/redisdb.py index d0071f72c..6cc4e46d1 100644 --- a/searx/shared/redisdb.py +++ b/searx/shared/redisdb.py @@ -26,20 +26,31 @@ import redis from searx import get_setting +OLD_REDIS_URL_DEFAULT_URL = 'unix:///usr/local/searxng-redis/run/redis.sock?db=0' +"""This was the default Redis URL in settings.yml.""" + +_CLIENT = None logger = logging.getLogger('searx.shared.redisdb') -_client = None def client() -> redis.Redis: - return _client + return _CLIENT def initialize(): - global _client # pylint: disable=global-statement + global _CLIENT # pylint: disable=global-statement + redis_url = get_setting('redis.url') try: - _client = redis.Redis.from_url(get_setting('redis.url')) - logger.info("connected redis: %s", get_setting('redis.url')) - except redis.exceptions.ConnectionError as exc: + if redis_url: + _CLIENT = redis.Redis.from_url(redis_url) + logger.info("connected redis: %s", redis_url) + return True + except redis.exceptions.ConnectionError: _pw = pwd.getpwuid(os.getuid()) - logger.error("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid) - logger.error(" %s", exc) + logger.exception("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid) + if redis_url == OLD_REDIS_URL_DEFAULT_URL: + logger.info( + "You can safely ignore the above Redis error if you don't use Redis." + "You can remove this error by setting redis.url to false in your settings.yml." + ) + return False diff --git a/utils/searxng.sh b/utils/searxng.sh index 58f220e1f..e7aa8ed15 100755 --- a/utils/searxng.sh +++ b/utils/searxng.sh @@ -295,7 +295,7 @@ In your instance, redis DB connector is configured at: ${redis_url} " - if searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + if searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.initialize() or exit(42)"; then info_msg "SearXNG instance is able to connect redis DB." return fi @@ -684,7 +684,7 @@ To install uWSGI use:: die 42 "SearXNG's uWSGI app not available" fi - if ! searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + if ! searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.initialize() or exit(42)"; then rst_para "\ The configured redis DB is not available: If your server is public to the internet, you should setup a bot protection to block excessively bot queries. diff --git a/utils/searxng_check.py b/utils/searxng_check.py index 39e774340..bd2d60288 100644 --- a/utils/searxng_check.py +++ b/utils/searxng_check.py @@ -29,6 +29,6 @@ if os.path.isfile(OLD_SETTING): from searx.shared import redisdb from searx import get_setting -if not redisdb.init(): +if not redisdb.initialize(): warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2) warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2)