mirror of
https://github.com/searxng/searxng.git
synced 2024-06-11 01:39:35 +00:00
1184caa0fd
This patch implements a table of ranges. A *range* is a series of integers, e.g. 0-10 with start value 0 and end value 10. Start and end values are stored in a two-column table, the RangeTable. This table is stored in Redis in a sorted list and a ZRANGEBYSCORE query can be used to determine if a particular integer is contained in one of the ranges in the table. The ZRANGEBYSCORE query scales with O(log(N)). To make it more clear: a query in a table with 1 million ranges takes just twice as long as the query in a table with 1000 ranges. A RangeTable is excellent for IP ranges and ultra-fast queries to see if an IP (which is nothing but an integer) is contained in one of the ranges of the table. Typical applications are e.g. large IP block lists or network ranges. Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
71 lines
2.1 KiB
Python
71 lines
2.1 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# lint: pylint
|
|
"""Implementation of the redis client (redis-py_).
|
|
|
|
.. _redis-py: https://github.com/redis/redis-py
|
|
|
|
This implementation uses the :ref:`settings redis` setup from ``settings.yml``.
|
|
A redis DB connect can be tested by::
|
|
|
|
>>> from searx import redisdb
|
|
>>> redisdb.initialize()
|
|
True
|
|
>>> db = redisdb.client()
|
|
>>> db.set("foo", "bar")
|
|
True
|
|
>>> db.get("foo")
|
|
b'bar'
|
|
>>>
|
|
|
|
"""
|
|
|
|
import os
|
|
import pwd
|
|
import logging
|
|
import redis
|
|
from searx import get_setting
|
|
|
|
|
|
OLD_REDIS_URL_DEFAULT_URL = 'unix:///usr/local/searxng-redis/run/redis.sock?db=0'
|
|
"""This was the default Redis URL in settings.yml."""
|
|
|
|
_CLIENT: redis.Redis = None # type: ignore
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def client() -> redis.Redis:
|
|
return _CLIENT
|
|
|
|
|
|
def initialize():
|
|
global _CLIENT # pylint: disable=global-statement
|
|
redis_url = get_setting('redis.url')
|
|
if not redis_url:
|
|
return False
|
|
try:
|
|
# create a client, but no connection is done
|
|
_CLIENT = redis.Redis.from_url(redis_url) # type: ignore
|
|
|
|
# log the parameters as seen by the redis lib, without the password
|
|
kwargs = _CLIENT.get_connection_kwargs().copy()
|
|
kwargs.pop('password', None)
|
|
kwargs = ' '.join([f'{k}={v!r}' for k, v in kwargs.items()])
|
|
logger.info("connecting to Redis %s", kwargs)
|
|
|
|
# check the connection
|
|
_CLIENT.ping()
|
|
|
|
# no error: the redis connection is working
|
|
logger.info("connected to Redis")
|
|
return True
|
|
except redis.exceptions.RedisError as e:
|
|
_CLIENT = None # type: ignore
|
|
_pw = pwd.getpwuid(os.getuid())
|
|
logger.exception("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid)
|
|
if redis_url == OLD_REDIS_URL_DEFAULT_URL and isinstance(e, redis.exceptions.ConnectionError):
|
|
logger.info(
|
|
"You can safely ignore the above Redis error if you don't use Redis. "
|
|
"You can remove this error by setting redis.url to false in your settings.yml."
|
|
)
|
|
return False
|