From 2134703b4bb9847d0efeac3b28ceefb7d1f26271 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 26 Dec 2021 22:44:46 +0100 Subject: [PATCH] [enh] settings.yml: implement general.enable_metrics * allow not to record metrics (response time, etc...) * this commit doesn't change the UI. If the metrics are disabled /stats and /stats/errors will return empty response. in /preferences, the columns response time and reliability will be empty. --- docs/admin/engines/settings.rst | 3 +++ searx/metrics/__init__.py | 12 ++++++++---- searx/metrics/error_recorder.py | 6 +++++- searx/metrics/models.py | 17 ++++++++++++++--- searx/search/__init__.py | 4 ++-- searx/settings.yml | 1 + searx/settings_defaults.py | 1 + searx/webapp.py | 2 +- 8 files changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index b04de7cb3..80fb183c0 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -81,6 +81,9 @@ Global Settings ``contact_url``: Contact ``mailto:`` address or WEB form. +``enable_metrics``: + Enabled by default. Record various anonymous metrics availabled at ``/stats``, + ``/stats/errors`` and ``/preferences``. .. _settings global server: diff --git a/searx/metrics/__init__.py b/searx/metrics/__init__.py index 37f0ba121..bc755b96b 100644 --- a/searx/metrics/__init__.py +++ b/searx/metrics/__init__.py @@ -9,7 +9,7 @@ from timeit import default_timer from operator import itemgetter from searx.engines import engines -from .models import HistogramStorage, CounterStorage +from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage from .error_recorder import count_error, count_exception, errors_per_engines __all__ = [ @@ -69,14 +69,18 @@ def counter(*args): return counter_storage.get(*args) -def initialize(engine_names=None): +def initialize(engine_names=None, enabled=True): """ Initialize metrics """ global counter_storage, histogram_storage # pylint: disable=global-statement - counter_storage = CounterStorage() - histogram_storage = HistogramStorage() + if enabled: + counter_storage = CounterStorage() + histogram_storage = HistogramStorage() + else: + counter_storage = VoidCounterStorage() + histogram_storage = HistogramStorage(histogram_class=VoidHistogram) # max_timeout = max of all the engine.timeout max_timeout = 2 diff --git a/searx/metrics/error_recorder.py b/searx/metrics/error_recorder.py index 76d27f64f..1d0d6e7a3 100644 --- a/searx/metrics/error_recorder.py +++ b/searx/metrics/error_recorder.py @@ -9,7 +9,7 @@ from searx.exceptions import ( SearxEngineAPIException, SearxEngineAccessDeniedException, ) -from searx import searx_parent_dir +from searx import searx_parent_dir, settings from searx.engines import engines @@ -165,6 +165,8 @@ def get_error_context(framerecords, exception_classname, log_message, log_parame def count_exception(engine_name: str, exc: Exception, secondary: bool = False) -> None: + if not settings['general']['enable_metrics']: + return framerecords = inspect.trace() try: exception_classname = get_exception_classname(exc) @@ -178,6 +180,8 @@ def count_exception(engine_name: str, exc: Exception, secondary: bool = False) - def count_error( engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None, secondary: bool = False ) -> None: + if not settings['general']['enable_metrics']: + return framerecords = list(reversed(inspect.stack()[1:])) try: error_context = get_error_context(framerecords, None, log_message, log_parameters or (), secondary) diff --git a/searx/metrics/models.py b/searx/metrics/models.py index d42569b7f..900a7fa93 100644 --- a/searx/metrics/models.py +++ b/searx/metrics/models.py @@ -102,16 +102,17 @@ class Histogram: class HistogramStorage: - __slots__ = 'measures' + __slots__ = 'measures', 'histogram_class' - def __init__(self): + def __init__(self, histogram_class=Histogram): self.clear() + self.histogram_class = histogram_class def clear(self): self.measures = {} def configure(self, width, size, *args): - measure = Histogram(width, size) + measure = self.histogram_class(width, size) self.measures[args] = measure return measure @@ -154,3 +155,13 @@ class CounterStorage: logger.debug("Counters:") for k in ks: logger.debug("- %-60s %s", '|'.join(k), self.counters[k]) + + +class VoidHistogram(Histogram): + def observe(self, value): + pass + + +class VoidCounterStorage(CounterStorage): + def add(self, value, *args): + pass diff --git a/searx/search/__init__.py b/searx/search/__init__.py index d66f3362d..e790bd05e 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -24,13 +24,13 @@ from searx.search.checker import initialize as initialize_checker logger = logger.getChild('search') -def initialize(settings_engines=None, enable_checker=False, check_network=False): +def initialize(settings_engines=None, enable_checker=False, check_network=False, enable_metrics=True): settings_engines = settings_engines or settings['engines'] load_engines(settings_engines) initialize_network(settings_engines, settings['outgoing']) if check_network: check_network_configuration() - initialize_metrics([engine['name'] for engine in settings_engines]) + initialize_metrics([engine['name'] for engine in settings_engines], enable_metrics) initialize_processors(settings_engines) if enable_checker: initialize_checker() diff --git a/searx/settings.yml b/searx/settings.yml index 3227a5a55..4ac9c8943 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -2,6 +2,7 @@ general: debug: false # Debug mode, only for development instance_name: "SearXNG" # displayed name contact_url: false # mailto:contact@example.com + enable_metrics: true # record stats brand: new_issue_url: https://github.com/searxng/searxng/issues/new diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 9c4711bfc..95d482a0e 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -142,6 +142,7 @@ SCHEMA = { 'debug': SettingsValue(bool, False, 'SEARXNG_DEBUG'), 'instance_name': SettingsValue(str, 'SearXNG'), 'contact_url': SettingsValue((None, False, str), None), + 'enable_metrics': SettingsValue(bool, True), }, 'brand': { 'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'), diff --git a/searx/webapp.py b/searx/webapp.py index a2aa84d9d..410c005cb 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -1333,7 +1333,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai # initialize the engines except on the first run of the werkzeug server. if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"): plugin_initialize(app) - search_initialize(enable_checker=True, check_network=True) + search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics']) def run():