Merge 3d0b224d84 into ac430a9eaf

docs/admin/installation-searxng.rst

@@ -96,7 +96,7 @@ Modify the ``/etc/searxng/settings.yml`` to your needs:
 
      .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml
         :language: yaml
-        :end-before: # hostname_replace:
+        :end-before: # hostnames:
 
      To see the entire file jump to :origin:`utils/templates/etc/searxng/settings.yml`
 
@@ -104,7 +104,7 @@ Modify the ``/etc/searxng/settings.yml`` to your needs:
 
      .. literalinclude:: ../../searx/settings.yml
         :language: yaml
-        :end-before: # hostname_replace:
+        :end-before: # hostnames:
 
      To see the entire file jump to :origin:`searx/settings.yml`
 

searx/plugins/hostname_replace.py

@@ -1,49 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# pylint: disable=missing-module-docstring
-
-import re
-from urllib.parse import urlunparse, urlparse
-
-from flask_babel import gettext
-
-from searx import settings
-from searx.plugins import logger
-
-name = gettext('Hostname replace')
-description = gettext('Rewrite result hostnames or remove results based on the hostname')
-default_on = False
-preference_section = 'general'
-
-plugin_id = 'hostname_replace'
-
-replacements = {re.compile(p): r for (p, r) in settings[plugin_id].items()} if plugin_id in settings else {}
-
-logger = logger.getChild(plugin_id)
-parsed = 'parsed_url'
-_url_fields = ['iframe_src', 'audio_src']
-
-
-def on_result(_request, _search, result):
-
-    for pattern, replacement in replacements.items():
-
-        if parsed in result:
-            if pattern.search(result[parsed].netloc):
-                # to keep or remove this result from the result list depends
-                # (only) on the 'parsed_url'
-                if not replacement:
-                    return False
-                result[parsed] = result[parsed]._replace(netloc=pattern.sub(replacement, result[parsed].netloc))
-                result['url'] = urlunparse(result[parsed])
-
-        for url_field in _url_fields:
-            if result.get(url_field):
-                url_src = urlparse(result[url_field])
-                if pattern.search(url_src.netloc):
-                    if not replacement:
-                        del result[url_field]
-                    else:
-                        url_src = url_src._replace(netloc=pattern.sub(replacement, url_src.netloc))
-                        result[url_field] = urlunparse(url_src)
-
-    return True

searx/plugins/hostnames.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=missing-module-docstring, too-many-branches
+
+import re
+from urllib.parse import urlunparse, urlparse
+
+from flask_babel import gettext
+
+from searx import settings
+from searx.plugins import logger
+from searx.settings_loader import get_yaml_file
+
+name = gettext('Hostnames plugin')
+description = gettext('Rewrite hostnames, remove results or prioritize them based on the hostname')
+default_on = False
+preference_section = 'general'
+
+plugin_id = 'hostnames'
+
+logger = logger.getChild(plugin_id)
+parsed = 'parsed_url'
+_url_fields = ['iframe_src', 'audio_src']
+
+
+def _load_regular_expressions(settings_key):
+    setting_value = settings.get(plugin_id, {}).get(settings_key)
+
+    if not setting_value:
+        return {}
+
+    # load external file with configuration
+    if isinstance(setting_value, str):
+        setting_value = get_yaml_file(setting_value)
+
+    if isinstance(setting_value, list):
+        return {re.compile(r) for r in setting_value}
+
+    if isinstance(setting_value, dict):
+        return {re.compile(p): r for (p, r) in setting_value.items()}
+
+    return {}
+
+
+replacements = _load_regular_expressions('replace')
+removables = _load_regular_expressions('remove')
+high_priority = _load_regular_expressions('high_priority')
+low_priority = _load_regular_expressions('low_priority')
+
+
+def _matches_parsed_url(result, pattern):
+    return parsed in result and pattern.search(result[parsed].netloc)
+
+
+def on_result(_request, _search, result):
+    for pattern, replacement in replacements.items():
+        if _matches_parsed_url(result, pattern):
+            logger.debug(result['url'])
+            result[parsed] = result[parsed]._replace(netloc=pattern.sub(replacement, result[parsed].netloc))
+            result['url'] = urlunparse(result[parsed])
+            logger.debug(result['url'])
+
+        for url_field in _url_fields:
+            if not result.get(url_field):
+                continue
+
+            url_src = urlparse(result[url_field])
+            if pattern.search(url_src.netloc):
+                url_src = url_src._replace(netloc=pattern.sub(replacement, url_src.netloc))
+                result[url_field] = urlunparse(url_src)
+
+    for pattern in removables:
+        if _matches_parsed_url(result, pattern):
+            return False
+
+        for url_field in _url_fields:
+            if not result.get(url_field):
+                continue
+
+            url_src = urlparse(result[url_field])
+            if pattern.search(url_src.netloc):
+                del result[url_field]
+
+    for pattern in low_priority:
+        if _matches_parsed_url(result, pattern):
+            result['score'] = -3
+
+    for pattern in high_priority:
+        if _matches_parsed_url(result, pattern):
+            result['score'] = 3
+
+    return True

searx/results.py

@@ -355,7 +355,7 @@ class ResultContainer:
 
         for result in self._merged_results:
             score = result_score(result)
-            result['score'] = score
+            result['score'] = result.get('score', 0) + score
 
             # removing html content and whitespace duplications
             if result.get('content'):

searx/settings.yml

@@ -219,7 +219,7 @@ outgoing:
 #   - 'Tracker URL remover'
 #   - 'Ahmia blacklist'  # activation depends on outgoing.using_tor_proxy
 #   # these plugins are disabled if nothing is configured ..
-#   - 'Hostname replace'  # see hostname_replace configuration below
+#   - 'Hostnames plugin'  # see 'hostnames' configuration below
 #   - 'Open Access DOI rewrite'
 #   - 'Tor check plugin'
 #   # Read the docs before activate: auto-detection of the language could be
@@ -227,17 +227,31 @@ outgoing:
 #   # preferences if they want.
 #   - 'Autodetect search language'
 
-# Configuration of the "Hostname replace" plugin:
+# Configuration of the "Hostnames plugin":
+#
+# hostnames:
+#   replace:
+#     '(.*\.)?youtube\.com$': 'invidious.example.com'
+#     '(.*\.)?youtu\.be$': 'invidious.example.com'
+#     '(.*\.)?reddit\.com$': 'teddit.example.com'
+#     '(.*\.)?redd\.it$': 'teddit.example.com'
+#     '(www\.)?twitter\.com$': 'nitter.example.com'
+#   remove:
+#     - '(.*\.)?facebook.com$'
+#   low_priority:
+#     - '(.*\.)?google\.com$'
+#   high_priority:
+#     - '(.*\.)?wikipedia.org$'
+#
+# Alternatively you can use external files for configuring the "Hostnames plugin":
+#
+# hostnames:
+#  replace: 'rewrite-hosts.yml'
+#
+# Content of 'rewrite-hosts.yml' (place the file in the same directory as 'settings.yml'):
+# '(.*\.)?youtube\.com$': 'invidious.example.com'
+# '(.*\.)?youtu\.be$': 'invidious.example.com'
 #
-# hostname_replace:
-#   '(.*\.)?youtube\.com$': 'invidious.example.com'
-#   '(.*\.)?youtu\.be$': 'invidious.example.com'
-#   '(.*\.)?youtube-noocookie\.com$': 'yotter.example.com'
-#   '(.*\.)?reddit\.com$': 'teddit.example.com'
-#   '(.*\.)?redd\.it$': 'teddit.example.com'
-#   '(www\.)?twitter\.com$': 'nitter.example.com'
-#   # to remove matching host names from result list, set value to false
-#   'spam\.example\.com': false
 
 checker:
   # disable checker when in debug mode

searx/settings_loader.py

@@ -31,6 +31,14 @@ def load_yaml(file_name):
         raise SearxSettingsException(e, file_name) from e
 
 
+def get_yaml_file(file_name):
+    path = existing_filename_or_none(join(searx_dir, file_name))
+    if path is None:
+        raise FileNotFoundError(f"File {file_name} does not exist!")
+
+    return load_yaml(path)
+
+
 def get_default_settings_path():
     return existing_filename_or_none(join(searx_dir, 'settings.yml'))
 

utils/templates/etc/searxng/settings.yml

@@ -36,16 +36,22 @@ enabled_plugins:
   - 'Self Informations'
   - 'Tracker URL remover'
   - 'Ahmia blacklist'
-  # - 'Hostname replace'  # see hostname_replace configuration below
+  # - 'Hostnames plugin'  # see 'hostnames' configuration below
   # - 'Open Access DOI rewrite'
 
 # plugins:
 #   - only_show_green_results
 
-# hostname_replace:
-#
-#   # twitter --> nitter
-#   '(www\.)?twitter\.com$': 'nitter.net'
+# hostnames:
+#   replace:
+#     '(.*\.)?youtube\.com$': 'invidious.example.com'
+#     '(.*\.)?youtu\.be$': 'invidious.example.com'
+#   remove:
+#     - '(.*\.)?facebook.com$'
+#   low_priority:
+#     - '(.*\.)?google\.com$'
+#   high_priority:
+#     - '(.*\.)?wikipedia.org$'
 
 engines: