Add searx_extra package

Split the utils directory into:
* searx_extra contains update scripts, standalone_searx.py
* utils contains the files to build and setup searx.
This commit is contained in:
Alexandre Flament 2021-02-25 17:42:52 +01:00
parent 111d38cd8f
commit b8cd326464
21 changed files with 47 additions and 75 deletions

View file

@ -11,12 +11,12 @@ jobs:
strategy:
matrix:
fetch:
- ahmia_blacklist
- currencies
- external_bangs
- firefox_version
- languages
- wikidata_units
- update_ahmia_blacklist.py
- update_currencies.py
- update_external_bangs.py
- update_firefox_version.py
- update_languages.py
- update_wikidata_units.py
steps:
- name: Checkout
uses: actions/checkout@v2
@ -45,10 +45,10 @@ jobs:
- name: Fetch data
env:
FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py
FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
run: |
source local/py3/bin/activate
python $FETCH_SCRIPT
$FETCH_SCRIPT
- name: Create Pull Request
id: cpr

View file

@ -195,8 +195,8 @@ PYLINT_FILES=\
searx/engines/google_videos.py \
searx/engines/google_images.py \
searx/engines/mediathekviewweb.py \
utils/fetch_external_bangs.py \
searx/engines/google_scholar.py
searx/engines/google_scholar.py \
searx_extra/update/update_external_bangs.py
test.pylint: pyenvinstall
$(call cmd,pylint,$(PYLINT_FILES))

View file

@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
user/index
admin/index
dev/index
searx_extra/index
utils/index
blog/index

View file

@ -0,0 +1,14 @@
.. _searx_extra:
======================================================
Tooling box ``searx_extra`` for developers and users
======================================================
In the folder :origin:`searx_extra/` we maintain some tools useful for
developers and users.
.. toctree::
:maxdepth: 2
:caption: Contents
standalone_searx.py

View file

@ -0,0 +1,9 @@
.. _standalone_searx.py:
===================================
``searx_extra/standalone_searx.py``
===================================
.. automodule:: searx_extra.standalone_searx
:members:

View file

@ -1,12 +1,11 @@
.. _searx_utils:
.. _toolboxing:
=======================
Tooling box ``utils/*``
=======================
========================================
Tooling box ``utils`` for administrators
========================================
In the folder :origin:`utils/` we maintain some tools useful for admins and
developers.
In the folder :origin:`utils/` we maintain some tools useful for administrators.
.. toctree::
:maxdepth: 2
@ -16,7 +15,6 @@ developers.
filtron.sh
morty.sh
lxc.sh
standalone_searx.py
.. _toolboxing common:

View file

@ -1,11 +0,0 @@
.. _standalone_searx.py:
=============================
``utils/standalone_searx.py``
=============================
.. automodule:: standalone_searx
:members:

0
searx_extra/__init__.py Normal file
View file

View file

@ -15,7 +15,7 @@ Example to use this script:
.. code:: bash
$ python3 utils/standalone_searx.py rain
$ python3 searx_extra/standalone_searx.py rain
Example to run it from python:

View file

View file

@ -4,11 +4,8 @@
# More info in https://ahmia.fi/blacklist/
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from os.path import join
#
import requests
from searx import searx_dir

View file

@ -7,7 +7,6 @@ import json
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir, settings
from searx.engines.wikidata import send_wikidata_query

View file

@ -3,13 +3,9 @@
import sys
import json
from urllib.parse import quote, urlparse
from os.path import realpath, dirname
import cld3
import detect_language
from lxml.html import fromstring
# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx.engines.wikidata import send_wikidata_query
from searx.utils import extract_text
import searx

View file

@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
"""
# pylint: disable=C0116
import sys
import json
import re
from os.path import realpath, dirname, join
from os.path import join
import requests
# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
from searx import searx_dir # pylint: disable=E0401 C0413

View file

@ -1,14 +1,9 @@
#!/usr/bin/env python
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
#
import json
import requests
import re
from os.path import dirname, join
from urllib.parse import urlparse, urljoin
from distutils.version import LooseVersion, StrictVersion
from lxml import html

View file

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
#!/usr/bin/env python
# This script generates languages.py from intersecting each engine's supported languages.
#
@ -7,11 +7,9 @@
import json
from pathlib import Path
from pprint import pformat
from sys import path
from babel import Locale, UnknownLocaleError
from babel.languages import get_global
path.append('../searx') # noqa
from searx import settings, searx_dir
from searx.engines import initialize_engines, engines

View file

@ -4,9 +4,7 @@ import json
import collections
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from os.path import join
from searx import searx_dir
from searx.engines.wikidata import send_wikidata_query

View file

@ -41,7 +41,7 @@ setup(
author='Adam Tauber',
author_email='asciimoo@gmail.com',
license='GNU Affero General Public License',
packages=find_packages(exclude=["tests*"]),
packages=find_packages(exclude=["tests*", "searx_extra"]),
zip_safe=False,
install_requires=requirements,
extras_require={

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Test utils/standalone_searx.py"""
import datetime
import importlib.util
import io
import sys
@ -10,16 +9,7 @@ from nose2.tools import params
from searx.search import SearchQuery, EngineRef, initialize
from searx.testing import SearxTestCase
def get_standalone_searx_module():
"""Get standalone_searx module."""
module_name = 'utils.standalone_searx'
filename = 'utils/standalone_searx.py'
spec = importlib.util.spec_from_file_location(module_name, filename)
sas = importlib.util.module_from_spec(spec)
spec.loader.exec_module(sas)
return sas
from searx_extra import standalone_searx as sas
class StandaloneSearx(SearxTestCase):
@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):
def test_parse_argument_no_args(self):
"""Test parse argument without args."""
sas = get_standalone_searx_module()
with patch.object(sys, 'argv', ['standalone_searx']), \
self.assertRaises(SystemExit):
sys.stderr = io.StringIO()
@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):
def test_parse_argument_basic_args(self):
"""Test parse argument with basic args."""
sas = get_standalone_searx_module()
query = 'red box'
exp_dict = {
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict(self):
"""test to_dict."""
sas = get_standalone_searx_module()
self.assertEqual(
sas.to_dict(
sas.get_search_query(sas.parse_argument(['red box']))),
@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):
def test_to_dict_with_mock(self):
"""test to dict."""
sas = get_standalone_searx_module()
with patch.object(sas.searx.search, 'Search') as mock_s:
m_search = mock_s().search()
m_sq = Mock()
@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):
def test_get_search_query(self):
"""test get_search_query."""
sas = get_standalone_searx_module()
args = sas.parse_argument(['rain', ])
search_q = sas.get_search_query(args)
self.assertTrue(search_q)
@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):
def test_no_parsed_url(self):
"""test no_parsed_url func"""
sas = get_standalone_searx_module()
self.assertEqual(
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
[{}]
@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
)
def test_json_serial(self, arg, exp_res):
"""test json_serial func"""
sas = get_standalone_searx_module()
self.assertEqual(sas.json_serial(arg), exp_res)
def test_json_serial_error(self):
"""test error on json_serial."""
sas = get_standalone_searx_module()
with self.assertRaises(TypeError):
sas.json_serial('a')