searxng/searx/engines/yacy.py

162 lines
4 KiB
Python
Raw Normal View History

# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""YaCy_ is a free distributed search engine, built on the principles of
peer-to-peer (P2P) networks.
API: Dev:APIyacysearch_
Releases:
- https://github.com/yacy/yacy_search_server/tags
- https://download.yacy.net/
.. _Yacy: https://yacy.net/
.. _Dev:APIyacysearch: https://wiki.yacy.net/index.php/Dev:APIyacysearch
Configuration
=============
The engine has the following (additional) settings:
.. code:: yaml
- name: yacy
engine: yacy
shortcut: ya
base_url: http://localhost:8090
# Yacy search mode. 'global' or 'local'.
search_mode: 'global'
number_of_results: 5
http_digest_auth_user: ""
http_digest_auth_pass: ""
Implementations
===============
"""
# pylint: disable=fixme
from json import loads
from urllib.parse import urlencode
from dateutil import parser
2021-03-18 18:59:01 +00:00
from httpx import DigestAuth
2016-12-11 13:05:07 +00:00
from searx.utils import html_to_text
# about
about = {
"website": 'https://yacy.net/',
"wikidata_id": 'Q1759675',
"official_api_documentation": 'https://wiki.yacy.net/index.php/Dev:API',
"use_official_api": True,
"require_api_key": False,
"results": 'JSON',
}
# engine dependent config
categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
paging = True
number_of_results = 5
http_digest_auth_user = ""
http_digest_auth_pass = ""
search_mode = 'global'
"""Yacy search mode ``global`` or ``local``. By default, Yacy operates in ``global``
mode.
``global``
Peer-to-Peer search
``local``
Privacy or Stealth mode, restricts the search to local yacy instance.
"""
# search-url
base_url = 'http://localhost:8090'
search_url = (
'/yacysearch.json?{query}'
'&startRecord={offset}'
'&maximumRecords={limit}'
'&contentdom={search_type}'
2023-08-02 08:35:08 +00:00
'&resource={resource}'
)
# yacy specific type-definitions
search_types = {'general': 'text', 'images': 'image', 'files': 'app', 'music': 'audio', 'videos': 'video'}
2014-01-20 01:31:20 +00:00
def request(query, params):
offset = (params['pageno'] - 1) * number_of_results
2015-02-09 15:55:01 +00:00
search_type = search_types.get(params.get('category'), '0')
params['url'] = base_url + search_url.format(
2023-08-02 08:35:08 +00:00
query=urlencode({'query': query}),
offset=offset,
limit=number_of_results,
search_type=search_type,
resource=search_mode,
)
if http_digest_auth_user and http_digest_auth_pass:
2021-03-18 18:59:01 +00:00
params['auth'] = DigestAuth(http_digest_auth_user, http_digest_auth_pass)
# add language tag if specified
if params['language'] != 'all':
params['url'] += '&lr=lang_' + params['language'].split('-')[0]
return params
2014-01-20 01:31:20 +00:00
def response(resp):
results = []
raw_search_results = loads(resp.text)
# return empty array if there are no results
2014-02-11 12:13:51 +00:00
if not raw_search_results:
return []
2015-02-09 15:55:01 +00:00
search_results = raw_search_results.get('channels', [])
2015-02-09 15:55:01 +00:00
if len(search_results) == 0:
return []
for result in search_results[0].get('items', []):
2015-02-01 10:48:15 +00:00
# parse image results
if resp.search_params.get('category') == 'images':
2018-01-06 13:52:14 +00:00
result_url = ''
if 'url' in result:
result_url = result['url']
elif 'link' in result:
result_url = result['link']
else:
continue
2015-02-01 10:48:15 +00:00
# append result
results.append(
{
'url': result_url,
'title': result['title'],
'content': '',
'img_src': result['image'],
'template': 'images.html',
}
)
2015-02-01 10:48:15 +00:00
# parse general results
2015-02-01 10:48:15 +00:00
else:
publishedDate = parser.parse(result['pubDate'])
# append result
results.append(
{
'url': result['link'],
'title': result['title'],
'content': html_to_text(result['description']),
'publishedDate': publishedDate,
}
)
2015-02-09 15:55:01 +00:00
# TODO parse video, audio and file results
return results