searxng/searx/engines/yahoo_news.py

104 lines
2.7 KiB
Python
Raw Permalink Normal View History

# SPDX-License-Identifier: AGPL-3.0-or-later
"""Yahoo (News)
Yahoo News is "English only" and do not offer localized nor language queries.
"""
# pylint: disable=invalid-name
2014-03-04 12:11:53 +00:00
2016-11-30 17:43:03 +00:00
import re
from urllib.parse import urlencode
from datetime import datetime, timedelta
2014-03-18 12:19:50 +00:00
from dateutil import parser
from lxml import html
from searx.utils import (
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.yahoo import parse_url
# about
about = {
"website": 'https://news.yahoo.com',
"wikidata_id": 'Q3044717',
"official_api_documentation": 'https://developer.yahoo.com/api/',
"use_official_api": False,
"require_api_key": False,
"results": 'HTML',
}
language_support = False
time_range_support = False
safesearch = False
2014-09-01 14:17:29 +00:00
paging = True
categories = ['news']
2014-09-01 14:17:29 +00:00
# search-url
search_url = (
# fmt: off
'https://news.search.yahoo.com/search'
'?{query}&b={offset}'
# fmt: on
)
AGO_RE = re.compile(r'([0-9]+)\s*(year|month|week|day|minute|hour)')
AGO_TIMEDELTA = {
'minute': timedelta(minutes=1),
'hour': timedelta(hours=1),
'day': timedelta(days=1),
'week': timedelta(days=7),
'month': timedelta(days=30),
'year': timedelta(days=365),
}
2014-03-04 12:11:53 +00:00
2014-03-04 12:11:53 +00:00
def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
2014-09-01 14:17:29 +00:00
params['url'] = search_url.format(offset=offset, query=urlencode({'p': query}))
logger.debug("query_url --> %s", params['url'])
2014-03-04 12:11:53 +00:00
return params
2014-03-04 12:11:53 +00:00
def response(resp):
results = []
dom = html.fromstring(resp.text)
2014-09-01 14:17:29 +00:00
# parse results
for result in eval_xpath_list(dom, '//ol[contains(@class,"searchCenterMiddle")]//li'):
url = eval_xpath_getindex(result, './/h4/a/@href', 0, None)
if url is None:
2015-05-02 19:08:56 +00:00
continue
url = parse_url(url)
title = extract_text(result.xpath('.//h4/a'))
content = extract_text(result.xpath('.//p'))
thumbnail = eval_xpath_getindex(result, './/img/@data-src', 0, None)
item = {'url': url, 'title': title, 'content': content, 'thumbnail': thumbnail}
pub_date = extract_text(result.xpath('.//span[contains(@class,"s-time")]'))
ago = AGO_RE.search(pub_date)
if ago:
number = int(ago.group(1))
delta = AGO_TIMEDELTA[ago.group(2)]
pub_date = datetime.now() - delta * number
else:
2017-04-08 17:42:50 +00:00
try:
pub_date = parser.parse(pub_date)
except parser.ParserError:
pub_date = None
if pub_date is not None:
item['publishedDate'] = pub_date
results.append(item)
for suggestion in eval_xpath_list(dom, '//div[contains(@class,"AlsoTry")]//td'):
results.append({'suggestion': extract_text(suggestion)})
2014-03-04 12:11:53 +00:00
return results