From 7c13d630e4531630ce3c392a7d60752715742291 Mon Sep 17 00:00:00 2001 From: dalf Date: Sun, 7 Dec 2014 16:37:56 +0100 Subject: [PATCH] [fix] pep8 : engines (errors E121, E127, E128 and E501 still exist) --- searx/engines/bing.py | 15 ++++++++------- searx/engines/bing_images.py | 20 +++++++++++--------- searx/engines/bing_news.py | 19 ++++++++++--------- searx/engines/currency_convert.py | 2 +- searx/engines/dailymotion.py | 5 ++--- searx/engines/deviantart.py | 4 ++-- searx/engines/duckduckgo.py | 12 +++++++----- searx/engines/dummy.py | 2 +- searx/engines/faroo.py | 11 ++++++----- searx/engines/generalfile.py | 4 ++-- searx/engines/github.py | 4 ++-- searx/engines/google_images.py | 7 ++++--- searx/engines/google_news.py | 7 ++++--- searx/engines/openstreetmap.py | 30 +++++++++++++++--------------- searx/engines/piratebay.py | 6 +++--- searx/engines/soundcloud.py | 4 ++-- searx/engines/stackoverflow.py | 8 ++++---- searx/engines/twitter.py | 4 ++-- searx/engines/vimeo.py | 12 +++++++----- searx/engines/yacy.py | 11 ++++++----- searx/engines/yahoo.py | 15 ++++++++------- searx/engines/youtube.py | 4 ++-- 22 files changed, 109 insertions(+), 97 deletions(-) diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 56c6b36c1..5de461cfe 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -1,8 +1,9 @@ ## Bing (Web) -# +# # @website https://www.bing.com -# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month -# +# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), +# max. 5000 query/month +# # @using-api no (because of query limit) # @results HTML (using search portal) # @stable no (HTML can change) @@ -58,8 +59,8 @@ def response(resp): content = escape(' '.join(result.xpath('.//p//text()'))) # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'content': content}) # return results if something is found @@ -74,8 +75,8 @@ def response(resp): content = escape(' '.join(result.xpath('.//p//text()'))) # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'content': content}) # return results diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index b3eabba45..6c5e49bc9 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -1,17 +1,19 @@ ## Bing (Images) -# +# # @website https://www.bing.com/images -# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month -# +# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), +# max. 5000 query/month +# # @using-api no (because of query limit) # @results HTML (using search portal) # @stable no (HTML can change) # @parse url, title, img_src # -# @todo currently there are up to 35 images receive per page, because bing does not parse count=10. limited response to 10 images +# @todo currently there are up to 35 images receive per page, +# because bing does not parse count=10. +# limited response to 10 images from urllib import urlencode -from cgi import escape from lxml import html from yaml import load import re @@ -51,15 +53,15 @@ def response(resp): dom = html.fromstring(resp.content) # init regex for yaml-parsing - p = re.compile( '({|,)([a-z]+):(")') + p = re.compile('({|,)([a-z]+):(")') # parse results for result in dom.xpath('//div[@class="dg_u"]'): link = result.xpath('./a')[0] # parse yaml-data (it is required to add a space, to make it parsable) - yaml_data = load(p.sub( r'\1\2: \3', link.attrib.get('m'))) - + yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m'))) + title = link.attrib.get('t1') #url = 'http://' + link.attrib.get('t3') url = yaml_data.get('surl') @@ -69,7 +71,7 @@ def response(resp): results.append({'template': 'images.html', 'url': url, 'title': title, - 'content': '', + 'content': '', 'img_src': img_src}) # TODO stop parsing if 10 images are found diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 279f0d698..5dce4a2b2 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -1,8 +1,9 @@ ## Bing (News) -# +# # @website https://www.bing.com/news -# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month -# +# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), +# max. 5000 query/month +# # @using-api no (because of query limit) # @results HTML (using search portal) # @stable no (HTML can change) @@ -57,12 +58,12 @@ def response(resp): url = link.attrib.get('href') title = ' '.join(link.xpath('.//text()')) contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()') - if contentXPath != None: + if contentXPath is not None: content = escape(' '.join(contentXPath)) - + # parse publishedDate publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()') - if publishedDateXPath != None: + if publishedDateXPath is not None: publishedDate = escape(' '.join(publishedDateXPath)) if re.match("^[0-9]+ minute(s|) ago$", publishedDate): @@ -89,10 +90,10 @@ def response(resp): except TypeError: # FIXME publishedDate = datetime.now() - + # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'publishedDate': publishedDate, 'content': content}) diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py index b5f0953d8..d8841c1d1 100644 --- a/searx/engines/currency_convert.py +++ b/searx/engines/currency_convert.py @@ -55,6 +55,6 @@ def response(resp): resp.search_params['to'].lower() ) - results.append({'answer' : answer, 'url': url}) + results.append({'answer': answer, 'url': url}) return results diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py index 75c2e5071..a5bffa866 100644 --- a/searx/engines/dailymotion.py +++ b/searx/engines/dailymotion.py @@ -1,8 +1,8 @@ ## Dailymotion (Videos) -# +# # @website https://www.dailymotion.com # @provide-api yes (http://www.dailymotion.com/developer) -# +# # @using-api yes # @results JSON # @stable yes @@ -12,7 +12,6 @@ from urllib import urlencode from json import loads -from lxml import html # engine dependent config categories = ['videos'] diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index ff5e1d465..d436e8163 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -1,8 +1,8 @@ ## Deviantart (Images) -# +# # @website https://www.deviantart.com/ # @provide-api yes (https://www.deviantart.com/developers/) (RSS) -# +# # @using-api no (TODO, rewrite to api) # @results HTML # @stable no (HTML can change) diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 296dd9b2d..583e33f73 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -1,15 +1,17 @@ ## DuckDuckGo (Web) -# +# # @website https://duckduckgo.com/ -# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site -# +# @provide-api yes (https://duckduckgo.com/api), +# but not all results from search-site +# # @using-api no # @results HTML (using search portal) # @stable no (HTML can change) # @parse url, title, content # # @todo rewrite to api -# @todo language support (the current used site does not support language-change) +# @todo language support +# (the current used site does not support language-change) from urllib import urlencode from lxml.html import fromstring @@ -37,7 +39,7 @@ def request(query, params): if params['language'] == 'all': locale = 'en-us' else: - locale = params['language'].replace('_','-').lower() + locale = params['language'].replace('_', '-').lower() params['url'] = url.format( query=urlencode({'q': query, 'kl': locale}), diff --git a/searx/engines/dummy.py b/searx/engines/dummy.py index 5a2cdf6b5..c60b7a5d2 100644 --- a/searx/engines/dummy.py +++ b/searx/engines/dummy.py @@ -1,5 +1,5 @@ ## Dummy -# +# # @results empty array # @stable yes diff --git a/searx/engines/faroo.py b/searx/engines/faroo.py index 8c69c5bee..dada4758d 100644 --- a/searx/engines/faroo.py +++ b/searx/engines/faroo.py @@ -1,8 +1,8 @@ ## Faroo (Web, News) -# +# # @website http://www.faroo.com # @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key -# +# # @using-api yes # @results JSON # @stable yes @@ -24,9 +24,10 @@ api_key = None url = 'http://www.faroo.com/' search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}' -search_category = {'general': 'web', +search_category = {'general': 'web', 'news': 'news'} + # do search-request def request(query, params): offset = (params['pageno']-1) * number_of_results + 1 @@ -48,7 +49,7 @@ def request(query, params): query=urlencode({'q': query}), language=language, categorie=categorie, - api_key=api_key ) + api_key=api_key) # using searx User-Agent params['headers']['User-Agent'] = searx_useragent() @@ -101,7 +102,7 @@ def response(resp): results.append({'template': 'images.html', 'url': result['url'], 'title': result['title'], - 'content': result['kwic'], + 'content': result['kwic'], 'img_src': result['iurl']}) # return results diff --git a/searx/engines/generalfile.py b/searx/engines/generalfile.py index 11d8b6955..b7d716298 100644 --- a/searx/engines/generalfile.py +++ b/searx/engines/generalfile.py @@ -1,8 +1,8 @@ ## General Files (Files) -# +# # @website http://www.general-files.org # @provide-api no (nothing found) -# +# # @using-api no (because nothing found) # @results HTML (using search portal) # @stable no (HTML can change) diff --git a/searx/engines/github.py b/searx/engines/github.py index 53fec029f..a68aed141 100644 --- a/searx/engines/github.py +++ b/searx/engines/github.py @@ -1,8 +1,8 @@ ## Github (It) -# +# # @website https://github.com/ # @provide-api yes (https://developer.github.com/v3/) -# +# # @using-api yes # @results JSON # @stable yes (using api) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 6c99f2801..7b8b15b42 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -1,8 +1,9 @@ ## Google (Images) -# +# # @website https://www.google.com -# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! -# +# @provide-api yes (https://developers.google.com/web-search/docs/), +# deprecated! +# # @using-api yes # @results JSON # @stable yes (but deprecated) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index becc7e21d..eb114f9c9 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -1,8 +1,9 @@ ## Google (News) -# +# # @website https://www.google.com -# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! -# +# @provide-api yes (https://developers.google.com/web-search/docs/), +# deprecated! +# # @using-api yes # @results JSON # @stable yes (but deprecated) diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py index f727ca8ea..36b6011e7 100644 --- a/searx/engines/openstreetmap.py +++ b/searx/engines/openstreetmap.py @@ -39,16 +39,16 @@ def response(resp): url = result_base_url.format(osm_type=osm_type, osm_id=r['osm_id']) - osm = {'type':osm_type, - 'id':r['osm_id']} + osm = {'type': osm_type, + 'id': r['osm_id']} - geojson = r.get('geojson') + geojson = r.get('geojson') # if no geojson is found and osm_type is a node, add geojson Point if not geojson and\ osm_type == 'node': - geojson = {u'type':u'Point', - u'coordinates':[r['lon'],r['lat']]} + geojson = {u'type': u'Point', + u'coordinates': [r['lon'], r['lat']]} address_raw = r.get('address') address = {} @@ -59,20 +59,20 @@ def response(resp): r['class'] == 'tourism' or\ r['class'] == 'leisure': if address_raw.get('address29'): - address = {'name':address_raw.get('address29')} + address = {'name': address_raw.get('address29')} else: - address = {'name':address_raw.get(r['type'])} + address = {'name': address_raw.get(r['type'])} # add rest of adressdata, if something is already found if address.get('name'): - address.update({'house_number':address_raw.get('house_number'), - 'road':address_raw.get('road'), - 'locality':address_raw.get('city', - address_raw.get('town', - address_raw.get('village'))), - 'postcode':address_raw.get('postcode'), - 'country':address_raw.get('country'), - 'country_code':address_raw.get('country_code')}) + address.update({'house_number': address_raw.get('house_number'), + 'road': address_raw.get('road'), + 'locality': address_raw.get('city', + address_raw.get('town', + address_raw.get('village'))), + 'postcode': address_raw.get('postcode'), + 'country': address_raw.get('country'), + 'country_code': address_raw.get('country_code')}) else: address = None diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py index 9533b629e..14905dc83 100644 --- a/searx/engines/piratebay.py +++ b/searx/engines/piratebay.py @@ -1,8 +1,8 @@ ## Piratebay (Videos, Music, Files) -# +# # @website https://thepiratebay.se # @provide-api no (nothing found) -# +# # @using-api no # @results HTML (using search portal) # @stable yes (HTML can change) @@ -23,7 +23,7 @@ url = 'https://thepiratebay.se/' search_url = url + 'search/{search_term}/{pageno}/99/{search_type}' # piratebay specific type-definitions -search_types = {'files': '0', +search_types = {'files': '0', 'music': '100', 'videos': '200'} diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py index aebea239f..390e7ca1f 100644 --- a/searx/engines/soundcloud.py +++ b/searx/engines/soundcloud.py @@ -1,8 +1,8 @@ ## Soundcloud (Music) -# +# # @website https://soundcloud.com # @provide-api yes (https://developers.soundcloud.com/) -# +# # @using-api yes # @results JSON # @stable yes diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py index edbe74a70..dcbb1890c 100644 --- a/searx/engines/stackoverflow.py +++ b/searx/engines/stackoverflow.py @@ -1,8 +1,8 @@ ## Stackoverflow (It) -# +# # @website https://stackoverflow.com/ # @provide-api not clear (https://api.stackexchange.com/docs/advanced-search) -# +# # @using-api no # @results HTML # @stable no (HTML can change) @@ -50,8 +50,8 @@ def response(resp): content = escape(' '.join(result.xpath(content_xpath))) # append result - results.append({'url': href, - 'title': title, + results.append({'url': href, + 'title': title, 'content': content}) # return results diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py index 8de78144e..0689150c8 100644 --- a/searx/engines/twitter.py +++ b/searx/engines/twitter.py @@ -1,8 +1,8 @@ ## Twitter (Social media) -# +# # @website https://www.bing.com/news # @provide-api yes (https://dev.twitter.com/docs/using-search) -# +# # @using-api no # @results HTML (using search portal) # @stable no (HTML can change) diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py index 2a91e76fa..c66c4148a 100644 --- a/searx/engines/vimeo.py +++ b/searx/engines/vimeo.py @@ -1,8 +1,9 @@ ## Vimeo (Videos) -# +# # @website https://vimeo.com/ -# @provide-api yes (http://developer.vimeo.com/api), they have a maximum count of queries/hour -# +# @provide-api yes (http://developer.vimeo.com/api), +# they have a maximum count of queries/hour +# # @using-api no (TODO, rewrite to api) # @results HTML (using search portal) # @stable no (HTML can change) @@ -35,11 +36,12 @@ publishedDate_xpath = './/p[@class="meta"]//attribute::datetime' # do search-request def request(query, params): - params['url'] = search_url.format(pageno=params['pageno'] , + params['url'] = search_url.format(pageno=params['pageno'], query=urlencode({'q': query})) # TODO required? - params['cookies']['__utma'] = '00000000.000#0000000.0000000000.0000000000.0000000000.0' + params['cookies']['__utma'] =\ + '00000000.000#0000000.0000000000.0000000000.0000000000.0' return params diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py index 2345b24f3..3ee0e91c7 100644 --- a/searx/engines/yacy.py +++ b/searx/engines/yacy.py @@ -1,8 +1,9 @@ ## Yacy (Web, Images, Videos, Music, Files) -# +# # @website http://yacy.net -# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) -# +# @provide-api yes +# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) +# # @using-api yes # @results JSON # @stable yes @@ -16,7 +17,7 @@ from urllib import urlencode from dateutil import parser # engine dependent config -categories = ['general', 'images'] #TODO , 'music', 'videos', 'files' +categories = ['general', 'images'] # TODO , 'music', 'videos', 'files' paging = True language_support = True number_of_results = 5 @@ -28,7 +29,7 @@ search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limi # yacy specific type-definitions search_types = {'general': 'text', 'images': 'image', - 'files': 'app', + 'files': 'app', 'music': 'audio', 'videos': 'video'} diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 5e34a2b07..938540ece 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -1,8 +1,9 @@ ## Yahoo (Web) -# +# # @website https://search.yahoo.com/web -# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries -# +# @provide-api yes (https://developer.yahoo.com/boss/search/), +# $0.80/1000 queries +# # @using-api no (because pricing) # @results HTML (using search portal) # @stable no (HTML can change) @@ -40,8 +41,8 @@ def parse_url(url_string): if endpos > -1: endpositions.append(endpos) - if start==0 or len(endpositions) == 0: - return url_string + if start == 0 or len(endpositions) == 0: + return url_string else: end = min(endpositions) return unquote(url_string[start:end]) @@ -84,8 +85,8 @@ def response(resp): content = extract_text(result.xpath(content_xpath)[0]) # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'content': content}) # if no suggestion found, return results diff --git a/searx/engines/youtube.py b/searx/engines/youtube.py index 7d1c207f0..973e799f8 100644 --- a/searx/engines/youtube.py +++ b/searx/engines/youtube.py @@ -1,8 +1,8 @@ ## Youtube (Videos) -# +# # @website https://www.youtube.com/ # @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/) -# +# # @using-api yes # @results JSON # @stable yes