[enh] gevent/grequests changed to the built-in threading lib

This commit is contained in:
Adam Tauber 2014-12-05 19:24:11 +01:00
parent b1b0b82a2a
commit d959cb1c05
5 changed files with 20 additions and 23 deletions

View file

@ -1,6 +1,6 @@
flask flask
flask-babel flask-babel
grequests requests
lxml lxml
pyyaml pyyaml
python-dateutil python-dateutil

View file

@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com> (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
''' '''
import grequests import requests as requests_lib
import threading
import re import re
from itertools import izip_longest, chain from itertools import izip_longest, chain
from datetime import datetime from datetime import datetime
@ -32,6 +33,18 @@ from searx.query import Query
number_of_searches = 0 number_of_searches = 0
def threaded_requests(requests):
for fn, url, request_args in requests:
th = threading.Thread(
target=fn, args=(url,), kwargs=request_args, name=url,
)
th.start()
for th in threading.enumerate():
if th.name.startswith('http'):
th.join()
# get default reqest parameter # get default reqest parameter
def default_request_params(): def default_request_params():
return { return {
@ -471,9 +484,9 @@ class Search(object):
# specific type of request (GET or POST) # specific type of request (GET or POST)
if request_params['method'] == 'GET': if request_params['method'] == 'GET':
req = grequests.get req = requests_lib.get
else: else:
req = grequests.post req = requests_lib.post
request_args['data'] = request_params['data'] request_args['data'] = request_params['data']
# ignoring empty urls # ignoring empty urls
@ -481,10 +494,10 @@ class Search(object):
continue continue
# append request to list # append request to list
requests.append(req(request_params['url'], **request_args)) requests.append((req, request_params['url'], request_args))
# send all search-request # send all search-request
grequests.map(requests) threaded_requests(requests)
# update engine-specific stats # update engine-specific stats
for engine_name, engine_results in results.items(): for engine_name, engine_results in results.items():

View file

@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com> (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
''' '''
from gevent import monkey
monkey.patch_all()
if __name__ == '__main__': if __name__ == '__main__':
from sys import path from sys import path
from os.path import realpath, dirname from os.path import realpath, dirname

View file

@ -41,7 +41,7 @@ setup(
install_requires=[ install_requires=[
'flask', 'flask',
'flask-babel', 'flask-babel',
'grequests', 'requests',
'lxml', 'lxml',
'pyyaml', 'pyyaml',
'setuptools', 'setuptools',

View file

@ -36,18 +36,6 @@ zc.recipe.testrunner = 2.0.0
# WebTest==2.0.11 # WebTest==2.0.11
beautifulsoup4 = 4.3.2 beautifulsoup4 = 4.3.2
# Required by:
# grequests==0.2.0
gevent = 1.0
# Required by:
# gevent==1.0
greenlet = 0.4.2
# Required by:
# searx==0.1
grequests = 0.2.0
# Required by: # Required by:
# robotframework-httplibrary==0.4.2 # robotframework-httplibrary==0.4.2
jsonpatch = 1.3 jsonpatch = 1.3