From d959cb1c059008984554c129cb6e17b6c5394bfc Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Fri, 5 Dec 2014 19:24:11 +0100 Subject: [PATCH] [enh] gevent/grequests changed to the built-in threading lib --- requirements.txt | 2 +- searx/search.py | 23 ++++++++++++++++++----- searx/webapp.py | 4 ---- setup.py | 2 +- versions.cfg | 12 ------------ 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/requirements.txt b/requirements.txt index 88c1bc715..07b53d2ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flask flask-babel -grequests +requests lxml pyyaml python-dateutil diff --git a/searx/search.py b/searx/search.py index 0e7aaed8f..19469cc52 100644 --- a/searx/search.py +++ b/searx/search.py @@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, ''' -import grequests +import requests as requests_lib +import threading import re from itertools import izip_longest, chain from datetime import datetime @@ -32,6 +33,18 @@ from searx.query import Query number_of_searches = 0 +def threaded_requests(requests): + for fn, url, request_args in requests: + th = threading.Thread( + target=fn, args=(url,), kwargs=request_args, name=url, + ) + th.start() + + for th in threading.enumerate(): + if th.name.startswith('http'): + th.join() + + # get default reqest parameter def default_request_params(): return { @@ -471,9 +484,9 @@ class Search(object): # specific type of request (GET or POST) if request_params['method'] == 'GET': - req = grequests.get + req = requests_lib.get else: - req = grequests.post + req = requests_lib.post request_args['data'] = request_params['data'] # ignoring empty urls @@ -481,10 +494,10 @@ class Search(object): continue # append request to list - requests.append(req(request_params['url'], **request_args)) + requests.append((req, request_params['url'], request_args)) # send all search-request - grequests.map(requests) + threaded_requests(requests) # update engine-specific stats for engine_name, engine_results in results.items(): diff --git a/searx/webapp.py b/searx/webapp.py index 46061913e..57a16ff4f 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, ''' -from gevent import monkey -monkey.patch_all() - - if __name__ == '__main__': from sys import path from os.path import realpath, dirname diff --git a/setup.py b/setup.py index d62613882..d976a31f7 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( install_requires=[ 'flask', 'flask-babel', - 'grequests', + 'requests', 'lxml', 'pyyaml', 'setuptools', diff --git a/versions.cfg b/versions.cfg index e0d7f4514..2f5dae8ee 100644 --- a/versions.cfg +++ b/versions.cfg @@ -36,18 +36,6 @@ zc.recipe.testrunner = 2.0.0 # WebTest==2.0.11 beautifulsoup4 = 4.3.2 -# Required by: -# grequests==0.2.0 -gevent = 1.0 - -# Required by: -# gevent==1.0 -greenlet = 0.4.2 - -# Required by: -# searx==0.1 -grequests = 0.2.0 - # Required by: # robotframework-httplibrary==0.4.2 jsonpatch = 1.3