From bacc9a3df156605c49fb93ede6e6a730110d15db Mon Sep 17 00:00:00 2001 From: Kirill Isakov Date: Mon, 28 Mar 2016 19:15:03 +0600 Subject: [PATCH] Add paging support to XPath & Erowid engines --- searx/engines/xpath.py | 16 +++++++++++++++- searx/settings.yml | 5 ++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index f51634be0..e701c02bf 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -11,6 +11,14 @@ title_xpath = None suggestion_xpath = '' results_xpath = '' +# parameters for engines with paging support +# +# number of results on each page +# (only needed if the site requires not a page number, but an offset) +page_size = 1 +# number of the first page (usually 0 or 1) +first_page_num = 1 + ''' if xpath_results is list, extract the text from each result and concat the list @@ -76,8 +84,14 @@ def normalize_url(url): def request(query, params): query = urlencode({'q': query})[2:] - params['url'] = search_url.format(query=query) + + fp = {'query': query} + if paging and search_url.find('{pageno}') >= 0: + fp['pageno'] = (params['pageno'] + first_page_num - 1) * page_size + + params['url'] = search_url.format(**fp) params['query'] = query + return params diff --git a/searx/settings.yml b/searx/settings.yml index c0324976c..473b8612a 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -84,7 +84,10 @@ engines: - name : erowid engine : xpath - search_url : https://www.erowid.org/search.php?q={query} + paging : True + first_page_num : 0 + page_size : 30 + search_url : https://www.erowid.org/search.php?q={query}&s={pageno} url_xpath : //dl[@class="results-list"]/dt[@class="result-title"]/a/@href title_xpath : //dl[@class="results-list"]/dt[@class="result-title"]/a/text() content_xpath : //dl[@class="results-list"]/dd[@class="result-details"]