diff --git a/.nvmrc b/.nvmrc index 0a2cd987d..3f784dcf9 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v16.15.1 \ No newline at end of file +v16.20.2 \ No newline at end of file diff --git a/manage b/manage index 380560362..58bafae01 100755 --- a/manage +++ b/manage @@ -41,7 +41,7 @@ PATH="${REPO_ROOT}/node_modules/.bin:${PATH}" PYOBJECTS="searx" PY_SETUP_EXTRAS='[test]' -GECKODRIVER_VERSION="v0.30.0" +GECKODRIVER_VERSION="v0.33.0" # SPHINXOPTS= BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization") BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests") diff --git a/package.json b/package.json index 5eb3da891..dd24a34f8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "dependencies": { - "eslint": "^8.18.0", - "pyright": "^1.1.255" + "eslint": "^8.50.0", + "pyright": "^1.1.329" }, "scripts": { "clean": "rm -Rf node_modules package-lock.json" diff --git a/searx/botdetection/limiter.py b/searx/botdetection/limiter.py index f8509d5a6..9b3532f0d 100644 --- a/searx/botdetection/limiter.py +++ b/searx/botdetection/limiter.py @@ -51,7 +51,6 @@ from . import ( http_accept, http_accept_encoding, http_accept_language, - http_connection, http_user_agent, ip_limit, ip_lists, @@ -136,7 +135,6 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None: http_accept, http_accept_encoding, http_accept_language, - http_connection, http_user_agent, ip_limit, ]: diff --git a/searx/data/engine_traits.json b/searx/data/engine_traits.json index 7b498dd62..e13538aa1 100644 --- a/searx/data/engine_traits.json +++ b/searx/data/engine_traits.json @@ -130,363 +130,1579 @@ "regions": {} }, "bing": { - "all_locale": null, + "all_locale": "clear", "custom": {}, "data_type": "traits_v1", "languages": { - "ar": "ar", - "bg": "bg", - "bn": "bn", - "ca": "ca", - "cs": "cs", - "da": "da", - "de": "de", - "en": "en", - "es": "es", - "et": "et", - "eu": "eu", - "fi": "fi", - "fr": "fr", - "gl": "gl", - "gu": "gu", - "he": "he", - "hi": "hi", - "hr": "hr", - "hu": "hu", - "is": "is", - "it": "it", - "ja": "jp", - "kn": "kn", - "ko": "ko", - "lt": "lt", - "lv": "lv", - "ml": "ml", - "mr": "mr", - "ms": "ms", - "nb": "nb", - "nl": "nl", - "pa": "pa", - "pl": "pl", - "pt": "pt-pt", - "ro": "ro", - "ru": "ru", - "sk": "sk", - "sl": "sl", - "sr": "sr", - "sv": "sv", - "ta": "ta", - "te": "te", - "th": "th", - "tr": "tr", - "uk": "uk", - "vi": "vi", + "af": "af-af", + "am": "am-am", + "ar": "ar-ar", + "as": "as-as", + "az_Latn": "az-latn", + "be": "be-be", + "bg": "bg-bg", + "bn": "bn-bn", + "bs_Latn": "bs-latn", + "ca": "ca-ca", + "chr": "chr-cher", + "cs": "cs-cs", + "cy": "cy-cy", + "da": "da-dk", + "de": "de-de", + "el": "el-el", + "en": "en-us", + "es": "es-es", + "et": "et-et", + "eu": "eu-eu", + "fa": "prs-prs", + "fi": "fi-fi", + "fil": "fil-fil", + "fr": "fr-fr", + "ga": "ga-ga", + "gd": "gd-gd", + "gl": "gl-gl", + "gu": "gu-gu", + "ha": "ha-latn", + "he": "he-he", + "hi": "hi-hi", + "hr": "hr-hr", + "hu": "hu-hu", + "hy": "hy-hy", + "id": "id-id", + "ig": "ig-ig", + "is": "is-is", + "it": "it-it", + "ja": "ja-ja", + "ka": "ka-ka", + "kk": "kk-kk", + "km": "km-km", + "kn": "kn-kn", + "ko": "ko-ko", + "kok": "kok-kok", + "ku": "ku-arab", + "ky": "ky-ky", + "lb": "lb-lb", + "lo": "lo-lo", + "lt": "lt-lt", + "lv": "lv-lv", + "mi": "mi-mi", + "mk": "mk-mk", + "ml": "ml-ml", + "mn": "mn-cyrl-mn", + "mr": "mr-mr", + "ms": "ms-ms", + "mt": "mt-mt", + "nb": "nb-nb", + "ne": "ne-ne", + "nl": "nl-nl", + "nn": "nn-nn", + "or": "or-or", + "pa_Arab": "pa-arab", + "pa_Guru": "pa-guru", + "pl": "pl-pl", + "pt": "pt-br", + "qu": "quz-quz", + "ro": "ro-ro", + "ru": "ru-ru", + "rw": "rw-rw", + "sd_Arab": "sd-arab", + "si": "si-si", + "sk": "sk-sk", + "sl": "sl-sl", + "sq": "sq-sq", + "sr_Cyrl": "sr-cyrl", + "sr_Latn": "sr-latn", + "sv": "sv-sv", + "sw": "sw-sw", + "ta": "ta-ta", + "te": "te-te", + "tg": "tg-cyrl", + "th": "th-th", + "ti": "ti-ti", + "tk": "tk-tk", + "tr": "tr-tr", + "tt": "tt-tt", + "ug": "ug-ug", + "uk": "uk-uk", + "ur": "ur-ur", + "uz_Latn": "uz-latn", + "vi": "vi-vi", + "wo": "wo-wo", + "xh": "xh-xh", + "yo": "yo-yo", "zh": "zh-hans", "zh_Hans": "zh-hans", - "zh_Hant": "zh-hant" + "zh_Hant": "zh-hant", + "zu": "zu-zu" }, "regions": { - "da-DK": "da-DK", - "de-AT": "de-AT", - "de-CH": "de-CH", - "de-DE": "de-DE", - "en-AU": "en-AU", - "en-CA": "en-CA", - "en-GB": "en-GB", - "en-IN": "en-IN", - "en-MY": "en-MY", - "en-NZ": "en-NZ", - "en-PH": "en-PH", - "en-US": "en-US", - "en-ZA": "en-ZA", - "es-AR": "es-AR", - "es-CL": "es-CL", - "es-ES": "es-ES", - "es-MX": "es-MX", - "es-US": "es-US", - "fi-FI": "fi-FI", - "fr-BE": "fr-BE", - "fr-CA": "fr-CA", - "fr-CH": "fr-CH", - "fr-FR": "fr-FR", - "id-ID": "en-ID", - "it-IT": "it-IT", - "ja-JP": "ja-JP", - "ko-KR": "ko-KR", - "nb-NO": "no-NO", - "nl-BE": "nl-BE", - "nl-NL": "nl-NL", - "pl-PL": "pl-PL", - "pt-BR": "pt-BR", - "ru-RU": "ru-RU", - "sv-SE": "sv-SE", - "tr-TR": "tr-TR", - "zh-CN": "zh-CN", - "zh-HK": "zh-HK", - "zh-TW": "zh-TW" + "am-ET": "am-et", + "ar-AE": "ar-ae", + "ar-BH": "ar-bh", + "ar-DJ": "ar-dj", + "ar-DZ": "ar-dz", + "ar-EG": "ar-eg", + "ar-ER": "ar-er", + "ar-IL": "ar-il", + "ar-IQ": "ar-iq", + "ar-JO": "ar-jo", + "ar-KM": "ar-km", + "ar-KW": "ar-kw", + "ar-LB": "ar-lb", + "ar-LY": "ar-ly", + "ar-MA": "ar-ma", + "ar-MR": "ar-mr", + "ar-OM": "ar-om", + "ar-PS": "ar-ps", + "ar-QA": "ar-qa", + "ar-SA": "ar-sa", + "ar-SD": "ar-sd", + "ar-SO": "ar-so", + "ar-SY": "ar-sy", + "ar-TD": "ar-td", + "ar-TN": "ar-tn", + "ar-YE": "ar-ye", + "be-BY": "be-by", + "bg-BG": "bg-bg", + "bn-BD": "bn-bd", + "ca-AD": "ca-ad", + "cs-CZ": "cs-cz", + "da-DK": "da-dk", + "de-AT": "de-at", + "de-BE": "de-be", + "de-CH": "de-ch", + "de-DE": "de-de", + "de-LI": "de-li", + "de-LU": "de-lu", + "el-CY": "el-cy", + "el-GR": "el-gr", + "en-AG": "en-ag", + "en-AI": "en-ai", + "en-AS": "en-as", + "en-AU": "en-au", + "en-BB": "en-bb", + "en-BI": "en-bi", + "en-BM": "en-bm", + "en-BS": "en-bs", + "en-BW": "en-bw", + "en-BZ": "en-bz", + "en-CA": "en-ca", + "en-CC": "en-cc", + "en-CK": "en-ck", + "en-CM": "en-cm", + "en-CX": "en-cx", + "en-DM": "en-dm", + "en-ER": "en-er", + "en-FJ": "en-fj", + "en-FK": "en-fk", + "en-FM": "en-fm", + "en-GB": "en-gb", + "en-GD": "en-gd", + "en-GG": "en-gg", + "en-GH": "en-gh", + "en-GI": "en-gi", + "en-GM": "en-gm", + "en-GU": "en-gu", + "en-GY": "en-gy", + "en-HK": "en-hk", + "en-IE": "en-ie", + "en-IN": "en-in", + "en-JE": "en-je", + "en-JM": "en-jm", + "en-KE": "en-ke", + "en-KI": "en-ki", + "en-KN": "en-kn", + "en-KY": "en-ky", + "en-LC": "en-lc", + "en-LR": "en-lr", + "en-LS": "en-ls", + "en-MG": "en-mg", + "en-MH": "en-mh", + "en-MP": "en-mp", + "en-MS": "en-ms", + "en-MT": "en-mt", + "en-MU": "en-mu", + "en-MW": "en-mw", + "en-NA": "en-na", + "en-NF": "en-nf", + "en-NG": "en-ng", + "en-NR": "en-nr", + "en-NU": "en-nu", + "en-NZ": "en-nz", + "en-PG": "en-pg", + "en-PH": "en-ph", + "en-PK": "en-pk", + "en-PN": "en-pn", + "en-PR": "en-pr", + "en-PW": "en-pw", + "en-RW": "en-rw", + "en-SB": "en-sb", + "en-SC": "en-sc", + "en-SD": "en-sd", + "en-SG": "en-sg", + "en-SH": "en-sh", + "en-SL": "en-sl", + "en-SS": "en-ss", + "en-SX": "en-sx", + "en-SZ": "en-sz", + "en-TC": "en-tc", + "en-TK": "en-tk", + "en-TO": "en-to", + "en-TT": "en-tt", + "en-TV": "en-tv", + "en-TZ": "en-tz", + "en-UG": "en-ug", + "en-US": "en-us", + "en-VC": "en-vc", + "en-VG": "en-vg", + "en-VI": "en-vi", + "en-VU": "en-vu", + "en-WS": "en-ws", + "en-ZA": "en-za", + "en-ZM": "en-zm", + "en-ZW": "en-zw", + "es-AR": "es-ar", + "es-BO": "es-bo", + "es-CL": "es-cl", + "es-CO": "es-co", + "es-CR": "es-cr", + "es-DO": "es-do", + "es-EC": "es-ec", + "es-ES": "es-es", + "es-GQ": "es-gq", + "es-GT": "es-gt", + "es-HN": "es-hn", + "es-MX": "es-mx", + "es-NI": "es-ni", + "es-PA": "es-pa", + "es-PE": "es-pe", + "es-PR": "es-pr", + "es-PY": "es-py", + "es-SV": "es-sv", + "es-UY": "es-uy", + "es-VE": "es-ve", + "et-EE": "et-ee", + "fa-AF": "fa-af", + "fa-IR": "fa-ir", + "fi-FI": "fi-fi", + "fil-PH": "fil-ph", + "fr-BE": "fr-be", + "fr-BF": "fr-bf", + "fr-BI": "fr-bi", + "fr-BJ": "fr-bj", + "fr-BL": "fr-bl", + "fr-CA": "fr-ca", + "fr-CD": "fr-cd", + "fr-CF": "fr-cf", + "fr-CG": "fr-cg", + "fr-CH": "fr-ch", + "fr-CI": "fr-ci", + "fr-CM": "fr-cm", + "fr-DJ": "fr-dj", + "fr-DZ": "fr-dz", + "fr-FR": "fr-fr", + "fr-GA": "fr-ga", + "fr-GF": "fr-gf", + "fr-GN": "fr-gn", + "fr-GP": "fr-gp", + "fr-GQ": "fr-gq", + "fr-HT": "fr-ht", + "fr-KM": "fr-km", + "fr-LU": "fr-lu", + "fr-MA": "fr-ma", + "fr-MC": "fr-mc", + "fr-MF": "fr-mf", + "fr-MG": "fr-mg", + "fr-ML": "fr-ml", + "fr-MQ": "fr-mq", + "fr-MU": "fr-mu", + "fr-NC": "fr-nc", + "fr-NE": "fr-ne", + "fr-PF": "fr-pf", + "fr-PM": "fr-pm", + "fr-RE": "fr-re", + "fr-RW": "fr-rw", + "fr-SC": "fr-sc", + "fr-SN": "fr-sn", + "fr-SY": "fr-sy", + "fr-TD": "fr-td", + "fr-TG": "fr-tg", + "fr-TN": "fr-tn", + "fr-VU": "fr-vu", + "fr-WF": "fr-wf", + "fr-YT": "fr-yt", + "ga-IE": "ga-ie", + "he-IL": "he-il", + "hi-IN": "hi-in", + "hr-BA": "hr-ba", + "hr-HR": "hr-hr", + "hu-HU": "hu-hu", + "hy-AM": "hy-am", + "id-ID": "id-id", + "is-IS": "is-is", + "it-CH": "it-ch", + "it-IT": "it-it", + "it-SM": "it-sm", + "it-VA": "it-va", + "ja-JP": "ja-jp", + "ka-GE": "ka-ge", + "kk-KZ": "kk-kz", + "km-KH": "km-kh", + "ko-KR": "ko-kr", + "ky-KG": "ky-kg", + "lb-LU": "lb-lu", + "lo-LA": "lo-la", + "lt-LT": "lt-lt", + "lv-LV": "lv-lv", + "mi-NZ": "mi-nz", + "mk-MK": "mk-mk", + "mn-MN": "mn-mn", + "ms-BN": "ms-bn", + "ms-MY": "ms-my", + "ms-SG": "ms-sg", + "mt-MT": "mt-mt", + "nb-NO": "nb-no", + "ne-NP": "ne-np", + "nl-AW": "nl-aw", + "nl-BE": "nl-be", + "nl-CW": "nl-cw", + "nl-NL": "nl-nl", + "nl-SR": "nl-sr", + "nl-SX": "nl-sx", + "nn-NO": "nn-no", + "pl-PL": "pl-pl", + "pt-AO": "pt-ao", + "pt-BR": "pt-br", + "pt-CV": "pt-cv", + "pt-GQ": "pt-gq", + "pt-GW": "pt-gw", + "pt-MO": "pt-mo", + "pt-MZ": "pt-mz", + "pt-PT": "pt-pt", + "pt-ST": "pt-st", + "qu-BO": "qu-bo", + "qu-EC": "qu-ec", + "qu-PE": "qu-pe", + "ro-MD": "ro-md", + "ro-RO": "ro-ro", + "ru-BY": "ru-by", + "ru-KG": "ru-kg", + "ru-KZ": "ru-kz", + "ru-RU": "ru-ru", + "ru-UA": "ru-ua", + "rw-RW": "rw-rw", + "si-LK": "si-lk", + "sk-SK": "sk-sk", + "sl-SI": "sl-si", + "sq-AL": "sq-al", + "sr-BA": "sr-ba", + "sr-ME": "sr-me", + "sr-RS": "sr-rs", + "sv-FI": "sv-fi", + "sv-SE": "sv-se", + "sw-KE": "sw-ke", + "sw-TZ": "sw-tz", + "sw-UG": "sw-ug", + "ta-LK": "ta-lk", + "ta-SG": "ta-sg", + "tg-TJ": "tg-tj", + "th-TH": "th-th", + "ti-ER": "ti-er", + "tk-TM": "tk-tm", + "tr-CY": "tr-cy", + "tr-TR": "tr-tr", + "uk-UA": "uk-ua", + "ur-PK": "ur-pk", + "vi-VN": "vi-vn", + "wo-SN": "wo-sn", + "yo-NG": "yo-ng", + "zh-CN": "zh-cn", + "zh-HK": "en-hk", + "zh-MO": "zh-mo", + "zh-SG": "zh-sg", + "zh-TW": "zh-tw" } }, "bing images": { - "all_locale": null, + "all_locale": "clear", "custom": {}, "data_type": "traits_v1", "languages": { - "ar": "ar", - "bg": "bg", - "bn": "bn", - "ca": "ca", - "cs": "cs", - "da": "da", - "de": "de", - "en": "en", - "es": "es", - "et": "et", - "eu": "eu", - "fi": "fi", - "fr": "fr", - "gl": "gl", - "gu": "gu", - "he": "he", - "hi": "hi", - "hr": "hr", - "hu": "hu", - "is": "is", - "it": "it", - "ja": "jp", - "kn": "kn", - "ko": "ko", - "lt": "lt", - "lv": "lv", - "ml": "ml", - "mr": "mr", - "ms": "ms", - "nb": "nb", - "nl": "nl", - "pa": "pa", - "pl": "pl", - "pt": "pt-pt", - "ro": "ro", - "ru": "ru", - "sk": "sk", - "sl": "sl", - "sr": "sr", - "sv": "sv", - "ta": "ta", - "te": "te", - "th": "th", - "tr": "tr", - "uk": "uk", - "vi": "vi", + "af": "af-af", + "am": "am-am", + "ar": "ar-ar", + "as": "as-as", + "az_Latn": "az-latn", + "be": "be-be", + "bg": "bg-bg", + "bn": "bn-bn", + "bs_Latn": "bs-latn", + "ca": "ca-ca", + "chr": "chr-cher", + "cs": "cs-cs", + "cy": "cy-cy", + "da": "da-dk", + "de": "de-de", + "el": "el-el", + "en": "en-us", + "es": "es-es", + "et": "et-et", + "eu": "eu-eu", + "fa": "prs-prs", + "fi": "fi-fi", + "fil": "fil-fil", + "fr": "fr-fr", + "ga": "ga-ga", + "gd": "gd-gd", + "gl": "gl-gl", + "gu": "gu-gu", + "ha": "ha-latn", + "he": "he-he", + "hi": "hi-hi", + "hr": "hr-hr", + "hu": "hu-hu", + "hy": "hy-hy", + "id": "id-id", + "ig": "ig-ig", + "is": "is-is", + "it": "it-it", + "ja": "ja-ja", + "ka": "ka-ka", + "kk": "kk-kk", + "km": "km-km", + "kn": "kn-kn", + "ko": "ko-ko", + "kok": "kok-kok", + "ku": "ku-arab", + "ky": "ky-ky", + "lb": "lb-lb", + "lo": "lo-lo", + "lt": "lt-lt", + "lv": "lv-lv", + "mi": "mi-mi", + "mk": "mk-mk", + "ml": "ml-ml", + "mn": "mn-cyrl-mn", + "mr": "mr-mr", + "ms": "ms-ms", + "mt": "mt-mt", + "nb": "nb-nb", + "ne": "ne-ne", + "nl": "nl-nl", + "nn": "nn-nn", + "or": "or-or", + "pa_Arab": "pa-arab", + "pa_Guru": "pa-guru", + "pl": "pl-pl", + "pt": "pt-br", + "qu": "quz-quz", + "ro": "ro-ro", + "ru": "ru-ru", + "rw": "rw-rw", + "sd_Arab": "sd-arab", + "si": "si-si", + "sk": "sk-sk", + "sl": "sl-sl", + "sq": "sq-sq", + "sr_Cyrl": "sr-cyrl", + "sr_Latn": "sr-latn", + "sv": "sv-sv", + "sw": "sw-sw", + "ta": "ta-ta", + "te": "te-te", + "tg": "tg-cyrl", + "th": "th-th", + "ti": "ti-ti", + "tk": "tk-tk", + "tr": "tr-tr", + "tt": "tt-tt", + "ug": "ug-ug", + "uk": "uk-uk", + "ur": "ur-ur", + "uz_Latn": "uz-latn", + "vi": "vi-vi", + "wo": "wo-wo", + "xh": "xh-xh", + "yo": "yo-yo", "zh": "zh-hans", "zh_Hans": "zh-hans", - "zh_Hant": "zh-hant" + "zh_Hant": "zh-hant", + "zu": "zu-zu" }, "regions": { - "da-DK": "da-DK", - "de-AT": "de-AT", - "de-CH": "de-CH", - "de-DE": "de-DE", - "en-AU": "en-AU", - "en-CA": "en-CA", - "en-GB": "en-GB", - "en-IN": "en-IN", - "en-MY": "en-MY", - "en-NZ": "en-NZ", - "en-PH": "en-PH", - "en-US": "en-US", - "en-ZA": "en-ZA", - "es-AR": "es-AR", - "es-CL": "es-CL", - "es-ES": "es-ES", - "es-MX": "es-MX", - "es-US": "es-US", - "fi-FI": "fi-FI", - "fr-BE": "fr-BE", - "fr-CA": "fr-CA", - "fr-CH": "fr-CH", - "fr-FR": "fr-FR", - "id-ID": "en-ID", - "it-IT": "it-IT", - "ja-JP": "ja-JP", - "ko-KR": "ko-KR", - "nb-NO": "no-NO", - "nl-BE": "nl-BE", - "nl-NL": "nl-NL", - "pl-PL": "pl-PL", - "pt-BR": "pt-BR", - "ru-RU": "ru-RU", - "sv-SE": "sv-SE", - "tr-TR": "tr-TR", - "zh-CN": "zh-CN", - "zh-HK": "zh-HK", - "zh-TW": "zh-TW" + "am-ET": "am-et", + "ar-AE": "ar-ae", + "ar-BH": "ar-bh", + "ar-DJ": "ar-dj", + "ar-DZ": "ar-dz", + "ar-EG": "ar-eg", + "ar-ER": "ar-er", + "ar-IL": "ar-il", + "ar-IQ": "ar-iq", + "ar-JO": "ar-jo", + "ar-KM": "ar-km", + "ar-KW": "ar-kw", + "ar-LB": "ar-lb", + "ar-LY": "ar-ly", + "ar-MA": "ar-ma", + "ar-MR": "ar-mr", + "ar-OM": "ar-om", + "ar-PS": "ar-ps", + "ar-QA": "ar-qa", + "ar-SA": "ar-sa", + "ar-SD": "ar-sd", + "ar-SO": "ar-so", + "ar-SY": "ar-sy", + "ar-TD": "ar-td", + "ar-TN": "ar-tn", + "ar-YE": "ar-ye", + "be-BY": "be-by", + "bg-BG": "bg-bg", + "bn-BD": "bn-bd", + "ca-AD": "ca-ad", + "cs-CZ": "cs-cz", + "da-DK": "da-dk", + "de-AT": "de-at", + "de-BE": "de-be", + "de-CH": "de-ch", + "de-DE": "de-de", + "de-LI": "de-li", + "de-LU": "de-lu", + "el-CY": "el-cy", + "el-GR": "el-gr", + "en-AG": "en-ag", + "en-AI": "en-ai", + "en-AS": "en-as", + "en-AU": "en-au", + "en-BB": "en-bb", + "en-BI": "en-bi", + "en-BM": "en-bm", + "en-BS": "en-bs", + "en-BW": "en-bw", + "en-BZ": "en-bz", + "en-CA": "en-ca", + "en-CC": "en-cc", + "en-CK": "en-ck", + "en-CM": "en-cm", + "en-CX": "en-cx", + "en-DM": "en-dm", + "en-ER": "en-er", + "en-FJ": "en-fj", + "en-FK": "en-fk", + "en-FM": "en-fm", + "en-GB": "en-gb", + "en-GD": "en-gd", + "en-GG": "en-gg", + "en-GH": "en-gh", + "en-GI": "en-gi", + "en-GM": "en-gm", + "en-GU": "en-gu", + "en-GY": "en-gy", + "en-HK": "en-hk", + "en-IE": "en-ie", + "en-IN": "en-in", + "en-JE": "en-je", + "en-JM": "en-jm", + "en-KE": "en-ke", + "en-KI": "en-ki", + "en-KN": "en-kn", + "en-KY": "en-ky", + "en-LC": "en-lc", + "en-LR": "en-lr", + "en-LS": "en-ls", + "en-MG": "en-mg", + "en-MH": "en-mh", + "en-MP": "en-mp", + "en-MS": "en-ms", + "en-MT": "en-mt", + "en-MU": "en-mu", + "en-MW": "en-mw", + "en-NA": "en-na", + "en-NF": "en-nf", + "en-NG": "en-ng", + "en-NR": "en-nr", + "en-NU": "en-nu", + "en-NZ": "en-nz", + "en-PG": "en-pg", + "en-PH": "en-ph", + "en-PK": "en-pk", + "en-PN": "en-pn", + "en-PR": "en-pr", + "en-PW": "en-pw", + "en-RW": "en-rw", + "en-SB": "en-sb", + "en-SC": "en-sc", + "en-SD": "en-sd", + "en-SG": "en-sg", + "en-SH": "en-sh", + "en-SL": "en-sl", + "en-SS": "en-ss", + "en-SX": "en-sx", + "en-SZ": "en-sz", + "en-TC": "en-tc", + "en-TK": "en-tk", + "en-TO": "en-to", + "en-TT": "en-tt", + "en-TV": "en-tv", + "en-TZ": "en-tz", + "en-UG": "en-ug", + "en-US": "en-us", + "en-VC": "en-vc", + "en-VG": "en-vg", + "en-VI": "en-vi", + "en-VU": "en-vu", + "en-WS": "en-ws", + "en-ZA": "en-za", + "en-ZM": "en-zm", + "en-ZW": "en-zw", + "es-AR": "es-ar", + "es-BO": "es-bo", + "es-CL": "es-cl", + "es-CO": "es-co", + "es-CR": "es-cr", + "es-DO": "es-do", + "es-EC": "es-ec", + "es-ES": "es-es", + "es-GQ": "es-gq", + "es-GT": "es-gt", + "es-HN": "es-hn", + "es-MX": "es-mx", + "es-NI": "es-ni", + "es-PA": "es-pa", + "es-PE": "es-pe", + "es-PR": "es-pr", + "es-PY": "es-py", + "es-SV": "es-sv", + "es-UY": "es-uy", + "es-VE": "es-ve", + "et-EE": "et-ee", + "fa-AF": "fa-af", + "fa-IR": "fa-ir", + "fi-FI": "fi-fi", + "fil-PH": "fil-ph", + "fr-BE": "fr-be", + "fr-BF": "fr-bf", + "fr-BI": "fr-bi", + "fr-BJ": "fr-bj", + "fr-BL": "fr-bl", + "fr-CA": "fr-ca", + "fr-CD": "fr-cd", + "fr-CF": "fr-cf", + "fr-CG": "fr-cg", + "fr-CH": "fr-ch", + "fr-CI": "fr-ci", + "fr-CM": "fr-cm", + "fr-DJ": "fr-dj", + "fr-DZ": "fr-dz", + "fr-FR": "fr-fr", + "fr-GA": "fr-ga", + "fr-GF": "fr-gf", + "fr-GN": "fr-gn", + "fr-GP": "fr-gp", + "fr-GQ": "fr-gq", + "fr-HT": "fr-ht", + "fr-KM": "fr-km", + "fr-LU": "fr-lu", + "fr-MA": "fr-ma", + "fr-MC": "fr-mc", + "fr-MF": "fr-mf", + "fr-MG": "fr-mg", + "fr-ML": "fr-ml", + "fr-MQ": "fr-mq", + "fr-MU": "fr-mu", + "fr-NC": "fr-nc", + "fr-NE": "fr-ne", + "fr-PF": "fr-pf", + "fr-PM": "fr-pm", + "fr-RE": "fr-re", + "fr-RW": "fr-rw", + "fr-SC": "fr-sc", + "fr-SN": "fr-sn", + "fr-SY": "fr-sy", + "fr-TD": "fr-td", + "fr-TG": "fr-tg", + "fr-TN": "fr-tn", + "fr-VU": "fr-vu", + "fr-WF": "fr-wf", + "fr-YT": "fr-yt", + "ga-IE": "ga-ie", + "he-IL": "he-il", + "hi-IN": "hi-in", + "hr-BA": "hr-ba", + "hr-HR": "hr-hr", + "hu-HU": "hu-hu", + "hy-AM": "hy-am", + "id-ID": "id-id", + "is-IS": "is-is", + "it-CH": "it-ch", + "it-IT": "it-it", + "it-SM": "it-sm", + "it-VA": "it-va", + "ja-JP": "ja-jp", + "ka-GE": "ka-ge", + "kk-KZ": "kk-kz", + "km-KH": "km-kh", + "ko-KR": "ko-kr", + "ky-KG": "ky-kg", + "lb-LU": "lb-lu", + "lo-LA": "lo-la", + "lt-LT": "lt-lt", + "lv-LV": "lv-lv", + "mi-NZ": "mi-nz", + "mk-MK": "mk-mk", + "mn-MN": "mn-mn", + "ms-BN": "ms-bn", + "ms-MY": "ms-my", + "ms-SG": "ms-sg", + "mt-MT": "mt-mt", + "nb-NO": "nb-no", + "ne-NP": "ne-np", + "nl-AW": "nl-aw", + "nl-BE": "nl-be", + "nl-CW": "nl-cw", + "nl-NL": "nl-nl", + "nl-SR": "nl-sr", + "nl-SX": "nl-sx", + "nn-NO": "nn-no", + "pl-PL": "pl-pl", + "pt-AO": "pt-ao", + "pt-BR": "pt-br", + "pt-CV": "pt-cv", + "pt-GQ": "pt-gq", + "pt-GW": "pt-gw", + "pt-MO": "pt-mo", + "pt-MZ": "pt-mz", + "pt-PT": "pt-pt", + "pt-ST": "pt-st", + "qu-BO": "qu-bo", + "qu-EC": "qu-ec", + "qu-PE": "qu-pe", + "ro-MD": "ro-md", + "ro-RO": "ro-ro", + "ru-BY": "ru-by", + "ru-KG": "ru-kg", + "ru-KZ": "ru-kz", + "ru-RU": "ru-ru", + "ru-UA": "ru-ua", + "rw-RW": "rw-rw", + "si-LK": "si-lk", + "sk-SK": "sk-sk", + "sl-SI": "sl-si", + "sq-AL": "sq-al", + "sr-BA": "sr-ba", + "sr-ME": "sr-me", + "sr-RS": "sr-rs", + "sv-FI": "sv-fi", + "sv-SE": "sv-se", + "sw-KE": "sw-ke", + "sw-TZ": "sw-tz", + "sw-UG": "sw-ug", + "ta-LK": "ta-lk", + "ta-SG": "ta-sg", + "tg-TJ": "tg-tj", + "th-TH": "th-th", + "ti-ER": "ti-er", + "tk-TM": "tk-tm", + "tr-CY": "tr-cy", + "tr-TR": "tr-tr", + "uk-UA": "uk-ua", + "ur-PK": "ur-pk", + "vi-VN": "vi-vn", + "wo-SN": "wo-sn", + "yo-NG": "yo-ng", + "zh-CN": "zh-cn", + "zh-HK": "en-hk", + "zh-MO": "zh-mo", + "zh-SG": "zh-sg", + "zh-TW": "zh-tw" } }, "bing news": { - "all_locale": "en-WW", + "all_locale": "clear", "custom": {}, "data_type": "traits_v1", "languages": { - "ar": "ar", - "bg": "bg", - "bn": "bn", - "ca": "ca", - "cs": "cs", - "da": "da", - "de": "de", - "en": "en", - "es": "es", - "et": "et", - "eu": "eu", - "fi": "fi", - "fr": "fr", - "gl": "gl", - "gu": "gu", - "he": "he", - "hi": "hi", - "hr": "hr", - "hu": "hu", - "is": "is", - "it": "it", - "ja": "jp", - "kn": "kn", - "ko": "ko", - "lt": "lt", - "lv": "lv", - "ml": "ml", - "mr": "mr", - "ms": "ms", - "nb": "nb", - "nl": "nl", - "pa": "pa", - "pl": "pl", - "pt": "pt-pt", - "ro": "ro", - "ru": "ru", - "sk": "sk", - "sl": "sl", - "sr": "sr", - "sv": "sv", - "ta": "ta", - "te": "te", - "th": "th", - "tr": "tr", - "uk": "uk", - "vi": "vi", + "af": "af-af", + "am": "am-am", + "ar": "ar-ar", + "as": "as-as", + "az_Latn": "az-latn", + "be": "be-be", + "bg": "bg-bg", + "bn": "bn-bn", + "bs_Latn": "bs-latn", + "ca": "ca-ca", + "chr": "chr-cher", + "cs": "cs-cs", + "cy": "cy-cy", + "da": "da-dk", + "de": "de-de", + "el": "el-el", + "en": "en-us", + "es": "es-es", + "et": "et-et", + "eu": "eu-eu", + "fa": "prs-prs", + "fi": "fi-fi", + "fil": "fil-fil", + "fr": "fr-fr", + "ga": "ga-ga", + "gd": "gd-gd", + "gl": "gl-gl", + "gu": "gu-gu", + "ha": "ha-latn", + "he": "he-he", + "hi": "hi-hi", + "hr": "hr-hr", + "hu": "hu-hu", + "hy": "hy-hy", + "id": "id-id", + "ig": "ig-ig", + "is": "is-is", + "it": "it-it", + "ja": "ja-ja", + "ka": "ka-ka", + "kk": "kk-kk", + "km": "km-km", + "kn": "kn-kn", + "ko": "ko-ko", + "kok": "kok-kok", + "ku": "ku-arab", + "ky": "ky-ky", + "lb": "lb-lb", + "lo": "lo-lo", + "lt": "lt-lt", + "lv": "lv-lv", + "mi": "mi-mi", + "mk": "mk-mk", + "ml": "ml-ml", + "mn": "mn-cyrl-mn", + "mr": "mr-mr", + "ms": "ms-ms", + "mt": "mt-mt", + "nb": "nb-nb", + "ne": "ne-ne", + "nl": "nl-nl", + "nn": "nn-nn", + "or": "or-or", + "pa_Arab": "pa-arab", + "pa_Guru": "pa-guru", + "pl": "pl-pl", + "pt": "pt-br", + "qu": "quz-quz", + "ro": "ro-ro", + "ru": "ru-ru", + "rw": "rw-rw", + "sd_Arab": "sd-arab", + "si": "si-si", + "sk": "sk-sk", + "sl": "sl-sl", + "sq": "sq-sq", + "sr_Cyrl": "sr-cyrl", + "sr_Latn": "sr-latn", + "sv": "sv-sv", + "sw": "sw-sw", + "ta": "ta-ta", + "te": "te-te", + "tg": "tg-cyrl", + "th": "th-th", + "ti": "ti-ti", + "tk": "tk-tk", + "tr": "tr-tr", + "tt": "tt-tt", + "ug": "ug-ug", + "uk": "uk-uk", + "ur": "ur-ur", + "uz_Latn": "uz-latn", + "vi": "vi-vi", + "wo": "wo-wo", + "xh": "xh-xh", + "yo": "yo-yo", "zh": "zh-hans", "zh_Hans": "zh-hans", - "zh_Hant": "zh-hant" + "zh_Hant": "zh-hant", + "zu": "zu-zu" }, "regions": { - "da-DK": "da-DK", - "de-DE": "de-DE", - "en-AU": "en-AU", - "en-GB": "en-GB", - "en-US": "en-US", - "es-CL": "es-CL", - "es-MX": "es-MX", - "es-US": "es-US", - "fi-FI": "fi-FI", - "fr-CA": "fr-CA", - "fr-FR": "fr-FR", - "it-IT": "it-IT", - "pt-BR": "pt-BR", - "zh-CN": "zh-CN" + "am-ET": "am-et", + "ar-AE": "ar-ae", + "ar-BH": "ar-bh", + "ar-DJ": "ar-dj", + "ar-DZ": "ar-dz", + "ar-EG": "ar-eg", + "ar-ER": "ar-er", + "ar-IL": "ar-il", + "ar-IQ": "ar-iq", + "ar-JO": "ar-jo", + "ar-KM": "ar-km", + "ar-KW": "ar-kw", + "ar-LB": "ar-lb", + "ar-LY": "ar-ly", + "ar-MA": "ar-ma", + "ar-MR": "ar-mr", + "ar-OM": "ar-om", + "ar-PS": "ar-ps", + "ar-QA": "ar-qa", + "ar-SA": "ar-sa", + "ar-SD": "ar-sd", + "ar-SO": "ar-so", + "ar-SY": "ar-sy", + "ar-TD": "ar-td", + "ar-TN": "ar-tn", + "ar-YE": "ar-ye", + "be-BY": "be-by", + "bg-BG": "bg-bg", + "bn-BD": "bn-bd", + "ca-AD": "ca-ad", + "cs-CZ": "cs-cz", + "da-DK": "da-dk", + "de-AT": "de-at", + "de-BE": "de-be", + "de-CH": "de-ch", + "de-DE": "de-de", + "de-LI": "de-li", + "de-LU": "de-lu", + "el-CY": "el-cy", + "el-GR": "el-gr", + "en-AG": "en-ag", + "en-AI": "en-ai", + "en-AS": "en-as", + "en-AU": "en-au", + "en-BB": "en-bb", + "en-BI": "en-bi", + "en-BM": "en-bm", + "en-BS": "en-bs", + "en-BW": "en-bw", + "en-BZ": "en-bz", + "en-CA": "en-ca", + "en-CC": "en-cc", + "en-CK": "en-ck", + "en-CM": "en-cm", + "en-CX": "en-cx", + "en-DM": "en-dm", + "en-ER": "en-er", + "en-FJ": "en-fj", + "en-FK": "en-fk", + "en-FM": "en-fm", + "en-GB": "en-gb", + "en-GD": "en-gd", + "en-GG": "en-gg", + "en-GH": "en-gh", + "en-GI": "en-gi", + "en-GM": "en-gm", + "en-GU": "en-gu", + "en-GY": "en-gy", + "en-HK": "en-hk", + "en-IE": "en-ie", + "en-IN": "en-in", + "en-JE": "en-je", + "en-JM": "en-jm", + "en-KE": "en-ke", + "en-KI": "en-ki", + "en-KN": "en-kn", + "en-KY": "en-ky", + "en-LC": "en-lc", + "en-LR": "en-lr", + "en-LS": "en-ls", + "en-MG": "en-mg", + "en-MH": "en-mh", + "en-MP": "en-mp", + "en-MS": "en-ms", + "en-MT": "en-mt", + "en-MU": "en-mu", + "en-MW": "en-mw", + "en-NA": "en-na", + "en-NF": "en-nf", + "en-NG": "en-ng", + "en-NR": "en-nr", + "en-NU": "en-nu", + "en-NZ": "en-nz", + "en-PG": "en-pg", + "en-PH": "en-ph", + "en-PK": "en-pk", + "en-PN": "en-pn", + "en-PR": "en-pr", + "en-PW": "en-pw", + "en-RW": "en-rw", + "en-SB": "en-sb", + "en-SC": "en-sc", + "en-SD": "en-sd", + "en-SG": "en-sg", + "en-SH": "en-sh", + "en-SL": "en-sl", + "en-SS": "en-ss", + "en-SX": "en-sx", + "en-SZ": "en-sz", + "en-TC": "en-tc", + "en-TK": "en-tk", + "en-TO": "en-to", + "en-TT": "en-tt", + "en-TV": "en-tv", + "en-TZ": "en-tz", + "en-UG": "en-ug", + "en-US": "en-us", + "en-VC": "en-vc", + "en-VG": "en-vg", + "en-VI": "en-vi", + "en-VU": "en-vu", + "en-WS": "en-ws", + "en-ZA": "en-za", + "en-ZM": "en-zm", + "en-ZW": "en-zw", + "es-AR": "es-ar", + "es-BO": "es-bo", + "es-CL": "es-cl", + "es-CO": "es-co", + "es-CR": "es-cr", + "es-DO": "es-do", + "es-EC": "es-ec", + "es-ES": "es-es", + "es-GQ": "es-gq", + "es-GT": "es-gt", + "es-HN": "es-hn", + "es-MX": "es-mx", + "es-NI": "es-ni", + "es-PA": "es-pa", + "es-PE": "es-pe", + "es-PR": "es-pr", + "es-PY": "es-py", + "es-SV": "es-sv", + "es-UY": "es-uy", + "es-VE": "es-ve", + "et-EE": "et-ee", + "fa-AF": "fa-af", + "fa-IR": "fa-ir", + "fi-FI": "fi-fi", + "fil-PH": "fil-ph", + "fr-BE": "fr-be", + "fr-BF": "fr-bf", + "fr-BI": "fr-bi", + "fr-BJ": "fr-bj", + "fr-BL": "fr-bl", + "fr-CA": "fr-ca", + "fr-CD": "fr-cd", + "fr-CF": "fr-cf", + "fr-CG": "fr-cg", + "fr-CH": "fr-ch", + "fr-CI": "fr-ci", + "fr-CM": "fr-cm", + "fr-DJ": "fr-dj", + "fr-DZ": "fr-dz", + "fr-FR": "fr-fr", + "fr-GA": "fr-ga", + "fr-GF": "fr-gf", + "fr-GN": "fr-gn", + "fr-GP": "fr-gp", + "fr-GQ": "fr-gq", + "fr-HT": "fr-ht", + "fr-KM": "fr-km", + "fr-LU": "fr-lu", + "fr-MA": "fr-ma", + "fr-MC": "fr-mc", + "fr-MF": "fr-mf", + "fr-MG": "fr-mg", + "fr-ML": "fr-ml", + "fr-MQ": "fr-mq", + "fr-MU": "fr-mu", + "fr-NC": "fr-nc", + "fr-NE": "fr-ne", + "fr-PF": "fr-pf", + "fr-PM": "fr-pm", + "fr-RE": "fr-re", + "fr-RW": "fr-rw", + "fr-SC": "fr-sc", + "fr-SN": "fr-sn", + "fr-SY": "fr-sy", + "fr-TD": "fr-td", + "fr-TG": "fr-tg", + "fr-TN": "fr-tn", + "fr-VU": "fr-vu", + "fr-WF": "fr-wf", + "fr-YT": "fr-yt", + "ga-IE": "ga-ie", + "he-IL": "he-il", + "hi-IN": "hi-in", + "hr-BA": "hr-ba", + "hr-HR": "hr-hr", + "hu-HU": "hu-hu", + "hy-AM": "hy-am", + "id-ID": "id-id", + "is-IS": "is-is", + "it-CH": "it-ch", + "it-IT": "it-it", + "it-SM": "it-sm", + "it-VA": "it-va", + "ja-JP": "ja-jp", + "ka-GE": "ka-ge", + "kk-KZ": "kk-kz", + "km-KH": "km-kh", + "ko-KR": "ko-kr", + "ky-KG": "ky-kg", + "lb-LU": "lb-lu", + "lo-LA": "lo-la", + "lt-LT": "lt-lt", + "lv-LV": "lv-lv", + "mi-NZ": "mi-nz", + "mk-MK": "mk-mk", + "mn-MN": "mn-mn", + "ms-BN": "ms-bn", + "ms-MY": "ms-my", + "ms-SG": "ms-sg", + "mt-MT": "mt-mt", + "nb-NO": "nb-no", + "ne-NP": "ne-np", + "nl-AW": "nl-aw", + "nl-BE": "nl-be", + "nl-CW": "nl-cw", + "nl-NL": "nl-nl", + "nl-SR": "nl-sr", + "nl-SX": "nl-sx", + "nn-NO": "nn-no", + "pl-PL": "pl-pl", + "pt-AO": "pt-ao", + "pt-BR": "pt-br", + "pt-CV": "pt-cv", + "pt-GQ": "pt-gq", + "pt-GW": "pt-gw", + "pt-MO": "pt-mo", + "pt-MZ": "pt-mz", + "pt-PT": "pt-pt", + "pt-ST": "pt-st", + "qu-BO": "qu-bo", + "qu-EC": "qu-ec", + "qu-PE": "qu-pe", + "ro-MD": "ro-md", + "ro-RO": "ro-ro", + "ru-BY": "ru-by", + "ru-KG": "ru-kg", + "ru-KZ": "ru-kz", + "ru-RU": "ru-ru", + "ru-UA": "ru-ua", + "rw-RW": "rw-rw", + "si-LK": "si-lk", + "sk-SK": "sk-sk", + "sl-SI": "sl-si", + "sq-AL": "sq-al", + "sr-BA": "sr-ba", + "sr-ME": "sr-me", + "sr-RS": "sr-rs", + "sv-FI": "sv-fi", + "sv-SE": "sv-se", + "sw-KE": "sw-ke", + "sw-TZ": "sw-tz", + "sw-UG": "sw-ug", + "ta-LK": "ta-lk", + "ta-SG": "ta-sg", + "tg-TJ": "tg-tj", + "th-TH": "th-th", + "ti-ER": "ti-er", + "tk-TM": "tk-tm", + "tr-CY": "tr-cy", + "tr-TR": "tr-tr", + "uk-UA": "uk-ua", + "ur-PK": "ur-pk", + "vi-VN": "vi-vn", + "wo-SN": "wo-sn", + "yo-NG": "yo-ng", + "zh-CN": "en-hk", + "zh-HK": "en-hk", + "zh-MO": "zh-mo", + "zh-SG": "zh-sg", + "zh-TW": "zh-tw" } }, "bing videos": { - "all_locale": null, + "all_locale": "clear", "custom": {}, "data_type": "traits_v1", "languages": { - "ar": "ar", - "bg": "bg", - "bn": "bn", - "ca": "ca", - "cs": "cs", - "da": "da", - "de": "de", - "en": "en", - "es": "es", - "et": "et", - "eu": "eu", - "fi": "fi", - "fr": "fr", - "gl": "gl", - "gu": "gu", - "he": "he", - "hi": "hi", - "hr": "hr", - "hu": "hu", - "is": "is", - "it": "it", - "ja": "jp", - "kn": "kn", - "ko": "ko", - "lt": "lt", - "lv": "lv", - "ml": "ml", - "mr": "mr", - "ms": "ms", - "nb": "nb", - "nl": "nl", - "pa": "pa", - "pl": "pl", - "pt": "pt-pt", - "ro": "ro", - "ru": "ru", - "sk": "sk", - "sl": "sl", - "sr": "sr", - "sv": "sv", - "ta": "ta", - "te": "te", - "th": "th", - "tr": "tr", - "uk": "uk", - "vi": "vi", + "af": "af-af", + "am": "am-am", + "ar": "ar-ar", + "as": "as-as", + "az_Latn": "az-latn", + "be": "be-be", + "bg": "bg-bg", + "bn": "bn-bn", + "bs_Latn": "bs-latn", + "ca": "ca-ca", + "chr": "chr-cher", + "cs": "cs-cs", + "cy": "cy-cy", + "da": "da-dk", + "de": "de-de", + "el": "el-el", + "en": "en-us", + "es": "es-es", + "et": "et-et", + "eu": "eu-eu", + "fa": "prs-prs", + "fi": "fi-fi", + "fil": "fil-fil", + "fr": "fr-fr", + "ga": "ga-ga", + "gd": "gd-gd", + "gl": "gl-gl", + "gu": "gu-gu", + "ha": "ha-latn", + "he": "he-he", + "hi": "hi-hi", + "hr": "hr-hr", + "hu": "hu-hu", + "hy": "hy-hy", + "id": "id-id", + "ig": "ig-ig", + "is": "is-is", + "it": "it-it", + "ja": "ja-ja", + "ka": "ka-ka", + "kk": "kk-kk", + "km": "km-km", + "kn": "kn-kn", + "ko": "ko-ko", + "kok": "kok-kok", + "ku": "ku-arab", + "ky": "ky-ky", + "lb": "lb-lb", + "lo": "lo-lo", + "lt": "lt-lt", + "lv": "lv-lv", + "mi": "mi-mi", + "mk": "mk-mk", + "ml": "ml-ml", + "mn": "mn-cyrl-mn", + "mr": "mr-mr", + "ms": "ms-ms", + "mt": "mt-mt", + "nb": "nb-nb", + "ne": "ne-ne", + "nl": "nl-nl", + "nn": "nn-nn", + "or": "or-or", + "pa_Arab": "pa-arab", + "pa_Guru": "pa-guru", + "pl": "pl-pl", + "pt": "pt-br", + "qu": "quz-quz", + "ro": "ro-ro", + "ru": "ru-ru", + "rw": "rw-rw", + "sd_Arab": "sd-arab", + "si": "si-si", + "sk": "sk-sk", + "sl": "sl-sl", + "sq": "sq-sq", + "sr_Cyrl": "sr-cyrl", + "sr_Latn": "sr-latn", + "sv": "sv-sv", + "sw": "sw-sw", + "ta": "ta-ta", + "te": "te-te", + "tg": "tg-cyrl", + "th": "th-th", + "ti": "ti-ti", + "tk": "tk-tk", + "tr": "tr-tr", + "tt": "tt-tt", + "ug": "ug-ug", + "uk": "uk-uk", + "ur": "ur-ur", + "uz_Latn": "uz-latn", + "vi": "vi-vi", + "wo": "wo-wo", + "xh": "xh-xh", + "yo": "yo-yo", "zh": "zh-hans", "zh_Hans": "zh-hans", - "zh_Hant": "zh-hant" + "zh_Hant": "zh-hant", + "zu": "zu-zu" }, "regions": { - "da-DK": "da-DK", - "de-AT": "de-AT", - "de-CH": "de-CH", - "de-DE": "de-DE", - "en-AU": "en-AU", - "en-CA": "en-CA", - "en-GB": "en-GB", - "en-IN": "en-IN", - "en-MY": "en-MY", - "en-NZ": "en-NZ", - "en-PH": "en-PH", - "en-US": "en-US", - "en-ZA": "en-ZA", - "es-AR": "es-AR", - "es-CL": "es-CL", - "es-ES": "es-ES", - "es-MX": "es-MX", - "es-US": "es-US", - "fi-FI": "fi-FI", - "fr-BE": "fr-BE", - "fr-CA": "fr-CA", - "fr-CH": "fr-CH", - "fr-FR": "fr-FR", - "id-ID": "en-ID", - "it-IT": "it-IT", - "ja-JP": "ja-JP", - "ko-KR": "ko-KR", - "nb-NO": "no-NO", - "nl-BE": "nl-BE", - "nl-NL": "nl-NL", - "pl-PL": "pl-PL", - "pt-BR": "pt-BR", - "ru-RU": "ru-RU", - "sv-SE": "sv-SE", - "tr-TR": "tr-TR", - "zh-CN": "zh-CN", - "zh-HK": "zh-HK", - "zh-TW": "zh-TW" + "am-ET": "am-et", + "ar-AE": "ar-ae", + "ar-BH": "ar-bh", + "ar-DJ": "ar-dj", + "ar-DZ": "ar-dz", + "ar-EG": "ar-eg", + "ar-ER": "ar-er", + "ar-IL": "ar-il", + "ar-IQ": "ar-iq", + "ar-JO": "ar-jo", + "ar-KM": "ar-km", + "ar-KW": "ar-kw", + "ar-LB": "ar-lb", + "ar-LY": "ar-ly", + "ar-MA": "ar-ma", + "ar-MR": "ar-mr", + "ar-OM": "ar-om", + "ar-PS": "ar-ps", + "ar-QA": "ar-qa", + "ar-SA": "ar-sa", + "ar-SD": "ar-sd", + "ar-SO": "ar-so", + "ar-SY": "ar-sy", + "ar-TD": "ar-td", + "ar-TN": "ar-tn", + "ar-YE": "ar-ye", + "be-BY": "be-by", + "bg-BG": "bg-bg", + "bn-BD": "bn-bd", + "ca-AD": "ca-ad", + "cs-CZ": "cs-cz", + "da-DK": "da-dk", + "de-AT": "de-at", + "de-BE": "de-be", + "de-CH": "de-ch", + "de-DE": "de-de", + "de-LI": "de-li", + "de-LU": "de-lu", + "el-CY": "el-cy", + "el-GR": "el-gr", + "en-AG": "en-ag", + "en-AI": "en-ai", + "en-AS": "en-as", + "en-AU": "en-au", + "en-BB": "en-bb", + "en-BI": "en-bi", + "en-BM": "en-bm", + "en-BS": "en-bs", + "en-BW": "en-bw", + "en-BZ": "en-bz", + "en-CA": "en-ca", + "en-CC": "en-cc", + "en-CK": "en-ck", + "en-CM": "en-cm", + "en-CX": "en-cx", + "en-DM": "en-dm", + "en-ER": "en-er", + "en-FJ": "en-fj", + "en-FK": "en-fk", + "en-FM": "en-fm", + "en-GB": "en-gb", + "en-GD": "en-gd", + "en-GG": "en-gg", + "en-GH": "en-gh", + "en-GI": "en-gi", + "en-GM": "en-gm", + "en-GU": "en-gu", + "en-GY": "en-gy", + "en-HK": "en-hk", + "en-IE": "en-ie", + "en-IN": "en-in", + "en-JE": "en-je", + "en-JM": "en-jm", + "en-KE": "en-ke", + "en-KI": "en-ki", + "en-KN": "en-kn", + "en-KY": "en-ky", + "en-LC": "en-lc", + "en-LR": "en-lr", + "en-LS": "en-ls", + "en-MG": "en-mg", + "en-MH": "en-mh", + "en-MP": "en-mp", + "en-MS": "en-ms", + "en-MT": "en-mt", + "en-MU": "en-mu", + "en-MW": "en-mw", + "en-NA": "en-na", + "en-NF": "en-nf", + "en-NG": "en-ng", + "en-NR": "en-nr", + "en-NU": "en-nu", + "en-NZ": "en-nz", + "en-PG": "en-pg", + "en-PH": "en-ph", + "en-PK": "en-pk", + "en-PN": "en-pn", + "en-PR": "en-pr", + "en-PW": "en-pw", + "en-RW": "en-rw", + "en-SB": "en-sb", + "en-SC": "en-sc", + "en-SD": "en-sd", + "en-SG": "en-sg", + "en-SH": "en-sh", + "en-SL": "en-sl", + "en-SS": "en-ss", + "en-SX": "en-sx", + "en-SZ": "en-sz", + "en-TC": "en-tc", + "en-TK": "en-tk", + "en-TO": "en-to", + "en-TT": "en-tt", + "en-TV": "en-tv", + "en-TZ": "en-tz", + "en-UG": "en-ug", + "en-US": "en-us", + "en-VC": "en-vc", + "en-VG": "en-vg", + "en-VI": "en-vi", + "en-VU": "en-vu", + "en-WS": "en-ws", + "en-ZA": "en-za", + "en-ZM": "en-zm", + "en-ZW": "en-zw", + "es-AR": "es-ar", + "es-BO": "es-bo", + "es-CL": "es-cl", + "es-CO": "es-co", + "es-CR": "es-cr", + "es-DO": "es-do", + "es-EC": "es-ec", + "es-ES": "es-es", + "es-GQ": "es-gq", + "es-GT": "es-gt", + "es-HN": "es-hn", + "es-MX": "es-mx", + "es-NI": "es-ni", + "es-PA": "es-pa", + "es-PE": "es-pe", + "es-PR": "es-pr", + "es-PY": "es-py", + "es-SV": "es-sv", + "es-UY": "es-uy", + "es-VE": "es-ve", + "et-EE": "et-ee", + "fa-AF": "fa-af", + "fa-IR": "fa-ir", + "fi-FI": "fi-fi", + "fil-PH": "fil-ph", + "fr-BE": "fr-be", + "fr-BF": "fr-bf", + "fr-BI": "fr-bi", + "fr-BJ": "fr-bj", + "fr-BL": "fr-bl", + "fr-CA": "fr-ca", + "fr-CD": "fr-cd", + "fr-CF": "fr-cf", + "fr-CG": "fr-cg", + "fr-CH": "fr-ch", + "fr-CI": "fr-ci", + "fr-CM": "fr-cm", + "fr-DJ": "fr-dj", + "fr-DZ": "fr-dz", + "fr-FR": "fr-fr", + "fr-GA": "fr-ga", + "fr-GF": "fr-gf", + "fr-GN": "fr-gn", + "fr-GP": "fr-gp", + "fr-GQ": "fr-gq", + "fr-HT": "fr-ht", + "fr-KM": "fr-km", + "fr-LU": "fr-lu", + "fr-MA": "fr-ma", + "fr-MC": "fr-mc", + "fr-MF": "fr-mf", + "fr-MG": "fr-mg", + "fr-ML": "fr-ml", + "fr-MQ": "fr-mq", + "fr-MU": "fr-mu", + "fr-NC": "fr-nc", + "fr-NE": "fr-ne", + "fr-PF": "fr-pf", + "fr-PM": "fr-pm", + "fr-RE": "fr-re", + "fr-RW": "fr-rw", + "fr-SC": "fr-sc", + "fr-SN": "fr-sn", + "fr-SY": "fr-sy", + "fr-TD": "fr-td", + "fr-TG": "fr-tg", + "fr-TN": "fr-tn", + "fr-VU": "fr-vu", + "fr-WF": "fr-wf", + "fr-YT": "fr-yt", + "ga-IE": "ga-ie", + "he-IL": "he-il", + "hi-IN": "hi-in", + "hr-BA": "hr-ba", + "hr-HR": "hr-hr", + "hu-HU": "hu-hu", + "hy-AM": "hy-am", + "id-ID": "id-id", + "is-IS": "is-is", + "it-CH": "it-ch", + "it-IT": "it-it", + "it-SM": "it-sm", + "it-VA": "it-va", + "ja-JP": "ja-jp", + "ka-GE": "ka-ge", + "kk-KZ": "kk-kz", + "km-KH": "km-kh", + "ko-KR": "ko-kr", + "ky-KG": "ky-kg", + "lb-LU": "lb-lu", + "lo-LA": "lo-la", + "lt-LT": "lt-lt", + "lv-LV": "lv-lv", + "mi-NZ": "mi-nz", + "mk-MK": "mk-mk", + "mn-MN": "mn-mn", + "ms-BN": "ms-bn", + "ms-MY": "ms-my", + "ms-SG": "ms-sg", + "mt-MT": "mt-mt", + "nb-NO": "nb-no", + "ne-NP": "ne-np", + "nl-AW": "nl-aw", + "nl-BE": "nl-be", + "nl-CW": "nl-cw", + "nl-NL": "nl-nl", + "nl-SR": "nl-sr", + "nl-SX": "nl-sx", + "nn-NO": "nn-no", + "pl-PL": "pl-pl", + "pt-AO": "pt-ao", + "pt-BR": "pt-br", + "pt-CV": "pt-cv", + "pt-GQ": "pt-gq", + "pt-GW": "pt-gw", + "pt-MO": "pt-mo", + "pt-MZ": "pt-mz", + "pt-PT": "pt-pt", + "pt-ST": "pt-st", + "qu-BO": "qu-bo", + "qu-EC": "qu-ec", + "qu-PE": "qu-pe", + "ro-MD": "ro-md", + "ro-RO": "ro-ro", + "ru-BY": "ru-by", + "ru-KG": "ru-kg", + "ru-KZ": "ru-kz", + "ru-RU": "ru-ru", + "ru-UA": "ru-ua", + "rw-RW": "rw-rw", + "si-LK": "si-lk", + "sk-SK": "sk-sk", + "sl-SI": "sl-si", + "sq-AL": "sq-al", + "sr-BA": "sr-ba", + "sr-ME": "sr-me", + "sr-RS": "sr-rs", + "sv-FI": "sv-fi", + "sv-SE": "sv-se", + "sw-KE": "sw-ke", + "sw-TZ": "sw-tz", + "sw-UG": "sw-ug", + "ta-LK": "ta-lk", + "ta-SG": "ta-sg", + "tg-TJ": "tg-tj", + "th-TH": "th-th", + "ti-ER": "ti-er", + "tk-TM": "tk-tm", + "tr-CY": "tr-cy", + "tr-TR": "tr-tr", + "uk-UA": "uk-ua", + "ur-PK": "ur-pk", + "vi-VN": "vi-vn", + "wo-SN": "wo-sn", + "yo-NG": "yo-ng", + "zh-CN": "zh-cn", + "zh-HK": "en-hk", + "zh-MO": "zh-mo", + "zh-SG": "zh-sg", + "zh-TW": "zh-tw" } }, "brave": { diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 9086623ea..ae001ffac 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -8,19 +8,18 @@ implementations are shared by other engines: - :ref:`bing videos engine` On the `preference page`_ Bing offers a lot of languages an regions (see section -'Search results languages' and 'Country/region'). However, the abundant choice -does not correspond to reality, where Bing has a full-text indexer only for a -limited number of languages. By example: you can select a language like Māori -but you never get a result in this language. +LANGUAGE and COUNTRY/REGION). The Language is the language of the UI, we need +in SearXNG to get the translations of data such as *"published last week"*. -What comes a bit closer to the truth are the `search-APIs`_ but they don`t seem -to be completely correct either (if you take a closer look you will find some -inaccuracies there too): +There is a description of the offical search-APIs_, unfortunately this is not +the API we can use or that bing itself would use. You can look up some things +in the API to get a better picture of bing, but the value specifications like +the market codes are usually outdated or at least no longer used by bing itself. -- :py:obj:`searx.engines.bing.bing_traits_url` -- :py:obj:`searx.engines.bing_videos.bing_traits_url` -- :py:obj:`searx.engines.bing_images.bing_traits_url` -- :py:obj:`searx.engines.bing_news.bing_traits_url` +The market codes have been harmonized and are identical for web, video and +images. The news area has also been harmonized with the other categories. Only +political adjustments still seem to be made -- for example, there is no news +category for the Chinese market. .. _preference page: https://www.bing.com/account/general .. _search-APIs: https://learn.microsoft.com/en-us/bing/search-apis/ @@ -30,9 +29,8 @@ inaccuracies there too): from typing import TYPE_CHECKING import base64 -import datetime import re -import uuid +import time from urllib.parse import parse_qs, urlencode, urlparse from lxml import html import babel @@ -45,7 +43,7 @@ from searx.enginelib.traits import EngineTraits if TYPE_CHECKING: import logging - logger: logging.Logger + logger = logging.getLogger() traits: EngineTraits @@ -58,124 +56,63 @@ about = { "results": 'HTML', } -send_accept_language_header = True -"""Bing tries to guess user's language and territory from the HTTP -Accept-Language. Optional the user can select a search-language (can be -different to the UI language) and a region (market code).""" - # engine dependent config categories = ['general', 'web'] paging = True time_range_support = True safesearch = True -safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'} # cookie: ADLT=STRICT +"""Bing results are always SFW. To get NSFW links from bing some age +verification by a cookie is needed / thats not possible in SearXNG. +""" base_url = 'https://www.bing.com/search' """Bing (Web) search URL""" -bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes' -"""Bing (Web) search API description""" + +def _page_offset(pageno): + return (int(pageno) - 1) * 10 + 1 -def _get_offset_from_pageno(pageno): - return (pageno - 1) * 10 + 1 - - -def set_bing_cookies(params, engine_language, engine_region, SID): - - # set cookies - # ----------- - - params['cookies']['_EDGE_V'] = '1' - - # _EDGE_S: F=1&SID=3A5253BD6BCA609509B741876AF961CA&mkt=zh-tw - _EDGE_S = [ - 'F=1', - 'SID=%s' % SID, - 'mkt=%s' % engine_region.lower(), - 'ui=%s' % engine_language.lower(), - ] - params['cookies']['_EDGE_S'] = '&'.join(_EDGE_S) - logger.debug("cookie _EDGE_S=%s", params['cookies']['_EDGE_S']) - - # "_EDGE_CD": "m=zh-tw", - - _EDGE_CD = [ # pylint: disable=invalid-name - 'm=%s' % engine_region.lower(), # search region: zh-cn - 'u=%s' % engine_language.lower(), # UI: en-us - ] - - params['cookies']['_EDGE_CD'] = '&'.join(_EDGE_CD) + ';' - logger.debug("cookie _EDGE_CD=%s", params['cookies']['_EDGE_CD']) - - SRCHHPGUSR = [ # pylint: disable=invalid-name - 'SRCHLANG=%s' % engine_language, - # Trying to set ADLT cookie here seems not to have any effect, I assume - # there is some age verification by a cookie (and/or session ID) needed, - # to disable the SafeSearch. - 'ADLT=%s' % safesearch_types.get(params['safesearch'], 'DEMOTE'), - ] - params['cookies']['SRCHHPGUSR'] = '&'.join(SRCHHPGUSR) - logger.debug("cookie SRCHHPGUSR=%s", params['cookies']['SRCHHPGUSR']) +def set_bing_cookies(params, engine_language, engine_region): + params['cookies']['_EDGE_CD'] = f'm={engine_region}&u={engine_language}' + params['cookies']['_EDGE_S'] = f'mkt={engine_region}&ui={engine_language}' + logger.debug("bing cookies: %s", params['cookies']) def request(query, params): """Assemble a Bing-Web request.""" - engine_region = traits.get_region(params['searxng_locale'], 'en-US') - engine_language = traits.get_language(params['searxng_locale'], 'en') + engine_region = traits.get_region(params['searxng_locale'], traits.all_locale) # type: ignore + engine_language = traits.get_language(params['searxng_locale'], 'en') # type: ignore + set_bing_cookies(params, engine_language, engine_region) - SID = uuid.uuid1().hex.upper() - CVID = uuid.uuid1().hex.upper() - - set_bing_cookies(params, engine_language, engine_region, SID) - - # build URL query - # --------------- - - # query term - page = int(params.get('pageno', 1)) + page = params.get('pageno', 1) query_params = { - # fmt: off 'q': query, + # if arg 'pq' is missed, somtimes on page 4 we get results from page 1, + # don't ask why it is only sometimes / its M$ and they have never been + # deterministic ;) 'pq': query, - 'cvid': CVID, - 'qs': 'n', - 'sp': '-1' - # fmt: on } - # page + # To get correct page, arg first and this arg FORM is needed, the value PERE + # is on page 2, on page 3 its PERE1 and on page 4 its PERE2 .. and so forth. + # The 'first' arg should never send on page 1. + if page > 1: - referer = base_url + '?' + urlencode(query_params) - params['headers']['Referer'] = referer - logger.debug("headers.Referer --> %s", referer) - - query_params['first'] = _get_offset_from_pageno(page) - + query_params['first'] = _page_offset(page) # see also arg FORM if page == 2: query_params['FORM'] = 'PERE' elif page > 2: query_params['FORM'] = 'PERE%s' % (page - 2) - filters = '' - if params['time_range']: - query_params['filt'] = 'custom' + params['url'] = f'{base_url}?{urlencode(query_params)}' - if params['time_range'] == 'day': - filters = 'ex1:"ez1"' - elif params['time_range'] == 'week': - filters = 'ex1:"ez2"' - elif params['time_range'] == 'month': - filters = 'ex1:"ez3"' - elif params['time_range'] == 'year': - epoch_1970 = datetime.date(1970, 1, 1) - today_no = (datetime.date.today() - epoch_1970).days - filters = 'ex1:"ez5_%s_%s"' % (today_no - 365, today_no) + if params.get('time_range'): + unix_day = int(time.time() / 86400) + time_ranges = {'day': '1', 'week': '2', 'month': '3', 'year': f'5_{unix_day-365}_{unix_day}'} + params['url'] += f'&filters=ex1:"ez{time_ranges[params["time_range"]]}"' - params['url'] = base_url + '?' + urlencode(query_params) - if filters: - params['url'] = params['url'] + '&filters=' + filters return params @@ -197,10 +134,11 @@ def response(resp): url = link.attrib.get('href') title = extract_text(link) - content = eval_xpath(result, '(.//p)[1]') + content = eval_xpath(result, './/p') for p in content: - # Make sure that the element is free of links - for e in p.xpath('.//a'): + # Make sure that the element is free of: + # Web + for e in p.xpath('.//span[@class="algoSlug_icon"]'): e.getparent().remove(e) content = extract_text(content) @@ -236,7 +174,7 @@ def response(resp): except Exception as e: # pylint: disable=broad-except logger.debug('result error :\n%s', e) - if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: + if result_len and _page_offset(resp.search_params.get("pageno", 0)) > result_len: # Avoid reading more results than avalaible. # For example, if there is 100 results from some search and we try to get results from 120 to 130, # Bing will send back the results from 0 to 10 and no error. @@ -249,72 +187,76 @@ def response(resp): def fetch_traits(engine_traits: EngineTraits): """Fetch languages and regions from Bing-Web.""" - - xpath_market_codes = '//table[1]/tbody/tr/td[3]' - # xpath_country_codes = '//table[2]/tbody/tr/td[2]' - xpath_language_codes = '//table[3]/tbody/tr/td[2]' - - _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes) - - -def _fetch_traits(engine_traits: EngineTraits, url: str, xpath_language_codes: str, xpath_market_codes: str): - # pylint: disable=too-many-locals,import-outside-toplevel + # pylint: disable=import-outside-toplevel from searx.network import get # see https://github.com/searxng/searxng/issues/762 - # insert alias to map from a language (zh) to a language + script (zh_Hans) - engine_traits.languages['zh'] = 'zh-hans' - - resp = get(url) - + resp = get("https://www.bing.com/account/general") if not resp.ok: # type: ignore - print("ERROR: response from peertube is not OK.") + print("ERROR: response from bing is not OK.") dom = html.fromstring(resp.text) # type: ignore - map_lang = {'jp': 'ja'} - for td in eval_xpath(dom, xpath_language_codes): - eng_lang = td.text + # languages - if eng_lang in ('en-gb', 'pt-br'): - # language 'en' is already in the list and a language 'en-gb' can't - # be handled in SearXNG, same with pt-br which is covered by pt-pt. - continue + engine_traits.languages['zh'] = 'zh-hans' - babel_lang = map_lang.get(eng_lang, eng_lang).replace('-', '_') - try: - sxng_tag = language_tag(babel.Locale.parse(babel_lang)) - except babel.UnknownLocaleError: - print("ERROR: language (%s) is unknown by babel" % (eng_lang)) - continue - conflict = engine_traits.languages.get(sxng_tag) - if conflict: - if conflict != eng_lang: - print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_lang)) - continue - engine_traits.languages[sxng_tag] = eng_lang - - map_region = { - 'en-ID': 'id_ID', - 'no-NO': 'nb_NO', + map_lang = {'prs': 'fa-AF', 'en': 'en-us'} + bing_ui_lang_map = { + # HINT: this list probably needs to be supplemented + 'en': 'us', # en --> en-us + 'da': 'dk', # da --> da-dk } - for td in eval_xpath(dom, xpath_market_codes): - eng_region = td.text - babel_region = map_region.get(eng_region, eng_region).replace('-', '_') - - if eng_region == 'en-WW': - engine_traits.all_locale = eng_region - continue - + for href in eval_xpath(dom, '//div[@id="language-section"]//li/a/@href'): + eng_lang = parse_qs(urlparse(href).query)['setlang'][0] + babel_lang = map_lang.get(eng_lang, eng_lang) try: - sxng_tag = region_tag(babel.Locale.parse(babel_region)) + sxng_tag = language_tag(babel.Locale.parse(babel_lang.replace('-', '_'))) except babel.UnknownLocaleError: - print("ERROR: region (%s) is unknown by babel" % (eng_region)) + print("ERROR: language (%s) is unknown by babel" % (babel_lang)) continue - conflict = engine_traits.regions.get(sxng_tag) + # Language (e.g. 'en' or 'de') from https://www.bing.com/account/general + # is converted by bing to 'en-us' or 'de-de'. But only if there is not + # already a '-' delemitter in the language. For instance 'pt-PT' --> + # 'pt-pt' and 'pt-br' --> 'pt-br' + bing_ui_lang = eng_lang.lower() + if '-' not in bing_ui_lang: + bing_ui_lang = bing_ui_lang + '-' + bing_ui_lang_map.get(bing_ui_lang, bing_ui_lang) + + conflict = engine_traits.languages.get(sxng_tag) if conflict: - if conflict != eng_region: - print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_region)) + if conflict != bing_ui_lang: + print(f"CONFLICT: babel {sxng_tag} --> {conflict}, {bing_ui_lang}") continue - engine_traits.regions[sxng_tag] = eng_region + engine_traits.languages[sxng_tag] = bing_ui_lang + + # regions (aka "market codes") + + engine_traits.regions['zh-CN'] = 'zh-cn' + + map_market_codes = { + 'zh-hk': 'en-hk', # not sure why, but at M$ this is the market code for Hongkong + } + for href in eval_xpath(dom, '//div[@id="region-section"]//li/a/@href'): + cc_tag = parse_qs(urlparse(href).query)['cc'][0] + if cc_tag == 'clear': + engine_traits.all_locale = cc_tag + continue + + # add market codes from official languages of the country .. + for lang_tag in babel.languages.get_official_languages(cc_tag, de_facto=True): + if lang_tag not in engine_traits.languages.keys(): + # print("ignore lang: %s <-- %s" % (cc_tag, lang_tag)) + continue + lang_tag = lang_tag.split('_')[0] # zh_Hant --> zh + market_code = f"{lang_tag}-{cc_tag}" # zh-tw + + market_code = map_market_codes.get(market_code, market_code) + sxng_tag = region_tag(babel.Locale.parse('%s_%s' % (lang_tag, cc_tag.upper()))) + conflict = engine_traits.regions.get(sxng_tag) + if conflict: + if conflict != market_code: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, market_code)) + continue + engine_traits.regions[sxng_tag] = market_code diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index bd3a34aa5..faf0941c8 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -6,23 +6,20 @@ from typing import TYPE_CHECKING -import uuid import json from urllib.parse import urlencode from lxml import html from searx.enginelib.traits import EngineTraits -from searx.engines.bing import ( - set_bing_cookies, - _fetch_traits, -) -from searx.engines.bing import send_accept_language_header # pylint: disable=unused-import +from searx.engines.bing import set_bing_cookies +from searx.engines.bing import fetch_traits # pylint: disable=unused-import + if TYPE_CHECKING: import logging - logger: logging.Logger + logger = logging.getLogger() traits: EngineTraits @@ -45,39 +42,29 @@ time_range_support = True base_url = 'https://www.bing.com/images/async' """Bing (Images) search URL""" -bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-image-search/reference/market-codes' -"""Bing (Images) search API description""" - time_map = { - # fmt: off 'day': 60 * 24, 'week': 60 * 24 * 7, 'month': 60 * 24 * 31, 'year': 60 * 24 * 365, - # fmt: on } def request(query, params): """Assemble a Bing-Image request.""" - engine_region = traits.get_region(params['searxng_locale'], 'en-US') - engine_language = traits.get_language(params['searxng_locale'], 'en') - - SID = uuid.uuid1().hex.upper() - set_bing_cookies(params, engine_language, engine_region, SID) + engine_region = traits.get_region(params['searxng_locale'], traits.all_locale) # type: ignore + engine_language = traits.get_language(params['searxng_locale'], 'en') # type: ignore + set_bing_cookies(params, engine_language, engine_region) # build URL query - # - example: https://www.bing.com/images/async?q=foo&first=155&count=35 - + # - example: https://www.bing.com/images/async?q=foo&async=content&first=1&count=35 query_params = { - # fmt: off 'q': query, - 'async' : 'content', + 'async': '1', # to simplify the page count lets use the default of 35 images per page - 'first' : (int(params.get('pageno', 1)) - 1) * 35 + 1, - 'count' : 35, - # fmt: on + 'first': (int(params.get('pageno', 1)) - 1) * 35 + 1, + 'count': 35, } # time range @@ -120,13 +107,3 @@ def response(resp): } ) return results - - -def fetch_traits(engine_traits: EngineTraits): - """Fetch languages and regions from Bing-News.""" - - xpath_market_codes = '//table[1]/tbody/tr/td[3]' - # xpath_country_codes = '//table[2]/tbody/tr/td[2]' - xpath_language_codes = '//table[3]/tbody/tr/td[2]' - - _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes) diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 18992e2d1..3585a1cd1 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -1,22 +1,23 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint """Bing-News: description see :py:obj:`searx.engines.bing`. + +.. hint:: + + Bing News is *different* in some ways! + """ # pylint: disable=invalid-name from typing import TYPE_CHECKING -import uuid from urllib.parse import urlencode from lxml import html +from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_getindex from searx.enginelib.traits import EngineTraits -from searx.engines.bing import ( - set_bing_cookies, - _fetch_traits, -) -from searx.engines.bing import send_accept_language_header # pylint: disable=unused-import +from searx.engines.bing import set_bing_cookies if TYPE_CHECKING: import logging @@ -39,58 +40,48 @@ about = { # engine dependent config categories = ['news'] paging = True +"""If go through the pages and there are actually no new results for another +page, then bing returns the results from the last page again.""" + time_range_support = True time_map = { - 'day': '4', - 'week': '8', - 'month': '9', + 'day': 'interval="4"', + 'week': 'interval="7"', + 'month': 'interval="9"', } -"""A string '4' means *last hour*. We use *last hour* for ``day`` here since the +"""A string '4' means *last hour*. We use *last hour* for ``day`` here since the difference of *last day* and *last week* in the result list is just marginally. -""" +Bing does not have news range ``year`` / we use ``month`` instead.""" base_url = 'https://www.bing.com/news/infinitescrollajax' """Bing (News) search URL""" -bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/reference/market-codes' -"""Bing (News) search API description""" - -mkt_alias = { - 'zh': 'en-WW', - 'zh-CN': 'en-WW', -} -"""Bing News has an official market code 'zh-CN' but we won't get a result with -this market code. For 'zh' and 'zh-CN' we better use the *Worldwide aggregate* -market code (en-WW). -""" - def request(query, params): """Assemble a Bing-News request.""" - sxng_locale = params['searxng_locale'] - engine_region = traits.get_region(mkt_alias.get(sxng_locale, sxng_locale), traits.all_locale) - engine_language = traits.get_language(sxng_locale, 'en') - - SID = uuid.uuid1().hex.upper() - set_bing_cookies(params, engine_language, engine_region, SID) + engine_region = traits.get_region(params['searxng_locale'], traits.all_locale) # type: ignore + engine_language = traits.get_language(params['searxng_locale'], 'en') # type: ignore + set_bing_cookies(params, engine_language, engine_region) # build URL query # # example: https://www.bing.com/news/infinitescrollajax?q=london&first=1 + page = int(params.get('pageno', 1)) - 1 query_params = { - # fmt: off 'q': query, 'InfiniteScroll': 1, # to simplify the page count lets use the default of 10 images per page - 'first' : (int(params.get('pageno', 1)) - 1) * 10 + 1, - # fmt: on + 'first': page * 10 + 1, + 'SFX': page, + 'form': 'PTFTNR', + 'setlang': engine_region.split('-')[0], + 'cc': engine_region.split('-')[-1], } if params['time_range']: - # qft=interval:"7" - query_params['qft'] = 'qft=interval="%s"' % time_map.get(params['time_range'], '9') + query_params['qft'] = time_map.get(params['time_range'], 'interval="9"') params['url'] = base_url + '?' + urlencode(query_params) @@ -106,18 +97,34 @@ def response(resp): dom = html.fromstring(resp.text) - for newsitem in dom.xpath('//div[contains(@class, "newsitem")]'): + for newsitem in eval_xpath_list(dom, '//div[contains(@class, "newsitem")]'): + + link = eval_xpath_getindex(newsitem, './/a[@class="title"]', 0, None) + if link is None: + continue + url = link.attrib.get('href') + title = extract_text(link) + content = extract_text(eval_xpath(newsitem, './/div[@class="snippet"]')) + + metadata = [] + source = eval_xpath_getindex(newsitem, './/div[contains(@class, "source")]', 0, None) + if source is not None: + for item in ( + eval_xpath_getindex(source, './/span[@aria-label]/@aria-label', 0, None), + # eval_xpath_getindex(source, './/a', 0, None), + # eval_xpath_getindex(source, './div/span', 3, None), + link.attrib.get('data-author'), + ): + if item is not None: + t = extract_text(item) + if t and t.strip(): + metadata.append(t.strip()) + metadata = ' | '.join(metadata) - url = newsitem.xpath('./@url')[0] - title = ' '.join(newsitem.xpath('.//div[@class="caption"]//a[@class="title"]/text()')).strip() - content = ' '.join(newsitem.xpath('.//div[@class="snippet"]/text()')).strip() thumbnail = None - author = newsitem.xpath('./@data-author')[0] - metadata = ' '.join(newsitem.xpath('.//div[@class="source"]/span/text()')).strip() - - img_src = newsitem.xpath('.//a[@class="imagelink"]//img/@src') - if img_src: - thumbnail = 'https://www.bing.com/' + img_src[0] + imagelink = eval_xpath_getindex(newsitem, './/a[@class="imagelink"]//img', 0, None) + if imagelink is not None: + thumbnail = 'https://www.bing.com/' + imagelink.attrib.get('src') results.append( { @@ -125,7 +132,6 @@ def response(resp): 'title': title, 'content': content, 'img_src': thumbnail, - 'author': author, 'metadata': metadata, } ) @@ -134,17 +140,20 @@ def response(resp): def fetch_traits(engine_traits: EngineTraits): - """Fetch languages and regions from Bing-News. + """Fetch languages and regions from Bing-News.""" + # pylint: disable=import-outside-toplevel - The :py:obj:`description ` of the - first table says *"query parameter when calling the Video Search API."* - .. that's why I use the 4. table "News Category API markets" for the - ``xpath_market_codes``. + from searx.engines.bing import fetch_traits as _f - """ + _f(engine_traits) - xpath_market_codes = '//table[4]/tbody/tr/td[3]' - # xpath_country_codes = '//table[2]/tbody/tr/td[2]' - xpath_language_codes = '//table[3]/tbody/tr/td[2]' + # fix market codes not known by bing news: - _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes) + # In bing the market code 'zh-cn' exists, but there is no 'news' category in + # bing for this market. Alternatively we use the the market code from Honk + # Kong. Even if this is not correct, it is better than having no hits at + # all, or sending false queries to bing that could raise the suspicion of a + # bot. + + # HINT: 'en-hk' is the region code it does not indicate the language en!! + engine_traits.regions['zh-CN'] = 'en-hk' diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py index 8ee0bb66e..41bf2d3e1 100644 --- a/searx/engines/bing_videos.py +++ b/searx/engines/bing_videos.py @@ -5,18 +5,15 @@ # pylint: disable=invalid-name from typing import TYPE_CHECKING -import uuid import json from urllib.parse import urlencode from lxml import html from searx.enginelib.traits import EngineTraits -from searx.engines.bing import ( - set_bing_cookies, - _fetch_traits, -) -from searx.engines.bing import send_accept_language_header # pylint: disable=unused-import +from searx.engines.bing import set_bing_cookies +from searx.engines.bing import fetch_traits # pylint: disable=unused-import +from searx.engines.bing_images import time_map if TYPE_CHECKING: import logging @@ -44,40 +41,24 @@ time_range_support = True base_url = 'https://www.bing.com/videos/asyncv2' """Bing (Videos) async search URL.""" -bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-video-search/reference/market-codes' -"""Bing (Video) search API description""" - -time_map = { - # fmt: off - 'day': 60 * 24, - 'week': 60 * 24 * 7, - 'month': 60 * 24 * 31, - 'year': 60 * 24 * 365, - # fmt: on -} - def request(query, params): """Assemble a Bing-Video request.""" - engine_region = traits.get_region(params['searxng_locale'], 'en-US') - engine_language = traits.get_language(params['searxng_locale'], 'en') - - SID = uuid.uuid1().hex.upper() - set_bing_cookies(params, engine_language, engine_region, SID) + engine_region = traits.get_region(params['searxng_locale'], traits.all_locale) # type: ignore + engine_language = traits.get_language(params['searxng_locale'], 'en') # type: ignore + set_bing_cookies(params, engine_language, engine_region) # build URL query # # example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35 query_params = { - # fmt: off 'q': query, - 'async' : 'content', + 'async': 'content', # to simplify the page count lets use the default of 35 images per page - 'first' : (int(params.get('pageno', 1)) - 1) * 35 + 1, - 'count' : 35, - # fmt: on + 'first': (int(params.get('pageno', 1)) - 1) * 35 + 1, + 'count': 35, } # time range @@ -116,13 +97,3 @@ def response(resp): ) return results - - -def fetch_traits(engine_traits: EngineTraits): - """Fetch languages and regions from Bing-Videos.""" - - xpath_market_codes = '//table[1]/tbody/tr/td[3]' - # xpath_country_codes = '//table[2]/tbody/tr/td[2]' - xpath_language_codes = '//table[3]/tbody/tr/td[2]' - - _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes) diff --git a/searx/engines/emojipedia.py b/searx/engines/emojipedia.py index 020bf689b..a1f3ef7b8 100644 --- a/searx/engines/emojipedia.py +++ b/searx/engines/emojipedia.py @@ -14,7 +14,6 @@ from lxml import html from searx.utils import ( eval_xpath_list, - eval_xpath_getindex, extract_text, ) @@ -28,11 +27,9 @@ about = { } categories = [] -paging = False -time_range_support = False base_url = 'https://emojipedia.org' -search_url = base_url + '/search/?{query}' +search_url = base_url + '/search?{query}' def request(query, params): @@ -47,20 +44,10 @@ def response(resp): dom = html.fromstring(resp.text) - for result in eval_xpath_list(dom, "//ol[@class='search-results']/li"): + for result in eval_xpath_list(dom, '//div[starts-with(@class, "EmojisList")]/a'): - extracted_desc = extract_text(eval_xpath_getindex(result, './/p', 0)) - - if 'No results found.' in extracted_desc: - break - - link = eval_xpath_getindex(result, './/h2/a', 0) - - url = base_url + link.attrib.get('href') - title = extract_text(link) - content = extracted_desc - - res = {'url': url, 'title': title, 'content': content} + url = base_url + result.attrib.get('href') + res = {'url': url, 'title': extract_text(result), 'content': ''} results.append(res) diff --git a/searx/engines/matrixrooms.py b/searx/engines/matrixrooms.py new file mode 100644 index 000000000..06c4d99a9 --- /dev/null +++ b/searx/engines/matrixrooms.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Matrixrooms.info (social media) +""" + +from urllib.parse import quote_plus + +about = { + "website": 'https://matrixrooms.info', + "wikidata_id": 'Q107565255', + "official_api_documentation": None, + "use_official_api": False, + "require_api_key": False, + "results": 'JSON', +} +paging = True +categories = ['social media'] + +base_url = "https://apicdn.matrixrooms.info" +matrix_url = "https://matrix.to" +page_size = 20 + + +def request(query, params): + params['url'] = f"{base_url}/search/{quote_plus(query)}/{page_size}/{(params['pageno']-1)*page_size}" + return params + + +def response(resp): + results = [] + + for result in resp.json(): + results.append( + { + 'url': matrix_url + '/#/' + result['alias'], + 'title': result['name'], + 'content': result['topic'] + + f" // {result['members']} members" + + f" // {result['alias']}" + + f" // {result['server']}", + 'thumbnail': result['avatar_url'], + } + ) + + return results diff --git a/searx/engines/moviepilot.py b/searx/engines/moviepilot.py index 6bd2b2290..f9206c74c 100644 --- a/searx/engines/moviepilot.py +++ b/searx/engines/moviepilot.py @@ -105,11 +105,11 @@ def response(resp): item['metadata'] = html_to_text(result.get('meta_short', '')) if result.get('image'): - item['thumbnail'] = image_url.format(image_id=result['image'], filename=result['image_filename']) + item['img_src'] = image_url.format(image_id=result['image'], filename=result['image_filename']) else: item['url'] = result['url'] item['content'] = ', '.join([result['class'], result['info'], result['more']]) - item['thumbnail'] = result['image'] + item['img_src'] = result['image'] results.append(item) diff --git a/searx/engines/pinterest.py b/searx/engines/pinterest.py new file mode 100644 index 000000000..4ca316698 --- /dev/null +++ b/searx/engines/pinterest.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Pinterest (images) +""" + +from json import dumps + +about = { + "website": 'https://www.pinterest.com/', + "wikidata_id": 'Q255381', + "official_api_documentation": 'https://developers.pinterest.com/docs/api/v5/', + "use_official_api": False, + "require_api_key": False, + "results": 'JSON', +} + +categories = ['images'] +paging = True + +base_url = 'https://www.pinterest.com' + + +def request(query, params): + args = { + 'options': { + 'query': query, + 'bookmarks': [params['engine_data'].get('bookmark', '')], + }, + 'context': {}, + } + params['url'] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}" + + return params + + +def response(resp): + results = [] + + json_resp = resp.json() + + results.append( + { + 'engine_data': json_resp['resource_response']['bookmark'], + # it's called bookmark by pinterest, but it's rather a nextpage + # parameter to get the next results + 'key': 'bookmark', + } + ) + + for result in json_resp['resource_response']['data']['results']: + results.append( + { + 'template': 'images.html', + 'url': result['link'] or f"{base_url}/pin/{result['id']}/", + 'title': result.get('title') or result.get('grid_title'), + 'content': (result.get('rich_summary') or {}).get('display_description') or "", + 'img_src': result['images']['orig']['url'], + 'thumbnail_src': result['images']['236x']['url'], + 'source': (result.get('rich_summary') or {}).get('site_name'), + } + ) + + return results diff --git a/searx/engines/tagesschau.py b/searx/engines/tagesschau.py index e50be90d0..cfa184025 100644 --- a/searx/engines/tagesschau.py +++ b/searx/engines/tagesschau.py @@ -42,6 +42,17 @@ paging = True results_per_page = 10 base_url = "https://www.tagesschau.de" +use_source_url = True +"""When set to false, display URLs from Tagesschau, and not the actual source +(e.g. NDR, WDR, SWR, HR, ...) + +.. note:: + + The actual source may contain additional content, such as commentary, that is + not displayed in the Tagesschau. + +""" + def request(query, params): args = { @@ -78,7 +89,7 @@ def _story(item): 'thumbnail': item.get('teaserImage', {}).get('imageVariants', {}).get('16x9-256'), 'publishedDate': datetime.strptime(item['date'][:19], '%Y-%m-%dT%H:%M:%S'), 'content': item['firstSentence'], - 'url': item['shareURL'], + 'url': item['shareURL'] if use_source_url else item['detailsweb'], } diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 0fdeacec2..53dc6f806 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -133,12 +133,7 @@ def response(resp): continue url = parse_url(url) - title = eval_xpath_getindex(result, './/h3/a', 0, default=None) - if title is None: - continue - offset = len(extract_text(title.xpath('span'))) - title = extract_text(title)[offset:] - + title = extract_text(result.xpath('.//h3//a/@aria-label')) content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='') content = extract_text(content, allow_none=True) @@ -164,7 +159,7 @@ def fetch_traits(engine_traits: EngineTraits): resp = network.get('https://search.yahoo.com/preferences/languages') if not resp.ok: - print("ERROR: response from peertube is not OK.") + print("ERROR: response from yahoo is not OK.") dom = html.fromstring(resp.text) offset = len('lang_') diff --git a/searx/settings.yml b/searx/settings.yml index 404145f89..518e61d53 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1018,6 +1018,11 @@ engines: require_api_key: false results: HTML + - name: matrixrooms + engine: matrixrooms + shortcut: mtrx + disabled: true + - name: metacpan engine: metacpan shortcut: cpan @@ -1201,6 +1206,10 @@ engines: engine: photon shortcut: ph + - name: pinterest + engine: pinterest + shortcut: pin + - name: piped engine: piped shortcut: ppd @@ -1483,6 +1492,9 @@ engines: - name: tagesschau engine: tagesschau + # when set to false, display URLs from Tagesschau, and not the actual source + # (e.g. NDR, WDR, SWR, HR, ...) + use_source_url: true shortcut: ts disabled: true