diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 7c2bc1b63..b73fc5044 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -28,17 +28,8 @@ Special features of the no-JS services (DDG-lite & DDG-html): HTML ``
``, HTTP-Headers & DDG's bot Blocker: - The HTTP *Sec-Fetch* headers (and the User-Agent_, see below) are generated by - the WEB-client and are checked by DDG's bot blocker. - -At least the follow-up page (content) is accessed by sending form data with -*Sec-Fetch* headers. If those HTTP headers are incorrect or missed, DDG's bot -protection puts the IP on it's block list:: - - Sec-Fetch-Dest: document - Sec-Fetch-Mode: navigate - Sec-Fetch-Site: same-origin - Sec-Fetch-User: ?1 + The HTTP User-Agent_ (see below) is generated by the WEB-client and are + checked by DDG's bot blocker. To simulate the behavior of a real browser session, it might be necessary to evaluate additional headers. For example, in the response from DDG, the @@ -390,13 +381,6 @@ def request(query: str, params: "OnlineParams") -> None: # The vqd value is generated from the query and the UA header. To be able to # reuse the vqd value, the UA header must be static. headers["User-Agent"] = _HTTP_User_Agent - - # Sec-Fetch-* headers are already set by the ``OnlineProcessor``. - # headers["Sec-Fetch-Dest"] = "document" - # headers["Sec-Fetch-Mode"] = "navigate" - # headers["Sec-Fetch-Site"] = "same-origin" - # headers["Sec-Fetch-User"] = "?1" - headers["Referer"] = "https://html.duckduckgo.com/" ui_lang = params["searxng_locale"] diff --git a/searx/engines/duckduckgo_extra.py b/searx/engines/duckduckgo_extra.py index 80719a1be..4e32ba118 100644 --- a/searx/engines/duckduckgo_extra.py +++ b/searx/engines/duckduckgo_extra.py @@ -87,12 +87,6 @@ def request(query: str, params: "OnlineParams") -> None: params["url"] = None return - if params["pageno"] > 1 and ddg_category in ["videos", "news"]: - # DDG has limited results for videos and news and we got already all - # results from the first request. - params["url"] = None - return - # HTTP headers # ============ @@ -102,11 +96,6 @@ def request(query: str, params: "OnlineParams") -> None: headers["User-Agent"] = _HTTP_User_Agent vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params) - # Sec-Fetch-* headers are already set by the ``OnlineProcessor``. - # Overwrite the default Sec-Fetch-* headers to fit to a XHTMLRequest - headers["Sec-Fetch-Dest"] = "empty" - headers["Sec-Fetch-Mode"] = "cors" - headers["Accept"] = "*/*" headers["Referer"] = "https://duckduckgo.com/" headers["Host"] = "duckduckgo.com" diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index 53389ad04..427aff6e0 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -159,14 +159,6 @@ class OnlineProcessor(EngineProcessor): headers["Accept-Language"] = f"{_l},{_l}-{_t};q=0.7,en;q=0.3" self.logger.debug("HTTP Accept-Language: %s", headers.get("Accept-Language", "")) - # https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header - headers["Sec-Fetch-Dest"] = "document" - headers["Sec-Fetch-Mode"] = "navigate" - headers["Sec-Fetch-Site"] = "same-origin" - headers["Sec-Fetch-User"] = "?1" - # Sec-GPC is in an experimental state (FFox only) - # headers["Sec-GPC"] = "1" - return params def _send_http_request(self, params: OnlineParams):