[fix] online engines: remove HTTP Sec-Fetch-* headers

The Sec-Fetch-* headers seem to cause more problems than they solve. They will
be removed for now.

Related:

- https://github.com/searxng/searxng/pull/5758#pullrequestreview-3834221131

Suggested-by: @Bnyro
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/duckduckgo.py

@@ -28,17 +28,8 @@ Special features of the no-JS services (DDG-lite & DDG-html):
 
 HTML ``<form>``, HTTP-Headers & DDG's bot Blocker:
 
-  The HTTP *Sec-Fetch* headers (and the User-Agent_, see below) are generated by
-  the WEB-client and are checked by DDG's bot blocker.
-
-At least the follow-up page (content) is accessed by sending form data with
-*Sec-Fetch* headers.  If those HTTP headers are incorrect or missed, DDG's bot
-protection puts the IP on it's block list::
-
-    Sec-Fetch-Dest: document
-    Sec-Fetch-Mode: navigate
-    Sec-Fetch-Site: same-origin
-    Sec-Fetch-User: ?1
+  The HTTP User-Agent_ (see below) is generated by the WEB-client and are
+  checked by DDG's bot blocker.
 
 To simulate the behavior of a real browser session, it might be necessary to
 evaluate additional headers.  For example, in the response from DDG, the
@@ -390,13 +381,6 @@ def request(query: str, params: "OnlineParams") -> None:
     # The vqd value is generated from the query and the UA header. To be able to
     # reuse the vqd value, the UA header must be static.
     headers["User-Agent"] = _HTTP_User_Agent
-
-    # Sec-Fetch-* headers are already set by the ``OnlineProcessor``.
-    # headers["Sec-Fetch-Dest"] = "document"
-    # headers["Sec-Fetch-Mode"] = "navigate"
-    # headers["Sec-Fetch-Site"] = "same-origin"
-    # headers["Sec-Fetch-User"] = "?1"
-
     headers["Referer"] = "https://html.duckduckgo.com/"
 
     ui_lang = params["searxng_locale"]

searx/engines/duckduckgo_extra.py

@@ -87,12 +87,6 @@ def request(query: str, params: "OnlineParams") -> None:
         params["url"] = None
         return
 
-    if params["pageno"] > 1 and ddg_category in ["videos", "news"]:
-        # DDG has limited results for videos and news and we got already all
-        # results from the first request.
-        params["url"] = None
-        return
-
     # HTTP headers
     # ============
 
@@ -102,11 +96,6 @@ def request(query: str, params: "OnlineParams") -> None:
     headers["User-Agent"] = _HTTP_User_Agent
     vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
 
-    # Sec-Fetch-* headers are already set by the ``OnlineProcessor``.
-    # Overwrite the default Sec-Fetch-* headers to fit to a XHTMLRequest
-    headers["Sec-Fetch-Dest"] = "empty"
-    headers["Sec-Fetch-Mode"] = "cors"
-
     headers["Accept"] = "*/*"
     headers["Referer"] = "https://duckduckgo.com/"
     headers["Host"] = "duckduckgo.com"

searx/search/processors/online.py

@@ -159,14 +159,6 @@ class OnlineProcessor(EngineProcessor):
             headers["Accept-Language"] = f"{_l},{_l}-{_t};q=0.7,en;q=0.3"
         self.logger.debug("HTTP Accept-Language: %s", headers.get("Accept-Language", ""))
 
-        # https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header
-        headers["Sec-Fetch-Dest"] = "document"
-        headers["Sec-Fetch-Mode"] = "navigate"
-        headers["Sec-Fetch-Site"] = "same-origin"
-        headers["Sec-Fetch-User"] = "?1"
-        # Sec-GPC is in an experimental state (FFox only)
-        # headers["Sec-GPC"] = "1"
-
         return params
 
     def _send_http_request(self, params: OnlineParams):