# SPDX-License-Identifier: AGPL-3.0-or-later """ Arch Linux Wiki ~~~~~~~~~~~~~~~ This implementation does not use a official API: Mediawiki provides API, but Arch Wiki blocks access to it. """ from urllib.parse import urlencode, urljoin, urlparse import babel import lxml from searx.enginelib.traits import EngineTraits from searx.locales import language_tag from searx.utils import ( eval_xpath_getindex, eval_xpath_list, extract_text, searxng_useragent, ) about = { "website": "https://wiki.archlinux.org/", "wikidata_id": "Q101445877", "official_api_documentation": None, "use_official_api": False, "require_api_key": False, "results": "HTML", } # engine dependent config categories = ["it", "software wikis"] paging = True main_wiki = "wiki.archlinux.org" def request(query, params): sxng_lang = params["searxng_locale"].split("-")[0] netloc: str = traits.custom["wiki_netloc"].get(sxng_lang, main_wiki) # type: ignore title: str = traits.custom["title"].get(sxng_lang, "Special:Search") # type: ignore base_url = "https://" + netloc + "/index.php?" offset = (params["pageno"] - 1) * 20 if netloc == main_wiki: eng_lang: str = traits.get_language(sxng_lang, "English") # type: ignore query += " (" + eng_lang + ")" # wiki.archlinux.org is protected by anubis # - https://github.com/searxng/searxng/issues/4646#issuecomment-2817848019 params["headers"]["User-Agent"] = searxng_useragent() elif netloc == "wiki.archlinuxcn.org": base_url = "https://" + netloc + "/wzh/index.php?" args = { "search": query, "title": title, "limit": 20, "offset": offset, "profile": "default", } params["url"] = base_url + urlencode(args) return params def response(resp): results = [] dom = lxml.html.fromstring(resp.text) # type: ignore # get the base URL for the language in which request was made sxng_lang = resp.search_params["searxng_locale"].split("-")[0] netloc: str = traits.custom["wiki_netloc"].get(sxng_lang, main_wiki) # type: ignore base_url = "https://" + netloc + "/index.php?" for result in eval_xpath_list(dom, '//ul[@class="mw-search-results"]/li'): link = eval_xpath_getindex(result, './/div[@class="mw-search-result-heading"]/a', 0) content = extract_text(result.xpath('.//div[@class="searchresult"]')) results.append( { "url": urljoin(base_url, link.get("href")), # type: ignore "title": extract_text(link), "content": content, } ) return results def fetch_traits(engine_traits: EngineTraits): """Fetch languages from Archlinux-Wiki. The location of the Wiki address of a language is mapped in a :py:obj:`custom field ` (``wiki_netloc``). Depending on the location, the ``title`` argument in the request is translated. .. code:: python "custom": { "wiki_netloc": { "de": "wiki.archlinux.de", # ... "zh": "wiki.archlinuxcn.org" } "title": { "de": "Spezial:Suche", # ... "zh": "Special:\u641c\u7d22" }, }, """ # pylint: disable=import-outside-toplevel from searx.network import get # see https://github.com/searxng/searxng/issues/762 engine_traits.custom["wiki_netloc"] = {} engine_traits.custom["title"] = {} title_map = { "de": "Spezial:Suche", "fa": "ویژه:جستجو", "ja": "特別:検索", "zh": "Special:搜索", } resp = get("https://wiki.archlinux.org/", timeout=5) if not resp.ok: raise RuntimeError("Response from Arch Linux Wiki is not OK.") dom = lxml.html.fromstring(resp.text) # type: ignore for a in eval_xpath_list(dom, "//a[@class='interlanguage-link-target']"): sxng_tag = language_tag(babel.Locale.parse(a.get("lang"), sep="-")) # zh_Hans --> zh sxng_tag = sxng_tag.split("_")[0] netloc = urlparse(a.get("href")).netloc if netloc != "wiki.archlinux.org": title = title_map.get(sxng_tag) if not title: print("ERROR: title tag from %s (%s) is unknown" % (netloc, sxng_tag)) continue engine_traits.custom["wiki_netloc"][sxng_tag] = netloc engine_traits.custom["title"][sxng_tag] = title # type: ignore eng_tag = extract_text(eval_xpath_list(a, ".//span")) engine_traits.languages[sxng_tag] = eng_tag # type: ignore engine_traits.languages["en"] = "English"