Scraper

code usage example

importasynciodefmain():
URL="http://localhost:8080/search"# searxngquery= (
"text here"
)
data=fetch_search_results(URL, query)
output=extract_relevant_fields(data)
urls=list(output["results"].values())
extracted_texts=asyncio.run(extract_text_from_urls(urls))
forurl, textinextracted_texts.items():
print(f"Extracted content from {url}:\n{text}\n")
if__name__=="__main__":
main()