diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 00000000..e8149ad0 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,102 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +# https://docs.coderabbit.ai/getting-started/configure-coderabbit/ + +language: en-US +tone_instructions: '' +early_access: true +enable_free_tier: true +reviews: + profile: chill + request_changes_workflow: false + high_level_summary: true + high_level_summary_placeholder: '@coderabbitai summary' + auto_title_placeholder: '@coderabbitai' + auto_title_instructions: '' + review_status: true + commit_status: true + collapse_walkthrough: false + changed_files_summary: true + sequence_diagrams: true + assess_linked_issues: true + related_issues: true + related_prs: true + suggested_labels: true + auto_apply_labels: false + suggested_reviewers: true + poem: false + labeling_instructions: [] + path_filters: [] + path_instructions: [] + abort_on_close: true + auto_review: + enabled: true + auto_incremental_review: true + ignore_title_keywords: [] + labels: [] + drafts: false + base_branches: [] + tools: + shellcheck: + enabled: true + ruff: + enabled: true + markdownlint: + enabled: true + github-checks: + enabled: true + timeout_ms: 90000 + languagetool: + enabled: true + enabled_only: false + level: default + biome: + enabled: true + hadolint: + enabled: true + swiftlint: + enabled: true + phpstan: + enabled: true + level: default + golangci-lint: + enabled: true + yamllint: + enabled: true + gitleaks: + enabled: true + checkov: + enabled: true + detekt: + enabled: true + eslint: + enabled: true + rubocop: + enabled: true + buf: + enabled: true + regal: + enabled: true + actionlint: + enabled: true + pmd: + enabled: true + cppcheck: + enabled: true + semgrep: + enabled: true + circleci: + enabled: true +chat: + auto_reply: true +knowledge_base: + opt_out: false + learnings: + scope: auto + issues: + scope: auto + jira: + project_keys: [] + linear: + team_keys: [] + pull_requests: + scope: auto diff --git a/README.md b/README.md index 603ee141..2c381e1b 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This repository contains the Infrastructure as Code (IaC) configuration and docu - [Cloud Services (External Dependencies)](#cloud-services-external-dependencies) - [Getting Started](#getting-started) - [Development](#development) + - [GitHub automations](#github-automations) - [EOL](#eol) ## Principles @@ -77,7 +78,20 @@ Perform the checks by running `task lint` (or `pre-commit run --all-files`). Create/update example `.env` files: `task docker:create-example-env`. -Updating the container images is automated with [Renovate](https://docs.renovatebot.com/). +### GitHub automations + +- Workflows configured (see in [`.github/workflows`](.github/workflows)): + - [Pre-commit checks](.github/workflows/pre-commit.yml) + - [Build dev & admin container](.github/workflows/devcontainer.yml) + - [Build and deploy documentation site](.github/workflows/docs-web.yml) +- Updating the container images and other components is automated with [Renovate](https://docs.renovatebot.com/) + - [GitHub Marketplace App](https://github.com/marketplace/renovate) + - Configured with [`renovate.json`](renovate.json) + - Free for public and private repositories +- AI Code Review for MRs is done with [Code Rabbit](https://www.coderabbit.ai/) + - [GitHub Marketplace App](https://github.com/marketplace/coderabbitai) + - Configured with [`.coderabbit.yaml`](.coderabbit.yaml) + - Free for public repositories ## EOL diff --git a/Taskfile.yaml b/Taskfile.yaml index 6a8ab3b4..68de4740 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -7,9 +7,10 @@ version: '3' includes: ansible: ./ansible/Taskfile.ansible.yaml docker: ./docker/Taskfile.docker.yaml + app: ./docker/Taskfile.apps.yaml + crowdsec: ./docker/security/Taskfile.crowdsec.yaml docs: ./docs/web/Taskfile.web.yaml devcontainer: ./.devcontainer/Taskfile.dev.yaml - crowdsec: ./docker/security/Taskfile.crowdsec.yaml tasks: default: diff --git a/config-example/docker/myhost/.env b/config-example/docker/myhost/.env index 9c82199c..43c69046 100644 --- a/config-example/docker/myhost/.env +++ b/config-example/docker/myhost/.env @@ -1,6 +1,7 @@ # Host-specific Docker Compose environment variables -# Storage - Located on main node +### Storage - Located on main node + STORAGE_ALL=/mnt STORAGE_LOCAL=/mnt/local/storage STORAGE_LOCAL_MEDIA=${STORAGE_LOCAL}/media @@ -14,7 +15,8 @@ DOCKER_VOLUMES=${STORAGE_LOCAL}/docker-volumes BEES_BTRFS_UUID=96c03041-7dde-48b4-8cc1-3f72e18e1bd2 BEES_DB_SIZE=1073741824 # 1 GiB -# Storage - Located on NAS +### Storage - Located on NAS + STORAGE_NAS=/mnt/nas STORAGE_DOWNLOADS=${STORAGE_NAS}/downloads STORAGE_TORRENT_DOWNLOADS=${STORAGE_NAS}/downloads/torrent-downloads @@ -23,13 +25,13 @@ STORAGE_MEDIA=${STORAGE_NAS}/media STORAGE_FAMILYMEDIA=${STORAGE_NAS}/family-media STORAGE_BACKUP_ON_NAS=${STORAGE_NAS}/backup -# For Homepage +### For Homepage LOCATION_CITY=Greenwich LOCATION_LATITUDE=51.48 LOCATION_LONGITUDE=0.00 -# Local accounts +### Local accounts KOPIA_B2_REPOSITORY_PASSWORD="use-some-very-secure-value-here" KOPIA_NAS_REPOSITORY_PASSWORD="use-some-very-secure-value-here" @@ -52,14 +54,30 @@ AUTHELIA_ADMIN_PASSWORD="use-some-very-secure-value-here" # Generate key within the crowdsec container: cscli bouncers add traefik-bouncer CROWDSEC_BOUNCER_API_KEY="use-some-very-secure-value-here" -# Cloud accounts - # Use a local account that has at least read privileges. Local account can be created on the Legacy Interface. # To temporally switch to the old interface: Settings -> System -> Legacy Interface -> Enable # Then create the user on Settings -> Admins UNIFI_LOCAL_VIEWONLY_USERNAME="viewonly" UNIFI_LOCAL_VIEWONLY_PASSWORD="use-some-very-secure-value-here" +# Generate a secure random key (min 32 chars) using: `openssl rand -base64 32` +OPEN_WEBUI_SECRET_KEY="use-some-very-secure-value-here" + +# Any random string +SEARXNG_SECRET="use-some-very-secure-value-here" + +# Any random string +QDRANT_API_KEY="use-some-very-secure-value-here" + +N8N_ENCRYPTION_KEY="use-some-very-secure-value-here" +N8N_USER_MANAGEMENT_JWT_SECRET="use-some-very-secure-value-here" + +N8N_POSTGRES_USER="root" +N8N_POSTGRES_PASSWORD="use-some-very-secure-value-here" +N8N_POSTGRES_DB="n8n" + +### Cloud accounts + CLOUDFLARE_DNS_API_TOKEN="use-some-very-secure-value-here" CLOUDFLARE_TUNNEL_TOKEN="use-some-very-secure-value-here" diff --git a/config-example/docker/myhost/apply.sh b/config-example/docker/myhost/apply.sh index b546314a..3c240df3 100755 --- a/config-example/docker/myhost/apply.sh +++ b/config-example/docker/myhost/apply.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -e + +### Common code ### + # shellcheck disable=SC2034,SC2046 HOST_CONFIG_DIR=$(dirname $(realpath -s "$0")) @@ -15,6 +18,9 @@ fi # shellcheck disable=SC1091 source "${DOCKER_STACKS_DIR}/common.sh" + +### Host-specific configuration ### + init "myhost" set +e @@ -78,13 +84,18 @@ up tools vaultwarden up tools openspeedtest up tools cyberchef up tools guacamole +up tools kasm +up tools stirling-pdf +up tools searxng # AI up ai ollama -up ai open-webui up ai litellm +up ai open-webui +up ai open-webui-pipelines up ai autogenstudio up ai sillytavern +up ai qdrant # Communication # down communication matrix-synapse @@ -104,4 +115,7 @@ up arr flaresolverr # Personal up personal personal-web + +### Common code ### + cleanup diff --git a/docker/Taskfile.apps.yaml b/docker/Taskfile.apps.yaml new file mode 100644 index 00000000..2e58e198 --- /dev/null +++ b/docker/Taskfile.apps.yaml @@ -0,0 +1,12 @@ +version: '3' + +tasks: + ollama-port-forward: + desc: Forward Ollama port + dotenv: ['config/docker/.env'] + env: + REMOTE_PORT: 11444 + LOCAL_PORT: 11434 + cmds: + - echo "Forwarding remote port (${REMOTE_PORT}) to local Ollama port (${LOCAL_PORT}). Press Ctrl+C to to stop." + - ssh -N -o 'ExitOnForwardFailure yes' -R "*:${REMOTE_PORT}:localhost:${LOCAL_PORT}" ${ADMIN_USER}@${MAIN_NODE} diff --git a/docker/Taskfile.docker.yaml b/docker/Taskfile.docker.yaml index 0589594d..82d9dabd 100644 --- a/docker/Taskfile.docker.yaml +++ b/docker/Taskfile.docker.yaml @@ -62,7 +62,7 @@ tasks: cmds: - scripts/create-example-env.py config/docker/.env > config-example/docker/.env - scripts/create-example-env.py config/docker/${MAIN_NODE}/.env > config-example/docker/myhost/.env - - cp config/docker/${MAIN_NODE}/apply.sh config-example/docker/myhost/apply.sh + - cat config/docker/${MAIN_NODE}/apply.sh | sed "s/${MAIN_NODE}/myhost/g" > config-example/docker/myhost/apply.sh sources: - config/docker/**/.env - config/docker/**/apply.sh diff --git a/docker/ai/README.md b/docker/ai/README.md index 1ee28417..25ca82c1 100644 --- a/docker/ai/README.md +++ b/docker/ai/README.md @@ -31,9 +31,9 @@ Man sshd -> `GatewayPorts`: > The argument may be no to force remote port forwardings to be available to the local host only, yes to force remote port forwardings to bind to the wildcard address, or clientspecified to allow the client to select the address to which the forwarding is bound. > The default is no. -My usage: +Task to start port forwarding (execute on the machine hosting Ollama): ```sh -ssh -N -o 'ExitOnForwardFailure yes' -R "*:11444:localhost:11434" buba@nest +task app:ollama-port-forward ``` ### Access Ollama with OpenAI compatible API diff --git a/docker/ai/litellm/config.yaml b/docker/ai/litellm/config.yaml index 84500289..0efefb1a 100644 --- a/docker/ai/litellm/config.yaml +++ b/docker/ai/litellm/config.yaml @@ -1,18 +1,38 @@ +# LiteLLM supports all the text / chat / vision models from OpenRouter +# https://docs.litellm.ai/docs/providers/openrouter +# https://openrouter.ai/models +# +# LiteLLM supports all anthropic models +# https://docs.litellm.ai/docs/providers/anthropic +# https://docs.anthropic.com/en/docs/about-claude/models +# +# LiteLLM supports all models from Ollama +# https://docs.litellm.ai/docs/providers/ollama +# https://github.com/ollama/ollama + +--- model_list: - model_name: claude-3-5-sonnet # RECEIVED MODEL NAME litellm_params: # All params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input - model: claude-3-5-sonnet-20240620 # MODEL NAME sent to `litellm.completion()` + model: anthropic/claude-3-5-sonnet-latest # MODEL NAME sent to `litellm.completion()` api_key: "os.environ/ANTHROPIC_API_KEY" # Does os.getenv("ANTHROPIC_API_KEY") + - model_name: gpt-4o + litellm_params: + model: openrouter/openai/gpt-4o-2024-11-20 + api_key: "os.environ/OPENROUTER_API_KEY" + - model_name: ollama-local-phi litellm_params: model: ollama_chat/phi3.5:3.8b api_base: "os.environ/LOCAL_OLLAMA_API_BASE" + api_key: "none" - model_name: ollama-mac-mistral litellm_params: model: ollama_chat/mistral:7b-instruct api_base: "os.environ/REMOTE_OLLAMA_API_BASE" + api_key: "none" router_settings: fallbacks: diff --git a/docker/ai/ollama.yaml b/docker/ai/ollama.yaml index cccd5d3c..3dba3fc7 100644 --- a/docker/ai/ollama.yaml +++ b/docker/ai/ollama.yaml @@ -28,7 +28,6 @@ services: traefik.enable: true traefik.http.routers.ollama.entrypoints: websecure traefik.http.routers.ollama.middlewares: https-local@file - traefik.http.routers.ollama.rule: Host(`ollama.${MYDOMAIN}`) traefik.http.services.ollama.loadbalancer.server.port: 11434 homepage.group: AI homepage.name: "Ollama" diff --git a/docker/ai/open-webui-pipelines.yaml b/docker/ai/open-webui-pipelines.yaml new file mode 100644 index 00000000..0af2f57f --- /dev/null +++ b/docker/ai/open-webui-pipelines.yaml @@ -0,0 +1,33 @@ +# Pipelines: Versatile, UI-Agnostic OpenAI-Compatible Plugin Framework +# +# > Welcome to Pipelines, an Open WebUI initiative. Pipelines bring modular, customizable workflows to any UI client supporting OpenAI API specs – and much more! +# > Easily extend functionalities, integrate unique logic, and create dynamic workflows with just a few lines of code. +# +# https://docs.openwebui.com/pipelines/ +# https://github.com/open-webui/pipelines +--- +name: open-webui-pipelines +services: + pipelines: + image: ghcr.io/open-webui/pipelines:main + container_name: open-webui-pipelines + restart: on-failure:10 + volumes: + - ${DOCKER_VOLUMES}/open-webui-pipelines:/app/pipelines + networks: + - proxy + labels: + traefik.enable: true + traefik.http.routers.open-webui-pipelines.entrypoints: websecure + traefik.http.routers.open-webui-pipelines.middlewares: https-local@file + traefik.http.routers.open-webui-pipelines.rule: Host(`open-webui-pipelines.${MYDOMAIN}`) + traefik.http.services.open-webui-pipelines.loadbalancer.server.port: 9099 + homepage.group: AI + homepage.name: "Open WebUI Pipelines" + homepage.icon: open-webui.png + homepage.href: https://open-webui-pipelines.${MYDOMAIN}/ + homepage.description: "UI-Agnostic OpenAI API Plugin Framework" + +networks: + proxy: + external: true diff --git a/docker/ai/open-webui.yaml b/docker/ai/open-webui.yaml index 97b0155e..f6c654fb 100644 --- a/docker/ai/open-webui.yaml +++ b/docker/ai/open-webui.yaml @@ -9,23 +9,32 @@ name: open-webui services: open-webui: - image: ghcr.io/open-webui/open-webui:v0.5.3 + image: ghcr.io/open-webui/open-webui:v0.5.4 container_name: open-webui restart: unless-stopped volumes: - ${DOCKER_VOLUMES}/open-webui:/app/backend/data networks: - proxy + # Environment Variable Configuration + # https://docs.openwebui.com/getting-started/advanced-topics/env-configuration/ environment: - OLLAMA_API_BASE_URL: https://ollama.${MYDOMAIN}/api - WEBUI_SECRET_KEY: '' + WEBUI_SECRET_KEY: ${OPEN_WEBUI_SECRET_KEY} + WEBUI_URL: "https://open-webui.${MYDOMAIN}" + OLLAMA_API_BASE_URL: "https://ollama.${MYDOMAIN}/api" + # Web search using SearXNG + # Details: https://github.com/open-webui/docs/blob/main/docs/tutorials/integrations/web_search.md + ENABLE_RAG_WEB_SEARCH: true + RAG_WEB_SEARCH_ENGINE: "searxng" + RAG_WEB_SEARCH_RESULT_COUNT: 5 + RAG_WEB_SEARCH_CONCURRENT_REQUESTS: 10 + SEARXNG_QUERY_URL: "https://searxng.${MYDOMAIN}/search?q=&format=json" extra_hosts: - host.docker.internal:host-gateway labels: traefik.enable: true traefik.http.routers.open-webui.entrypoints: websecure traefik.http.routers.open-webui.middlewares: https-local@file - traefik.http.routers.open-webui.rule: Host(`open-webui.${MYDOMAIN}`) traefik.http.services.open-webui.loadbalancer.server.port: 8080 homepage.group: AI homepage.name: "Open WebUI" diff --git a/docker/ai/qdrant.yaml b/docker/ai/qdrant.yaml new file mode 100644 index 00000000..8a0ebc53 --- /dev/null +++ b/docker/ai/qdrant.yaml @@ -0,0 +1,47 @@ +# Qdrant - High-performance, massive-scale Vector Database and Vector Search Engine for the next generation of AI. +# +# Note as of version 1.12.5 the Quickstart on the Dashboard are using incorrect endpoint URLs. +# To follow the tutorial, remove the `dashboard/` from the requests' URLs. Use e.g. F12 Developer Tools -> Edit and Resend feature. +# Issue: https://github.com/qdrant/qdrant-web-ui/issues/252 +# +# https://qdrant.tech/ +# https://github.com/qdrant/qdrant +--- +name: qdrant +services: + qdrant: + image: qdrant/qdrant:v1.12.5 + container_name: qdrant + restart: unless-stopped + healthcheck: + # Based on: https://github.com/qdrant/qdrant/issues/4250 + # Better would be to use `/healthz`: https://qdrant.tech/documentation/guides/monitoring/#kubernetes-health-endpoints + test: + - CMD-SHELL + - bash -c ':> /dev/tcp/127.0.0.1/6333' || exit 1 + interval: 30s + timeout: 10s + retries: 3 + volumes: + - ${DOCKER_VOLUMES}/qdrant/storage:/qdrant/storage + networks: + - proxy + environment: + # Reference: https://github.com/qdrant/qdrant/blob/master/config/config.yaml + QDRANT__LOG_LEVEL: INFO + QDRANT__SERVICE__API_KEY: ${QDRANT_API_KEY:-""} + QDRANT__SERVICE__ENABLE_CORS: false # TODO how do restrict to https://qdrant.${MYDOMAIN} ? + labels: + traefik.enable: true + traefik.http.routers.qdrant.entrypoints: websecure + traefik.http.routers.qdrant.middlewares: https-local@file + traefik.http.services.qdrant.loadbalancer.server.port: 6333 + homepage.group: AI + homepage.name: "Qdrant" + homepage.icon: qdrant.png + homepage.href: https://qdrant.${MYDOMAIN}/dashboard/ + homepage.description: "Open-Source Vector Database" + +networks: + proxy: + external: true diff --git a/docker/automation/n8n.yaml b/docker/automation/n8n.yaml index 11c43a08..f6662b06 100644 --- a/docker/automation/n8n.yaml +++ b/docker/automation/n8n.yaml @@ -6,6 +6,24 @@ --- name: n8n services: + n8n-postgres: + image: postgres:16-alpine + container_name: n8n-postgres + networks: + - n8n-backend + restart: unless-stopped + environment: + POSTGRES_USER: ${N8N_POSTGRES_USER} + POSTGRES_PASSWORD: ${N8N_POSTGRES_PASSWORD} + POSTGRES_DB: ${N8N_POSTGRES_DB} + volumes: + - n8n_postgres_storage:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -h localhost -U ${N8N_POSTGRES_USER} -d ${N8N_POSTGRES_DB}'] + interval: 5s + timeout: 5s + retries: 10 + initContainer: image: busybox container_name: n8n-init @@ -24,14 +42,23 @@ services: N8N_HOST: n8n.${MYDOMAIN} N8N_PORT: 5678 N8N_PROTOCOL: https + N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY} + N8N_USER_MANAGEMENT_JWT_SECRET: ${N8N_USER_MANAGEMENT_JWT_SECRET} + DB_TYPE: postgresdb + DB_POSTGRESDB_HOST: n8n-postgres + DB_POSTGRESDB_USER: ${N8N_POSTGRES_USER} + DB_POSTGRESDB_PASSWORD: ${N8N_POSTGRES_PASSWORD} NODE_ENV: production volumes: - ${DOCKER_VOLUMES}/n8n:/home/node/.n8n networks: + - n8n-backend - proxy depends_on: initContainer: condition: service_completed_successfully + n8n-postgres: + condition: service_healthy labels: traefik.enable: true traefik.http.routers.n8n.entrypoints: websecure @@ -44,5 +71,10 @@ services: homepage.description: Workflow automation platform networks: + n8n-backend: + external: false proxy: external: true + +volumes: + n8n_postgres_storage: diff --git a/docker/tools/searxng.yaml b/docker/tools/searxng.yaml new file mode 100644 index 00000000..c7b5d5bb --- /dev/null +++ b/docker/tools/searxng.yaml @@ -0,0 +1,83 @@ +# SearXNG is a free internet metasearch engine which aggregates results from various search services and databases. +# Users are neither tracked nor profiled. +# +# https://github.com/searxng/searxng +# https://github.com/searxng/searxng-docker +--- +name: searxng +services: + searxng-redis: + container_name: searxng-redis + image: docker.io/valkey/valkey:8-alpine + command: valkey-server --save 30 1 --loglevel warning + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s + retries: 3 + # Named volume is needed to avoid "chown - Operation not permitted" error + volumes: + - searxng-valkey-data:/data + cap_drop: + - ALL + cap_add: + - SETGID + - SETUID + - DAC_OVERRIDE + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + networks: + - searxng-backend + + searxng: + container_name: searxng + image: docker.io/searxng/searxng:latest + restart: unless-stopped + volumes: + - ${DOCKER_VOLUMES}/searxng:/etc/searxng + - ./searxng/settings.yml:/etc/searxng/settings.yml + environment: + SEARXNG_BASE_URL: https://searxng.${MYDOMAIN}/ + # https://docs.searxng.org/admin/settings/settings_server.html + SEARXNG_SECRET: ${SEARXNG_SECRET} + # https://docs.searxng.org/admin/settings/settings_redis.html + SEARXNG_REDIS_URL: "redis://searxng-redis:6379/0" + UWSGI_WORKERS: ${SEARXNG_UWSGI_WORKERS:-4} + UWSGI_THREADS: ${SEARXNG_UWSGI_THREADS:-4} + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + networks: + - searxng-backend + - proxy + labels: + traefik.enable: true + traefik.http.routers.searxng.entrypoints: websecure + traefik.http.routers.searxng.middlewares: https-local@file + traefik.http.services.searxng.loadbalancer.server.port: 8080 + homepage.group: Tools + homepage.name: SearXNG + homepage.icon: searxng.png + homepage.href: https://searxng.${MYDOMAIN}/ + homepage.description: "Internet metasearch engine" + +networks: + searxng-backend: + external: false + proxy: + external: true + +volumes: + searxng-valkey-data: diff --git a/docker/tools/searxng/settings.yml b/docker/tools/searxng/settings.yml new file mode 100644 index 00000000..ffe0906e --- /dev/null +++ b/docker/tools/searxng/settings.yml @@ -0,0 +1,2520 @@ +general: + # Debug mode, only for development. Is overwritten by ${SEARXNG_DEBUG} + debug: false + # displayed name + instance_name: "searxng" + # For example: https://example.com/privacy + privacypolicy_url: false + # use true to use your own donation page written in searx/info/en/donate.md + # use false to disable the donation link + donation_url: false + # mailto:contact@example.com + contact_url: false + # record stats + enable_metrics: true + # expose stats in open metrics format at /metrics + # leave empty to disable (no password set) + # open_metrics: + open_metrics: '' + +brand: + new_issue_url: https://github.com/searxng/searxng/issues/new + docs_url: https://docs.searxng.org/ + public_instances: https://searx.space + wiki_url: https://github.com/searxng/searxng/wiki + issue_url: https://github.com/searxng/searxng/issues + # custom: + # maintainer: "Jon Doe" + # # Custom entries in the footer: [title]: [link] + # links: + # Uptime: https://uptime.searxng.org/history/darmarit-org + # About: "https://searxng.org" + +search: + # Filter results. 0: None, 1: Moderate, 2: Strict + safe_search: 0 + # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "yandex", "mwmbl", + # "seznam", "startpage", "stract", "swisscows", "qwant", "wikipedia" - leave blank to turn it off + # by default. + autocomplete: "" + # minimun characters to type before autocompleter starts + autocomplete_min: 4 + # backend for the favicon near URL in search results. + # Available resolvers: "allesedv", "duckduckgo", "google", "yandex" - leave blank to turn it off by default. + favicon_resolver: "" + # Default search language - leave blank to detect from browser information or + # use codes from 'languages.py' + default_lang: "auto" + # max_page: 0 # if engine supports paging, 0 means unlimited numbers of pages + # Available languages + # languages: + # - all + # - en + # - en-US + # - de + # - it-IT + # - fr + # - fr-BE + # ban time in seconds after engine errors + ban_time_on_fail: 5 + # max ban time in seconds after engine errors + max_ban_time_on_fail: 120 + suspended_times: + # Engine suspension time after error (in seconds; set to 0 to disable) + # For error "Access denied" and "HTTP error [402, 403]" + SearxEngineAccessDenied: 86400 + # For error "CAPTCHA" + SearxEngineCaptcha: 86400 + # For error "Too many request" and "HTTP error 429" + SearxEngineTooManyRequests: 3600 + # Cloudflare CAPTCHA + cf_SearxEngineCaptcha: 1296000 + cf_SearxEngineAccessDenied: 86400 + # ReCAPTCHA + recaptcha_SearxEngineCaptcha: 604800 + + # remove format to deny access, use lower case. + # formats: [html, csv, json, rss] + formats: + - html + - json # Added for Open WebUI + +server: + # Is overwritten by ${SEARXNG_PORT} and ${SEARXNG_BIND_ADDRESS} + port: 8888 + bind_address: "127.0.0.1" + # public URL of the instance, to ensure correct inbound links. Is overwritten + # by ${SEARXNG_URL}. + base_url: / # "http://example.com/location" + # rate limit the number of request on the instance, block some bots. + # Is overwritten by ${SEARXNG_LIMITER} + limiter: false + # enable features designed only for public instances. + # Is overwritten by ${SEARXNG_PUBLIC_INSTANCE} + public_instance: false + + # If your instance owns a /etc/searxng/settings.yml file, then set the following + # values there. + + secret_key: "" # Is overwritten by ${SEARXNG_SECRET} + # Proxy image results through SearXNG. Is overwritten by ${SEARXNG_IMAGE_PROXY} + image_proxy: false + # 1.0 and 1.1 are supported + http_protocol_version: "1.0" + # POST queries are more secure as they don't show up in history but may cause + # problems when using Firefox containers + method: "POST" + default_http_headers: + X-Content-Type-Options: nosniff + X-Download-Options: noopen + X-Robots-Tag: noindex, nofollow + Referrer-Policy: no-referrer + +redis: + # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}. + # https://docs.searxng.org/admin/settings/settings_redis.html#settings-redis + url: false + +ui: + # Custom static path - leave it blank if you didn't change + static_path: "" + # Is overwritten by ${SEARXNG_STATIC_USE_HASH}. + static_use_hash: false + # Custom templates path - leave it blank if you didn't change + templates_path: "" + # query_in_title: When true, the result page's titles contains the query + # it decreases the privacy, since the browser can records the page titles. + query_in_title: false + # infinite_scroll: When true, automatically loads the next page when scrolling to bottom of the current page. + infinite_scroll: false + # ui theme + default_theme: simple + # center the results ? + center_alignment: false + # URL prefix of the internet archive, don't forget trailing slash (if needed). + # cache_url: "https://webcache.googleusercontent.com/search?q=cache:" + # Default interface locale - leave blank to detect from browser information or + # use codes from the 'locales' config section + default_locale: "" + # Open result links in a new tab by default + # results_on_new_tab: false + theme_args: + # style of simple theme: auto, light, dark + simple_style: auto + # Perform search immediately if a category selected. + # Disable to select multiple categories at once and start the search manually. + search_on_category_select: true + # Hotkeys: default or vim + hotkeys: default + # URL formatting: pretty, full or host + url_formatting: pretty + +# Lock arbitrary settings on the preferences page. To find the ID of the user +# setting you want to lock, check the ID of the form on the page "preferences". +# +# preferences: +# lock: +# - language +# - autocomplete +# - method +# - query_in_title + +# searx supports result proxification using an external service: +# https://github.com/asciimoo/morty uncomment below section if you have running +# morty proxy the key is base64 encoded (keep the !!binary notation) +# Note: since commit af77ec3, morty accepts a base64 encoded key. +# +# result_proxy: +# url: http://127.0.0.1:3000/ +# # the key is a base64 encoded string, the YAML !!binary prefix is optional +# key: !!binary "your_morty_proxy_key" +# # [true|false] enable the "proxy" button next to each result +# proxify_results: true + +# communication with search engines +# +outgoing: + # default timeout in seconds, can be override by engine + request_timeout: 3.0 + # the maximum timeout in seconds + # max_request_timeout: 10.0 + # suffix of searx_useragent, could contain information like an email address + # to the administrator + useragent_suffix: "" + # The maximum number of concurrent connections that may be established. + pool_connections: 100 + # Allow the connection pool to maintain keep-alive connections below this + # point. + pool_maxsize: 20 + # See https://www.python-httpx.org/http2/ + enable_http2: true + # uncomment below section if you want to use a custom server certificate + # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults + # and https://www.python-httpx.org/compatibility/#ssl-configuration + # verify: ~/.mitmproxy/mitmproxy-ca-cert.cer + # + # uncomment below section if you want to use a proxyq see: SOCKS proxies + # https://2.python-requests.org/en/latest/user/advanced/#proxies + # are also supported: see + # https://2.python-requests.org/en/latest/user/advanced/#socks + # + # proxies: + # all://: + # - http://proxy1:8080 + # - http://proxy2:8080 + # + # using_tor_proxy: true + # + # Extra seconds to add in order to account for the time taken by the proxy + # + # extra_proxy_timeout: 10 + # + # uncomment below section only if you have more than one network interface + # which can be the source of outgoing search requests + # + # source_ips: + # - 1.1.1.1 + # - 1.1.1.2 + # - fe80::/126 + +# External plugin configuration, for more details see +# https://docs.searxng.org/dev/plugins.html +# +# plugins: +# - plugin1 +# - plugin2 +# - ... + +# Comment or un-comment plugin to activate / deactivate by default. +# +# enabled_plugins: +# # these plugins are enabled if nothing is configured .. +# - 'Basic Calculator' +# - 'Hash plugin' +# - 'Self Information' +# - 'Tracker URL remover' +# - 'Unit converter plugin' +# - 'Ahmia blacklist' # activation depends on outgoing.using_tor_proxy +# # these plugins are disabled if nothing is configured .. +# - 'Hostnames plugin' # see 'hostnames' configuration below +# - 'Open Access DOI rewrite' +# - 'Tor check plugin' + +# Configuration of the "Hostnames plugin": +# +# hostnames: +# replace: +# '(.*\.)?youtube\.com$': 'invidious.example.com' +# '(.*\.)?youtu\.be$': 'invidious.example.com' +# '(.*\.)?reddit\.com$': 'teddit.example.com' +# '(.*\.)?redd\.it$': 'teddit.example.com' +# '(www\.)?twitter\.com$': 'nitter.example.com' +# remove: +# - '(.*\.)?facebook.com$' +# low_priority: +# - '(.*\.)?google(\..*)?$' +# high_priority: +# - '(.*\.)?wikipedia.org$' +# +# Alternatively you can use external files for configuring the "Hostnames plugin": +# +# hostnames: +# replace: 'rewrite-hosts.yml' +# +# Content of 'rewrite-hosts.yml' (place the file in the same directory as 'settings.yml'): +# '(.*\.)?youtube\.com$': 'invidious.example.com' +# '(.*\.)?youtu\.be$': 'invidious.example.com' +# + +checker: + # disable checker when in debug mode + off_when_debug: true + + # use "scheduling: false" to disable scheduling + # scheduling: interval or int + + # to activate the scheduler: + # * uncomment "scheduling" section + # * add "cache2 = name=searxngcache,items=2000,blocks=2000,blocksize=4096,bitmap=1" + # to your uwsgi.ini + + # scheduling: + # start_after: [300, 1800] # delay to start the first run of the checker + # every: [86400, 90000] # how often the checker runs + + # additional tests: only for the YAML anchors (see the engines section) + # + additional_tests: + rosebud: &test_rosebud + matrix: + query: rosebud + lang: en + result_container: + - not_empty + - ['one_title_contains', 'citizen kane'] + test: + - unique_results + + android: &test_android + matrix: + query: ['android'] + lang: ['en', 'de', 'fr', 'zh-CN'] + result_container: + - not_empty + - ['one_title_contains', 'google'] + test: + - unique_results + + # tests: only for the YAML anchors (see the engines section) + tests: + infobox: &tests_infobox + infobox: + matrix: + query: ["linux", "new york", "bbc"] + result_container: + - has_infobox + +categories_as_tabs: + general: + images: + videos: + news: + map: + music: + it: + science: + files: + social media: + +engines: + - name: 9gag + engine: 9gag + shortcut: 9g + disabled: true + + - name: adobe stock + engine: adobe_stock + shortcut: asi + categories: ["images"] + # https://docs.searxng.org/dev/engines/online/adobe_stock.html + adobe_order: relevance + adobe_content_types: ["photo", "illustration", "zip_vector", "template", "3d", "image"] + timeout: 6 + disabled: true + + - name: adobe stock video + engine: adobe_stock + shortcut: asv + network: adobe stock + categories: ["videos"] + adobe_order: relevance + adobe_content_types: ["video"] + timeout: 6 + disabled: true + + - name: adobe stock audio + engine: adobe_stock + shortcut: asa + network: adobe stock + categories: ["music"] + adobe_order: relevance + adobe_content_types: ["audio"] + timeout: 6 + disabled: true + + - name: alpine linux packages + engine: alpinelinux + disabled: true + shortcut: alp + + - name: annas archive + engine: annas_archive + disabled: true + shortcut: aa + + # - name: annas articles + # engine: annas_archive + # shortcut: aaa + # # https://docs.searxng.org/dev/engines/online/annas_archive.html + # aa_content: 'magazine' # book_fiction, book_unknown, book_nonfiction, book_comic + # aa_ext: 'pdf' # pdf, epub, .. + # aa_sort: oldest' # newest, oldest, largest, smallest + + - name: apk mirror + engine: apkmirror + timeout: 4.0 + shortcut: apkm + disabled: true + + - name: apple app store + engine: apple_app_store + shortcut: aps + disabled: true + + # Requires Tor + - name: ahmia + engine: ahmia + categories: onions + enable_http: true + shortcut: ah + + - name: anaconda + engine: xpath + paging: true + first_page_num: 0 + search_url: https://anaconda.org/search?q={query}&page={pageno} + results_xpath: //tbody/tr + url_xpath: ./td/h5/a[last()]/@href + title_xpath: ./td/h5 + content_xpath: ./td[h5]/text() + categories: it + timeout: 6.0 + shortcut: conda + disabled: true + + - name: arch linux wiki + engine: archlinux + shortcut: al + + - name: artic + engine: artic + shortcut: arc + timeout: 4.0 + + - name: arxiv + engine: arxiv + shortcut: arx + timeout: 4.0 + + - name: ask + engine: ask + shortcut: ask + disabled: true + + # tmp suspended: dh key too small + # - name: base + # engine: base + # shortcut: bs + + - name: bandcamp + engine: bandcamp + shortcut: bc + categories: music + + - name: wikipedia + engine: wikipedia + shortcut: wp + # add "list" to the array to get results in the results list + display_type: ["infobox"] + categories: [general] + + - name: bilibili + engine: bilibili + shortcut: bil + disabled: true + + - name: bing + engine: bing + shortcut: bi + disabled: true + + - name: bing images + engine: bing_images + shortcut: bii + + - name: bing news + engine: bing_news + shortcut: bin + + - name: bing videos + engine: bing_videos + shortcut: biv + + - name: bitbucket + engine: xpath + paging: true + search_url: https://bitbucket.org/repo/all/{pageno}?name={query} + url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href + title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"] + content_xpath: //article[@class="repo-summary"]/p + categories: [it, repos] + timeout: 4.0 + disabled: true + shortcut: bb + about: + website: https://bitbucket.org/ + wikidata_id: Q2493781 + official_api_documentation: https://developer.atlassian.com/bitbucket + use_official_api: false + require_api_key: false + results: HTML + + - name: bpb + engine: bpb + shortcut: bpb + disabled: true + + - name: btdigg + engine: btdigg + shortcut: bt + disabled: true + + - name: openverse + engine: openverse + categories: images + shortcut: opv + + - name: media.ccc.de + engine: ccc_media + shortcut: c3tv + # We don't set language: de here because media.ccc.de is not just + # for a German audience. It contains many English videos and many + # German videos have English subtitles. + disabled: true + + - name: chefkoch + engine: chefkoch + shortcut: chef + # to show premium or plus results too: + # skip_premium: false + + - name: cloudflareai + engine: cloudflareai + shortcut: cfai + # get api token and accont id from https://developers.cloudflare.com/workers-ai/get-started/rest-api/ + cf_account_id: 'your_cf_accout_id' + cf_ai_api: 'your_cf_api' + # create your ai gateway by https://developers.cloudflare.com/ai-gateway/get-started/creating-gateway/ + cf_ai_gateway: 'your_cf_ai_gateway_name' + # find the model name from https://developers.cloudflare.com/workers-ai/models/#text-generation + cf_ai_model: 'ai_model_name' + # custom your preferences + # cf_ai_model_display_name: 'Cloudflare AI' + # cf_ai_model_assistant: 'prompts_for_assistant_role' + # cf_ai_model_system: 'prompts_for_system_role' + timeout: 30 + disabled: true + + # - name: core.ac.uk + # engine: core + # categories: science + # shortcut: cor + # # get your API key from: https://core.ac.uk/api-keys/register/ + # api_key: 'unset' + + - name: cppreference + engine: cppreference + shortcut: cpp + paging: false + disabled: true + + - name: crossref + engine: crossref + shortcut: cr + timeout: 30 + disabled: true + + - name: crowdview + engine: json_engine + shortcut: cv + categories: general + paging: false + search_url: https://crowdview-next-js.onrender.com/api/search-v3?query={query} + results_query: results + url_query: link + title_query: title + content_query: snippet + title_html_to_text: true + content_html_to_text: true + disabled: true + about: + website: https://crowdview.ai/ + + - name: yep + engine: yep + shortcut: yep + categories: general + search_type: web + timeout: 5 + disabled: true + + - name: yep images + engine: yep + shortcut: yepi + categories: images + search_type: images + disabled: true + + - name: yep news + engine: yep + shortcut: yepn + categories: news + search_type: news + disabled: true + + - name: curlie + engine: xpath + shortcut: cl + categories: general + disabled: true + paging: true + lang_all: '' + search_url: https://curlie.org/search?q={query}&lang={lang}&start={pageno}&stime=92452189 + page_size: 20 + results_xpath: //div[@id="site-list-content"]/div[@class="site-item"] + url_xpath: ./div[@class="title-and-desc"]/a/@href + title_xpath: ./div[@class="title-and-desc"]/a/div + content_xpath: ./div[@class="title-and-desc"]/div[@class="site-descr"] + about: + website: https://curlie.org/ + wikidata_id: Q60715723 + use_official_api: false + require_api_key: false + results: HTML + + - name: currency + engine: currency_convert + categories: general + shortcut: cc + + - name: deezer + engine: deezer + shortcut: dz + disabled: true + + - name: destatis + engine: destatis + shortcut: destat + disabled: true + + - name: deviantart + engine: deviantart + shortcut: da + timeout: 3.0 + + - name: ddg definitions + engine: duckduckgo_definitions + shortcut: ddd + weight: 2 + disabled: true + tests: *tests_infobox + + # cloudflare protected + # - name: digbt + # engine: digbt + # shortcut: dbt + # timeout: 6.0 + # disabled: true + + - name: docker hub + engine: docker_hub + shortcut: dh + categories: [it, packages] + + - name: encyclosearch + engine: json_engine + shortcut: es + categories: general + paging: true + search_url: https://encyclosearch.org/encyclosphere/search?q={query}&page={pageno}&resultsPerPage=15 + results_query: Results + url_query: SourceURL + title_query: Title + content_query: Description + disabled: true + about: + website: https://encyclosearch.org + official_api_documentation: https://encyclosearch.org/docs/#/rest-api + use_official_api: true + require_api_key: false + results: JSON + + - name: erowid + engine: xpath + paging: true + first_page_num: 0 + page_size: 30 + search_url: https://www.erowid.org/search.php?q={query}&s={pageno} + url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href + title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text() + content_xpath: //dl[@class="results-list"]/dd[@class="result-details"] + categories: [] + shortcut: ew + disabled: true + about: + website: https://www.erowid.org/ + wikidata_id: Q1430691 + official_api_documentation: + use_official_api: false + require_api_key: false + results: HTML + + # - name: elasticsearch + # shortcut: es + # engine: elasticsearch + # base_url: http://localhost:9200 + # username: elastic + # password: changeme + # index: my-index + # # available options: match, simple_query_string, term, terms, custom + # query_type: match + # # if query_type is set to custom, provide your query here + # #custom_query_json: {"query":{"match_all": {}}} + # #show_metadata: false + # disabled: true + + - name: wikidata + engine: wikidata + shortcut: wd + timeout: 3.0 + weight: 2 + # add "list" to the array to get results in the results list + display_type: ["infobox"] + tests: *tests_infobox + categories: [general] + + - name: duckduckgo + engine: duckduckgo + shortcut: ddg + + - name: duckduckgo images + engine: duckduckgo_extra + categories: [images, web] + ddg_category: images + shortcut: ddi + disabled: true + + - name: duckduckgo videos + engine: duckduckgo_extra + categories: [videos, web] + ddg_category: videos + shortcut: ddv + disabled: true + + - name: duckduckgo news + engine: duckduckgo_extra + categories: [news, web] + ddg_category: news + shortcut: ddn + disabled: true + + - name: duckduckgo weather + engine: duckduckgo_weather + shortcut: ddw + disabled: true + + - name: apple maps + engine: apple_maps + shortcut: apm + disabled: true + timeout: 5.0 + + - name: emojipedia + engine: emojipedia + timeout: 4.0 + shortcut: em + disabled: true + + - name: tineye + engine: tineye + shortcut: tin + timeout: 9.0 + disabled: true + + - name: etymonline + engine: xpath + paging: true + search_url: https://etymonline.com/search?page={pageno}&q={query} + url_xpath: //a[contains(@class, "word__name--")]/@href + title_xpath: //a[contains(@class, "word__name--")] + content_xpath: //section[contains(@class, "word__defination")] + first_page_num: 1 + shortcut: et + categories: [dictionaries] + about: + website: https://www.etymonline.com/ + wikidata_id: Q1188617 + official_api_documentation: + use_official_api: false + require_api_key: false + results: HTML + + # - name: ebay + # engine: ebay + # shortcut: eb + # base_url: 'https://www.ebay.com' + # disabled: true + # timeout: 5 + + - name: 1x + engine: www1x + shortcut: 1x + timeout: 3.0 + disabled: true + + - name: fdroid + engine: fdroid + shortcut: fd + disabled: true + + - name: findthatmeme + engine: findthatmeme + shortcut: ftm + disabled: true + + - name: flickr + categories: images + shortcut: fl + # You can use the engine using the official stable API, but you need an API + # key, see: https://www.flickr.com/services/apps/create/ + # engine: flickr + # api_key: 'apikey' # required! + # Or you can use the html non-stable engine, activated by default + engine: flickr_noapi + + - name: free software directory + engine: mediawiki + shortcut: fsd + categories: [it, software wikis] + base_url: https://directory.fsf.org/ + search_type: title + timeout: 5.0 + disabled: true + about: + website: https://directory.fsf.org/ + wikidata_id: Q2470288 + + # - name: freesound + # engine: freesound + # shortcut: fnd + # disabled: true + # timeout: 15.0 + # API key required, see: https://freesound.org/docs/api/overview.html + # api_key: MyAPIkey + + - name: frinkiac + engine: frinkiac + shortcut: frk + disabled: true + + - name: fyyd + engine: fyyd + shortcut: fy + timeout: 8.0 + disabled: true + + - name: geizhals + engine: geizhals + shortcut: geiz + disabled: true + + - name: genius + engine: genius + shortcut: gen + + - name: gentoo + engine: mediawiki + shortcut: ge + categories: ["it", "software wikis"] + base_url: "https://wiki.gentoo.org/" + api_path: "api.php" + search_type: text + timeout: 10 + + - name: gitlab + engine: gitlab + base_url: https://gitlab.com + shortcut: gl + disabled: true + about: + website: https://gitlab.com/ + wikidata_id: Q16639197 + + # - name: gnome + # engine: gitlab + # base_url: https://gitlab.gnome.org + # shortcut: gn + # about: + # website: https://gitlab.gnome.org + # wikidata_id: Q44316 + + - name: github + engine: github + shortcut: gh + + - name: codeberg + # https://docs.searxng.org/dev/engines/online/gitea.html + engine: gitea + base_url: https://codeberg.org + shortcut: cb + disabled: true + + - name: gitea.com + engine: gitea + base_url: https://gitea.com + shortcut: gitea + disabled: true + + - name: goodreads + engine: goodreads + shortcut: good + timeout: 4.0 + disabled: true + + - name: google + engine: google + shortcut: go + # additional_tests: + # android: *test_android + + - name: google images + engine: google_images + shortcut: goi + # additional_tests: + # android: *test_android + # dali: + # matrix: + # query: ['Dali Christ'] + # lang: ['en', 'de', 'fr', 'zh-CN'] + # result_container: + # - ['one_title_contains', 'Salvador'] + + - name: google news + engine: google_news + shortcut: gon + # additional_tests: + # android: *test_android + + - name: google videos + engine: google_videos + shortcut: gov + # additional_tests: + # android: *test_android + + - name: google scholar + engine: google_scholar + shortcut: gos + + - name: google play apps + engine: google_play + categories: [files, apps] + shortcut: gpa + play_categ: apps + disabled: true + + - name: google play movies + engine: google_play + categories: videos + shortcut: gpm + play_categ: movies + disabled: true + + - name: material icons + engine: material_icons + categories: images + shortcut: mi + disabled: true + + - name: habrahabr + engine: xpath + paging: true + search_url: https://habr.com/en/search/page{pageno}/?q={query} + results_xpath: //article[contains(@class, "tm-articles-list__item")] + url_xpath: .//a[@class="tm-title__link"]/@href + title_xpath: .//a[@class="tm-title__link"] + content_xpath: .//div[contains(@class, "article-formatted-body")] + categories: it + timeout: 4.0 + disabled: true + shortcut: habr + about: + website: https://habr.com/ + wikidata_id: Q4494434 + official_api_documentation: https://habr.com/en/docs/help/api/ + use_official_api: false + require_api_key: false + results: HTML + + - name: hackernews + engine: hackernews + shortcut: hn + disabled: true + + - name: hex + engine: hex + shortcut: hex + disabled: true + # Valid values: name inserted_at updated_at total_downloads recent_downloads + sort_criteria: "recent_downloads" + page_size: 10 + + - name: crates.io + engine: crates + shortcut: crates + disabled: true + timeout: 6.0 + + - name: hoogle + engine: xpath + search_url: https://hoogle.haskell.org/?hoogle={query} + results_xpath: '//div[@class="result"]' + title_xpath: './/div[@class="ans"]//a' + url_xpath: './/div[@class="ans"]//a/@href' + content_xpath: './/div[@class="from"]' + page_size: 20 + categories: [it, packages] + shortcut: ho + about: + website: https://hoogle.haskell.org/ + wikidata_id: Q34010 + official_api_documentation: https://hackage.haskell.org/api + use_official_api: false + require_api_key: false + results: JSON + + - name: imdb + engine: imdb + shortcut: imdb + timeout: 6.0 + disabled: true + + - name: imgur + engine: imgur + shortcut: img + disabled: true + + - name: ina + engine: ina + shortcut: in + timeout: 6.0 + disabled: true + + - name: invidious + engine: invidious + # Instanes will be selected randomly, see https://api.invidious.io/ for + # instances that are stable (good uptime) and close to you. + base_url: + - https://invidious.io.lol + - https://invidious.fdn.fr + - https://yt.artemislena.eu + - https://invidious.tiekoetter.com + - https://invidious.flokinet.to + - https://vid.puffyan.us + - https://invidious.privacydev.net + - https://inv.tux.pizza + shortcut: iv + timeout: 3.0 + disabled: true + + - name: jisho + engine: jisho + shortcut: js + timeout: 3.0 + disabled: true + + - name: kickass + engine: kickass + base_url: + - https://kickasstorrents.to + - https://kickasstorrents.cr + - https://kickasstorrent.cr + - https://kickass.sx + - https://kat.am + shortcut: kc + timeout: 4.0 + + - name: lemmy communities + engine: lemmy + lemmy_type: Communities + shortcut: leco + + - name: lemmy users + engine: lemmy + network: lemmy communities + lemmy_type: Users + shortcut: leus + + - name: lemmy posts + engine: lemmy + network: lemmy communities + lemmy_type: Posts + shortcut: lepo + + - name: lemmy comments + engine: lemmy + network: lemmy communities + lemmy_type: Comments + shortcut: lecom + + - name: library genesis + engine: xpath + # search_url: https://libgen.is/search.php?req={query} + search_url: https://libgen.rs/search.php?req={query} + url_xpath: //a[contains(@href,"book/index.php?md5")]/@href + title_xpath: //a[contains(@href,"book/")]/text()[1] + content_xpath: //td/a[1][contains(@href,"=author")]/text() + categories: files + timeout: 7.0 + disabled: true + shortcut: lg + about: + website: https://libgen.fun/ + wikidata_id: Q22017206 + official_api_documentation: + use_official_api: false + require_api_key: false + results: HTML + + - name: z-library + engine: zlibrary + shortcut: zlib + categories: files + timeout: 7.0 + + - name: library of congress + engine: loc + shortcut: loc + categories: images + + - name: libretranslate + engine: libretranslate + # https://github.com/LibreTranslate/LibreTranslate?tab=readme-ov-file#mirrors + base_url: + - https://translate.terraprint.co + - https://trans.zillyhuhn.com + # api_key: abc123 + shortcut: lt + disabled: true + + - name: lingva + engine: lingva + shortcut: lv + # set lingva instance in url, by default it will use the official instance + # url: https://lingva.thedaviddelta.com + + - name: lobste.rs + engine: xpath + search_url: https://lobste.rs/search?q={query}&what=stories&order=relevance + results_xpath: //li[contains(@class, "story")] + url_xpath: .//a[@class="u-url"]/@href + title_xpath: .//a[@class="u-url"] + content_xpath: .//a[@class="domain"] + categories: it + shortcut: lo + timeout: 5.0 + disabled: true + about: + website: https://lobste.rs/ + wikidata_id: Q60762874 + official_api_documentation: + use_official_api: false + require_api_key: false + results: HTML + + - name: mastodon users + engine: mastodon + mastodon_type: accounts + base_url: https://mastodon.social + shortcut: mau + + - name: mastodon hashtags + engine: mastodon + mastodon_type: hashtags + base_url: https://mastodon.social + shortcut: mah + + # - name: matrixrooms + # engine: mrs + # # https://docs.searxng.org/dev/engines/online/mrs.html + # # base_url: https://mrs-api-host + # shortcut: mtrx + # disabled: true + + - name: mdn + shortcut: mdn + engine: json_engine + categories: [it] + paging: true + search_url: https://developer.mozilla.org/api/v1/search?q={query}&page={pageno} + results_query: documents + url_query: mdn_url + url_prefix: https://developer.mozilla.org + title_query: title + content_query: summary + about: + website: https://developer.mozilla.org + wikidata_id: Q3273508 + official_api_documentation: null + use_official_api: false + require_api_key: false + results: JSON + + - name: metacpan + engine: metacpan + shortcut: cpan + disabled: true + number_of_results: 20 + + # - name: meilisearch + # engine: meilisearch + # shortcut: mes + # enable_http: true + # base_url: http://localhost:7700 + # index: my-index + + - name: mixcloud + engine: mixcloud + shortcut: mc + + # MongoDB engine + # Required dependency: pymongo + # - name: mymongo + # engine: mongodb + # shortcut: md + # exact_match_only: false + # host: '127.0.0.1' + # port: 27017 + # enable_http: true + # results_per_page: 20 + # database: 'business' + # collection: 'reviews' # name of the db collection + # key: 'name' # key in the collection to search for + + - name: mozhi + engine: mozhi + base_url: + - https://mozhi.aryak.me + - https://translate.bus-hit.me + - https://nyc1.mz.ggtyler.dev + # mozhi_engine: google - see https://mozhi.aryak.me for supported engines + timeout: 4.0 + shortcut: mz + disabled: true + + - name: mwmbl + engine: mwmbl + # api_url: https://api.mwmbl.org + shortcut: mwm + disabled: true + + - name: npm + engine: npm + shortcut: npm + timeout: 5.0 + disabled: true + + - name: nyaa + engine: nyaa + shortcut: nt + disabled: true + + - name: mankier + engine: json_engine + search_url: https://www.mankier.com/api/v2/mans/?q={query} + results_query: results + url_query: url + title_query: name + content_query: description + categories: it + shortcut: man + about: + website: https://www.mankier.com/ + official_api_documentation: https://www.mankier.com/api + use_official_api: true + require_api_key: false + results: JSON + + # read https://docs.searxng.org/dev/engines/online/mullvad_leta.html + # - name: mullvadleta + # engine: mullvad_leta + # leta_engine: google # choose one of the following: google, brave + # use_cache: true # Only 100 non-cache searches per day, suggested only for private instances + # search_url: https://leta.mullvad.net + # categories: [general, web] + # shortcut: ml + + - name: odysee + engine: odysee + shortcut: od + disabled: true + + - name: openairedatasets + engine: json_engine + paging: true + search_url: https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} + results_query: response/results/result + url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ + title_query: metadata/oaf:entity/oaf:result/title/$ + content_query: metadata/oaf:entity/oaf:result/description/$ + content_html_to_text: true + categories: "science" + shortcut: oad + timeout: 5.0 + about: + website: https://www.openaire.eu/ + wikidata_id: Q25106053 + official_api_documentation: https://api.openaire.eu/ + use_official_api: false + require_api_key: false + results: JSON + + - name: openairepublications + engine: json_engine + paging: true + search_url: https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} + results_query: response/results/result + url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ + title_query: metadata/oaf:entity/oaf:result/title/$ + content_query: metadata/oaf:entity/oaf:result/description/$ + content_html_to_text: true + categories: science + shortcut: oap + timeout: 5.0 + about: + website: https://www.openaire.eu/ + wikidata_id: Q25106053 + official_api_documentation: https://api.openaire.eu/ + use_official_api: false + require_api_key: false + results: JSON + + - name: openlibrary + engine: openlibrary + shortcut: ol + timeout: 5 + disabled: true + + - name: openmeteo + engine: open_meteo + shortcut: om + disabled: true + + # - name: opensemanticsearch + # engine: opensemantic + # shortcut: oss + # base_url: 'http://localhost:8983/solr/opensemanticsearch/' + + - name: openstreetmap + engine: openstreetmap + shortcut: osm + + - name: openrepos + engine: xpath + paging: true + search_url: https://openrepos.net/search/node/{query}?page={pageno} + url_xpath: //li[@class="search-result"]//h3[@class="title"]/a/@href + title_xpath: //li[@class="search-result"]//h3[@class="title"]/a + content_xpath: //li[@class="search-result"]//div[@class="search-snippet-info"]//p[@class="search-snippet"] + categories: files + timeout: 4.0 + disabled: true + shortcut: or + about: + website: https://openrepos.net/ + wikidata_id: + official_api_documentation: + use_official_api: false + require_api_key: false + results: HTML + + - name: packagist + engine: json_engine + paging: true + search_url: https://packagist.org/search.json?q={query}&page={pageno} + results_query: results + url_query: url + title_query: name + content_query: description + categories: [it, packages] + disabled: true + timeout: 5.0 + shortcut: pack + about: + website: https://packagist.org + wikidata_id: Q108311377 + official_api_documentation: https://packagist.org/apidoc + use_official_api: true + require_api_key: false + results: JSON + + - name: pdbe + engine: pdbe + shortcut: pdb + # Hide obsolete PDB entries. Default is not to hide obsolete structures + # hide_obsolete: false + + - name: photon + engine: photon + shortcut: ph + + - name: pinterest + engine: pinterest + shortcut: pin + + - name: piped + engine: piped + shortcut: ppd + categories: videos + piped_filter: videos + timeout: 3.0 + + # URL to use as link and for embeds + frontend_url: https://srv.piped.video + # Instance will be selected randomly, for more see https://piped-instances.kavin.rocks/ + backend_url: + - https://pipedapi.kavin.rocks + - https://pipedapi-libre.kavin.rocks + - https://pipedapi.adminforge.de + + - name: piped.music + engine: piped + network: piped + shortcut: ppdm + categories: music + piped_filter: music_songs + timeout: 3.0 + + - name: piratebay + engine: piratebay + shortcut: tpb + # You may need to change this URL to a proxy if piratebay is blocked in your + # country + url: https://thepiratebay.org/ + timeout: 3.0 + + - name: pixiv + shortcut: pv + engine: pixiv + disabled: true + inactive: true + pixiv_image_proxies: + - https://pximg.example.org + # A proxy is required to load the images. Hosting an image proxy server + # for Pixiv: + # --> https://pixivfe.pages.dev/hosting-image-proxy-server/ + # Proxies from public instances. Ask the public instances owners if they + # agree to receive traffic from SearXNG! + # --> https://codeberg.org/VnPower/PixivFE#instances + # --> https://github.com/searxng/searxng/pull/3192#issuecomment-1941095047 + # image proxy of https://pixiv.cat + # - https://i.pixiv.cat + # image proxy of https://www.pixiv.pics + # - https://pximg.cocomi.eu.org + # image proxy of https://pixivfe.exozy.me + # - https://pximg.exozy.me + # image proxy of https://pixivfe.ducks.party + # - https://pixiv.ducks.party + # image proxy of https://pixiv.perennialte.ch + # - https://pximg.perennialte.ch + + - name: podcastindex + engine: podcastindex + shortcut: podcast + + # Required dependency: psychopg2 + # - name: postgresql + # engine: postgresql + # database: postgres + # username: postgres + # password: postgres + # limit: 10 + # query_str: 'SELECT * from my_table WHERE my_column = %(query)s' + # shortcut : psql + + - name: presearch + engine: presearch + search_type: search + categories: [general, web] + shortcut: ps + timeout: 4.0 + disabled: true + + - name: presearch images + engine: presearch + network: presearch + search_type: images + categories: [images, web] + timeout: 4.0 + shortcut: psimg + disabled: true + + - name: presearch videos + engine: presearch + network: presearch + search_type: videos + categories: [general, web] + timeout: 4.0 + shortcut: psvid + disabled: true + + - name: presearch news + engine: presearch + network: presearch + search_type: news + categories: [news, web] + timeout: 4.0 + shortcut: psnews + disabled: true + + - name: pub.dev + engine: xpath + shortcut: pd + search_url: https://pub.dev/packages?q={query}&page={pageno} + paging: true + results_xpath: //div[contains(@class,"packages-item")] + url_xpath: ./div/h3/a/@href + title_xpath: ./div/h3/a + content_xpath: ./div/div/div[contains(@class,"packages-description")]/span + categories: [packages, it] + timeout: 3.0 + disabled: true + first_page_num: 1 + about: + website: https://pub.dev/ + official_api_documentation: https://pub.dev/help/api + use_official_api: false + require_api_key: false + results: HTML + + - name: pubmed + engine: pubmed + shortcut: pub + timeout: 3.0 + + - name: pypi + shortcut: pypi + engine: pypi + + - name: qwant + qwant_categ: web + engine: qwant + shortcut: qw + categories: [general, web] + additional_tests: + rosebud: *test_rosebud + + - name: qwant news + qwant_categ: news + engine: qwant + shortcut: qwn + categories: news + network: qwant + + - name: qwant images + qwant_categ: images + engine: qwant + shortcut: qwi + categories: [images, web] + network: qwant + + - name: qwant videos + qwant_categ: videos + engine: qwant + shortcut: qwv + categories: [videos, web] + network: qwant + + # - name: library + # engine: recoll + # shortcut: lib + # base_url: 'https://recoll.example.org/' + # search_dir: '' + # mount_prefix: /export + # dl_prefix: 'https://download.example.org' + # timeout: 30.0 + # categories: files + # disabled: true + + # - name: recoll library reference + # engine: recoll + # base_url: 'https://recoll.example.org/' + # search_dir: reference + # mount_prefix: /export + # dl_prefix: 'https://download.example.org' + # shortcut: libr + # timeout: 30.0 + # categories: files + # disabled: true + + - name: radio browser + engine: radio_browser + shortcut: rb + + - name: reddit + engine: reddit + shortcut: re + page_size: 25 + disabled: true + + - name: right dao + engine: xpath + paging: true + page_size: 12 + search_url: https://rightdao.com/search?q={query}&start={pageno} + results_xpath: //div[contains(@class, "description")] + url_xpath: ../div[contains(@class, "title")]/a/@href + title_xpath: ../div[contains(@class, "title")] + content_xpath: . + categories: general + shortcut: rd + disabled: true + about: + website: https://rightdao.com/ + use_official_api: false + require_api_key: false + results: HTML + + - name: rottentomatoes + engine: rottentomatoes + shortcut: rt + disabled: true + + # Required dependency: redis + # - name: myredis + # shortcut : rds + # engine: redis_server + # exact_match_only: false + # host: '127.0.0.1' + # port: 6379 + # enable_http: true + # password: '' + # db: 0 + + # tmp suspended: bad certificate + # - name: scanr structures + # shortcut: scs + # engine: scanr_structures + # disabled: true + + - name: searchmysite + engine: xpath + shortcut: sms + categories: general + paging: true + search_url: https://searchmysite.net/search/?q={query}&page={pageno} + results_xpath: //div[contains(@class,'search-result')] + url_xpath: .//a[contains(@class,'result-link')]/@href + title_xpath: .//span[contains(@class,'result-title-txt')]/text() + content_xpath: ./p[@id='result-hightlight'] + disabled: true + about: + website: https://searchmysite.net + + - name: sepiasearch + engine: sepiasearch + shortcut: sep + + - name: soundcloud + engine: soundcloud + shortcut: sc + + - name: stackoverflow + engine: stackexchange + shortcut: st + api_site: 'stackoverflow' + categories: [it, q&a] + + - name: askubuntu + engine: stackexchange + shortcut: ubuntu + api_site: 'askubuntu' + categories: [it, q&a] + + - name: superuser + engine: stackexchange + shortcut: su + api_site: 'superuser' + categories: [it, q&a] + + - name: discuss.python + engine: discourse + shortcut: dpy + base_url: 'https://discuss.python.org' + categories: [it, q&a] + disabled: true + + - name: caddy.community + engine: discourse + shortcut: caddy + base_url: 'https://caddy.community' + categories: [it, q&a] + disabled: true + + - name: pi-hole.community + engine: discourse + shortcut: pi + categories: [it, q&a] + base_url: 'https://discourse.pi-hole.net' + disabled: true + + - name: searchcode code + engine: searchcode_code + shortcut: scc + disabled: true + + # - name: searx + # engine: searx_engine + # shortcut: se + # instance_urls : + # - http://127.0.0.1:8888/ + # - ... + # disabled: true + + - name: semantic scholar + engine: semantic_scholar + disabled: true + shortcut: se + + # Spotify needs API credentials + # - name: spotify + # engine: spotify + # shortcut: stf + # api_client_id: ******* + # api_client_secret: ******* + + # - name: solr + # engine: solr + # shortcut: slr + # base_url: http://localhost:8983 + # collection: collection_name + # sort: '' # sorting: asc or desc + # field_list: '' # comma separated list of field names to display on the UI + # default_fields: '' # default field to query + # query_fields: '' # query fields + # enable_http: true + + # - name: springer nature + # engine: springer + # # get your API key from: https://dev.springernature.com/signup + # # working API key, for test & debug: "a69685087d07eca9f13db62f65b8f601" + # api_key: 'unset' + # shortcut: springer + # timeout: 15.0 + + - name: startpage + engine: startpage + shortcut: sp + timeout: 6.0 + disabled: true + additional_tests: + rosebud: *test_rosebud + + - name: tokyotoshokan + engine: tokyotoshokan + shortcut: tt + timeout: 6.0 + disabled: true + + - name: solidtorrents + engine: solidtorrents + shortcut: solid + timeout: 4.0 + base_url: + - https://solidtorrents.to + - https://bitsearch.to + + # For this demo of the sqlite engine download: + # https://liste.mediathekview.de/filmliste-v2.db.bz2 + # and unpack into searx/data/filmliste-v2.db + # Query to test: "!demo concert" + # + # - name: demo + # engine: sqlite + # shortcut: demo + # categories: general + # result_template: default.html + # database: searx/data/filmliste-v2.db + # query_str: >- + # SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, + # COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, + # description AS content + # FROM film + # WHERE title LIKE :wildcard OR description LIKE :wildcard + # ORDER BY duration DESC + + - name: tagesschau + engine: tagesschau + # when set to false, display URLs from Tagesschau, and not the actual source + # (e.g. NDR, WDR, SWR, HR, ...) + use_source_url: true + shortcut: ts + disabled: true + + - name: tmdb + engine: xpath + paging: true + categories: movies + search_url: https://www.themoviedb.org/search?page={pageno}&query={query} + results_xpath: //div[contains(@class,"movie") or contains(@class,"tv")]//div[contains(@class,"card")] + url_xpath: .//div[contains(@class,"poster")]/a/@href + thumbnail_xpath: .//img/@src + title_xpath: .//div[contains(@class,"title")]//h2 + content_xpath: .//div[contains(@class,"overview")] + shortcut: tm + disabled: true + + # Requires Tor + - name: torch + engine: xpath + paging: true + search_url: + http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and + results_xpath: //table//tr + url_xpath: ./td[2]/a + title_xpath: ./td[2]/b + content_xpath: ./td[2]/small + categories: onions + enable_http: true + shortcut: tch + + # torznab engine lets you query any torznab compatible indexer. Using this + # engine in combination with Jackett opens the possibility to query a lot of + # public and private indexers directly from SearXNG. More details at: + # https://docs.searxng.org/dev/engines/online/torznab.html + # + # - name: Torznab EZTV + # engine: torznab + # shortcut: eztv + # base_url: http://localhost:9117/api/v2.0/indexers/eztv/results/torznab + # enable_http: true # if using localhost + # api_key: xxxxxxxxxxxxxxx + # show_magnet_links: true + # show_torrent_files: false + # # https://github.com/Jackett/Jackett/wiki/Jackett-Categories + # torznab_categories: # optional + # - 2000 + # - 5000 + + # tmp suspended - too slow, too many errors + # - name: urbandictionary + # engine : xpath + # search_url : https://www.urbandictionary.com/define.php?term={query} + # url_xpath : //*[@class="word"]/@href + # title_xpath : //*[@class="def-header"] + # content_xpath: //*[@class="meaning"] + # shortcut: ud + + - name: unsplash + engine: unsplash + shortcut: us + + - name: yandex + engine: yandex + categories: general + search_type: web + shortcut: yd + disabled: true + inactive: true + + - name: yandex images + engine: yandex + categories: images + search_type: images + shortcut: ydi + disabled: true + inactive: true + + - name: yandex music + engine: yandex_music + shortcut: ydm + disabled: true + # https://yandex.com/support/music/access.html + inactive: true + + - name: yahoo + engine: yahoo + shortcut: yh + disabled: true + + - name: yahoo news + engine: yahoo_news + shortcut: yhn + + - name: youtube + shortcut: yt + # You can use the engine using the official stable API, but you need an API + # key See: https://console.developers.google.com/project + # + # engine: youtube_api + # api_key: 'apikey' # required! + # + # Or you can use the html non-stable engine, activated by default + engine: youtube_noapi + + - name: dailymotion + engine: dailymotion + shortcut: dm + + - name: vimeo + engine: vimeo + shortcut: vm + + - name: wiby + engine: json_engine + paging: true + search_url: https://wiby.me/json/?q={query}&p={pageno} + url_query: URL + title_query: Title + content_query: Snippet + categories: [general, web] + shortcut: wib + disabled: true + about: + website: https://wiby.me/ + + - name: wikibooks + engine: mediawiki + weight: 0.5 + shortcut: wb + categories: [general, wikimedia] + base_url: "https://{language}.wikibooks.org/" + search_type: text + disabled: true + about: + website: https://www.wikibooks.org/ + wikidata_id: Q367 + + - name: wikinews + engine: mediawiki + shortcut: wn + categories: [news, wikimedia] + base_url: "https://{language}.wikinews.org/" + search_type: text + srsort: create_timestamp_desc + about: + website: https://www.wikinews.org/ + wikidata_id: Q964 + + - name: wikiquote + engine: mediawiki + weight: 0.5 + shortcut: wq + categories: [general, wikimedia] + base_url: "https://{language}.wikiquote.org/" + search_type: text + disabled: true + additional_tests: + rosebud: *test_rosebud + about: + website: https://www.wikiquote.org/ + wikidata_id: Q369 + + - name: wikisource + engine: mediawiki + weight: 0.5 + shortcut: ws + categories: [general, wikimedia] + base_url: "https://{language}.wikisource.org/" + search_type: text + disabled: true + about: + website: https://www.wikisource.org/ + wikidata_id: Q263 + + - name: wikispecies + engine: mediawiki + shortcut: wsp + categories: [general, science, wikimedia] + base_url: "https://species.wikimedia.org/" + search_type: text + disabled: true + about: + website: https://species.wikimedia.org/ + wikidata_id: Q13679 + tests: + wikispecies: + matrix: + query: "Campbell, L.I. et al. 2011: MicroRNAs" + lang: en + result_container: + - not_empty + - ['one_title_contains', 'Tardigrada'] + test: + - unique_results + + - name: wiktionary + engine: mediawiki + shortcut: wt + categories: [dictionaries, wikimedia] + base_url: "https://{language}.wiktionary.org/" + search_type: text + about: + website: https://www.wiktionary.org/ + wikidata_id: Q151 + + - name: wikiversity + engine: mediawiki + weight: 0.5 + shortcut: wv + categories: [general, wikimedia] + base_url: "https://{language}.wikiversity.org/" + search_type: text + disabled: true + about: + website: https://www.wikiversity.org/ + wikidata_id: Q370 + + - name: wikivoyage + engine: mediawiki + weight: 0.5 + shortcut: wy + categories: [general, wikimedia] + base_url: "https://{language}.wikivoyage.org/" + search_type: text + disabled: true + about: + website: https://www.wikivoyage.org/ + wikidata_id: Q373 + + - name: wikicommons.images + engine: wikicommons + shortcut: wc + categories: images + search_type: images + number_of_results: 10 + + - name: wikicommons.videos + engine: wikicommons + shortcut: wcv + categories: videos + search_type: videos + number_of_results: 10 + + - name: wikicommons.audio + engine: wikicommons + shortcut: wca + categories: music + search_type: audio + number_of_results: 10 + + - name: wikicommons.files + engine: wikicommons + shortcut: wcf + categories: files + search_type: files + number_of_results: 10 + + - name: wolframalpha + shortcut: wa + # You can use the engine using the official stable API, but you need an API + # key. See: https://products.wolframalpha.com/api/ + # + # engine: wolframalpha_api + # api_key: '' + # + # Or you can use the html non-stable engine, activated by default + engine: wolframalpha_noapi + timeout: 6.0 + categories: general + disabled: true + + - name: dictzone + engine: dictzone + shortcut: dc + + - name: mymemory translated + engine: translated + shortcut: tl + timeout: 5.0 + # You can use without an API key, but you are limited to 1000 words/day + # See: https://mymemory.translated.net/doc/usagelimits.php + # api_key: '' + + # Required dependency: mysql-connector-python + # - name: mysql + # engine: mysql_server + # database: mydatabase + # username: user + # password: pass + # limit: 10 + # query_str: 'SELECT * from mytable WHERE fieldname=%(query)s' + # shortcut: mysql + + # Required dependency: mariadb + # - name: mariadb + # engine: mariadb_server + # database: mydatabase + # username: user + # password: pass + # limit: 10 + # query_str: 'SELECT * from mytable WHERE fieldname=%(query)s' + # shortcut: mdb + + - name: 1337x + engine: 1337x + shortcut: 1337x + disabled: true + + - name: duden + engine: duden + shortcut: du + disabled: true + + - name: seznam + shortcut: szn + engine: seznam + disabled: true + + # - name: deepl + # engine: deepl + # shortcut: dpl + # # You can use the engine using the official stable API, but you need an API key + # # See: https://www.deepl.com/pro-api?cta=header-pro-api + # api_key: '' # required! + # timeout: 5.0 + # disabled: true + + - name: mojeek + shortcut: mjk + engine: mojeek + categories: [general, web] + disabled: true + + - name: mojeek images + shortcut: mjkimg + engine: mojeek + categories: [images, web] + search_type: images + paging: false + disabled: true + + - name: mojeek news + shortcut: mjknews + engine: mojeek + categories: [news, web] + search_type: news + paging: false + disabled: true + + - name: moviepilot + engine: moviepilot + shortcut: mp + disabled: true + + - name: naver + shortcut: nvr + categories: [general, web] + engine: xpath + paging: true + search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno} + url_xpath: //a[@class="link_tit"]/@href + title_xpath: //a[@class="link_tit"] + content_xpath: //div[@class="total_dsc_wrap"]/a + first_page_num: 1 + page_size: 10 + disabled: true + about: + website: https://www.naver.com/ + wikidata_id: Q485639 + official_api_documentation: https://developers.naver.com/docs/nmt/examples/ + use_official_api: false + require_api_key: false + results: HTML + language: ko + + - name: rubygems + shortcut: rbg + engine: xpath + paging: true + search_url: https://rubygems.org/search?page={pageno}&query={query} + results_xpath: /html/body/main/div/a[@class="gems__gem"] + url_xpath: ./@href + title_xpath: ./span/h2 + content_xpath: ./span/p + suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a + first_page_num: 1 + categories: [it, packages] + disabled: true + about: + website: https://rubygems.org/ + wikidata_id: Q1853420 + official_api_documentation: https://guides.rubygems.org/rubygems-org-api/ + use_official_api: false + require_api_key: false + results: HTML + + - name: peertube + engine: peertube + shortcut: ptb + paging: true + # alternatives see: https://instances.joinpeertube.org/instances + # base_url: https://tube.4aem.com + categories: videos + disabled: true + timeout: 6.0 + + - name: mediathekviewweb + engine: mediathekviewweb + shortcut: mvw + disabled: true + + - name: yacy + # https://docs.searxng.org/dev/engines/online/yacy.html + engine: yacy + categories: general + search_type: text + base_url: + - https://yacy.searchlab.eu + # see https://github.com/searxng/searxng/pull/3631#issuecomment-2240903027 + # - https://search.kyun.li + # - https://yacy.securecomcorp.eu + # - https://yacy.myserv.ca + # - https://yacy.nsupdate.info + # - https://yacy.electroncash.de + shortcut: ya + disabled: true + # if you aren't using HTTPS for your local yacy instance disable https + # enable_http: false + search_mode: 'global' + # timeout can be reduced in 'local' search mode + timeout: 5.0 + + - name: yacy images + engine: yacy + network: yacy + categories: images + search_type: image + shortcut: yai + disabled: true + # timeout can be reduced in 'local' search mode + timeout: 5.0 + + - name: rumble + engine: rumble + shortcut: ru + base_url: https://rumble.com/ + paging: true + categories: videos + disabled: true + + - name: livespace + engine: livespace + shortcut: ls + categories: videos + disabled: true + timeout: 5.0 + + - name: wordnik + engine: wordnik + shortcut: def + categories: [dictionaries] + timeout: 5.0 + + - name: woxikon.de synonyme + engine: xpath + shortcut: woxi + categories: [dictionaries] + timeout: 5.0 + disabled: true + search_url: https://synonyme.woxikon.de/synonyme/{query}.php + url_xpath: //div[@class="upper-synonyms"]/a/@href + content_xpath: //div[@class="synonyms-list-group"] + title_xpath: //div[@class="upper-synonyms"]/a + no_result_for_http_status: [404] + about: + website: https://www.woxikon.de/ + wikidata_id: # No Wikidata ID + use_official_api: false + require_api_key: false + results: HTML + language: de + + - name: seekr news + engine: seekr + shortcut: senews + categories: news + seekr_category: news + disabled: true + + - name: seekr images + engine: seekr + network: seekr news + shortcut: seimg + categories: images + seekr_category: images + disabled: true + + - name: seekr videos + engine: seekr + network: seekr news + shortcut: sevid + categories: videos + seekr_category: videos + disabled: true + + - name: stract + engine: stract + shortcut: str + disabled: true + + - name: svgrepo + engine: svgrepo + shortcut: svg + timeout: 10.0 + disabled: true + + - name: tootfinder + engine: tootfinder + shortcut: toot + + - name: voidlinux + engine: voidlinux + shortcut: void + disabled: true + + - name: wallhaven + engine: wallhaven + # api_key: abcdefghijklmnopqrstuvwxyz + shortcut: wh + + # wikimini: online encyclopedia for children + # The fulltext and title parameter is necessary for Wikimini because + # sometimes it will not show the results and redirect instead + - name: wikimini + engine: xpath + shortcut: wkmn + search_url: https://fr.wikimini.org/w/index.php?search={query}&title=Sp%C3%A9cial%3ASearch&fulltext=Search + url_xpath: //li/div[@class="mw-search-result-heading"]/a/@href + title_xpath: //li//div[@class="mw-search-result-heading"]/a + content_xpath: //li/div[@class="searchresult"] + categories: general + disabled: true + about: + website: https://wikimini.org/ + wikidata_id: Q3568032 + use_official_api: false + require_api_key: false + results: HTML + language: fr + + - name: wttr.in + engine: wttr + shortcut: wttr + timeout: 9.0 + + - name: yummly + engine: yummly + shortcut: yum + disabled: true + + - name: brave + engine: brave + shortcut: br + time_range_support: true + paging: true + categories: [general, web] + brave_category: search + # brave_spellcheck: true + + - name: brave.images + engine: brave + network: brave + shortcut: brimg + categories: [images, web] + brave_category: images + + - name: brave.videos + engine: brave + network: brave + shortcut: brvid + categories: [videos, web] + brave_category: videos + + - name: brave.news + engine: brave + network: brave + shortcut: brnews + categories: news + brave_category: news + + # - name: brave.goggles + # engine: brave + # network: brave + # shortcut: brgog + # time_range_support: true + # paging: true + # categories: [general, web] + # brave_category: goggles + # Goggles: # required! This should be a URL ending in .goggle + + - name: lib.rs + shortcut: lrs + engine: lib_rs + disabled: true + + - name: sourcehut + shortcut: srht + engine: xpath + paging: true + search_url: https://sr.ht/projects?page={pageno}&search={query} + results_xpath: (//div[@class="event-list"])[1]/div[@class="event"] + url_xpath: ./h4/a[2]/@href + title_xpath: ./h4/a[2] + content_xpath: ./p + first_page_num: 1 + categories: [it, repos] + disabled: true + about: + website: https://sr.ht + wikidata_id: Q78514485 + official_api_documentation: https://man.sr.ht/ + use_official_api: false + require_api_key: false + results: HTML + + - name: goo + shortcut: goo + engine: xpath + paging: true + search_url: https://search.goo.ne.jp/web.jsp?MT={query}&FR={pageno}0 + url_xpath: //div[@class="result"]/p[@class='title fsL1']/a/@href + title_xpath: //div[@class="result"]/p[@class='title fsL1']/a + content_xpath: //p[contains(@class,'url fsM')]/following-sibling::p + first_page_num: 0 + categories: [general, web] + disabled: true + timeout: 4.0 + about: + website: https://search.goo.ne.jp + wikidata_id: Q249044 + use_official_api: false + require_api_key: false + results: HTML + language: ja + + - name: bt4g + engine: bt4g + shortcut: bt4g + + - name: pkg.go.dev + engine: pkg_go_dev + shortcut: pgo + disabled: true + +# Doku engine lets you access to any Doku wiki instance: +# A public one or a privete/corporate one. +# - name: ubuntuwiki +# engine: doku +# shortcut: uw +# base_url: 'https://doc.ubuntu-fr.org' + +# Be careful when enabling this engine if you are +# running a public instance. Do not expose any sensitive +# information. You can restrict access by configuring a list +# of access tokens under tokens. +# - name: git grep +# engine: command +# command: ['git', 'grep', '{{QUERY}}'] +# shortcut: gg +# tokens: [] +# disabled: true +# delimiter: +# chars: ':' +# keys: ['filepath', 'code'] + +# Be careful when enabling this engine if you are +# running a public instance. Do not expose any sensitive +# information. You can restrict access by configuring a list +# of access tokens under tokens. +# - name: locate +# engine: command +# command: ['locate', '{{QUERY}}'] +# shortcut: loc +# tokens: [] +# disabled: true +# delimiter: +# chars: ' ' +# keys: ['line'] + +# Be careful when enabling this engine if you are +# running a public instance. Do not expose any sensitive +# information. You can restrict access by configuring a list +# of access tokens under tokens. +# - name: find +# engine: command +# command: ['find', '.', '-name', '{{QUERY}}'] +# query_type: path +# shortcut: fnd +# tokens: [] +# disabled: true +# delimiter: +# chars: ' ' +# keys: ['line'] + +# Be careful when enabling this engine if you are +# running a public instance. Do not expose any sensitive +# information. You can restrict access by configuring a list +# of access tokens under tokens. +# - name: pattern search in files +# engine: command +# command: ['fgrep', '{{QUERY}}'] +# shortcut: fgr +# tokens: [] +# disabled: true +# delimiter: +# chars: ' ' +# keys: ['line'] + +# Be careful when enabling this engine if you are +# running a public instance. Do not expose any sensitive +# information. You can restrict access by configuring a list +# of access tokens under tokens. +# - name: regex search in files +# engine: command +# command: ['grep', '{{QUERY}}'] +# shortcut: gr +# tokens: [] +# disabled: true +# delimiter: +# chars: ' ' +# keys: ['line'] + +doi_resolvers: + oadoi.org: 'https://oadoi.org/' + doi.org: 'https://doi.org/' + doai.io: 'https://dissem.in/' + sci-hub.se: 'https://sci-hub.se/' + sci-hub.st: 'https://sci-hub.st/' + sci-hub.ru: 'https://sci-hub.ru/' + +default_doi_resolver: 'oadoi.org' diff --git a/scripts/ensure-offline-data.sh b/scripts/ensure-offline-data.sh new file mode 100755 index 00000000..35af34ac --- /dev/null +++ b/scripts/ensure-offline-data.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +error() { + echo "Error: $1" >&2 + exit 1 +} + +ollama_pull() { + if [ -z "$1" ]; then + error "Model name is required" + fi + + # Check if ollama container is running + if ! docker ps --format '{{.Names}}' | grep -q '^ollama$'; then + error "Ollama container is not running" + fi + + docker exec ollama ollama pull "$1" +} + + +ollama_pull llama3.2:latest +ollama_pull qwen2.5-coder:7b + +# Embedding models - https://ollama.com/search?c=embedding +ollama_pull nomic-embed-text diff --git a/scripts/llm-gen/gen-streamlit.py b/scripts/llm-gen/gen-streamlit.py new file mode 100644 index 00000000..9e3c43e2 --- /dev/null +++ b/scripts/llm-gen/gen-streamlit.py @@ -0,0 +1,61 @@ +# Run with: streamlit run gen-streamlit.py --server.headless true + +import streamlit as st +from gen import LLMTextProcessor + +# Initialize the text processor +processor = LLMTextProcessor() + +# Load available templates from the "templates" folder +template_folder = "templates" +template_files = processor.get_template_names() + +# CSS to use fixed-width font for text areas +st.markdown(""" + +""", unsafe_allow_html=True) + +# Streamlit interface +st.title("LLM Text Processor") + +# Sidebar configuration +st.sidebar.header("Configuration") + +# Model selection +model = st.sidebar.text_input("Model", value="") # instruct + +# Temperature configuration +temperature = st.sidebar.slider("Temperature", min_value=0.0, max_value=1.0, value=0.7, step=0.1) + +# Template selection +template = st.sidebar.selectbox("Template", options=template_files) + +# Main input text +# st.header("Input Text") +input_text = st.text_area("Enter the input text:", height=200) + +# Process button +if st.button("Process"): + if not input_text.strip(): + st.error("Please enter some text.") + else: + # Get the response using the processor + try: + response = processor.respond( + input_text=input_text, + template=template, + model=model, + temperature=temperature, + ) + st.success("Processing Complete!") + st.text_area("Output:", response, height=300) + + # Display the response as markdown + st.markdown("### Formatted Output:") + st.markdown(response) + except Exception as e: + st.error(f"An error occurred: {str(e)}") diff --git a/scripts/llm-gen/gen.py b/scripts/llm-gen/gen.py index c2b49fed..19609eb8 100755 --- a/scripts/llm-gen/gen.py +++ b/scripts/llm-gen/gen.py @@ -44,6 +44,16 @@ def load_config(self) -> Dict: return {"model_types": model_types, "models": models} + def get_template_names(self) -> List[str]: + """Returns a list of YAML file names in the 'templates' folder without extension.""" + templates_dir = Path("templates") + if not templates_dir.is_dir(): + logging.warning(f"Templates directory not found: {templates_dir}") + return [] + + template_files = templates_dir.glob("*.yaml") + return [template.stem for template in template_files] + def get_indexed_filename(self, filename: str, index: int) -> str: """Returns filename with an index appended before the extension.""" base, ext = os.path.splitext(filename) diff --git a/scripts/llm-gen/templates/create-python-script.yaml b/scripts/llm-gen/templates/create-python-script.yaml index 6203ee54..dd94f5b7 100644 --- a/scripts/llm-gen/templates/create-python-script.yaml +++ b/scripts/llm-gen/templates/create-python-script.yaml @@ -2,6 +2,7 @@ params: model_type: code chunking: disabled temperature: 0.1 +description: Create a python script prompt: | Create a python script. The script must: . Use type hints in function definitions and separate the functions with two blank lines. diff --git a/scripts/llm-gen/templates/pick-cloud.yaml b/scripts/llm-gen/templates/pick-cloud.yaml index 2f1f2c3d..4fce3a0d 100644 --- a/scripts/llm-gen/templates/pick-cloud.yaml +++ b/scripts/llm-gen/templates/pick-cloud.yaml @@ -12,7 +12,7 @@ prompt: | Use JSON to answer in this format, do not use markdown formatting: { "cloud": "", - "reasoning": "" + "reasoning": "" } Message: diff --git a/scripts/llm-gen/templates/pick-model.yaml b/scripts/llm-gen/templates/pick-model.yaml index b061ab46..6e396520 100644 --- a/scripts/llm-gen/templates/pick-model.yaml +++ b/scripts/llm-gen/templates/pick-model.yaml @@ -4,14 +4,14 @@ params: prompt: | You have to route the LLM prompt to best fitting model type. Output only the model type based on the prompt. Possible model types: - - "code": any task involves applicatiop development, scripting, improvement of the source code of an application - - "creative": tasks involving writing stories, aboiut fictional characters or brainstorming marketing ideas + - "code": any task involves application development, scripting, improvement of the source code of an application + - "creative": tasks involving writing stories, about fictional characters or brainstorming marketing ideas - "instruct": any other generic task, or if unable to decide the model type Use JSON to answer in this format, do not use markdown formatting: { "model_type": "", - "reasoning": "" + "reasoning": "" } Prompt: diff --git a/scripts/llm-gen/templates/summarize-meeting.yaml b/scripts/llm-gen/templates/summarize-meeting.yaml index f6c1d6fd..716afdbb 100644 --- a/scripts/llm-gen/templates/summarize-meeting.yaml +++ b/scripts/llm-gen/templates/summarize-meeting.yaml @@ -5,7 +5,7 @@ params: prompt: | Read this transcript of an IT technical meeting provided in the and tags. - Create a sumary of the meeting in the exact structure provided in the and tags. + Create a summary of the meeting in the exact structure provided in the and tags. #