searx: filtron -> limiter

6 files changed, 180 insertions, 298 deletions

diff --git a/config/services/web/searx/default.nix b/config/services/web/searx/default.nix
index 46afcd8..6681366 100644
--- a/config/services/web/searx/default.nix
+++ b/config/services/web/searx/default.nix
@@ -1,37 +1,87 @@
 {
   config,
   lib,
+  pkgs,
   ...
 }:
 
 let
-  cfg = {
-    searx = config.services.searx;
-    filtron = config.services.filtron;
-  };
+  cfg = config.services.searx;
 in
 {
   imports = [
-    ./searx
-    ./filtron
+    ./engines.nix
+    ./limiter.nix
   ];
 
-  services.nginx.virtualHosts.searx = lib.mkIf (cfg.searx.enable && cfg.filtron.enable) {
-    serverName = "searx.${config.networking.domain}";
-    locations = {
-      "/" = {
-        proxyPass = "http://${cfg.filtron.listen.address}:${toString cfg.filtron.listen.port}";
-        extraConfig = ''
-          proxy_set_header Connection      $http_connection;
-          proxy_set_header X-Scheme        $scheme;
-          # proxy_set_header X-Script-Name   /;
-        '';
-      };
-      "/static/".alias = "${cfg.searx.package}/share/static/";
+  services.nginx.virtualHosts."searx.aristote.fr" = lib.mkIf (cfg.enable && cfg.configureNginx) {
+    locations."/" = {
+      extraConfig = ''
+        proxy_set_header Host            $host;
+        proxy_set_header Connection      $http_connection;
+        proxy_set_header X-Real-IP       $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Scheme        $scheme;
+        # proxy_set_header X-Script-Name   /;
+      '';
     };
     forceSSL = true;
     enableACME = true;
   };
 
-  systemd.services.uwsgi.personal.monitor = true;
+  services.searx = {
+    enable = true;
+    package = pkgs.searxng;
+
+    configureNginx = true;
+    domain = "searx.aristote.fr";
+
+    configureUwsgi = true;
+    uwsgiConfig = {
+      cache2 = "name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1";
+    };
+
+    environmentFile = "/etc/searx/secrets";
+    settings = {
+      use_default_settings = true;
+      general = {
+        debug = true;
+        contact_url = "mailto:quentin@${config.networking.domain}";
+        enable_stats = false;
+      };
+      search = {
+        autocomplete = "wikipedia";
+        default_lang = "all";
+        formats = [
+          "html"
+          "json"
+        ];
+      };
+      server = {
+        secret_key = "@SECRET_KEY@";
+        image_proxy = true;
+        http_protocol_version = "1.0";
+        method = "GET";
+      };
+      ui = {
+        theme_args = {
+          oscar_style = "pointhi";
+        };
+      };
+      enabled_plugins = [
+        "Open Access DOI rewrite"
+        "Hash plugin"
+        "HTTPS rewrite"
+        "Self Informations"
+        "Search on category select"
+        "Tracker URL remover"
+        "Vim-like hotkeys"
+      ];
+    };
+  };
+
+  systemd.services = {
+    searx-init.personal.monitor = true;
+    uwsgi.personal.monitor = true;
+  };
 }
diff --git a/config/services/web/searx/engines.nix b/config/services/web/searx/engines.nix
new file mode 100644
index 0000000..69cc861
--- /dev/null
+++ b/config/services/web/searx/engines.nix
@@ -0,0 +1,111 @@
+{ lib, ... }:
+
+with lib;
+let
+  overrideDisabled =
+    value: names:
+    map (name: {
+      inherit name;
+      disabled = value;
+    }) names;
+  enable = overrideDisabled false;
+  disable = overrideDisabled true;
+
+  makeSearchUrl =
+    {
+      baseUrl,
+      queryKeyword ? "query",
+      pageKeyword ? null,
+      preventRedirect ? false,
+      extraParameters ? { },
+    }:
+    baseUrl
+    + "?${queryKeyword}={query}"
+    # an invisible whitespace is added at the end of the query to prevent redirections
+    + (optionalString preventRedirect "%E2%80%8E")
+    + (optionalString (pageKeyword != null) "&${pageKeyword}={pageno}")
+    + (concatStrings (
+      mapAttrsToList (name: value: "&${name}=${builtins.toString value}") extraParameters
+    ));
+
+  wikipediaSearch = lang: {
+    name = "wikipedia search ${lang}";
+    engine = "xpath";
+    search_url = makeSearchUrl {
+      baseUrl = "https://${lang}.wikipedia.org/w/index.php";
+      queryKeyword = "search";
+      extraParameters = {
+        fulltext = 1;
+      };
+    };
+    results_xpath = ''//ul[@class="mw-search-results"]/li'';
+    url_xpath = ''.//div[@class="mw-search-result-heading"]/a/@href'';
+    title_xpath = ''.//div[@class="mw-search-result-heading"]/a/@title'';
+    content_xpath = ''.//div[@class="searchresult"]'';
+    shortcut = "w${lang}";
+    categories = "general";
+    disabled = true;
+    about = {
+      website = "https://${lang}.wikipedia.org/";
+      wikidata_id = "Q52";
+      official_api_documentation = "https://${lang}.wikipedia.org/api/";
+      use_official_api = false;
+      require_api_key = false;
+      results = "HTML";
+    };
+  };
+in
+{
+  services.searx.settings.engines =
+    (disable [
+      # general
+      "bing"
+      "brave"
+      "currency"
+      "dictzone"
+      "google"
+      # files
+      "btdigg"
+      # images
+      "openverse"
+      "bing images"
+      "library of congress"
+      "qwant images"
+      # it
+      "hoogle"
+      # map
+      "photon"
+    ])
+    ++ (enable [
+      # general
+      "duckduckgo"
+      "startpage"
+    ])
+    ++ [
+      {
+        name = "nlab";
+        engine = "xpath";
+        search_url = makeSearchUrl { baseUrl = "https://ncatlab.org/nlab/search"; };
+        results_xpath = "//li/a";
+        url_xpath = "./@href";
+        title_xpath = ".";
+        content_xpath = ".";
+        shortcut = "nlab";
+        timeout = 10.0;
+        categories = "science";
+        disabled = true;
+        about = {
+          website = "https://ncatlab.org/";
+          wikidata_id = "Q6954693";
+          official_api_documentation = "";
+          use_official_api = false;
+          require_api_key = false;
+          results = "HTML";
+        };
+      }
+    ]
+    ++ (map wikipediaSearch [
+      "fr"
+      "en"
+    ]);
+}
diff --git a/config/services/web/searx/filtron/default.nix b/config/services/web/searx/filtron/default.nix
deleted file mode 100644
index da4f1e6..0000000
--- a/config/services/web/searx/filtron/default.nix
+++ /dev/null
@@ -1,121 +0,0 @@
-{...}: {
-  services.filtron = {
-    enable = true;
-    target.port = 8000;
-    rules = [
-      {
-        name = "roboagent limit";
-        filters = [
-          "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)"
-        ];
-        limit = 0;
-        stop = true;
-        actions = [
-          {name = "log";}
-          {
-            name = "block";
-            params = {message = "Rate limit exceeded";};
-          }
-        ];
-      }
-      {
-        name = "botlimit";
-        filters = [
-          "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
-        ];
-        limit = 0;
-        stop = true;
-        actions = [
-          {name = "log";}
-          {
-            name = "block";
-            params = {message = "Rate limit exceeded";};
-          }
-        ];
-      }
-      {
-        name = "suspiciously frequent IP";
-        filters = [];
-        interval = 600;
-        limit = 30;
-        aggregations = ["Header:X-Forwarded-For"];
-        actions = [{name = "log";}];
-      }
-      {
-        name = "search request";
-        filters = ["Param:q" "Path=^(/|/search)$"];
-        interval = 61;
-        limit = 999;
-        subrules = [
-          {
-            name = "missing Accept-Language";
-            filters = ["!Header:Accept-Language"];
-            limit = 0;
-            stop = true;
-            actions = [
-              {name = "log";}
-              {
-                name = "block";
-                params = {message = "Rate limit exceeded";};
-              }
-            ];
-          }
-          # {
-          #   name = "suspiciously Connection=close header";
-          #   filters = [ "Header:Connection=close" ];
-          #   limit = 0;
-          #   stop = true;
-          #   actions = [
-          #     { name = "log"; }
-          #     {
-          #       name = "block";
-          #       params = { message = "Rate limit exceeded"; };
-          #     }
-          #   ];
-          # }
-          {
-            name = "IP limit";
-            interval = 61;
-            limit = 9;
-            stop = true;
-            aggregations = ["Header:X-Forwarded-For"];
-            actions = [
-              {name = "log";}
-              {
-                name = "block";
-                params = {message = "Rate limit exceeded";};
-              }
-            ];
-          }
-          {
-            name = "rss/json limit";
-            filters = ["Param:format=(csv|json|rss)"];
-            interval = 121;
-            limit = 2;
-            stop = true;
-            actions = [
-              {name = "log";}
-              {
-                name = "block";
-                params = {message = "Rate limit exceeded";};
-              }
-            ];
-          }
-          {
-            name = "useragent limit";
-            interval = 61;
-            limit = 199;
-            aggregations = ["Header:User-Agent"];
-            actions = [
-              {name = "log";}
-              {
-                name = "block";
-                params = {message = "Rate limit exceeded";};
-              }
-            ];
-          }
-        ];
-      }
-    ];
-  };
-}
diff --git a/config/services/web/searx/searx/limiter.nix b/config/services/web/searx/limiter.nix
index 785c7fe..785c7fe 100644
--- a/config/services/web/searx/searx/limiter.nix
+++ b/config/services/web/searx/limiter.nix
diff --git a/config/services/web/searx/searx/default.nix b/config/services/web/searx/searx/default.nix
deleted file mode 100644
index 003e7dc..0000000
--- a/config/services/web/searx/searx/default.nix
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  config,
-  pkgs,
-  ...
-}:
-
-let
-  cfg.filtron = config.services.filtron;
-in
-{
-  imports = [ ./engines.nix ];
-
-  services.searx = {
-    enable = true;
-    package = pkgs.searxng;
-
-    configureUwsgi = true;
-    uwsgiConfig = {
-      cache2 = "name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1";
-      http = ":${toString cfg.filtron.target.port}";
-    };
-
-    environmentFile = "/etc/searx/secrets";
-    settings = {
-      use_default_settings = true;
-      general = {
-        debug = true;
-        contact_url = "mailto:quentin@${config.networking.domain}";
-        enable_stats = false;
-      };
-      search = {
-        autocomplete = "wikipedia";
-        default_lang = "all";
-        formats = [
-          "html"
-          "json"
-        ];
-      };
-      server = {
-        secret_key = "@SECRET_KEY@";
-        image_proxy = true;
-        http_protocol_version = "1.0";
-        method = "GET";
-      };
-      ui = {
-        theme_args = {
-          oscar_style = "pointhi";
-        };
-      };
-      enabled_plugins = [
-        "Open Access DOI rewrite"
-        "Hash plugin"
-        "HTTPS rewrite"
-        "Self Informations"
-        "Search on category select"
-        "Tracker URL remover"
-        "Vim-like hotkeys"
-      ];
-    };
-  };
-
-  systemd.services.searx-init.personal.monitor = true;
-}
diff --git a/config/services/web/searx/searx/engines.nix b/config/services/web/searx/searx/engines.nix
deleted file mode 100644
index 1e9c34c..0000000
--- a/config/services/web/searx/searx/engines.nix
+++ /dev/null
@@ -1,95 +0,0 @@
-{ lib, ... }:
-
-with lib;
-let
-  overrideDisabled = value: names:
-    map (name: {
-      inherit name;
-      disabled = value;
-    }) names;
-  enable = overrideDisabled false;
-  disable = overrideDisabled true;
-
-  makeSearchUrl = { baseUrl, queryKeyword ? "query", pageKeyword ? null
-    , preventRedirect ? false, extraParameters ? { } }:
-    baseUrl + "?${queryKeyword}={query}"
-    # an invisible whitespace is added at the end of the query to prevent redirections
-    + (optionalString preventRedirect "%E2%80%8E")
-    + (optionalString (pageKeyword != null) "&${pageKeyword}={pageno}")
-    + (concatStrings
-      (mapAttrsToList (name: value: "&${name}=${builtins.toString value}")
-        extraParameters));
-
-  wikipediaSearch = lang: {
-    name = "wikipedia search ${lang}";
-    engine = "xpath";
-    search_url = makeSearchUrl {
-      baseUrl = "https://${lang}.wikipedia.org/w/index.php";
-      queryKeyword = "search";
-      extraParameters = { fulltext = 1; };
-    };
-    results_xpath = ''//ul[@class="mw-search-results"]/li'';
-    url_xpath = ''.//div[@class="mw-search-result-heading"]/a/@href'';
-    title_xpath = ''.//div[@class="mw-search-result-heading"]/a/@title'';
-    content_xpath = ''.//div[@class="searchresult"]'';
-    shortcut = "w${lang}";
-    categories = "general";
-    disabled = true;
-    about = {
-      website = "https://${lang}.wikipedia.org/";
-      wikidata_id = "Q52";
-      official_api_documentation = "https://${lang}.wikipedia.org/api/";
-      use_official_api = false;
-      require_api_key = false;
-      results = "HTML";
-    };
-  };
-in {
-  services.searx.settings.engines = (disable [
-    # general
-    "bing"
-    "brave"
-    "currency"
-    "dictzone"
-    "google"
-    # files
-    "btdigg"
-    # images
-    "openverse"
-    "bing images"
-    "library of congress"
-    "qwant images"
-    # it
-    "hoogle"
-    # map
-    "photon"
-  ]) ++ (enable [
-    # general
-    "duckduckgo"
-    "gigablast"
-    "startpage"
-  ]) ++ [
-    {
-      name = "nlab";
-      engine = "xpath";
-      search_url =
-        makeSearchUrl { baseUrl = "https://ncatlab.org/nlab/search"; };
-      results_xpath = "//li/a";
-      url_xpath = "./@href";
-      title_xpath = ".";
-      content_xpath = ".";
-      shortcut = "nlab";
-      timeout = 10.0;
-      categories = "science";
-      disabled = true;
-      about = {
-        website = "https://ncatlab.org/";
-        wikidata_id = "Q6954693";
-        official_api_documentation = "";
-        use_official_api = false;
-        require_api_key = false;
-        results = "HTML";
-      };
-    }
-  ] ++ (map wikipediaSearch [ "fr" "en" ]);
-}