move to probelist

the probelist allows to finely allow / dissalow certain paths.
tighter checks for ipset install
2026-09-05 08:37:47 -04:00 · 2026-09-03 18:40:17 +02:00 · 2026-08-28 00:34:23 +02:00 · 2026-08-28 00:34:23 +02:00 · 2026-08-28 00:34:23 +02:00 · 2026-08-28 00:34:23 +02:00
17 changed files with 874 additions and 316 deletions
@@ -8,7 +8,12 @@ jobs:
  build:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Format
+      run: rustup component add rustfmt clippy
+
    - name: Run CI
      run: make ci

@@ -12,6 +12,9 @@ jobs:
    - name: Checkout
      uses: actions/checkout@v4

+    - name: Format
+      run: rustup component add rustfmt clippy
+
    - name: Run tests
      run: make ci

@@ -1,2 +1,3 @@
 /target
 builds/**
+probelist.json
@@ -170,12 +170,6 @@ dependencies = [
 "once_cell",
 ]

-[[package]]
-name = "dtoa"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
-
 [[package]]
 name = "filetime"
 version = "0.2.17"
@@ -256,9 +250,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"

 [[package]]
 name = "itoa"
-version = "0.3.4"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"

 [[package]]
 name = "kqueue"
@@ -348,24 +342,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "num-traits"
-version = "0.1.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
-dependencies = [
- "num-traits 0.2.19",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "num_cpus"
 version = "1.13.1"
@@ -449,20 +425,44 @@ dependencies = [

 [[package]]
 name = "serde"
-version = "1.0.185"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]

 [[package]]
 name = "serde_json"
-version = "1.0.1"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c62115693d0a9ed8c32d1c760f0fdbe7d4b05cb13c135b9b54137ac0d59fccb"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
- "dtoa",
 "itoa",
- "num-traits 0.1.43",
+ "memchr",
 "serde",
+ "serde_core",
+ "zmij",
 ]

 [[package]]
@@ -703,3 +703,9 @@ name = "windows_x86_64_msvc"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "zmij"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
@@ -12,5 +12,5 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 lazy_static = "1.4.0"
 regex = "1.5.4"
 anyhow = "1.0.44"
-serde_json = "=1.0.1"
+serde_json = "1"
 clap = { version = "4.5.27", features = ["derive"] }
@@ -1,19 +1,18 @@
-BLOCKFAST_VERS := $(shell date '+%Y-%m-%d') / $(shell git rev-parse --short HEAD)
+BLOCKFAST_VERS := $(shell git show -s --format=%cd --date=format:%Y-%m-%d HEAD) / $(shell git rev-parse --short HEAD)
 export BLOCKFAST_VERS

 build::
-	echo $(BLOCKFAST_VERS)
+	@echo $(BLOCKFAST_VERS)
 	cargo build
 	cargo clippy --all
 	cargo fmt --all

 run::
-	touch /tmp/sshdtest
 	touch /tmp/clftest
-	touch /tmp/jsontest
+	touch /tmp/caddytest
 	touch /tmp/generictest
 	cargo build
-	sudo target/debug/blockfast -v -s=/tmp/sshdtest -c=/tmp/clftest -j=/tmp/jsontest --generic-logpath=/tmp/generictest --generic-ip='from ([0-9a-fA-F:.]+) port' --generic-positive='Failed password'
+	sudo target/debug/blockfast -v --clf-logpath=/tmp/clftest --caddy-logpath=/tmp/caddytest --generic-logpath=/tmp/generictest --generic-ip='from ([0-9a-fA-F:.]+) port' --generic-positive='Failed password'

 ci:: test
 	cargo fmt --all -- --check
@@ -21,14 +20,19 @@ ci:: test

 build-all::
 	mkdir -p builds
-	rustc --version > builds/buildout
 	cross build --release --target x86_64-unknown-linux-musl
 	cross build --release --target aarch64-unknown-linux-musl
 	cross build --release --target armv7-unknown-linux-musleabihf
 	cp target/x86_64-unknown-linux-musl/release/blockfast builds/blockfast-x86_64-linux
 	cp target/aarch64-unknown-linux-musl/release/blockfast builds/blockfast-aarch64-linux
 	cp target/armv7-unknown-linux-musleabihf/release/blockfast builds/blockfast-arm7-linux
+	chmod +x builds/blockfast-*
+	echo '```' > builds/buildout
+	echo $(BLOCKFAST_VERS) >> builds/buildout
+	rustc --version >> builds/buildout
 	sha256sum builds/* >> builds/buildout
+	echo '```' >> builds/buildout
+	cat builds/buildout

 watch::
 	ls src/*.rs | entr -rc -- make run
@@ -39,12 +43,6 @@ test::
 release::
 	cargo build --target x86_64-unknown-linux-musl --release

-hit-sshd::
-	echo "Sep 26 06:25:32 livecompute sshd[23254]: Invalid user neal from 9.124.36.195" >> /tmp/sshdtest
-
-ok-sshd::
-	echo "Sep 26 06:25:19 livecompute sshd[23246]: successful login 8.124.36.195 port 41883 ssh2" >> /tmp/sshdtest
-
 hit-generic::
 	echo "Sep 26 06:25:19 livecompute sshd[23246]: Failed password for root from 179.124.36.195 port 41883 ssh2"  >> /tmp/generictest

@@ -52,14 +50,14 @@ ok-generic::
 	echo "Sep 26 06:25:19 livecompute sshd[23246]: Successful login for root from 179.124.36.195 port 41883 ssh2"  >> /tmp/generictest

 hit-clf::
-	echo "1.124.36.195 - p [25/Sep/2021:13:49:56 +0200] \"POST /some/rpc HTTP/2.0\" 401 923" >> /tmp/clftest
+	echo "1.124.36.195 - p [25/Sep/2021:13:49:56 +0200] \"GET /.env HTTP/2.0\" 200 923" >> /tmp/clftest

 ok-clf::
 	echo "2.124.36.195 - p [25/Sep/2021:13:49:56 +0200] \"POST /some/rpc HTTP/2.0\" 200 23012" >> /tmp/clftest

-hit-json::
-	echo "{\"request\":{\"remote_ip\":\"1.124.36.19\"}, \"status\": 400}" >> /tmp/jsontest
+hit-caddy::
+	echo "{\"request\":{\"remote_ip\":\"1.124.36.19\",\"uri\":\"/.env\"}, \"status\": 200}" >> /tmp/caddytest

-ok-json::
-	echo "{\"request\":{\"remote_ip\":\"2.124.36.19\"}, \"status\": 200}" >> /tmp/jsontest
+ok-caddy::
+	echo "{\"request\":{\"remote_ip\":\"2.124.36.19\",\"uri\":\"/\"}, \"status\": 200}" >> /tmp/caddytest

@@ -3,10 +3,11 @@
 Block internets scanners fast 🍶

 Features:
-  - SSH log parser
-  - Common Log Format parser (apache logs, etc...)
-  - JSON log parser (caddy logs)
-  - Generic log parser
+  - Built-in scanner detection, built for well-known probe paths (`/.env`, `/phpinfo.php`, etc...) 
+  - Extends built-in scanners using your own logs. See probelist.example.json
+  - Supports logs from Common Log Format (apache, nginx logs, etc...)
+  - Caddy JSON log parser
+  - Generic log parser (regexp)
  - Sane defaults
  - Fast ip ban with `ipset`
  - Static release builds, no libc dependency
@@ -14,16 +15,15 @@ Features:

 ## example
 ```txt
-$ ./blockfast -s=/var/log/auth.log -j=/caddy/logs
-1737927469 - starting with sshd parsing at "/tmp/sshdtest"
-1737927469 - starting with json parsing at "/tmp/jsontest"
+$ ./blockfast --caddy-logpath=/caddy/logs
+1737927469 - starting with caddy parsing at "/tmp/caddytest"
 1737927469 - jail setup, allowance 5, time 21600s
-1737927477 - sshd logged offence for 9.124.36.195
-1737927478 - sshd logged offence for 9.124.36.195
-1737927479 - sshd logged offence for 9.124.36.195
-1737927479 - sshd logged offence for 9.124.36.195
-1737927480 - sshd logged offence for 9.124.36.195
-1737927480 - sshd jailtime for 9.124.36.195
+1737927477 - caddy logged offence for 9.124.36.195
+1737927478 - caddy logged offence for 9.124.36.195
+1737927479 - caddy logged offence for 9.124.36.195
+1737927479 - caddy logged offence for 9.124.36.195
+1737927480 - caddy logged offence for 9.124.36.195
+1737927480 - caddy jailtime for 9.124.36.195
 ```

 ## build
@@ -36,17 +36,16 @@ Blockfast - block internets scanners fast 🍶
 Author: pierre dubouilh <pldubouilh@gmail.com>

 Blockfast reads logs from various sources and blocks the offending IPs using iptables and ipset.
-It supports logs from sshd, Common-Log-Format (Apache, etc..), JSON (Caddy) and a generic logs parser.
+It supports logs in Common-Log-Format (Apache, nginx, etc..), Caddy JSON and a generic logs parser.

 Example:
-    # block invalid sshd attempts & invalid http statuses from caddy
-    ./blockfast -s=/var/log/auth.log -j=/caddy/logs
+    # block invalid http statuses from caddy
+    ./blockfast --caddy-logpath=/caddy/logs

    # generic log parser example with a log text to flag, and a regex to parse the offending IP.
    ./blockfast --generic-logpath=/tmp/generictest --generic-positive='Failed password' --generic-ip='from ([0-9a-fA-F:.]+) port'

 Usage: blockfast [OPTIONS]
-Usage: blockfast [OPTIONS]

 Options:
      --jailtime <JAILTIME>
@@ -55,22 +54,22 @@ Options:
          how many offences allowed (max 255) [default: 5]
  -v, --verbose
          log all offences
-  -s, --sshd-logpath <SSHD_LOGPATH>
-          path of sshd logfile
-  -c, --clf-logpath <CLF_LOGPATH>
-          path of Common-Log-Format logfile (Apache, etc..)
-  -j, --json-logpath <JSON_LOGPATH>
-          path of JSON HTTP logfile (Caddy)
+      --clf-logpath <CLF_LOGPATH>
+          path of Common-Log-Format logfile (Apache, nginx, etc..), can be repeated
+      --caddy-logpath <CADDY_LOGPATH>
+          path of Caddy JSON logfile, can be repeated
+      --probelist <PROBELIST>
+          path of a probelist JSON file, replaces the built-in probe list (see README)
      --generic-logpath <GENERIC_LOGPATH>
-          generic parser log file path
+          generic parser log file path, can be repeated
      --generic-ip <GENERIC_IP>
          generic parser ip regex
      --generic-positive <GENERIC_POSITIVE>
          generic parser positive - if a logline contains this, it is considered bad, the rest is good
      --generic-negative <GENERIC_NEGATIVE>
          generic parser negative - if a logline contains this, it is considered good, the rest is bad
-      --valid-http-statuses <VALID_HTTP_STATUSES>
-          valid http statuses (for CLF and JSON logs) [default: 200 101]
+      --invalid-http-statuses <INVALID_HTTP_STATUSES>
+          also flag these http statuses (for CLF and Caddy logs), on top of the built-in scanner-path detection. Coma separated list, accepts ranges with XX, e.g. "403,5xx".
  -h, --help
          Print help
  -V, --version
@@ -0,0 +1,108 @@
+{
+  "comment": "blockfast probelist - TEMPLATE. This file documents the format and how to generate a real one from server logs. blockfast loads it with --probelist=<file>; it then REPLACES the built-in probe list entirely, so a generated file must include generic scanner rules too, not only setup-specific ones. All `comment` fields (and this whole header) are ignored by the loader; entries containing only a `comment` are section separators. Everything else must follow the format below.",
+
+  "format": [
+    "each probe entry: { path, match?, status?, allowance?, comment? }",
+    "path (required): the string to match against the request URI, matched case-insensitively",
+    "match (optional, default 'contains'): 'contains' matches anywhere in the full URI including the query string; 'prefix' and 'exact' match against the query-stripped path only",
+    "status (optional): restricts the probe to these response statuses, as a comma-separated string with trailing-x ranges, e.g. '401' or '403,5xx' or '40x'. Without it the probe matches ANY status - important, since SPAs with catch-all routes answer 200 to probe paths",
+    "allowance (optional, 1-255): overrides blockfast's global --allowance for this probe, i.e. how many offences an IP may accumulate (within the jailtime window, shared across all probes) before being banned. Use a HIGHER value for endpoints legitimate users occasionally trip, a LOW value is implicit for pure probe paths"
+  ],
+
+  "howto_generate": [
+    "You are generating a ban-rule file for blockfast, a fail2ban-like daemon: it tails web server logs (Common Log Format or Caddy JSON), counts offences per client IP, and firewalls IPs that exceed their allowance. A wrong rule bans legitimate users, so precision beats recall.",
+    "1. Group the log lines by client IP. Classify each IP: legitimate clients use real app routes and mostly get 2xx/3xx; scanners enumerate paths (/.env, /phpinfo.php, /wp-*, credential/config filenames) that the app never serves.",
+    "2. Paths requested ONLY by scanners and NEVER by legitimate clients become unconditional probes (no status filter). Do not rely on the response status for these: a catch-all route may answer 200.",
+    "3. Endpoints that legitimate users DO touch, but that indicate abuse in volume (login endpoints, session checks answering 401, method-probing answering 405), become narrow rules: match 'prefix' or 'exact' + a status filter + a generous allowance. Never add such an endpoint without a status filter.",
+    "4. NEVER add: paths served by the app (assets, API routes), /favicon.ico, /robots.txt, /.well-known (ACME!), or anything a browser or well-behaved bot requests on its own. When unsure, leave it out.",
+    "5. Prefer 'exact' for short or generic names ('/env', '/i.php') - as substrings they would match legitimate paths ('/i.php' is inside '/api.php'). Use 'contains' for names that are unambiguous anywhere in a URI ('/.env', 'phpinfo').",
+    "6. Keep the generic scanner families in the output (see sections below), then append the setup-specific rules derived from the logs, each with a comment explaining the evidence.",
+    "7. Sanity-check the finished list by replaying the logs against it: no IP classified as a legitimate client may match any rule."
+  ],
+
+  "probes": [
+    { "comment": "--- setup-specific rules, derived from the logs (EXAMPLES - replace with real findings) ---" },
+    { "path": "/api/auth", "match": "prefix", "status": "401", "allowance": 10, "comment": "example: the SPA answers 401 on session checks when logged out - normal a few times, credential stuffing in bulk. prefix + status + generous allowance" },
+    { "path": "/", "match": "exact", "status": "405", "comment": "example: scanners POST to the root fishing for handlers; browsers never trigger 405 there" },
+    { "path": "/backup.tar.gz", "match": "exact", "comment": "example: artifact hunted by scanners in these logs, never linked by the app" },
+
+    { "comment": "--- generic: secrets & dotfiles ---" },
+    { "path": "/.env", "comment": "also matches /.env.bak, /backend/.env, ..." },
+    { "path": "%2eenv", "comment": "url-encoded .env" },
+    { "path": "/.git", "comment": "/.git/config, /.git/HEAD, /.gitconfig" },
+    { "path": "/.svn" },
+    { "path": "/.hg/" },
+    { "path": "/.aws" },
+    { "path": "/.ssh" },
+    { "path": "/.docker" },
+
+    { "comment": "--- generic: cloud credentials & config dumps ---" },
+    { "path": "credentials.json" },
+    { "path": "-key.json", "comment": "/gcp-key.json, /firebase-key.json, ..." },
+    { "path": "/keyfile.json" },
+    { "path": "/sa.json" },
+    { "path": "service-account.json" },
+    { "path": "firebase-adminsdk.json" },
+    { "path": "gcp-sa.json" },
+    { "path": "/docker-compose.yml" },
+    { "path": "/appsettings.json" },
+    { "path": "application.yml" },
+    { "path": "parameters.yml" },
+    { "path": "/web.config" },
+    { "path": "/settings.py" },
+    { "path": "/wp-config.php", "comment": "never served, only probed - even on real wordpress sites" },
+
+    { "comment": "--- generic: php probes ---" },
+    { "path": "phpinfo" },
+    { "path": "phpmyadmin" },
+    { "path": "adminer.php" },
+    { "path": ".php.bak" },
+    { "path": ".php.old" },
+    { "path": ".php.save" },
+    { "path": ".php~" },
+    { "path": "eval-stdin.php", "comment": "phpunit RCE" },
+    { "path": "/vendor/phpunit" },
+    { "path": "/test.php", "match": "exact" },
+    { "path": "/info.php", "match": "exact" },
+    { "path": "/pinfo.php", "match": "exact" },
+    { "path": "/pi.php", "match": "exact" },
+    { "path": "/i.php", "match": "exact" },
+    { "path": "/p.php", "match": "exact" },
+    { "path": "/php.php", "match": "exact" },
+    { "path": "/debug.php", "match": "exact" },
+    { "path": "/database.php", "match": "exact" },
+    { "path": "/config.php", "match": "exact" },
+    { "path": "/shell.php", "match": "exact" },
+    { "path": "/upload.php", "match": "exact" },
+
+    { "comment": "--- generic: fingerprinting & framework debug endpoints ---" },
+    { "path": "wlwmanifest.xml", "comment": "wordpress fingerprinting, legit wp traffic never touches it" },
+    { "path": "/actuator/env", "comment": "spring boot - keep narrow, bare /actuator would catch legit health checks" },
+    { "path": "/_profiler", "comment": "symfony" },
+    { "path": "/_ignition", "comment": "laravel RCE" },
+    { "path": "/_environment", "comment": "cakephp" },
+    { "path": "laravel.log" },
+    { "path": "/env", "match": "exact" },
+
+    { "comment": "--- generic: server status, traversal, IoT/router botnets ---" },
+    { "path": "server-status" },
+    { "path": "server-info" },
+    { "path": "/../" },
+    { "path": "%2e%2e" },
+    { "path": "/etc/passwd" },
+    { "path": "/cgi-bin/" },
+    { "path": "/boaform" },
+    { "path": "/hnap1" },
+    { "path": "/gponform" },
+
+    { "comment": "--- aggressive extras: correct for many setups, but NOT in blockfast's built-in default because they match legit traffic on some stacks. Include them only when the logs show the stack does not use them ---" },
+    { "path": "/wp-login.php", "comment": "legit logins on real wordpress sites - only include if not hosting wordpress" },
+    { "path": "/xmlrpc.php", "comment": "used by jetpack & wordpress mobile apps" },
+    { "path": "rest_route=", "comment": "wordpress REST access with permalinks disabled" },
+    { "path": "/autodiscover/", "comment": "outlook clients innocently probe this on any domain" },
+    { "path": "/manager/html", "comment": "tomcat console - legit for tomcat admins" },
+    { "path": "/solr/admin" },
+    { "path": "/geoserver/web" },
+    { "path": "/telescope/requests", "comment": "laravel telescope - legit for its admins" }
+  ]
+}
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.81.0"
+channel = "1.95.0"
@@ -1,8 +1,9 @@
+use crate::probes::ProbeList;
 use crate::utils::ParsingStatus;
 use anyhow::*;
 use std::{net::IpAddr, str::FromStr};

-pub fn parse(line: &str, valid_statuses: &[u32]) -> Result<ParsingStatus> {
+pub fn parse(line: &str, probelist: &ProbeList, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
    let json: serde_json::Value = serde_json::from_str(line)?;

    let remote_ip = json
@@ -12,14 +13,24 @@ pub fn parse(line: &str, valid_statuses: &[u32]) -> Result<ParsingStatus> {
        .and_then(|r| IpAddr::from_str(r).ok())
        .ok_or_else(|| anyhow!("cant parse json line - remote_ip"))?;

+    let uri = json
+        .get("request")
+        .and_then(|r| r.get("uri"))
+        .and_then(|r| r.as_str())
+        .ok_or_else(|| anyhow!("cant parse json line - uri"))?;
+
    let status = json
        .get("status")
        .and_then(|r| r.as_u64())
-        .ok_or_else(|| anyhow!("cant parse json line - status"))?;
+        .ok_or_else(|| anyhow!("cant parse json line - status"))? as u32;

-    let is_good_status = valid_statuses.iter().any(|s| s == &(status as u32));
-    if !is_good_status {
-        return Ok(ParsingStatus::BadEntry(remote_ip));
+    if let Some(probe) = probelist.check(uri, status) {
+        return Ok(ParsingStatus::BadEntry(remote_ip, probe.allowance));
+    }
+
+    let is_bad_status = invalid_statuses.iter().any(|s| s == &status);
+    if is_bad_status {
+        return Ok(ParsingStatus::BadEntry(remote_ip, None));
    }

    Ok(ParsingStatus::OkEntry)
@@ -37,9 +48,9 @@ mod tests {
        ];

        vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![200, 404]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]).unwrap();
            match ret {
-                ParsingStatus::BadEntry(_) => {}
+                ParsingStatus::BadEntry(..) => {}
                _ => panic!("bad parsing"),
            }
        })
@@ -53,7 +64,7 @@ mod tests {
        ];

        vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![200, 404]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]).unwrap();
            match ret {
                ParsingStatus::OkEntry => {}
                _ => panic!("bad parsing"),
@@ -61,6 +72,22 @@ mod tests {
        })
    }

+    #[test]
+    fn probe_uri() {
+        // a probe path is an offence even with a 200 status and no status list
+        let bad = r#"{"request":{"remote_ip":"1.2.3.4","uri":"/.env"},"status":200}"#;
+        match parse(bad, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::BadEntry(..) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ok = r#"{"request":{"remote_ip":"1.2.3.4","uri":"/api/auth/me"},"status":200}"#;
+        match parse(ok, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
    #[test]
    fn malformed() {
        let vectors = [
@@ -68,7 +95,7 @@ mod tests {
        ];

        vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![200, 404]);
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]);
            assert!(ret.is_err());
        })
    }
@@ -1,3 +1,4 @@
+use crate::probes::ProbeList;
 use crate::utils::ParsingStatus;
 use anyhow::*;
 use lazy_static::lazy_static;
@@ -5,29 +6,44 @@ use regex::Regex;
 use std::{net::IpAddr, str::FromStr};

 lazy_static! {
-    static ref RE_IP: Regex = Regex::new(r"^(\S+)\s").unwrap();
-    static ref RE_STATUS: Regex = Regex::new(r"(\d+)\s(\w+)$").unwrap();
+    // anchored on the left side of the line: host, ident, authuser, [date],
+    // the quoted request (honouring backslash escapes, so a quote injected in
+    // the URL cannot shift the match), then the status right after it. this
+    // covers plain CLF and the combined format - the trailing attacker
+    // controlled "referer" "user-agent" fields are never scanned
+    static ref RE_CLF: Regex =
+        Regex::new(r#"^(\S+)\s+\S+\s+\S+\s+\[[^\]]*\]\s+"((?:[^"\\]|\\.)*)"\s+(\d{3})(?:\s|$)"#)
+            .unwrap();
 }

-#[allow(clippy::bind_instead_of_map)]
-pub fn parse(line: &str, valid_statuses: &[u32]) -> Result<ParsingStatus> {
-    let ip = RE_IP
+pub fn parse(line: &str, probelist: &ProbeList, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
+    let caps = RE_CLF
        .captures(line)
-        .and_then(|c| c.get(1))
-        .and_then(|g| Some(g.as_str()))
-        .and_then(|e| IpAddr::from_str(e).ok())
+        .ok_or_else(|| anyhow!("cant parse clf line"))?;
+
+    let ip = caps
+        .get(1)
+        .and_then(|g| IpAddr::from_str(g.as_str()).ok())
        .ok_or_else(|| anyhow!("cant parse clf line - ip"))?;

-    let status = RE_STATUS
-        .captures(line)
-        .and_then(|c| c.get(1))
-        .and_then(|g| Some(g.as_str()))
-        .and_then(|e| e.parse::<u32>().ok())
+    let status = caps
+        .get(3)
+        .and_then(|g| g.as_str().parse::<u32>().ok())
        .ok_or_else(|| anyhow!("cant parse clf line - status"))?;

-    let is_good_status = valid_statuses.iter().any(|s| s == &status);
-    if !is_good_status {
-        return Ok(ParsingStatus::BadEntry(ip));
+    // the uri is the second token of the request field ("GET /uri HTTP/1.1")
+    let uri = caps
+        .get(2)
+        .and_then(|g| g.as_str().split_whitespace().nth(1));
+    if let Some(u) = uri {
+        if let Some(probe) = probelist.check(u, status) {
+            return Ok(ParsingStatus::BadEntry(ip, probe.allowance));
+        }
+    }
+
+    let is_bad_status = invalid_statuses.iter().any(|s| s == &status);
+    if is_bad_status {
+        return Ok(ParsingStatus::BadEntry(ip, None));
    }

    Ok(ParsingStatus::OkEntry)
@@ -45,9 +61,9 @@ mod tests {
        ];

        vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![200, 404]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![401, 429]).unwrap();
            match ret {
-                ParsingStatus::BadEntry(_) => {}
+                ParsingStatus::BadEntry(..) => {}
                _ => panic!("bad parsing"),
            }
        })
@@ -61,7 +77,7 @@ mod tests {
        ];

        vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![200, 404]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![401, 429]).unwrap();
            match ret {
                ParsingStatus::OkEntry => {}
                _ => panic!("bad parsing"),
@@ -69,6 +85,66 @@ mod tests {
        })
    }

+    #[test]
+    fn combined_format() {
+        // combined log format appends "referer" "user-agent" - the old parser
+        // read digits out of the user-agent as the status and silently missed these
+        let bad = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /admin HTTP/1.1" 401 923 "https://example.com/" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0""#;
+        match parse(bad, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
+            ParsingStatus::BadEntry(..) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ok = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET / HTTP/1.1" 200 923 "https://example.com/" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0""#;
+        match parse(ok, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
+    #[test]
+    fn probe_uri() {
+        // a probe path is an offence even with a 200 status and no status list
+        let bad = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /.env HTTP/1.1" 200 923"#;
+        match parse(bad, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::BadEntry(..) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ok = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /index.html HTTP/1.1" 200 923"#;
+        match parse(ok, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
+    #[test]
+    fn quote_injection() {
+        // servers escape quotes in the logged request - an escaped `\" 401 `
+        // inside the URL must not be mistaken for the end of the request field
+        let ok = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /x?a=\" 401 - HTTP/1.1" 200 923"#;
+        match parse(ok, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
+    #[test]
+    fn bodyless() {
+        // CLF uses `-` for absent body bytes, both branches must still parse
+        let bad = "8.8.8.8 - p [25/Sep/2021:13:49:56 +0200] \"GET / HTTP/2.0\" 401 -";
+        match parse(bad, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
+            ParsingStatus::BadEntry(..) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ok = "8.8.8.8 - p [25/Sep/2021:13:49:56 +0200] \"GET / HTTP/2.0\" 304 -";
+        match parse(ok, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
    #[test]
    fn malformed() {
        let vectors = [
@@ -77,7 +153,7 @@ mod tests {
        ];

        vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![200, 404]);
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]);
            assert!(ret.is_err());
        })
    }
@@ -30,7 +30,7 @@ pub fn parse(
        .and_then(|e| IpAddr::from_str(e).ok())
        .ok_or_else(|| anyhow!("cant parse clf line - ip"))?;

-    Ok(ParsingStatus::BadEntry(ip))
+    Ok(ParsingStatus::BadEntry(ip, None))
 }

 #[cfg(test)]
@@ -51,7 +51,7 @@ mod tests {

        let ret = parse(FAILED, Some(&ip), Some(&positive), None).unwrap();
        match ret {
-            ParsingStatus::BadEntry(_) => {}
+            ParsingStatus::BadEntry(..) => {}
            _ => panic!("bad parsing"),
        }

@@ -76,7 +76,7 @@ mod tests {

        let ret = parse(FAILED, Some(&ip), None, Some(&negative)).unwrap();
        match ret {
-            ParsingStatus::BadEntry(_) => {}
+            ParsingStatus::BadEntry(..) => {}
            _ => panic!("bad parsing"),
        }
    }
@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::net::IpAddr;
 use std::process::Command;
+use std::result::Result::Ok;
 use std::sync::Mutex;

 use anyhow::*;
@@ -11,7 +12,7 @@ pub struct Jail {
    name: String,
    allowance: u8,
    jailtime: u32,
-    remand: Mutex<HashMap<IpAddr, (u8, u64)>>,
+    remand: Mutex<HashMap<IpAddr, (u8, u64)>>, // ip -> (hits, timestamp)
 }

 fn exec(program: &str, cmd: &str, err: &str) -> Result<(), Error> {
@@ -22,22 +23,43 @@ fn exec(program: &str, cmd: &str, err: &str) -> Result<(), Error> {
    Ok(())
 }

+fn check_installed(program: &str) -> Result<()> {
+    match Command::new(program).arg("--version").output() {
+        Ok(_) => Ok(()),
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+            bail!("`{}` not found, please install it first", program)
+        }
+        Err(e) => Err(e).context(format!("cant execute `{}`", program)),
+    }
+}
+
+fn exec_ok(program: &str, cmd: &str) -> Result<bool> {
+    let parts: Vec<&str> = cmd.split_whitespace().collect();
+    let out = Command::new(program).args(parts).output()?;
+    Ok(out.status.code() == Some(0))
+}
+
 impl Jail {
    pub fn new(allowance: u8, jailtime: u32) -> Result<Jail> {
-        const ERR_MSG: &str = "error using ipset/iptables, maybe it's not installed, or this program isn't running as root ?";
+        const ERR_MSG: &str =
+            "error using ipset/iptables, maybe this program isn't running as root ?";
+        check_installed("ipset")?;
+        check_installed("iptables")?;
        let n = format!("blockfast_jail_{}", jailtime);

        // create
        let cmd = format!("create -exist {} hash:ip timeout {}", n, jailtime);
        exec("ipset", &cmd, ERR_MSG)?;

-        // setup input
-        let cmd = format!("-I INPUT 1 -m set -j DROP --match-set {} src", n);
-        exec("iptables", &cmd, ERR_MSG)?;
-
-        // setup fwd
-        let cmd = format!("-I FORWARD 1 -m set -j DROP --match-set {} src", n);
-        exec("iptables", &cmd, ERR_MSG)?;
+        // install drop rule on INPUT and FORWARD, but only if not already present
+        let rule_spec = format!("-m set -j DROP --match-set {} src", n);
+        for chain in &["INPUT", "FORWARD"] {
+            let check = format!("-C {} {}", chain, rule_spec);
+            if !exec_ok("iptables", &check)? {
+                let install = format!("-I {} 1 {}", chain, rule_spec);
+                exec("iptables", &install, ERR_MSG)?;
+            }
+        }

        log!("jail setup, allowance {}, time {}s", allowance, jailtime);
        Ok(Jail {
@@ -48,8 +70,10 @@ impl Jail {
        })
    }

-    pub fn sentence(&self, ip: IpAddr) -> Result<bool> {
+    // allowance_override comes from a matched probe, else the global setting applies
+    pub fn sentence(&self, ip: IpAddr, allowance_override: Option<u8>) -> Result<bool> {
        let now = get_epoch();
+        let allowance = allowance_override.unwrap_or(self.allowance);

        let should_ban = {
            let mut locked_map = self.remand.lock().map_err(|_| anyhow!("cant lock"))?;
@@ -66,7 +90,7 @@ impl Jail {
                    }
                })
                .or_insert((1, now));
-            if hits < self.allowance {
+            if hits < allowance {
                false
            } else {
                locked_map.remove_entry(&ip);
@@ -4,10 +4,10 @@ use anyhow::*;
 use clap::Parser;
 use linemux::{Line, MuxedLines};

+mod caddy;
 mod clf;
 mod generic;
-mod json;
-mod sshd;
+mod probes;
 mod utils;

 mod jail;
@@ -18,55 +18,49 @@ async fn run() -> Result<()> {
    let args = utils::Args::parse();
    let mut ml = MuxedLines::new()?;

-    // HTTP statuses
-    let ok_statuses = args.valid_http_statuses.clone();
-    let ok_statuses_parsed = parse_statuses(&ok_statuses)?;
-    let ok_statuses_ref = ok_statuses_parsed.as_ref();
+    // HTTP statuses - opt-in, on top of the probe detection
+    let invalid_statuses_parsed = match &args.invalid_http_statuses {
+        Some(s) => parse_statuses(s)?,
+        None => vec![],
+    };
+    let invalid_statuses_ref: &[u32] = invalid_statuses_parsed.as_ref();
+
+    // probes - loaded from --probelist if given, built-in list otherwise
+    let probelist = match args.probelist.as_ref() {
+        Some(p) => {
+            let pl = probes::ProbeList::load(p)?;
+            log!("loaded {} probes from {:?}", pl.len(), p);
+            pl
+        }
+        None => probes::ProbeList::builtin(),
+    };
+    let probelist = &probelist;

    // generic parser
-    let generic_path = args.generic_logpath.as_ref();
+    let generic_paths = &args.generic_logpath;
    let generic_ip_re = args.generic_ip.as_ref();
    let generic_positive = args.generic_positive.as_ref();
    let generic_negative = args.generic_negative.as_ref();
-    if args.generic_ip.is_some()
-        || args.generic_logpath.is_some()
-        || args.generic_positive.is_some()
-        || args.generic_negative.is_some()
-    {
-        if args.generic_ip.is_none() || args.generic_logpath.is_none() {
-            bail!("generic parser needs both ip regex and log file path");
-        }
-        if !(args.generic_positive.is_some() ^ args.generic_negative.is_some()) {
-            bail!("generic parser requires either a positive or a negative regex");
-        }
-        if let Some(p) = generic_path.as_ref() {
+    for p in generic_paths {
        ml.add_file(&p).await?;
        log!("starting with generic parsing at {:?}", &p);
    }
-    }
-
-    // sshd
-    let sshd_logpath = args.sshd_logpath.as_ref();
-    if let Some(p) = sshd_logpath {
-        ml.add_file(&p).await?;
-        log!("starting with sshd parsing at {:?}", &p);
-    }

    // common log format
-    let clf_logpath = args.clf_logpath.as_ref();
-    if let Some(p) = clf_logpath {
+    let clf_logpaths = &args.clf_logpath;
+    for p in clf_logpaths {
        ml.add_file(&p).await?;
        log!("starting with clf parsing at {:?}", &p);
    }

-    // json
-    let json_logpath = args.json_logpath.as_ref();
-    if let Some(p) = json_logpath {
+    // caddy json
+    let caddy_logpaths = &args.caddy_logpath;
+    for p in caddy_logpaths {
        ml.add_file(&p).await?;
-        log!("starting with json parsing at {:?}", &p);
+        log!("starting with caddy parsing at {:?}", &p);
    }

-    if json_logpath.is_none() && clf_logpath.is_none() && sshd_logpath.is_none() {
+    if caddy_logpaths.is_empty() && clf_logpaths.is_empty() && generic_paths.is_empty() {
        bail!("no log files to parse, see --help");
    }

@@ -78,13 +72,14 @@ async fn run() -> Result<()> {
        let path_buf = Some(line.source().to_path_buf());
        let path = path_buf.as_ref();

-        let (target, ret) = if path == sshd_logpath {
-            ("sshd", sshd::parse(payload)?)
-        } else if path == clf_logpath {
-            ("clf", clf::parse(payload, ok_statuses_ref)?)
-        } else if path == json_logpath {
-            ("json", json::parse(payload, ok_statuses_ref)?)
-        } else if path == generic_path {
+        let (target, ret) = if path.is_some_and(|p| clf_logpaths.contains(p)) {
+            ("clf", clf::parse(payload, probelist, invalid_statuses_ref)?)
+        } else if path.is_some_and(|p| caddy_logpaths.contains(p)) {
+            (
+                "caddy",
+                caddy::parse(payload, probelist, invalid_statuses_ref)?,
+            )
+        } else if path.is_some_and(|p| generic_paths.contains(p)) {
            (
                "generic",
                generic::parse(payload, generic_ip_re, generic_positive, generic_negative)?,
@@ -93,11 +88,11 @@ async fn run() -> Result<()> {
            bail!("file {:?} unknown ?", path)
        };

-        if let ParsingStatus::BadEntry(ip) = ret {
+        if let ParsingStatus::BadEntry(ip, allowance) = ret {
            if args.verbose {
                log!("{} logged offence for {}", target, ip);
            }
-            let banned = jail.sentence(ip)?;
+            let banned = jail.sentence(ip, allowance)?;
            if banned {
                log!("{} jailtime for {}", target, ip);
            }
@@ -106,18 +101,23 @@ async fn run() -> Result<()> {
        Ok(())
    };

-    while let Ok(Some(line)) = ml.next_line().await {
+    loop {
+        match ml.next_line().await {
+            Ok(Some(line)) => {
                if let Err(e) = assess_line(line) {
                    log!("ERR: {:?}", e);
                }
            }
-
-    Ok(())
+            Ok(None) => {
+                log!("log stream ended, exiting");
+                return Ok(());
+            }
+            Err(e) => return Err(e.into()),
+        }
+    }
 }

 #[tokio::main]
 async fn main() -> Result<()> {
-    run().await?;
-    eprintln!("\n");
-    Ok(())
+    run().await
 }
@@ -0,0 +1,351 @@
+// scanner/vulnerability-probe detection for the clf and caddy parsers.
+//
+// probes come either from the built-in list below (distilled from real caddy
+// logs + well-known probe paths), or from a probelist JSON file (--probelist),
+// meant to be generated per-setup, e.g. by feeding logs to an AI system:
+//
+// {
+//   "probes": [
+//     { "path": "/.env", "match": "contains" },
+//     { "path": "/test.php", "match": "exact" },
+//     { "path": "/api/auth", "match": "prefix", "status": "401", "allowance": 10 }
+//   ]
+// }
+//
+// `match` is exact | prefix | contains (default contains). exact and prefix
+// apply to the query-stripped path, contains to the whole uri - all lowercased.
+// `status` restricts the probe to those response statuses (same syntax as
+// --invalid-http-statuses, e.g. "401,4xx"); without it any status matches.
+// `allowance` overrides the global --allowance for this probe. a `comment`
+// field is allowed anywhere and ignored.
+
+use anyhow::*;
+use std::path::Path;
+use std::result::Result::Ok;
+
+use crate::utils::parse_statuses;
+
+// built-in probes, matched anywhere in the lowercased uri (query included).
+// deliberately conservative: only paths that no legitimate client of ANY
+// common stack ever requests. more aggressive, setup-specific rules (e.g.
+// wp-login.php, /actuator, hosted admin consoles) belong in a --probelist file.
+const PROBE_PARTS: &[&str] = &[
+    // secrets & dotfiles
+    "/.env",  // /.env, /.env.bak, /backend/.env, ...
+    "%2eenv", // url-encoded .env probes
+    "/.git",  // /.git/config, /.git/HEAD, /.gitconfig
+    "/.svn",
+    "/.hg/",
+    "/.aws", // /.aws/credentials
+    "/.ssh",
+    "/.docker",
+    // cloud credentials & config dumps
+    "credentials.json", // /google-credentials.json, /application_default_credentials.json
+    "-key.json",        // /gcp-key.json, /firebase-key.json
+    "/keyfile.json",
+    "/sa.json",
+    "service-account.json",
+    "firebase-adminsdk.json",
+    "gcp-sa.json",
+    "/docker-compose.yml",
+    "/appsettings.json",
+    "application.yml",
+    "parameters.yml",
+    "/web.config",
+    "/settings.py",
+    "/wp-config.php", // never served, only probed - even on real wordpress sites
+    // php probes
+    "phpinfo", // /phpinfo.php, /admin/phpinfo.php, /?phpinfo=1
+    "phpmyadmin",
+    "adminer.php",
+    ".php.bak",
+    ".php.old",
+    ".php.save",
+    ".php~",
+    "eval-stdin.php", // phpunit RCE
+    "/vendor/phpunit",
+    // wordpress fingerprinting (legit wordpress traffic never touches this)
+    "wlwmanifest.xml",
+    // framework debug/env endpoints
+    "/actuator/env", // spring boot (bare /actuator would catch legit health checks)
+    "/_profiler",    // symfony
+    "/_ignition",    // laravel RCE
+    "/_environment", // cakephp
+    "laravel.log",
+    // server status
+    "server-status",
+    "server-info",
+    // traversal & IoT/router botnets
+    "/../",
+    "%2e%2e",
+    "/etc/passwd",
+    "/cgi-bin/",
+    "/boaform",
+    "/hnap1",
+    "/gponform",
+];
+
+// built-in exact matches on the query-stripped path: names too short or
+// generic to be safe as substrings (/i.php would match /api.php, etc..)
+const PROBE_EXACT: &[&str] = &[
+    "/test.php",
+    "/info.php",
+    "/pinfo.php",
+    "/pi.php",
+    "/i.php",
+    "/p.php",
+    "/php.php",
+    "/debug.php",
+    "/database.php",
+    "/config.php",
+    "/shell.php",
+    "/upload.php",
+    "/env",
+];
+
+pub enum MatchKind {
+    Exact,
+    Prefix,
+    Contains,
+}
+
+pub struct Probe {
+    pub path: String,
+    pub kind: MatchKind,
+    pub statuses: Option<Vec<u32>>,
+    pub allowance: Option<u8>,
+}
+
+pub struct ProbeList {
+    probes: Vec<Probe>,
+}
+
+impl ProbeList {
+    pub fn builtin() -> ProbeList {
+        let mut probes: Vec<Probe> = vec![];
+        for p in PROBE_PARTS {
+            probes.push(Probe {
+                path: p.to_string(),
+                kind: MatchKind::Contains,
+                statuses: None,
+                allowance: None,
+            });
+        }
+        for p in PROBE_EXACT {
+            probes.push(Probe {
+                path: p.to_string(),
+                kind: MatchKind::Exact,
+                statuses: None,
+                allowance: None,
+            });
+        }
+        ProbeList { probes }
+    }
+
+    pub fn load(path: &Path) -> Result<ProbeList> {
+        let raw = std::fs::read_to_string(path)
+            .with_context(|| format!("cant read probelist {:?}", path))?;
+        ProbeList::from_json(&raw).with_context(|| format!("cant parse probelist {:?}", path))
+    }
+
+    fn from_json(raw: &str) -> Result<ProbeList> {
+        let json: serde_json::Value = serde_json::from_str(raw)?;
+        let arr = json
+            .get("probes")
+            .and_then(|p| p.as_array())
+            .ok_or_else(|| anyhow!("missing `probes` array"))?;
+
+        let mut probes = vec![];
+        for (i, p) in arr.iter().enumerate() {
+            // entries carrying only a comment are section separators, skip them
+            if p.get("path").is_none() && p.get("comment").is_some() {
+                continue;
+            }
+
+            let path = p
+                .get("path")
+                .and_then(|v| v.as_str())
+                .filter(|s| !s.is_empty())
+                .ok_or_else(|| anyhow!("probe #{}: missing `path`", i))?
+                .to_ascii_lowercase();
+
+            let kind = match p
+                .get("match")
+                .and_then(|v| v.as_str())
+                .unwrap_or("contains")
+            {
+                "exact" => MatchKind::Exact,
+                "prefix" => MatchKind::Prefix,
+                "contains" => MatchKind::Contains,
+                other => bail!("probe #{}: unknown match kind `{}`", i, other),
+            };
+
+            let statuses = match p.get("status") {
+                None => None,
+                Some(v) => {
+                    let s = v
+                        .as_str()
+                        .ok_or_else(|| anyhow!("probe #{}: `status` must be a string", i))?;
+                    Some(parse_statuses(s).with_context(|| format!("probe #{}", i))?)
+                }
+            };
+
+            let allowance = match p.get("allowance") {
+                None => None,
+                Some(v) => {
+                    let a = v
+                        .as_u64()
+                        .filter(|a| (1..=255).contains(a))
+                        .ok_or_else(|| anyhow!("probe #{}: `allowance` must be 1-255", i))?;
+                    Some(a as u8)
+                }
+            };
+
+            probes.push(Probe {
+                path,
+                kind,
+                statuses,
+                allowance,
+            });
+        }
+
+        Ok(ProbeList { probes })
+    }
+
+    pub fn len(&self) -> usize {
+        self.probes.len()
+    }
+
+    // returns the first probe matching this uri + response status
+    pub fn check(&self, uri: &str, status: u32) -> Option<&Probe> {
+        let uri = uri.to_ascii_lowercase();
+        let path = uri.split(['?', '#']).next().unwrap_or(&uri);
+
+        self.probes.iter().find(|p| {
+            let path_hit = match p.kind {
+                MatchKind::Exact => path == p.path,
+                MatchKind::Prefix => path.starts_with(&p.path),
+                MatchKind::Contains => uri.contains(&p.path),
+            };
+            path_hit && p.statuses.as_ref().is_none_or(|s| s.contains(&status))
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn builtin_positive() {
+        // all straight out of real scanner traffic
+        let vectors = [
+            "/.env",
+            "/.env.backup",
+            "/config/.env",
+            "/%2E%2E%2f%2Eenv",
+            "/.git/config",
+            "/.aws/credentials",
+            "/phpinfo.php",
+            "/admin/phpinfo.php",
+            "/?phpinfo=1",
+            "/wp-includes/wlwmanifest.xml",
+            "/blog/wp-includes/wlwmanifest.xml",
+            "/wp-config.php.bak",
+            "/config.php",
+            "/google-credentials.json",
+            "/gcp-key.json",
+            "/docker-compose.yml",
+            "/_profiler/phpinfo",
+            "/_ignition/health-check",
+            "/storage/logs/laravel.log",
+            "/actuator/env",
+            "/webroot/index.php/_environment",
+            "/test.php",
+            "/i.php",
+            "/env",
+            "/phpMyAdmin/index.php",
+            "/cgi-bin/luci",
+            "/../.env",
+            "/server-status.php",
+        ];
+
+        let pl = ProbeList::builtin();
+        vectors.iter().for_each(|e| {
+            assert!(pl.check(e, 200).is_some(), "should flag {}", e);
+        })
+    }
+
+    #[test]
+    fn builtin_negative() {
+        // real user traffic from the same logs
+        let vectors = [
+            "/",
+            "/db",
+            "/ui/",
+            "/ui/static/js/utils.js",
+            "/api/auth",
+            "/api/users/1",
+            "/api/channels/5/messages?after=1620&limit=200",
+            "/app.js",
+            "/config.json",
+            "/favicon.ico",
+            "/robots.txt",
+            "/.well-known/acme-challenge/token123",
+            "/api.php",
+            "/information",
+            "/environment-report",
+            // legit traffic on stacks the tight default must not break:
+            "/wp-login.php",                  // wordpress logins
+            "/xmlrpc.php",                    // wordpress apps/jetpack
+            "/?rest_route=/wp/v2/posts",      // wordpress REST
+            "/actuator/health",               // spring boot health checks
+            "/autodiscover/autodiscover.xml", // outlook probes any domain
+            "/solr/admin/ping",               // hosted consoles
+            "/composer.json",
+        ];
+
+        let pl = ProbeList::builtin();
+        vectors.iter().for_each(|e| {
+            assert!(pl.check(e, 200).is_none(), "should not flag {}", e);
+        })
+    }
+
+    #[test]
+    fn probelist_file() {
+        let raw = r#"{
+            "comment": "generated probelist",
+            "probes": [
+                { "comment": "--- section separator, ignored ---" },
+                { "path": "/.env" },
+                { "path": "/test.php", "match": "exact" },
+                { "path": "/api/auth", "match": "prefix", "status": "401,4xx", "allowance": 10, "comment": "stuffers" }
+            ]
+        }"#;
+        let pl = ProbeList::from_json(raw).unwrap();
+        assert_eq!(pl.len(), 3);
+
+        // contains, any status
+        assert!(pl.check("/backend/.env", 200).is_some());
+        // exact, query-stripped
+        assert!(pl.check("/test.php?x=1", 200).is_some());
+        assert!(pl.check("/xtest.php", 200).is_none());
+        // prefix + status filter + allowance override
+        let hit = pl.check("/api/auth/me", 401).unwrap();
+        assert_eq!(hit.allowance, Some(10));
+        assert!(pl.check("/api/auth/me", 200).is_none());
+        assert!(pl.check("/api/authless", 401).is_some()); // prefix is a plain str prefix
+    }
+
+    #[test]
+    fn probelist_rejects_garbage() {
+        assert!(ProbeList::from_json("{}").is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": ""}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "match": "regex"}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "status": 401}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "status": "9xx"}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "allowance": 300}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "allowance": 0}]}"#).is_err());
+    }
+}
@@ -1,96 +0,0 @@
-use anyhow::*;
-use lazy_static::lazy_static;
-use regex::Regex;
-use std::{net::IpAddr, str::FromStr};
-
-use crate::utils::ParsingStatus;
-
-struct Rule {
-    matcher: String,
-    extractor: Regex,
-}
-
-lazy_static! {
-    static ref SSHD_BAD: [Rule; 3] = [
-        Rule {
-            matcher: "Failed password".to_string(),
-            extractor: Regex::new(r"(from.)(\S+)").unwrap(),
-        },
-        Rule {
-            matcher: "Invalid user ".to_string(),
-            extractor: Regex::new(r"(from.)(\S+)").unwrap(),
-        },
-        Rule {
-            matcher: "authentication failure".to_string(),
-            extractor: Regex::new(r"(rhost=)(\S+)").unwrap()
-        },
-    ];
-}
-
-pub fn parse(line: &str) -> Result<ParsingStatus> {
-    let hits = SSHD_BAD
-        .iter()
-        .find(|rule| line.contains(&rule.matcher))
-        .and_then(|r| r.extractor.captures(line));
-
-    if hits.is_none() {
-        return Ok(ParsingStatus::OkEntry);
-    }
-
-    let ip = hits
-        .and_then(|c| c.get(2))
-        .and_then(|m| IpAddr::from_str(m.as_str()).ok())
-        .ok_or_else(|| anyhow!("cant parse sshd line"))?;
-
-    Ok(ParsingStatus::BadEntry(ip))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn positive() {
-        let vectors = [
-            "Sep 26 06:25:19 livecompute sshd[23246]: Failed password for root from 179.124.36.195 port 41883 ssh2",
-            "Sep 26 06:26:14 livecompute sshd[23292]: pam_unix(sshd:auth): authentication failure; logname= u =0 tty=ssh ruser= rhost=5.101.107.190",
-            "Sep 26 06:25:32 livecompute sshd[23254]: Invalid user neal from 35.184.211.144"
-        ];
-
-        vectors.iter().for_each(|e| {
-            let ret = parse(*e).unwrap();
-            match ret {
-                ParsingStatus::BadEntry(_) => {}
-                _ => panic!("bad parsing"),
-            }
-        })
-    }
-
-    #[test]
-    fn negative() {
-        let vectors = [
-            "Sep 26 06:25:19 livecompute sshd[23246]: successful login 179.124.36.195 port 41883 ssh2",
-            "Sep 26 06:26:14 livecompute sshd[23292]: pam_unix(sshd:auth): authentication total success; logname= u =0 tty=ssh ruser= rhost=5.101.107.190",
-            "Sep 26 06:25:32 livecompute sshd[23254]: very good user neal from 35.184.211.144"
-        ];
-
-        vectors.iter().for_each(|e| {
-            let ret = parse(*e).unwrap();
-            match ret {
-                ParsingStatus::OkEntry => {}
-                _ => panic!("bad parsing"),
-            }
-        })
-    }
-
-    #[test]
-    fn malformed() {
-        let vectors = [
-            "Sep 26 06:25:19 livecompute sshd[23246]: Failed password for root from 179.124.36.195.232 port 41883 ssh2",
-        ];
-
-        vectors.iter().for_each(|e| {
-            parse(*e).expect_err("");
-        })
-    }
-}
@@ -1,5 +1,5 @@
-use anyhow::{anyhow, Context, Result};
-use clap::Parser;
+use anyhow::{anyhow, bail, Context, Result};
+use clap::{ArgGroup, Parser};
 use regex::Regex;
 use std::{
    net::IpAddr,
@@ -9,7 +9,8 @@ use std::{
 #[derive(Debug)]
 pub enum ParsingStatus {
    OkEntry,
-    BadEntry(IpAddr),
+    // offending ip, plus an optional per-probe allowance override
+    BadEntry(IpAddr, Option<u8>),
 }

 pub fn get_epoch() -> u64 {
@@ -43,28 +44,30 @@ pub fn parse_regex(a: &str) -> Result<Regex> {
    Ok(r)
 }

+/// Parse a comma-separated list of HTTP statuses with trailing-`x` wildcards.
+///
+/// Accepted forms per token: `NNN` (literal), `NNx` (10-wide range), `Nxx` (100-wide range).
+/// All expanded codes must fall in the valid HTTP range 100..=599.
 pub fn parse_statuses(a: &str) -> Result<Vec<u32>> {
+    if a.trim().is_empty() {
+        bail!("invalid_http_statuses: empty input");
+    }
    let mut statuses = vec![];
-    for s in a.split(',') {
-        if s.contains("xx") {
-            let range = s.replace("xx", "");
-            let range = range.parse::<u32>().context("invalid range")?;
-            let range = range * 100;
-            for i in 0..100 {
-                let status = range + i;
-                statuses.push(status);
+    for raw in a.split(',') {
+        let s = raw.trim();
+        let xs = s.chars().rev().take_while(|c| *c == 'x').count();
+        let digits = &s[..s.len() - xs];
+        if digits.is_empty() || !digits.chars().all(|c| c.is_ascii_digit()) {
+            bail!("invalid http status `{}`", s);
        }
-        } else if s.contains("x") {
-            let range = s.replace("x", "");
-            let range = range.parse::<u32>().context("invalid range")?;
-            let range = range * 10;
-            for i in 0..10 {
-                let status = range + i;
-                statuses.push(status);
+        let base: u32 = digits.parse().context("invalid status")?;
+        let span = 10u32.pow(xs as u32);
+        let from = base * span;
+        for code in from..(from + span) {
+            if !(100..=599).contains(&code) {
+                bail!("status {} out of valid HTTP range (100..=599)", code);
            }
-        } else {
-            let status = s.parse::<u32>().context("invalid status")?;
-            statuses.push(status);
+            statuses.push(code);
        }
    }
    Ok(statuses)
@@ -84,17 +87,17 @@ Blockfast - block internets scanners fast 🍶
 Author: pierre dubouilh <pldubouilh@gmail.com>

 Blockfast reads logs from various sources and blocks the offending IPs using iptables and ipset.
-It supports logs from sshd, Common-Log-Format (Apache, etc..), JSON (Caddy) and a generic logs parser.
+It supports logs in Common-Log-Format (Apache, nginx, etc..), Caddy JSON and a generic logs parser.

 Example:
-    # block invalid sshd attempts & invalid http statuses from caddy
-    ./blockfast -s=/var/log/auth.log -j=/caddy/logs
+    # block invalid http statuses from caddy
+    ./blockfast --caddy-logpath=/caddy/logs

    # generic log parser example with a log text to flag, and a regex to parse the offending IP.
    ./blockfast --generic-logpath=/tmp/generictest --generic-positive='Failed password' --generic-ip='from ([0-9a-fA-F:.]+) port'",
    verbatim_doc_comment,
+    group(ArgGroup::new("generic_match").args(["generic_positive", "generic_negative"])),
 )]
-
 pub struct Args {
    /// jail time (seconds)
    #[clap(long, default_value = "21600")]
@@ -108,35 +111,88 @@ pub struct Args {
    #[clap(short, long)]
    pub verbose: bool,

-    /// path of sshd logfile
-    #[clap(short, long, value_parser = resolve_path)]
-    pub sshd_logpath: Option<PathBuf>,
-
-    /// path of Common-Log-Format logfile (Apache, etc..)
-    #[clap(short, long, value_parser = resolve_path)]
-    pub clf_logpath: Option<PathBuf>,
-
-    /// path of JSON logfile (works with Caddy)
-    #[clap(short, long, value_parser = resolve_path)]
-    pub json_logpath: Option<PathBuf>,
-
-    /// generic parser log file path
+    /// path of Common-Log-Format logfile (Apache, nginx, etc..), can be repeated
    #[clap(long, value_parser = resolve_path)]
-    pub generic_logpath: Option<PathBuf>,
+    pub clf_logpath: Vec<PathBuf>,
+
+    /// path of Caddy JSON logfile, can be repeated
+    #[clap(long, value_parser = resolve_path)]
+    pub caddy_logpath: Vec<PathBuf>,
+
+    /// path of a probelist JSON file, replaces the built-in probe list (see README)
+    #[clap(long, value_parser = resolve_path)]
+    pub probelist: Option<PathBuf>,
+
+    /// generic parser log file path, can be repeated
+    #[clap(long, value_parser = resolve_path, requires_all = ["generic_ip", "generic_match"])]
+    pub generic_logpath: Vec<PathBuf>,

    /// generic parser ip regex
-    #[clap(long , value_parser = parse_regex)]
+    #[clap(long, value_parser = parse_regex, requires = "generic_logpath")]
    pub generic_ip: Option<Regex>,

    /// generic parser positive - if a logline contains this, it is considered bad, the rest is good
-    #[clap(long)]
+    #[clap(long, requires = "generic_logpath")]
    pub generic_positive: Option<String>,

    /// generic parser negative - if a logline contains this, it is considered good, the rest is bad
-    #[clap(long)]
+    #[clap(long, requires = "generic_logpath")]
    pub generic_negative: Option<String>,

-    /// valid http statuses (for CLF and JSON logs). Coma separated list, accepts ranges with XX.
-    #[clap(long, default_value = "10x,20x,30x,404,408")]
-    pub valid_http_statuses: String,
+    /// also flag these http statuses (for CLF and Caddy logs), on top of the built-in
+    /// scanner-path detection. Coma separated list, accepts ranges with XX, e.g. "403,5xx".
+    /// Careful: many apps serve 4xx statuses to legitimate clients.
+    #[clap(long)]
+    pub invalid_http_statuses: Option<String>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::parse_statuses;
+
+    #[test]
+    fn literal() {
+        assert_eq!(parse_statuses("401").unwrap(), vec![401]);
+        assert_eq!(parse_statuses("401,404,429").unwrap(), vec![401, 404, 429]);
+    }
+
+    #[test]
+    fn ten_range() {
+        assert_eq!(
+            parse_statuses("40x").unwrap(),
+            (400..410).collect::<Vec<_>>()
+        );
+    }
+
+    #[test]
+    fn hundred_range() {
+        assert_eq!(
+            parse_statuses("4xx").unwrap(),
+            (400..500).collect::<Vec<_>>()
+        );
+    }
+
+    #[test]
+    fn mixed() {
+        let got = parse_statuses("401, 40x, 5xx").unwrap();
+        assert_eq!(got.len(), 1 + 10 + 100);
+        assert_eq!(got[0], 401);
+        assert!(got.contains(&500));
+        assert!(got.contains(&599));
+    }
+
+    #[test]
+    fn rejects_out_of_range() {
+        // 5x → 50..60, none of which are valid HTTP
+        assert!(parse_statuses("5x").is_err());
+    }
+
+    #[test]
+    fn rejects_garbage() {
+        assert!(parse_statuses("").is_err());
+        assert!(parse_statuses("xxx").is_err());
+        assert!(parse_statuses("1x2").is_err());
+        assert!(parse_statuses("abc").is_err());
+        assert!(parse_statuses("401,,402").is_err());
+    }
 }
Author	SHA1	Message	Date
Pierre Dubouilh	1299a41b6a	move to probelist the probelist allows to finely allow / dissalow certain paths.	2026-09-03 18:40:17 +02:00
Pierre Dubouilh	27e78fe349	tighter checks for ipset install	2026-08-28 00:34:23 +02:00
Pierre Dubouilh	2df9519224	rename json to caddy it was misleading	2026-08-28 00:34:23 +02:00
Pierre Dubouilh	e8437d33fb	tighten CLF	2026-08-28 00:34:23 +02:00
Pierre Dubouilh	e63e926ccf	drop sshd support	2026-08-28 00:34:23 +02:00
Pierre Dubouilh	ac17c11158	multiple config per target	2026-08-28 00:34:23 +02:00
Pierre Dubouilh	43392ace03	fixup readme & bump rust release	2026-05-27 18:10:50 +02:00
Pierre Dubouilh	9d0bc470cc	use clap to avoid CLI settings conflicts	2026-05-27 18:10:50 +02:00
Pierre Dubouilh	956d56716e	bump serde & rework status parsing	2026-05-27 18:10:50 +02:00
Pierre Dubouilh	9d677a13df	gracefully introduce ipsets	2026-05-27 18:10:50 +02:00
Pierre Dubouilh	38a374858e	tighten clf regex	2026-05-27 18:10:50 +02:00
Pierre Dubouilh	02f5bea4c0	fixup CLI flags	2026-05-27 18:10:50 +02:00
Pierre Dubouilh	1b9385e415	whitelist => blacklist	2026-05-27 18:10:50 +02:00