move to probelist

the probelist allows to finely allow / dissalow certain paths.

.gitignore

@@ -1,2 +1,3 @@
 /target
 builds/**
+probelist.json

Makefile

@@ -50,14 +50,14 @@ ok-generic::
 	echo "Sep 26 06:25:19 livecompute sshd[23246]: Successful login for root from 179.124.36.195 port 41883 ssh2"  >> /tmp/generictest
 
 hit-clf::
-	echo "1.124.36.195 - p [25/Sep/2021:13:49:56 +0200] \"POST /some/rpc HTTP/2.0\" 401 923" >> /tmp/clftest
+	echo "1.124.36.195 - p [25/Sep/2021:13:49:56 +0200] \"GET /.env HTTP/2.0\" 200 923" >> /tmp/clftest
 
 ok-clf::
 	echo "2.124.36.195 - p [25/Sep/2021:13:49:56 +0200] \"POST /some/rpc HTTP/2.0\" 200 23012" >> /tmp/clftest
 
 hit-caddy::
-	echo "{\"request\":{\"remote_ip\":\"1.124.36.19\"}, \"status\": 400}" >> /tmp/caddytest
+	echo "{\"request\":{\"remote_ip\":\"1.124.36.19\",\"uri\":\"/.env\"}, \"status\": 200}" >> /tmp/caddytest
 
 ok-caddy::
-	echo "{\"request\":{\"remote_ip\":\"2.124.36.19\"}, \"status\": 200}" >> /tmp/caddytest
+	echo "{\"request\":{\"remote_ip\":\"2.124.36.19\",\"uri\":\"/\"}, \"status\": 200}" >> /tmp/caddytest
 

README.md

@@ -3,9 +3,11 @@
 Block internets scanners fast 🍶
 
 Features:
-  - Common Log Format parser (apache, nginx logs, etc...)
+  - Built-in scanner detection, built for well-known probe paths (`/.env`, `/phpinfo.php`, etc...) 
+  - Extends built-in scanners using your own logs. See probelist.example.json
+  - Supports logs from Common Log Format (apache, nginx logs, etc...)
   - Caddy JSON log parser
-  - Generic log parser
+  - Generic log parser (regexp)
   - Sane defaults
   - Fast ip ban with `ipset`
   - Static release builds, no libc dependency
@@ -56,6 +58,8 @@ Options:
           path of Common-Log-Format logfile (Apache, nginx, etc..), can be repeated
       --caddy-logpath <CADDY_LOGPATH>
           path of Caddy JSON logfile, can be repeated
+      --probelist <PROBELIST>
+          path of a probelist JSON file, replaces the built-in probe list (see README)
       --generic-logpath <GENERIC_LOGPATH>
           generic parser log file path, can be repeated
       --generic-ip <GENERIC_IP>
@@ -65,7 +69,7 @@ Options:
       --generic-negative <GENERIC_NEGATIVE>
           generic parser negative - if a logline contains this, it is considered good, the rest is bad
       --invalid-http-statuses <INVALID_HTTP_STATUSES>
-          invalid http statuses (for CLF and Caddy logs). Coma separated list, accepts ranges with XX [default: 400,401,402,403]
+          also flag these http statuses (for CLF and Caddy logs), on top of the built-in scanner-path detection. Coma separated list, accepts ranges with XX, e.g. "403,5xx".
   -h, --help
           Print help
   -V, --version

probelist.example.json

@@ -0,0 +1,108 @@
+{
+  "comment": "blockfast probelist - TEMPLATE. This file documents the format and how to generate a real one from server logs. blockfast loads it with --probelist=<file>; it then REPLACES the built-in probe list entirely, so a generated file must include generic scanner rules too, not only setup-specific ones. All `comment` fields (and this whole header) are ignored by the loader; entries containing only a `comment` are section separators. Everything else must follow the format below.",
+
+  "format": [
+    "each probe entry: { path, match?, status?, allowance?, comment? }",
+    "path (required): the string to match against the request URI, matched case-insensitively",
+    "match (optional, default 'contains'): 'contains' matches anywhere in the full URI including the query string; 'prefix' and 'exact' match against the query-stripped path only",
+    "status (optional): restricts the probe to these response statuses, as a comma-separated string with trailing-x ranges, e.g. '401' or '403,5xx' or '40x'. Without it the probe matches ANY status - important, since SPAs with catch-all routes answer 200 to probe paths",
+    "allowance (optional, 1-255): overrides blockfast's global --allowance for this probe, i.e. how many offences an IP may accumulate (within the jailtime window, shared across all probes) before being banned. Use a HIGHER value for endpoints legitimate users occasionally trip, a LOW value is implicit for pure probe paths"
+  ],
+
+  "howto_generate": [
+    "You are generating a ban-rule file for blockfast, a fail2ban-like daemon: it tails web server logs (Common Log Format or Caddy JSON), counts offences per client IP, and firewalls IPs that exceed their allowance. A wrong rule bans legitimate users, so precision beats recall.",
+    "1. Group the log lines by client IP. Classify each IP: legitimate clients use real app routes and mostly get 2xx/3xx; scanners enumerate paths (/.env, /phpinfo.php, /wp-*, credential/config filenames) that the app never serves.",
+    "2. Paths requested ONLY by scanners and NEVER by legitimate clients become unconditional probes (no status filter). Do not rely on the response status for these: a catch-all route may answer 200.",
+    "3. Endpoints that legitimate users DO touch, but that indicate abuse in volume (login endpoints, session checks answering 401, method-probing answering 405), become narrow rules: match 'prefix' or 'exact' + a status filter + a generous allowance. Never add such an endpoint without a status filter.",
+    "4. NEVER add: paths served by the app (assets, API routes), /favicon.ico, /robots.txt, /.well-known (ACME!), or anything a browser or well-behaved bot requests on its own. When unsure, leave it out.",
+    "5. Prefer 'exact' for short or generic names ('/env', '/i.php') - as substrings they would match legitimate paths ('/i.php' is inside '/api.php'). Use 'contains' for names that are unambiguous anywhere in a URI ('/.env', 'phpinfo').",
+    "6. Keep the generic scanner families in the output (see sections below), then append the setup-specific rules derived from the logs, each with a comment explaining the evidence.",
+    "7. Sanity-check the finished list by replaying the logs against it: no IP classified as a legitimate client may match any rule."
+  ],
+
+  "probes": [
+    { "comment": "--- setup-specific rules, derived from the logs (EXAMPLES - replace with real findings) ---" },
+    { "path": "/api/auth", "match": "prefix", "status": "401", "allowance": 10, "comment": "example: the SPA answers 401 on session checks when logged out - normal a few times, credential stuffing in bulk. prefix + status + generous allowance" },
+    { "path": "/", "match": "exact", "status": "405", "comment": "example: scanners POST to the root fishing for handlers; browsers never trigger 405 there" },
+    { "path": "/backup.tar.gz", "match": "exact", "comment": "example: artifact hunted by scanners in these logs, never linked by the app" },
+
+    { "comment": "--- generic: secrets & dotfiles ---" },
+    { "path": "/.env", "comment": "also matches /.env.bak, /backend/.env, ..." },
+    { "path": "%2eenv", "comment": "url-encoded .env" },
+    { "path": "/.git", "comment": "/.git/config, /.git/HEAD, /.gitconfig" },
+    { "path": "/.svn" },
+    { "path": "/.hg/" },
+    { "path": "/.aws" },
+    { "path": "/.ssh" },
+    { "path": "/.docker" },
+
+    { "comment": "--- generic: cloud credentials & config dumps ---" },
+    { "path": "credentials.json" },
+    { "path": "-key.json", "comment": "/gcp-key.json, /firebase-key.json, ..." },
+    { "path": "/keyfile.json" },
+    { "path": "/sa.json" },
+    { "path": "service-account.json" },
+    { "path": "firebase-adminsdk.json" },
+    { "path": "gcp-sa.json" },
+    { "path": "/docker-compose.yml" },
+    { "path": "/appsettings.json" },
+    { "path": "application.yml" },
+    { "path": "parameters.yml" },
+    { "path": "/web.config" },
+    { "path": "/settings.py" },
+    { "path": "/wp-config.php", "comment": "never served, only probed - even on real wordpress sites" },
+
+    { "comment": "--- generic: php probes ---" },
+    { "path": "phpinfo" },
+    { "path": "phpmyadmin" },
+    { "path": "adminer.php" },
+    { "path": ".php.bak" },
+    { "path": ".php.old" },
+    { "path": ".php.save" },
+    { "path": ".php~" },
+    { "path": "eval-stdin.php", "comment": "phpunit RCE" },
+    { "path": "/vendor/phpunit" },
+    { "path": "/test.php", "match": "exact" },
+    { "path": "/info.php", "match": "exact" },
+    { "path": "/pinfo.php", "match": "exact" },
+    { "path": "/pi.php", "match": "exact" },
+    { "path": "/i.php", "match": "exact" },
+    { "path": "/p.php", "match": "exact" },
+    { "path": "/php.php", "match": "exact" },
+    { "path": "/debug.php", "match": "exact" },
+    { "path": "/database.php", "match": "exact" },
+    { "path": "/config.php", "match": "exact" },
+    { "path": "/shell.php", "match": "exact" },
+    { "path": "/upload.php", "match": "exact" },
+
+    { "comment": "--- generic: fingerprinting & framework debug endpoints ---" },
+    { "path": "wlwmanifest.xml", "comment": "wordpress fingerprinting, legit wp traffic never touches it" },
+    { "path": "/actuator/env", "comment": "spring boot - keep narrow, bare /actuator would catch legit health checks" },
+    { "path": "/_profiler", "comment": "symfony" },
+    { "path": "/_ignition", "comment": "laravel RCE" },
+    { "path": "/_environment", "comment": "cakephp" },
+    { "path": "laravel.log" },
+    { "path": "/env", "match": "exact" },
+
+    { "comment": "--- generic: server status, traversal, IoT/router botnets ---" },
+    { "path": "server-status" },
+    { "path": "server-info" },
+    { "path": "/../" },
+    { "path": "%2e%2e" },
+    { "path": "/etc/passwd" },
+    { "path": "/cgi-bin/" },
+    { "path": "/boaform" },
+    { "path": "/hnap1" },
+    { "path": "/gponform" },
+
+    { "comment": "--- aggressive extras: correct for many setups, but NOT in blockfast's built-in default because they match legit traffic on some stacks. Include them only when the logs show the stack does not use them ---" },
+    { "path": "/wp-login.php", "comment": "legit logins on real wordpress sites - only include if not hosting wordpress" },
+    { "path": "/xmlrpc.php", "comment": "used by jetpack & wordpress mobile apps" },
+    { "path": "rest_route=", "comment": "wordpress REST access with permalinks disabled" },
+    { "path": "/autodiscover/", "comment": "outlook clients innocently probe this on any domain" },
+    { "path": "/manager/html", "comment": "tomcat console - legit for tomcat admins" },
+    { "path": "/solr/admin" },
+    { "path": "/geoserver/web" },
+    { "path": "/telescope/requests", "comment": "laravel telescope - legit for its admins" }
+  ]
+}

src/caddy.rs

@@ -1,8 +1,9 @@
+use crate::probes::ProbeList;
 use crate::utils::ParsingStatus;
 use anyhow::*;
 use std::{net::IpAddr, str::FromStr};
 
-pub fn parse(line: &str, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
+pub fn parse(line: &str, probelist: &ProbeList, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
     let json: serde_json::Value = serde_json::from_str(line)?;
 
     let remote_ip = json
@@ -12,14 +13,24 @@ pub fn parse(line: &str, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
         .and_then(|r| IpAddr::from_str(r).ok())
         .ok_or_else(|| anyhow!("cant parse json line - remote_ip"))?;
 
+    let uri = json
+        .get("request")
+        .and_then(|r| r.get("uri"))
+        .and_then(|r| r.as_str())
+        .ok_or_else(|| anyhow!("cant parse json line - uri"))?;
+
     let status = json
         .get("status")
         .and_then(|r| r.as_u64())
-        .ok_or_else(|| anyhow!("cant parse json line - status"))?;
+        .ok_or_else(|| anyhow!("cant parse json line - status"))? as u32;
 
-    let is_bad_status = invalid_statuses.iter().any(|s| s == &(status as u32));
+    if let Some(probe) = probelist.check(uri, status) {
+        return Ok(ParsingStatus::BadEntry(remote_ip, probe.allowance));
+    }
+
+    let is_bad_status = invalid_statuses.iter().any(|s| s == &status);
     if is_bad_status {
-        return Ok(ParsingStatus::BadEntry(remote_ip));
+        return Ok(ParsingStatus::BadEntry(remote_ip, None));
     }
 
     Ok(ParsingStatus::OkEntry)
@@ -37,9 +48,9 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![429, 401]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]).unwrap();
             match ret {
-                ParsingStatus::BadEntry(_) => {}
+                ParsingStatus::BadEntry(..) => {}
                 _ => panic!("bad parsing"),
             }
         })
@@ -53,7 +64,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![429, 401]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]).unwrap();
             match ret {
                 ParsingStatus::OkEntry => {}
                 _ => panic!("bad parsing"),
@@ -61,6 +72,22 @@ mod tests {
         })
     }
 
+    #[test]
+    fn probe_uri() {
+        // a probe path is an offence even with a 200 status and no status list
+        let bad = r#"{"request":{"remote_ip":"1.2.3.4","uri":"/.env"},"status":200}"#;
+        match parse(bad, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::BadEntry(..) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ok = r#"{"request":{"remote_ip":"1.2.3.4","uri":"/api/auth/me"},"status":200}"#;
+        match parse(ok, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
     #[test]
     fn malformed() {
         let vectors = [
@@ -68,7 +95,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![429, 401]);
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]);
             assert!(ret.is_err());
         })
     }

src/clf.rs

@@ -1,3 +1,4 @@
+use crate::probes::ProbeList;
 use crate::utils::ParsingStatus;
 use anyhow::*;
 use lazy_static::lazy_static;
@@ -11,11 +12,11 @@ lazy_static! {
     // covers plain CLF and the combined format - the trailing attacker
     // controlled "referer" "user-agent" fields are never scanned
     static ref RE_CLF: Regex =
-        Regex::new(r#"^(\S+)\s+\S+\s+\S+\s+\[[^\]]*\]\s+"(?:[^"\\]|\\.)*"\s+(\d{3})(?:\s|$)"#)
+        Regex::new(r#"^(\S+)\s+\S+\s+\S+\s+\[[^\]]*\]\s+"((?:[^"\\]|\\.)*)"\s+(\d{3})(?:\s|$)"#)
             .unwrap();
 }
 
-pub fn parse(line: &str, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
+pub fn parse(line: &str, probelist: &ProbeList, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
     let caps = RE_CLF
         .captures(line)
         .ok_or_else(|| anyhow!("cant parse clf line"))?;
@@ -26,13 +27,23 @@ pub fn parse(line: &str, invalid_statuses: &[u32]) -> Result<ParsingStatus> {
         .ok_or_else(|| anyhow!("cant parse clf line - ip"))?;
 
     let status = caps
-        .get(2)
+        .get(3)
         .and_then(|g| g.as_str().parse::<u32>().ok())
         .ok_or_else(|| anyhow!("cant parse clf line - status"))?;
 
+    // the uri is the second token of the request field ("GET /uri HTTP/1.1")
+    let uri = caps
+        .get(2)
+        .and_then(|g| g.as_str().split_whitespace().nth(1));
+    if let Some(u) = uri {
+        if let Some(probe) = probelist.check(u, status) {
+            return Ok(ParsingStatus::BadEntry(ip, probe.allowance));
+        }
+    }
+
     let is_bad_status = invalid_statuses.iter().any(|s| s == &status);
     if is_bad_status {
-        return Ok(ParsingStatus::BadEntry(ip));
+        return Ok(ParsingStatus::BadEntry(ip, None));
     }
 
     Ok(ParsingStatus::OkEntry)
@@ -50,9 +61,9 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![401, 429]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![401, 429]).unwrap();
             match ret {
-                ParsingStatus::BadEntry(_) => {}
+                ParsingStatus::BadEntry(..) => {}
                 _ => panic!("bad parsing"),
             }
         })
@@ -66,7 +77,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![401, 429]).unwrap();
+            let ret = parse(*e, &ProbeList::builtin(), &vec![401, 429]).unwrap();
             match ret {
                 ParsingStatus::OkEntry => {}
                 _ => panic!("bad parsing"),
@@ -79,13 +90,29 @@ mod tests {
         // combined log format appends "referer" "user-agent" - the old parser
         // read digits out of the user-agent as the status and silently missed these
         let bad = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /admin HTTP/1.1" 401 923 "https://example.com/" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0""#;
-        match parse(bad, &vec![401, 429]).unwrap() {
+        match parse(bad, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
-            ParsingStatus::BadEntry(_) => {}
+            ParsingStatus::BadEntry(..) => {}
             _ => panic!("bad parsing"),
         }
 
         let ok = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET / HTTP/1.1" 200 923 "https://example.com/" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0""#;
-        match parse(ok, &vec![401, 429]).unwrap() {
+        match parse(ok, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
+    #[test]
+    fn probe_uri() {
+        // a probe path is an offence even with a 200 status and no status list
+        let bad = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /.env HTTP/1.1" 200 923"#;
+        match parse(bad, &ProbeList::builtin(), &[]).unwrap() {
+            ParsingStatus::BadEntry(..) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ok = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /index.html HTTP/1.1" 200 923"#;
+        match parse(ok, &ProbeList::builtin(), &[]).unwrap() {
             ParsingStatus::OkEntry => {}
             _ => panic!("bad parsing"),
         }
@@ -96,7 +123,7 @@ mod tests {
         // servers escape quotes in the logged request - an escaped `\" 401 `
         // inside the URL must not be mistaken for the end of the request field
         let ok = r#"8.8.8.8 - - [25/Sep/2021:13:49:56 +0200] "GET /x?a=\" 401 - HTTP/1.1" 200 923"#;
-        match parse(ok, &vec![401, 429]).unwrap() {
+        match parse(ok, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
             ParsingStatus::OkEntry => {}
             _ => panic!("bad parsing"),
         }
@@ -106,13 +133,13 @@ mod tests {
     fn bodyless() {
         // CLF uses `-` for absent body bytes, both branches must still parse
         let bad = "8.8.8.8 - p [25/Sep/2021:13:49:56 +0200] \"GET / HTTP/2.0\" 401 -";
-        match parse(bad, &vec![401, 429]).unwrap() {
+        match parse(bad, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
-            ParsingStatus::BadEntry(_) => {}
+            ParsingStatus::BadEntry(..) => {}
             _ => panic!("bad parsing"),
         }
 
         let ok = "8.8.8.8 - p [25/Sep/2021:13:49:56 +0200] \"GET / HTTP/2.0\" 304 -";
-        match parse(ok, &vec![401, 429]).unwrap() {
+        match parse(ok, &ProbeList::builtin(), &vec![401, 429]).unwrap() {
             ParsingStatus::OkEntry => {}
             _ => panic!("bad parsing"),
         }
@@ -126,7 +153,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e, &vec![429, 401]);
+            let ret = parse(*e, &ProbeList::builtin(), &vec![429, 401]);
             assert!(ret.is_err());
         })
     }

src/generic.rs

@@ -30,7 +30,7 @@ pub fn parse(
         .and_then(|e| IpAddr::from_str(e).ok())
         .ok_or_else(|| anyhow!("cant parse clf line - ip"))?;
 
-    Ok(ParsingStatus::BadEntry(ip))
+    Ok(ParsingStatus::BadEntry(ip, None))
 }
 
 #[cfg(test)]
@@ -51,7 +51,7 @@ mod tests {
 
         let ret = parse(FAILED, Some(&ip), Some(&positive), None).unwrap();
         match ret {
-            ParsingStatus::BadEntry(_) => {}
+            ParsingStatus::BadEntry(..) => {}
             _ => panic!("bad parsing"),
         }
 
@@ -76,7 +76,7 @@ mod tests {
 
         let ret = parse(FAILED, Some(&ip), None, Some(&negative)).unwrap();
         match ret {
-            ParsingStatus::BadEntry(_) => {}
+            ParsingStatus::BadEntry(..) => {}
             _ => panic!("bad parsing"),
         }
     }

src/jail.rs

@@ -70,8 +70,10 @@ impl Jail {
         })
     }
 
-    pub fn sentence(&self, ip: IpAddr) -> Result<bool> {
+    // allowance_override comes from a matched probe, else the global setting applies
+    pub fn sentence(&self, ip: IpAddr, allowance_override: Option<u8>) -> Result<bool> {
         let now = get_epoch();
+        let allowance = allowance_override.unwrap_or(self.allowance);
 
         let should_ban = {
             let mut locked_map = self.remand.lock().map_err(|_| anyhow!("cant lock"))?;
@@ -88,7 +90,7 @@ impl Jail {
                     }
                 })
                 .or_insert((1, now));
-            if hits < self.allowance {
+            if hits < allowance {
                 false
             } else {
                 locked_map.remove_entry(&ip);

src/main.rs

@@ -7,6 +7,7 @@ use linemux::{Line, MuxedLines};
 mod caddy;
 mod clf;
 mod generic;
+mod probes;
 mod utils;
 
 mod jail;
@@ -17,10 +18,23 @@ async fn run() -> Result<()> {
     let args = utils::Args::parse();
     let mut ml = MuxedLines::new()?;
 
-    // HTTP statuses
+    // HTTP statuses - opt-in, on top of the probe detection
-    let invalid_statuses = args.invalid_http_statuses.clone();
+    let invalid_statuses_parsed = match &args.invalid_http_statuses {
-    let invalid_statuses_parsed = parse_statuses(&invalid_statuses)?;
+        Some(s) => parse_statuses(s)?,
-    let invalid_statuses_ref = invalid_statuses_parsed.as_ref();
+        None => vec![],
+    };
+    let invalid_statuses_ref: &[u32] = invalid_statuses_parsed.as_ref();
+
+    // probes - loaded from --probelist if given, built-in list otherwise
+    let probelist = match args.probelist.as_ref() {
+        Some(p) => {
+            let pl = probes::ProbeList::load(p)?;
+            log!("loaded {} probes from {:?}", pl.len(), p);
+            pl
+        }
+        None => probes::ProbeList::builtin(),
+    };
+    let probelist = &probelist;
 
     // generic parser
     let generic_paths = &args.generic_logpath;
@@ -59,9 +73,12 @@ async fn run() -> Result<()> {
         let path = path_buf.as_ref();
 
         let (target, ret) = if path.is_some_and(|p| clf_logpaths.contains(p)) {
-            ("clf", clf::parse(payload, invalid_statuses_ref)?)
+            ("clf", clf::parse(payload, probelist, invalid_statuses_ref)?)
         } else if path.is_some_and(|p| caddy_logpaths.contains(p)) {
-            ("caddy", caddy::parse(payload, invalid_statuses_ref)?)
+            (
+                "caddy",
+                caddy::parse(payload, probelist, invalid_statuses_ref)?,
+            )
         } else if path.is_some_and(|p| generic_paths.contains(p)) {
             (
                 "generic",
@@ -71,11 +88,11 @@ async fn run() -> Result<()> {
             bail!("file {:?} unknown ?", path)
         };
 
-        if let ParsingStatus::BadEntry(ip) = ret {
+        if let ParsingStatus::BadEntry(ip, allowance) = ret {
             if args.verbose {
                 log!("{} logged offence for {}", target, ip);
             }
-            let banned = jail.sentence(ip)?;
+            let banned = jail.sentence(ip, allowance)?;
             if banned {
                 log!("{} jailtime for {}", target, ip);
             }

src/probes.rs

@@ -0,0 +1,351 @@
+// scanner/vulnerability-probe detection for the clf and caddy parsers.
+//
+// probes come either from the built-in list below (distilled from real caddy
+// logs + well-known probe paths), or from a probelist JSON file (--probelist),
+// meant to be generated per-setup, e.g. by feeding logs to an AI system:
+//
+// {
+//   "probes": [
+//     { "path": "/.env", "match": "contains" },
+//     { "path": "/test.php", "match": "exact" },
+//     { "path": "/api/auth", "match": "prefix", "status": "401", "allowance": 10 }
+//   ]
+// }
+//
+// `match` is exact | prefix | contains (default contains). exact and prefix
+// apply to the query-stripped path, contains to the whole uri - all lowercased.
+// `status` restricts the probe to those response statuses (same syntax as
+// --invalid-http-statuses, e.g. "401,4xx"); without it any status matches.
+// `allowance` overrides the global --allowance for this probe. a `comment`
+// field is allowed anywhere and ignored.
+
+use anyhow::*;
+use std::path::Path;
+use std::result::Result::Ok;
+
+use crate::utils::parse_statuses;
+
+// built-in probes, matched anywhere in the lowercased uri (query included).
+// deliberately conservative: only paths that no legitimate client of ANY
+// common stack ever requests. more aggressive, setup-specific rules (e.g.
+// wp-login.php, /actuator, hosted admin consoles) belong in a --probelist file.
+const PROBE_PARTS: &[&str] = &[
+    // secrets & dotfiles
+    "/.env",  // /.env, /.env.bak, /backend/.env, ...
+    "%2eenv", // url-encoded .env probes
+    "/.git",  // /.git/config, /.git/HEAD, /.gitconfig
+    "/.svn",
+    "/.hg/",
+    "/.aws", // /.aws/credentials
+    "/.ssh",
+    "/.docker",
+    // cloud credentials & config dumps
+    "credentials.json", // /google-credentials.json, /application_default_credentials.json
+    "-key.json",        // /gcp-key.json, /firebase-key.json
+    "/keyfile.json",
+    "/sa.json",
+    "service-account.json",
+    "firebase-adminsdk.json",
+    "gcp-sa.json",
+    "/docker-compose.yml",
+    "/appsettings.json",
+    "application.yml",
+    "parameters.yml",
+    "/web.config",
+    "/settings.py",
+    "/wp-config.php", // never served, only probed - even on real wordpress sites
+    // php probes
+    "phpinfo", // /phpinfo.php, /admin/phpinfo.php, /?phpinfo=1
+    "phpmyadmin",
+    "adminer.php",
+    ".php.bak",
+    ".php.old",
+    ".php.save",
+    ".php~",
+    "eval-stdin.php", // phpunit RCE
+    "/vendor/phpunit",
+    // wordpress fingerprinting (legit wordpress traffic never touches this)
+    "wlwmanifest.xml",
+    // framework debug/env endpoints
+    "/actuator/env", // spring boot (bare /actuator would catch legit health checks)
+    "/_profiler",    // symfony
+    "/_ignition",    // laravel RCE
+    "/_environment", // cakephp
+    "laravel.log",
+    // server status
+    "server-status",
+    "server-info",
+    // traversal & IoT/router botnets
+    "/../",
+    "%2e%2e",
+    "/etc/passwd",
+    "/cgi-bin/",
+    "/boaform",
+    "/hnap1",
+    "/gponform",
+];
+
+// built-in exact matches on the query-stripped path: names too short or
+// generic to be safe as substrings (/i.php would match /api.php, etc..)
+const PROBE_EXACT: &[&str] = &[
+    "/test.php",
+    "/info.php",
+    "/pinfo.php",
+    "/pi.php",
+    "/i.php",
+    "/p.php",
+    "/php.php",
+    "/debug.php",
+    "/database.php",
+    "/config.php",
+    "/shell.php",
+    "/upload.php",
+    "/env",
+];
+
+pub enum MatchKind {
+    Exact,
+    Prefix,
+    Contains,
+}
+
+pub struct Probe {
+    pub path: String,
+    pub kind: MatchKind,
+    pub statuses: Option<Vec<u32>>,
+    pub allowance: Option<u8>,
+}
+
+pub struct ProbeList {
+    probes: Vec<Probe>,
+}
+
+impl ProbeList {
+    pub fn builtin() -> ProbeList {
+        let mut probes: Vec<Probe> = vec![];
+        for p in PROBE_PARTS {
+            probes.push(Probe {
+                path: p.to_string(),
+                kind: MatchKind::Contains,
+                statuses: None,
+                allowance: None,
+            });
+        }
+        for p in PROBE_EXACT {
+            probes.push(Probe {
+                path: p.to_string(),
+                kind: MatchKind::Exact,
+                statuses: None,
+                allowance: None,
+            });
+        }
+        ProbeList { probes }
+    }
+
+    pub fn load(path: &Path) -> Result<ProbeList> {
+        let raw = std::fs::read_to_string(path)
+            .with_context(|| format!("cant read probelist {:?}", path))?;
+        ProbeList::from_json(&raw).with_context(|| format!("cant parse probelist {:?}", path))
+    }
+
+    fn from_json(raw: &str) -> Result<ProbeList> {
+        let json: serde_json::Value = serde_json::from_str(raw)?;
+        let arr = json
+            .get("probes")
+            .and_then(|p| p.as_array())
+            .ok_or_else(|| anyhow!("missing `probes` array"))?;
+
+        let mut probes = vec![];
+        for (i, p) in arr.iter().enumerate() {
+            // entries carrying only a comment are section separators, skip them
+            if p.get("path").is_none() && p.get("comment").is_some() {
+                continue;
+            }
+
+            let path = p
+                .get("path")
+                .and_then(|v| v.as_str())
+                .filter(|s| !s.is_empty())
+                .ok_or_else(|| anyhow!("probe #{}: missing `path`", i))?
+                .to_ascii_lowercase();
+
+            let kind = match p
+                .get("match")
+                .and_then(|v| v.as_str())
+                .unwrap_or("contains")
+            {
+                "exact" => MatchKind::Exact,
+                "prefix" => MatchKind::Prefix,
+                "contains" => MatchKind::Contains,
+                other => bail!("probe #{}: unknown match kind `{}`", i, other),
+            };
+
+            let statuses = match p.get("status") {
+                None => None,
+                Some(v) => {
+                    let s = v
+                        .as_str()
+                        .ok_or_else(|| anyhow!("probe #{}: `status` must be a string", i))?;
+                    Some(parse_statuses(s).with_context(|| format!("probe #{}", i))?)
+                }
+            };
+
+            let allowance = match p.get("allowance") {
+                None => None,
+                Some(v) => {
+                    let a = v
+                        .as_u64()
+                        .filter(|a| (1..=255).contains(a))
+                        .ok_or_else(|| anyhow!("probe #{}: `allowance` must be 1-255", i))?;
+                    Some(a as u8)
+                }
+            };
+
+            probes.push(Probe {
+                path,
+                kind,
+                statuses,
+                allowance,
+            });
+        }
+
+        Ok(ProbeList { probes })
+    }
+
+    pub fn len(&self) -> usize {
+        self.probes.len()
+    }
+
+    // returns the first probe matching this uri + response status
+    pub fn check(&self, uri: &str, status: u32) -> Option<&Probe> {
+        let uri = uri.to_ascii_lowercase();
+        let path = uri.split(['?', '#']).next().unwrap_or(&uri);
+
+        self.probes.iter().find(|p| {
+            let path_hit = match p.kind {
+                MatchKind::Exact => path == p.path,
+                MatchKind::Prefix => path.starts_with(&p.path),
+                MatchKind::Contains => uri.contains(&p.path),
+            };
+            path_hit && p.statuses.as_ref().is_none_or(|s| s.contains(&status))
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn builtin_positive() {
+        // all straight out of real scanner traffic
+        let vectors = [
+            "/.env",
+            "/.env.backup",
+            "/config/.env",
+            "/%2E%2E%2f%2Eenv",
+            "/.git/config",
+            "/.aws/credentials",
+            "/phpinfo.php",
+            "/admin/phpinfo.php",
+            "/?phpinfo=1",
+            "/wp-includes/wlwmanifest.xml",
+            "/blog/wp-includes/wlwmanifest.xml",
+            "/wp-config.php.bak",
+            "/config.php",
+            "/google-credentials.json",
+            "/gcp-key.json",
+            "/docker-compose.yml",
+            "/_profiler/phpinfo",
+            "/_ignition/health-check",
+            "/storage/logs/laravel.log",
+            "/actuator/env",
+            "/webroot/index.php/_environment",
+            "/test.php",
+            "/i.php",
+            "/env",
+            "/phpMyAdmin/index.php",
+            "/cgi-bin/luci",
+            "/../.env",
+            "/server-status.php",
+        ];
+
+        let pl = ProbeList::builtin();
+        vectors.iter().for_each(|e| {
+            assert!(pl.check(e, 200).is_some(), "should flag {}", e);
+        })
+    }
+
+    #[test]
+    fn builtin_negative() {
+        // real user traffic from the same logs
+        let vectors = [
+            "/",
+            "/db",
+            "/ui/",
+            "/ui/static/js/utils.js",
+            "/api/auth",
+            "/api/users/1",
+            "/api/channels/5/messages?after=1620&limit=200",
+            "/app.js",
+            "/config.json",
+            "/favicon.ico",
+            "/robots.txt",
+            "/.well-known/acme-challenge/token123",
+            "/api.php",
+            "/information",
+            "/environment-report",
+            // legit traffic on stacks the tight default must not break:
+            "/wp-login.php",                  // wordpress logins
+            "/xmlrpc.php",                    // wordpress apps/jetpack
+            "/?rest_route=/wp/v2/posts",      // wordpress REST
+            "/actuator/health",               // spring boot health checks
+            "/autodiscover/autodiscover.xml", // outlook probes any domain
+            "/solr/admin/ping",               // hosted consoles
+            "/composer.json",
+        ];
+
+        let pl = ProbeList::builtin();
+        vectors.iter().for_each(|e| {
+            assert!(pl.check(e, 200).is_none(), "should not flag {}", e);
+        })
+    }
+
+    #[test]
+    fn probelist_file() {
+        let raw = r#"{
+            "comment": "generated probelist",
+            "probes": [
+                { "comment": "--- section separator, ignored ---" },
+                { "path": "/.env" },
+                { "path": "/test.php", "match": "exact" },
+                { "path": "/api/auth", "match": "prefix", "status": "401,4xx", "allowance": 10, "comment": "stuffers" }
+            ]
+        }"#;
+        let pl = ProbeList::from_json(raw).unwrap();
+        assert_eq!(pl.len(), 3);
+
+        // contains, any status
+        assert!(pl.check("/backend/.env", 200).is_some());
+        // exact, query-stripped
+        assert!(pl.check("/test.php?x=1", 200).is_some());
+        assert!(pl.check("/xtest.php", 200).is_none());
+        // prefix + status filter + allowance override
+        let hit = pl.check("/api/auth/me", 401).unwrap();
+        assert_eq!(hit.allowance, Some(10));
+        assert!(pl.check("/api/auth/me", 200).is_none());
+        assert!(pl.check("/api/authless", 401).is_some()); // prefix is a plain str prefix
+    }
+
+    #[test]
+    fn probelist_rejects_garbage() {
+        assert!(ProbeList::from_json("{}").is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": ""}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "match": "regex"}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "status": 401}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "status": "9xx"}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "allowance": 300}]}"#).is_err());
+        assert!(ProbeList::from_json(r#"{"probes": [{"path": "/x", "allowance": 0}]}"#).is_err());
+    }
+}

src/utils.rs

@@ -9,7 +9,8 @@ use std::{
 #[derive(Debug)]
 pub enum ParsingStatus {
     OkEntry,
-    BadEntry(IpAddr),
+    // offending ip, plus an optional per-probe allowance override
+    BadEntry(IpAddr, Option<u8>),
 }
 
 pub fn get_epoch() -> u64 {
@@ -118,6 +119,10 @@ pub struct Args {
     #[clap(long, value_parser = resolve_path)]
     pub caddy_logpath: Vec<PathBuf>,
 
+    /// path of a probelist JSON file, replaces the built-in probe list (see README)
+    #[clap(long, value_parser = resolve_path)]
+    pub probelist: Option<PathBuf>,
+
     /// generic parser log file path, can be repeated
     #[clap(long, value_parser = resolve_path, requires_all = ["generic_ip", "generic_match"])]
     pub generic_logpath: Vec<PathBuf>,
@@ -134,9 +139,11 @@ pub struct Args {
     #[clap(long, requires = "generic_logpath")]
     pub generic_negative: Option<String>,
 
-    /// invalid http statuses (for CLF and Caddy logs). Coma separated list, accepts ranges with XX.
+    /// also flag these http statuses (for CLF and Caddy logs), on top of the built-in
-    #[clap(long, default_value = "400,401,402,403")]
+    /// scanner-path detection. Coma separated list, accepts ranges with XX, e.g. "403,5xx".
-    pub invalid_http_statuses: String,
+    /// Careful: many apps serve 4xx statuses to legitimate clients.
+    #[clap(long)]
+    pub invalid_http_statuses: Option<String>,
 }
 
 #[cfg(test)]