json & generic parser support

also added verbose flag and configurable HTTP statuses

src/clf.rs

@@ -5,13 +5,12 @@ use regex::Regex;
 use std::{net::IpAddr, str::FromStr};
 
 lazy_static! {
-    static ref BAD_STATUSES: [u32; 2] = [401, 429];
     static ref RE_IP: Regex = Regex::new(r"^(\S+)\s").unwrap();
     static ref RE_STATUS: Regex = Regex::new(r"(\d+)\s(\w+)$").unwrap();
 }
 
 #[allow(clippy::bind_instead_of_map)]
-pub fn parse(line: &str) -> Result<ParsingStatus> {
+pub fn parse(line: &str, valid_statuses: &[u32]) -> Result<ParsingStatus> {
     let ip = RE_IP
         .captures(line)
         .and_then(|c| c.get(1))
@@ -26,9 +25,8 @@ pub fn parse(line: &str) -> Result<ParsingStatus> {
         .and_then(|e| e.parse::<u32>().ok())
         .ok_or_else(|| anyhow!("cant parse clf line - status"))?;
 
-    let is_bad_status = BAD_STATUSES.iter().any(|s| s == &status);
-
-    if is_bad_status {
+    let is_good_status = valid_statuses.iter().any(|s| s == &status);
+    if !is_good_status {
         return Ok(ParsingStatus::BadEntry(ip));
     }
 
@@ -47,7 +45,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e).unwrap();
+            let ret = parse(*e, &vec![200, 404]).unwrap();
             match ret {
                 ParsingStatus::BadEntry(_) => {}
                 _ => panic!("bad parsing"),
@@ -63,7 +61,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e).unwrap();
+            let ret = parse(*e, &vec![200, 404]).unwrap();
             match ret {
                 ParsingStatus::OkEntry => {}
                 _ => panic!("bad parsing"),
@@ -79,7 +77,7 @@ mod tests {
         ];
 
         vectors.iter().for_each(|e| {
-            let ret = parse(*e);
+            let ret = parse(*e, &vec![200, 404]);
             assert!(ret.is_err());
         })
     }

src/generic.rs

@@ -0,0 +1,83 @@
+use crate::utils::ParsingStatus;
+use anyhow::*;
+use regex::Regex;
+use std::{net::IpAddr, str::FromStr};
+
+#[allow(clippy::bind_instead_of_map)]
+pub fn parse(
+    line: &str,
+    ip: Option<&Regex>,
+    positive: Option<&String>,
+    negative: Option<&String>,
+) -> Result<ParsingStatus> {
+    if let Some(ne) = negative {
+        if line.contains(ne) {
+            return Ok(ParsingStatus::OkEntry);
+        }
+    }
+
+    if let Some(po) = positive {
+        if !line.contains(po) {
+            return Ok(ParsingStatus::OkEntry);
+        }
+    }
+
+    let ip = ip.unwrap().captures(line);
+
+    let ip = ip
+        .and_then(|c| c.get(1))
+        .and_then(|g| Some(g.as_str()))
+        .and_then(|e| IpAddr::from_str(e).ok())
+        .ok_or_else(|| anyhow!("cant parse clf line - ip"))?;
+
+    Ok(ParsingStatus::BadEntry(ip))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FAILED: &str =
+        "Sep 26 06:25:19 livecompute sshd[23246]: Failed password for root from 179.124.36.195 port 41883 ssh2";
+
+    const SUCCESS: &str =
+        "Sep 26 06:25:19 livecompute sshd[23246]: Successful login for root from 179.124.36.195 port 41883 ssh2";
+
+    // generic log positive regex - what's that's flagged by this is considered bad, the rest is good
+    #[test]
+    fn positive() {
+        let positive = "Failed password".to_string();
+        let ip = Regex::new(r"from ([0-9a-fA-F:.]+) port").unwrap();
+
+        let ret = parse(FAILED, Some(&ip), Some(&positive), None).unwrap();
+        match ret {
+            ParsingStatus::BadEntry(_) => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ret = parse(SUCCESS, Some(&ip), Some(&positive), None).unwrap();
+        match ret {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+
+    // generic log negative regex - what's that's flagged by this is considered good, the rest is bad
+    #[test]
+    fn negative() {
+        let negative = "Successful login".to_string();
+        let ip = Regex::new(r"from ([0-9a-fA-F:.]+) port").unwrap();
+
+        let ret = parse(SUCCESS, Some(&ip), None, Some(&negative)).unwrap();
+        match ret {
+            ParsingStatus::OkEntry => {}
+            _ => panic!("bad parsing"),
+        }
+
+        let ret = parse(FAILED, Some(&ip), None, Some(&negative)).unwrap();
+        match ret {
+            ParsingStatus::BadEntry(_) => {}
+            _ => panic!("bad parsing"),
+        }
+    }
+}

src/jail.rs

@@ -48,7 +48,7 @@ impl Jail {
         })
     }
 
-    pub fn sentence(&self, ip: IpAddr, target: &str) -> Result<()> {
+    pub fn sentence(&self, ip: IpAddr) -> Result<bool> {
         let now = get_epoch();
 
         let should_ban = {
@@ -57,7 +57,8 @@ impl Jail {
             let (hits, _ts) = *locked_map
                 .entry(ip)
                 .and_modify(|(hits, ts)| {
-                    if *ts + self.jailtime as u64 > now { // reset if we have a hit, but past the defined jailtime
+                    if now > *ts + self.jailtime as u64 {
+                        // reset if we have a hit, but past the defined jailtime
                         *ts = now;
                         *hits = 1;
                     } else {
@@ -65,21 +66,20 @@ impl Jail {
                     }
                 })
                 .or_insert((1, now));
-
             if hits < self.allowance {
                 false
             } else {
-                locked_map.remove_entry(&ip); // preserve space
+                locked_map.remove_entry(&ip);
                 true
             }
         };
 
         if should_ban {
-            log!("{} jailtime for: {}", target, ip);
             let cmd = format!("ipset add -exist {} {}", self.name, ip);
             exec(&cmd, "")?;
+            return Ok(true);
         }
 
-        Ok(())
+        Ok(false)
     }
 }

src/json.rs

@@ -0,0 +1,75 @@
+use crate::utils::ParsingStatus;
+use anyhow::*;
+use std::{net::IpAddr, str::FromStr};
+
+pub fn parse(line: &str, valid_statuses: &[u32]) -> Result<ParsingStatus> {
+    let json: serde_json::Value = serde_json::from_str(line)?;
+
+    let remote_ip = json
+        .get("request")
+        .and_then(|r| r.get("remote_ip"))
+        .and_then(|r| r.as_str())
+        .and_then(|r| IpAddr::from_str(r).ok())
+        .ok_or_else(|| anyhow!("cant parse json line - remote_ip"))?;
+
+    let status = json
+        .get("status")
+        .and_then(|r| r.as_u64())
+        .ok_or_else(|| anyhow!("cant parse json line - status"))?;
+
+    let is_good_status = valid_statuses.iter().any(|s| s == &(status as u32));
+    if !is_good_status {
+        return Ok(ParsingStatus::BadEntry(remote_ip));
+    }
+
+    Ok(ParsingStatus::OkEntry)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn positive() {
+        let vectors = [
+            r#"{"level":"info","ts":1738064403.2176833,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"46884","client_ip":"127.0.0.1","proto":"HTTP/1.1","method":"GET","host":"127.0.0.1:8009","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"Accept-Language":["en-US,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br, zstd"],"Connection":["keep-alive"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Site":["cross-site"],"Priority":["u=0, i"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]}},"bytes_read":0,"user_id":"","duration":0.002135063,"size":35133,"status":429,"resp_headers":{"Vary":["Accept, Accept-Encoding"],"Last-Modified":["Tue, 28 Jan 2025 12:40:02 GMT"],"Content-Type":["text/html; charset=utf-8"],"Server":["Caddy"]}}"#,
+            r#"{"level":"info","ts":1738064403.2176833,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"46884","client_ip":"127.0.0.1","proto":"HTTP/1.1","method":"GET","host":"127.0.0.1:8009","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"Accept-Language":["en-US,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br, zstd"],"Connection":["keep-alive"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Site":["cross-site"],"Priority":["u=0, i"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]}},"bytes_read":0,"user_id":"","duration":0.002135063,"size":35133,"status":401,"resp_headers":{"Vary":["Accept, Accept-Encoding"],"Last-Modified":["Tue, 28 Jan 2025 12:40:02 GMT"],"Content-Type":["text/html; charset=utf-8"],"Server":["Caddy"]}}"#,
+        ];
+
+        vectors.iter().for_each(|e| {
+            let ret = parse(*e, &vec![200, 404]).unwrap();
+            match ret {
+                ParsingStatus::BadEntry(_) => {}
+                _ => panic!("bad parsing"),
+            }
+        })
+    }
+
+    #[test]
+    fn negative() {
+        let vectors = [
+            r#"{"level":"info","ts":1738064403.2176833,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"46884","client_ip":"127.0.0.1","proto":"HTTP/1.1","method":"GET","host":"127.0.0.1:8009","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"Accept-Language":["en-US,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br, zstd"],"Connection":["keep-alive"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Site":["cross-site"],"Priority":["u=0, i"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]}},"bytes_read":0,"user_id":"","duration":0.002135063,"size":35133,"status":200,"resp_headers":{"Vary":["Accept, Accept-Encoding"],"Last-Modified":["Tue, 28 Jan 2025 12:40:02 GMT"],"Content-Type":["text/html; charset=utf-8"],"Server":["Caddy"]}}"#,
+            r#"{"level":"info","ts":1738064403.2176833,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"46884","client_ip":"127.0.0.1","proto":"HTTP/1.1","method":"GET","host":"127.0.0.1:8009","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Mode":["navigate"],"Accept-Language":["en-US,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br, zstd"],"Connection":["keep-alive"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Site":["cross-site"],"Priority":["u=0, i"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]}},"bytes_read":0,"user_id":"","duration":0.002135063,"size":35133,"status":404,"resp_headers":{"Vary":["Accept, Accept-Encoding"],"Last-Modified":["Tue, 28 Jan 2025 12:40:02 GMT"],"Content-Type":["text/html; charset=utf-8"],"Server":["Caddy"]}}"#,
+        ];
+
+        vectors.iter().for_each(|e| {
+            let ret = parse(*e, &vec![200, 404]).unwrap();
+            match ret {
+                ParsingStatus::OkEntry => {}
+                _ => panic!("bad parsing"),
+            }
+        })
+    }
+
+    #[test]
+    fn malformed() {
+        let vectors = [
+            r#"{"level":"info","ts":1738064403.2176833,"logger":"http.log.access.log0","msg":"handled request","requeto":"HTTP/1 x86_64; rv:133.0)"],"Server":["Caddy"]}}"#,
+        ];
+
+        vectors.iter().for_each(|e| {
+            let ret = parse(*e, &vec![200, 404]);
+            assert!(ret.is_err());
+        })
+    }
+}

src/main.rs

@@ -1,10 +1,12 @@
-use std::path::PathBuf;
 use std::result::Result::Ok;
 
 use anyhow::*;
+use clap::Parser;
 use linemux::{Line, MuxedLines};
 
 mod clf;
+mod generic;
+mod json;
 mod sshd;
 mod utils;
 
@@ -13,48 +15,91 @@ use crate::jail::Jail;
 use crate::utils::*;
 
 async fn run() -> Result<()> {
-    let args = utils::cli().get_matches();
+    let args = utils::Args::parse();
     let mut ml = MuxedLines::new()?;
 
-    // jail
-    let jailtime_str = args.value_of("jailtime").unwrap_or("");
-    let jailtime = jailtime_str.parse().context("parsing jailtime")?;
-
-    let allowance_str = args.value_of("allowance").unwrap_or("");
-    let allowance = allowance_str.parse().context("parsing allowance")?;
-
-    let jail = Jail::new(allowance, jailtime)?;
+    // generic parser
+    let generic_path = args.generic_logpath.as_ref();
+    let generic_ip_re = args.generic_ip.as_ref();
+    let generic_positive = args.generic_positive.as_ref();
+    let generic_negative = args.generic_negative.as_ref();
+    if args.generic_ip.is_some()
+        || args.generic_logpath.is_some()
+        || args.generic_positive.is_some()
+        || args.generic_negative.is_some()
+    {
+        if args.generic_ip.is_none() || args.generic_logpath.is_none() {
+            bail!("generic parser needs both ip regex and log file path");
+        }
+        if !(args.generic_positive.is_some() ^ args.generic_negative.is_some()) {
+            bail!("generic parser requires either a positive or a negative regex");
+        }
+        if let Some(p) = generic_path.as_ref() {
+            ml.add_file(&p).await?;
+            log!("starting with generic parsing at {:?}", &p);
+        }
+    }
 
     // sshd
-    let mut path_sshd: PathBuf = args.value_of("sshd_logpath").unwrap_or("").into();
-    if path_sshd.exists() {
-        path_sshd = std::fs::canonicalize(path_sshd)?;
-        ml.add_file(&path_sshd).await?;
-        log!("starting with sshd parsing at {:?}", &path_sshd);
+    let sshd_logpath = args.sshd_logpath.as_ref();
+    if let Some(p) = sshd_logpath {
+        ml.add_file(&p).await?;
+        log!("starting with sshd parsing at {:?}", &p);
     }
 
     // common log format
-    let mut path_clf: PathBuf = args.value_of("clf_logpath").unwrap_or("").into();
-    if path_clf.exists() {
-        path_clf = std::fs::canonicalize(path_clf)?;
-        ml.add_file(&path_clf).await?;
-        log!("starting with clf parsing at {:?}", &path_clf);
+    let clf_logpath = args.clf_logpath.as_ref();
+    if let Some(p) = clf_logpath {
+        ml.add_file(&p).await?;
+        log!("starting with clf parsing at {:?}", &p);
     }
 
+    // json
+    let json_logpath = args.json_logpath.as_ref();
+    if let Some(p) = json_logpath {
+        ml.add_file(&p).await?;
+        log!("starting with json parsing at {:?}", &p);
+    }
+
+    if json_logpath.is_none() && clf_logpath.is_none() && sshd_logpath.is_none() {
+        bail!("no log files to parse, see --help");
+    }
+
+    // HTTP statuses
+    let ok_statuses = args.valid_http_statuses.clone();
+    let ok_statuses_ref = ok_statuses.as_ref();
+
+    // jail
+    let jail = Jail::new(args.allowance, args.jailtime)?;
+
     let assess_line = |line: Line| {
         let payload = line.line();
-        let path = line.source();
+        let path_buf = Some(line.source().to_path_buf());
+        let path = path_buf.as_ref();
 
-        let (target, ret) = if path == path_sshd {
+        let (target, ret) = if path == sshd_logpath {
             ("sshd", sshd::parse(payload)?)
-        } else if path == path_clf {
-            ("clf", clf::parse(payload)?)
+        } else if path == clf_logpath {
+            ("clf", clf::parse(payload, ok_statuses_ref)?)
+        } else if path == json_logpath {
+            ("json", json::parse(payload, ok_statuses_ref)?)
+        } else if path == generic_path {
+            (
+                "generic",
+                generic::parse(payload, generic_ip_re, generic_positive, generic_negative)?,
+            )
         } else {
-            bail!("file {:?} unknown", path)
+            bail!("file {:?} unknown ?", path)
         };
 
         if let ParsingStatus::BadEntry(ip) = ret {
-            jail.sentence(ip, target)?;
+            if args.verbose {
+                log!("{} logged offence for {}", target, ip);
+            }
+            let banned = jail.sentence(ip)?;
+            if banned {
+                log!("{} jailtime for {}", target, ip);
+            }
         }
 
         Ok(())
@@ -73,6 +118,5 @@ async fn run() -> Result<()> {
 async fn main() -> Result<()> {
     run().await?;
     eprintln!("\n");
-    let _ = utils::cli().print_help();
     Ok(())
 }

src/utils.rs

@@ -1,5 +1,10 @@
-use clap::{App, Arg};
-use std::net::IpAddr;
+use anyhow::{anyhow, Context, Result};
+use clap::Parser;
+use regex::Regex;
+use std::{
+    net::IpAddr,
+    path::{Path, PathBuf},
+};
 
 #[derive(Debug)]
 pub enum ParsingStatus {
@@ -15,57 +20,97 @@ pub fn get_epoch() -> u64 {
 macro_rules! log{
     ($first:expr) => {
         let ts = crate::utils::get_epoch();
-        eprintln!("{} ~ {}", ts, $first);
+        eprintln!("{} - {}", ts, $first);
     };
     ($first:expr, $($others:expr),+) => {
         let ts = crate::utils::get_epoch();
         let formatted = format!($first, $($others), *);
-        eprintln!("{} ~ {}", ts, formatted);
+        eprintln!("{} - {}", ts, formatted);
     };
 }
 
+pub fn resolve_path(a: &str) -> Result<PathBuf> {
+    let p = Path::new(a);
+    if !p.exists() {
+        return Err(anyhow!("path {:?} does not exist", p));
+    }
+    let p = std::fs::canonicalize(p)?;
+    Ok(p)
+}
+
+pub fn parse_regex(a: &str) -> Result<Regex> {
+    println!("a {:?}", a);
+    let r: Regex = Regex::new(a).context("invalid regexp for generic parser")?;
+    Ok(r)
+}
+
 pub(crate) use log;
 
-pub fn cli() -> App<'static, 'static> {
-    App::new("ban internets scanner fast 🍶")
-        .version(env!("CARGO_PKG_VERSION"))
-        .author("pierre dubouilh <pldubouilh@gmail.com>")
-        // .arg(Arg::with_name("prune")
-        //     .short("prune")
-        //     .help("prune current logfiles to prefill banlist")
-        //     .default_value("false")
-        //     .takes_value(true))
-        .arg(
-            Arg::with_name("jailtime")
-                .short("j")
-                .help("jail time (seconds)")
-                .default_value("21600") // 6 hours
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("allowance")
-                .short("a")
-                .help("how many offences allowed (max 255")
-                .default_value("5")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("sshd_logpath")
-                .short("sshd_logpath")
-                .help("path of sshd logfile (disable with empty path)")
-                .default_value("/var/log/auth.log")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("clf_logpath")
-                .short("clf_logpath")
-                .help("path of Common-Log-Format (Apache, etc..) logfile")
-                .default_value("")
-                .takes_value(true),
-        )
-    // .arg(Arg::with_name("clf_bad_http_codes")
-    //     .short("cb")
-    //     .help("bad http statuses for CLF")
-    //     .default_value([401, 429])
-    //     .takes_value(true))
+#[derive(Parser, Debug)]
+#[command(
+    name = "Blockfast",
+    author = "pierre dubouilh <pldubouilh@gmail.com>",
+    arg_required_else_help = true,
+    version,
+    long_about = None,
+    about = "
+Blockfast - block internets scanners fast 🍶
+Author: pierre dubouilh <pldubouilh@gmail.com>
+
+Blockfast reads logs from various sources and blocks the offending IPs using iptables and ipset.
+It supports logs from sshd, Common-Log-Format (Apache, etc..), JSON (Caddy) and a generic logs parser.
+
+Example:
+    # block invalid sshd attempts & invalid http statuses from caddy
+    ./blockfast -s=/var/log/auth.log -j=/caddy/logs
+
+    # generic log parser example with a positive text, and a regex to parse the offending IP.
+    ./blockfast --generic-logpath=/tmp/generictest --generic-positive='Failed password' --generic-ip='from ([0-9a-fA-F:.]+) port'",
+    verbatim_doc_comment,
+)]
+
+pub struct Args {
+    /// jail time (seconds)
+    #[clap(long, default_value = "21600")]
+    pub jailtime: u32,
+
+    /// how many offences allowed (max 255)
+    #[clap(long, default_value = "5")]
+    pub allowance: u8,
+
+    /// log all offences
+    #[clap(short, long)]
+    pub verbose: bool,
+
+    /// path of sshd logfile
+    #[clap(short, long, value_parser = resolve_path)]
+    pub sshd_logpath: Option<PathBuf>,
+
+    /// path of Common-Log-Format logfile (Apache, etc..)
+    #[clap(short, long, value_parser = resolve_path)]
+    pub clf_logpath: Option<PathBuf>,
+
+    /// path of JSON logfile (works with Caddy)
+    #[clap(short, long, value_parser = resolve_path)]
+    pub json_logpath: Option<PathBuf>,
+
+    /// generic parser log file path
+    #[clap(long, value_parser = resolve_path)]
+    pub generic_logpath: Option<PathBuf>,
+
+    /// generic parser ip regex
+    #[clap(long , value_parser = parse_regex)]
+    pub generic_ip: Option<Regex>,
+
+    /// generic parser positive - if a logline contains this, it is considered bad, the rest is good
+    #[clap(long)]
+    pub generic_positive: Option<String>,
+
+    /// generic parser negative - if a logline contains this, it is considered good, the rest is bad
+    #[clap(long)]
+    pub generic_negative: Option<String>,
+
+    /// valid http statuses (for CLF and JSON logs)
+    #[clap(long, default_values_t = [200,101])]
+    pub valid_http_statuses: Vec<u32>,
 }