// prometheus

// Metrics output

prometheus.remote_write "prometheus" {
    endpoint {
        url = "https://{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}/api/v1/write"
    }
}

// System metrics relabel

prometheus.relabel "system_metrics" {
    forward_to = [prometheus.remote_write.prometheus.receiver]

    rule {
        target_label = "instance"
        replacement  = "{{ node['name'] }}"
    }
    rule {
        target_label = "job"
        replacement  = "system"
    }
    rule {
        source_labels = ["name"]
        regex         = "(.+)\\.service"
        target_label  = "name"
        replacement   = "$1"
    }
    rule {
        source_labels = ["name"]
        regex         = "(.+)\\.timer"
        target_label  = "name"
        replacement   = "$1.timer"
    }
    rule {
        source_labels = ["name"]
        regex         = "(.+)\\.socket"
        target_label  = "name"
        replacement   = "$1.socket"
    }
}

// Generic service metrics relabel

prometheus.relabel "service_metrics" {
    forward_to = [prometheus.remote_write.prometheus.receiver]

    rule {
        target_label = "instance"
        replacement  = "{{ node['name'] }}"
    }
    rule {
        source_labels = ["job"]
        regex         = "prometheus\\.scrape\\.(.+)"
        target_label  = "job"
        replacement   = "$1"
    }
    rule {
        source_labels = ["job"]
        regex         = "integrations\\/(.+)"
        target_label  = "job"
        replacement   = "$1"
    }
    rule {
        source_labels = ["name", "job"]
        separator     = ";"
        regex         = ";(.+)"
        target_label  = "name"
        replacement   = "$1"
    }
}

// Metrics input

// System metrics input

prometheus.exporter.unix "system" {
    enable_collectors = [
        "systemd",
        "cgroup",
        "processes",
        "cpu",
        "meminfo",
        "filesystem",
        "netdev",
    ]
    filesystem {
        mount_points_exclude = "^/(boot|sys|proc|dev|run|mnt|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
        fs_types_exclude     = "^(tmpfs|devtmpfs|devfs|iso9660|overlay|aufs|squashfs)$"
    }
}

prometheus.scrape "system" {
    targets    = prometheus.exporter.unix.system.targets
    forward_to = [prometheus.relabel.system_metrics.receiver]
}
{% if node['name'] == "fw" %}

// CrowdSec metrics input
prometheus.scrape "crowdsec" {
    targets = [
        {
            "__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:6060",
            "job"         = "crowdsec",
            "name"        = "crowdsec",
        },
        {
            "__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:60601",
            "job"         = "crowdsec",
            "name"        = "crowdsec-bouncer",
        },
    ]

    honor_labels = true
    forward_to   = [prometheus.relabel.service_metrics.receiver]
}
{% endif %}
{% if node['name'] == "infra" %}

// PostgreSQL metrics
prometheus.exporter.postgres "postgresql" {
    data_source_names = [
        "postgres://alloy@{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}/postgres?sslmode=verify-full",
    ]
}

prometheus.scrape "postgresql" {
    targets    = prometheus.exporter.postgres.postgresql.targets
    forward_to = [prometheus.relabel.service_metrics.receiver]
}

// x509-exporter metrics
prometheus.scrape "x509" {
    targets = [
        {
            "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:{{ services['x509-exporter']['ports']['http'] }}",
        },
    ]

    forward_to = [prometheus.relabel.service_metrics.receiver]
}
{% endif %}
{% if node['name'] in ["infra", "auth", "app"] %}

// Caddy metrics
prometheus.scrape "caddy" {
    targets = [
        {
            "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:443",
        },
    ]
    scheme       = "https"
    metrics_path = "/metrics"
    forward_to   = [prometheus.relabel.service_metrics.receiver]
}
{% endif %}

// Loki

// Logs output

loki.write "loki" {
    endpoint {
        url       = "https://{{ services['loki']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['loki']['ports']['https'] }}/loki/api/v1/push"
        tenant_id = "{{ domain['internal'] }}"
    }
}

// Journald logs relabel

loki.relabel "journal_logs" {
    forward_to = []
    rule {
        target_label = "instance"
        replacement  = "{{ node['name'] }}"
    }
    rule {
        target_label = "job"
        replacement  = "systemd-journal"
    }
    rule {
        source_labels = ["__journal__systemd_unit"]
        regex         = "(.+)\\.service"
        target_label  = "job"
        replacement   = "$1"
    }
    rule {
        source_labels = ["__journal__systemd_unit"]
        regex         = "(.+)\\.timer"
        target_label  = "job"
        replacement   = "$1.timer"
    }
    rule {
        source_labels = ["__journal__systemd_unit"]
        regex         = "(.+)\\.socket"
        target_label  = "job"
        replacement   = "$1.socket"
    }
    rule {
        source_labels = ["__journal_syslog_identifier"]
        regex         = "(.+)"
        target_label  = "job"
        replacement   = "$1"
    }
    rule {
        source_labels = ["job"]
        regex         = "user@\\d+"
        target_label  = "job"
        replacement   = "systemd-journal"
    }
    rule {
        source_labels = ["__journal__systemd_user_unit"]
        regex         = "(.+)\\.service"
        target_label  = "job"
        replacement   = "$1"
    }
    rule {
        source_labels = ["__journal__systemd_user_unit"]
        regex         = "(.+)\\.timer"
        target_label  = "job"
        replacement   = "$1.timer"
    }
    rule {
        source_labels = ["__journal__systemd_user_unit"]
        regex         = "(.+)\\.socket"
        target_label  = "job"
        replacement   = "$1.socket"
    }
    rule {
        source_labels = ["job"]
        regex         = "(.+)"
        target_label  = "service_name"
        replacement   = "$1"
    }
    rule {
        source_labels = ["__journal_priority_keyword"]
        regex         = "(.+)"
        target_label  = "journal_level"
        replacement   = "$1"
    }
    rule {
        target_label = "level"
        replacement  = "info"
    }
    rule {
        source_labels = ["__journal_priority_keyword"]
        regex         = "debug"
        target_label  = "level"
        replacement   = "debug"
    }
    rule {
        source_labels = ["__journal_priority_keyword"]
        regex         = "info|notice"
        target_label  = "level"
        replacement   = "info"
    }
    rule {
        source_labels = ["__journal_priority_keyword"]
        regex         = "warning"
        target_label  = "level"
        replacement   = "warn"
    }
    rule {
        source_labels = ["__journal_priority_keyword"]
        regex         = "err"
        target_label  = "level"
        replacement   = "error"
    }
    rule {
        source_labels = ["__journal_priority_keyword"]
        regex         = "crit|alert|emerg"
        target_label  = "level"
        replacement   = "fatal"
    }
}

// Journald logs parser

loki.process "journal_parser" {
    forward_to = [loki.write.loki.receiver]
    stage.match {
        selector = "{level!=\"fatal\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(fatal|critical|panic|uncaught|unhandled|assertion failed|out of memory|\\bboom\\b|killed process)\""
        stage.static_labels {
            values = {
                level = "fatal",
            }
        }
    }
    stage.match {
        selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\berror\\b|\\berr\\b|\\bfailed\\b|\\bfailure\\b|\\bexception\\b|traceback|segfault|exit-code|container died|permission denied|access denied|connection refused|connection reset|timed out|timeout exceeded|no such file|authentication failed|sqlstate|duplicate key|relation .* does not exist|column .* does not exist|database .* does not exist|relation .* already exists|column .* already exists|x509|unknown authority|certificate .*expired|tls handshake|bad gateway|upstream .*failed|upstream .*error)\""
        stage.static_labels {
            values = {
                level = "error",
            }
        }
    }
    stage.match {
        selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(Main process exited|Failed with result|start request repeated too quickly|Dependency failed|Unit .* entered failed state)\""

        stage.static_labels {
            values = {
                level = "error",
            }
        }
    }
    stage.match {
        selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\bwarn\\b|warning|deprecated|retrying|rate.?limit|too many requests|slow query|slow request)\""

        stage.static_labels {
            values = {
                level = "warn",
            }
        }
    }
    stage.match {
        selector = "{job=~\"ssh|sshd\"}"

        stage.regex {
            expression = "Accepted \\w+ for (?P<user>[^ ]+) from (?P<ip>[0-9a-fA-F:.]+)"
        }

        stage.labels {
            values = {
                user = "",
            }
        }
    }
}
{% if node['name'] == "fw" %}

// Suricata logs relabel

loki.relabel "suricata_logs" {
    forward_to = [loki.process.suricata_parser.receiver]
    
    rule {
        target_label = "instance"
        replacement  = "{{ node['name'] }}"
    }
    rule {
        target_label = "job"
        replacement  = "suricata_eve"
    }
    rule {
        target_label = "service_name"
        replacement  = "suricata_eve"
    }
    rule {
        target_label = "level"
        replacement  = "info"
    }
}

// Suricata logs parser

loki.process "suricata_parser" {
    forward_to = [loki.write.loki.receiver]

    stage.json {
        expressions = {
            event_type = "event_type",
        }
    }
    stage.labels {
        values = {
            event_type = "",
        }
    }
    stage.match {
        selector = "{event_type=\"alert\"}"
        stage.json {
            expressions = {
                severity = "alert.severity",
            }
        }
        stage.labels {
            values = {
                severity = "",
            }
        }
        stage.static_labels {
            values = {
                level = "warn",
            }
        }
    }
    stage.match {
        selector = "{event_type=\"alert\", severity=~\"1|2\"}"
        stage.static_labels {
            values = {
                level = "error",
            }
        }
    }
}

// Suricata eve.json input

local.file_match "suricata_logs" {
    path_targets = [
        { "__path__" = "/var/log/suricata/eve.json" },
    ]
}

loki.source.file "suricata" {
    targets    = local.file_match.suricata_logs.targets
    forward_to = [loki.relabel.suricata_logs.receiver]
}
{% endif %}
{% if node['name'] == "auth" %}

// Caddy access logs relabel

loki.relabel "caddy_logs" {
    forward_to = [loki.process.caddy_parser.receiver]

    rule {
        target_label = "instance"
        replacement  = "{{ node['name'] }}"
    }

    rule {
        target_label = "job"
        replacement  = "caddy_access"
    }

    rule {
        target_label = "service_name"
        replacement  = "caddy_access"
    }

    rule {
        target_label = "level"
        replacement  = "info"
    }
}

// Caddy access logs parser

loki.process "caddy_parser" {
    forward_to = [loki.write.loki.receiver]

    stage.json {
        expressions = {
            status    = "status",
            duration  = "duration",
            method    = "request.method",
            host      = "request.host",
            uri       = "request.uri",
            remote_ip = "request.remote_ip",
        }
    }
    stage.labels {
        values = {
            status = "",
            method = "",
            host   = "",
        }
    }
    stage.structured_metadata {
        values = {
            uri       = "",
            remote_ip = "",
            duration  = "",
        }
    }
    stage.match {
        selector = "{status=~\"401|403|429\"}"

        stage.static_labels {
            values = {
                level = "warn",
            }
        }
    }
    stage.match {
        selector = "{status=~\"5..\"}"

        stage.static_labels {
            values = {
                level = "error",
            }
        }
    }
}

// Caddy access log input

local.file_match "caddy_logs" {
    path_targets = [
        { "__path__" = "/var/log/caddy/access.log" },
    ]
}

loki.source.file "caddy" {
    targets    = local.file_match.caddy_logs.targets
    forward_to = [loki.relabel.caddy_logs.receiver]
}
{% endif %}

// Journald logs input

loki.source.journal "systemd" {
    forward_to    = [loki.process.journal_parser.receiver]
    relabel_rules = loki.relabel.journal_logs.rules
}