refactor(observation): update alloy, grafana, prometheus

ansible/roles/infra/tasks/services/set_grafana.yaml

@@ -12,6 +12,7 @@
     - "grafana/etc"
     - "grafana/etc/provisioning"
     - "grafana/etc/provisioning/datasources"
+    - "grafana/etc/provisioning/dashboards"
     - "grafana/etc/dashboards"
     - "grafana/ssl"
   become: true
@@ -58,10 +59,29 @@
   notify: "notification_restart_grafana"
   no_log: true
 
-- name: Deploy provisioing file
+- name: Deploy provisioing files
   ansible.builtin.template:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/provisioning/datasources/datasources.yaml.j2"
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/provisioning/{{ item.file }}"
-    dest: "{{ node['home_path'] }}/containers/grafana/etc/provisioning/datasources/datasources.yaml"
+    dest: "{{ node['home_path'] }}/containers/grafana/etc/provisioning/{{ item.dest }}"
+    owner: "{{ services['grafana']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0400"
+  loop:
+    - name: "datasource"
+      file: "datasources/datasources.yaml.j2"
+      dest: "datasources/datasources.yaml"
+    - name: "dashboard"
+      file: "dashboards/dashboards.yaml"
+      dest: "dashboards/dashboards.yaml"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  notify: "notification_restart_grafana"
+
+- name: Deploy dashboard files
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/dashboards/"
+    dest: "{{ node['home_path'] }}/containers/grafana/etc/dashboards/"
     owner: "{{ services['grafana']['subuid'] }}"
     group: "{{ node['group'] }}"
     mode: "0400"

config/services/containers/infra/grafana/etc/provisioning/dashboards/dashboards.yaml

@@ -0,0 +1,11 @@
+# https://github.com/grafana/grafana/blob/main/conf/provisioning/dashboards/sample.yaml
+apiVersion: 1
+
+providers:
+ - name: 'default'
+   orgId: 1
+   folder: 'My Dashboards'
+   folderUid: ''
+   type: file
+   options:
+     path: /etc/grafana/dashboards

config/services/containers/infra/prometheus/etc/prometheus.yaml.j2

@@ -24,9 +24,9 @@ scrape_configs:
     scheme: "https"
     tls_config:
       ca_file: "/etc/ssl/prometheus/{{ root_cert_filename }}"
-      server_name: "{{ ['internal'] }}.{{ domain['internal'] }}"
+      server_name: "{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}"
     static_configs:
-      - targets: ["localhost:9090"]
+      - targets: ["{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}"]
        # The label name is added as a label `label_name=<label_value>` to any timeseries scraped from this config.
         labels:
           instance: "{{ node['name'] }}"

config/services/containers/infra/prometheus/etc/rules.yaml.j2

@@ -1,7 +1,7 @@
 groups:
   - name: node_exporters_heartbeat
     rules:
-{% for instance in ["vmm", "fw", "infra", "auth", "app"] %}
+{% for instance in (((groups['hypervisor'] | default([])) + (groups['vms'] | default([]))) | difference(['vmm_init']) | sort) %}
     - alert: {{ instance }}_node_exporter_down
       expr: |
         (present_over_time(up{instance="{{ instance }}"}[5m]) or on() vector(0)) == 0
@@ -25,14 +25,14 @@ groups:
           description: "postgresql node is down for 5 mins."
   - name: Certificate_expiry_check
     rules:
-{% for filename in ["root.crt", "intermediate.crt", "crowdsec.crt", "blocky.crt", "kopia.crt", "postgresql.crt", "ldap.crt", "prometheus.crt", "loki.crt"] %}
+{% for filename in ['root', 'intermediate'] + (services | dict2items | selectattr('value.dependency.cert', 'defined') | selectattr('value.dependency.cert', 'equalto', true) | map(attribute='key') | list) %}
       - alert: {{ filename | replace('.', '_') }}_is_expired_soon
         expr: |
-          max(x509_cert_not_after{filename="{{ filename }}"}) - time() < 2592000
+          max(x509_cert_not_after{filename="{{ filename }}.crt"}) - time() < 2592000
         for: 1d
         labels:
           severity: critical
         annotations:
-          summary: "{{ filename }} is expired in 30 days"
+          summary: "{{ filename }}.crt is expired in 30 days"
-          description: "{{ filename }} is expired in 30 days."
+          description: "{{ filename }}.crt is expired in 30 days."
 {% endfor %}

config/services/containers/infra/prometheus/prometheus.container.j2

@@ -13,6 +13,8 @@ Image=docker.io/prom/prometheus:{{ version['containers']['prometheus'] }}
 ContainerName=prometheus
 HostName=prometheus
 
+AddHost={{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:host-gateway
+
 PublishPort={{ services['prometheus']['ports']['https'] }}:9090/tcp
 
 Volume=%h/containers/prometheus/data:/prometheus:rw

config/services/systemd/common/alloy/config.alloy.j2

@@ -1,28 +1,25 @@
-// The "name" and "job"
+// prometheus
-// job > prometheus: which exporter / loki: which service
+
-// name > prometheus: which service
+// Metrics output
-// service_name > loki: which service
+
-// Metric
-//// Metric ouput
 prometheus.remote_write "prometheus" {
     endpoint {
         url = "https://{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}/api/v1/write"
     }
 }
 
-//// Metric relabel
+// System metrics relabel
-////// For node metrics
+
-prometheus.relabel "system_relabel" {
+prometheus.relabel "system_metrics" {
     forward_to = [prometheus.remote_write.prometheus.receiver]
+
     rule {
         target_label = "instance"
         replacement  = "{{ node['name'] }}"
     }
     rule {
-        source_labels = ["job"]
-        regex         = "integrations\\/(.+)"
         target_label = "job"
-        replacement = "$1"
+        replacement  = "system"
     }
     rule {
         source_labels = ["name"]
@@ -30,11 +27,25 @@ prometheus.relabel "system_relabel" {
         target_label  = "name"
         replacement   = "$1"
     }
+    rule {
+        source_labels = ["name"]
+        regex         = "(.+)\\.timer"
+        target_label  = "name"
+        replacement   = "$1.timer"
+    }
+    rule {
+        source_labels = ["name"]
+        regex         = "(.+)\\.socket"
+        target_label  = "name"
+        replacement   = "$1.socket"
+    }
 }
 
-////// For service metrics
+// Generic service metrics relabel
-prometheus.relabel "default_label" {
+
+prometheus.relabel "service_metrics" {
     forward_to = [prometheus.remote_write.prometheus.receiver]
+
     rule {
         target_label = "instance"
         replacement  = "{{ node['name'] }}"
@@ -51,108 +62,153 @@ prometheus.relabel "default_label" {
         target_label  = "job"
         replacement   = "$1"
     }
+    rule {
+        source_labels = ["name", "job"]
+        separator     = ";"
+        regex         = ";(.+)"
+        target_label  = "name"
+        replacement   = "$1"
+    }
 }
 
-//// Metric input
+// Metrics input
-////// For node metrics
+
+// System metrics input
+
 prometheus.exporter.unix "system" {
-    enable_collectors = ["systemd", "cgroup", "processes", "cpu", "meminfo", "filesystem", "netdev"]
+    enable_collectors = [
+        "systemd",
+        "cgroup",
+        "processes",
+        "cpu",
+        "meminfo",
+        "filesystem",
+        "netdev",
+    ]
     filesystem {
-        mount_points_exclude = "^/(sys|proc|dev|run|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
+        mount_points_exclude = "^/(boot|sys|proc|dev|run|mnt|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
         fs_types_exclude     = "^(tmpfs|devtmpfs|devfs|iso9660|overlay|aufs|squashfs)$"
     }
 }
+
 prometheus.scrape "system" {
     targets    = prometheus.exporter.unix.system.targets
-    forward_to = [prometheus.relabel.system_relabel.receiver]
+    forward_to = [prometheus.relabel.system_metrics.receiver]
 }
+{% if node['name'] == "fw" %}
 
-{% if node['name'] == 'fw' %}
+// CrowdSec metrics input
-////// For Crowdsec metrics
 prometheus.scrape "crowdsec" {
     targets = [
-        { "__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:6060", "job" = "crowdsec" },
+        {
-        { "__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:60601", "job" = "crowdsec-bouncer" },
+            "__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:6060",
+            "job"         = "crowdsec",
+            "name"        = "crowdsec",
+        },
+        {
+            "__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:60601",
+            "job"         = "crowdsec",
+            "name"        = "crowdsec-bouncer",
+        },
     ]
+
     honor_labels = true
-    forward_to = [prometheus.relabel.default_label.receiver]
+    forward_to   = [prometheus.relabel.service_metrics.receiver]
 }
 {% endif %}
+{% if node['name'] == "infra" %}
 
-{% if node['name'] == 'infra' %}
+// PostgreSQL metrics
-////// For postgresql metrics
 prometheus.exporter.postgres "postgresql" {
     data_source_names = [
         "postgres://alloy@{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}/postgres?sslmode=verify-full",
     ]
 }
+
 prometheus.scrape "postgresql" {
     targets    = prometheus.exporter.postgres.postgresql.targets
-    forward_to = [prometheus.relabel.default_label.receiver]
+    forward_to = [prometheus.relabel.service_metrics.receiver]
 }
-///// For certificates metrics
+
+// x509-exporter metrics
 prometheus.scrape "x509" {
     targets = [
-        { "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:{{ services['x509-exporter']['ports']['http'] }}" },
+        {
+            "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:{{ services['x509-exporter']['ports']['http'] }}",
+        },
     ]
-    forward_to = [prometheus.relabel.default_label.receiver]
+
+    forward_to = [prometheus.relabel.service_metrics.receiver]
 }
 {% endif %}
-
 {% if node['name'] in ["infra", "auth", "app"] %}
-////// For Input Caddy metrics
+
+// Caddy metrics
 prometheus.scrape "caddy" {
     targets = [
-        { "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:443" },
+        {
+            "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:443",
+        },
     ]
     scheme       = "https"
-    forward_to = [prometheus.relabel.default_label.receiver]
+    metrics_path = "/metrics"
+    forward_to   = [prometheus.relabel.service_metrics.receiver]
 }
 {% endif %}
 
-// Log
+// Loki
-//// Logs output
+
+// Logs output
+
 loki.write "loki" {
     endpoint {
         url       = "https://{{ services['loki']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['loki']['ports']['https'] }}/loki/api/v1/push"
         tenant_id = "{{ domain['internal'] }}"
     }
 }
-//// Logs relabel
+
-///// journal
+// Journald logs relabel
-loki.relabel "journal_relabel" {
+
+loki.relabel "journal_logs" {
     forward_to = []
     rule {
         target_label = "instance"
         replacement  = "{{ node['name'] }}"
     }
-    // Default value
     rule {
         target_label = "job"
         replacement  = "systemd-journal"
     }
-    // if identifier exists
-    rule {
-        source_labels = ["__journal_syslog_identifier"]
-        regex         = "(.+)"
-        target_label  = "job"
-        replacement   = "$1"
-    }
-    // if systemd_unit exists
     rule {
         source_labels = ["__journal__systemd_unit"]
         regex         = "(.+)\\.service"
         target_label  = "job"
         replacement   = "$1"
     }
-    // if systemd_unit is "user@$UID"
+    rule {
+        source_labels = ["__journal__systemd_unit"]
+        regex         = "(.+)\\.timer"
+        target_label  = "job"
+        replacement   = "$1.timer"
+    }
+    rule {
+        source_labels = ["__journal__systemd_unit"]
+        regex         = "(.+)\\.socket"
+        target_label  = "job"
+        replacement   = "$1.socket"
+    }
+    rule {
+        source_labels = ["__journal_syslog_identifier"]
+        regex         = "(.+)"
+        target_label  = "job"
+        replacement   = "$1"
+    }
     rule {
         source_labels = ["job"]
         regex         = "user@\\d+"
         target_label  = "job"
         replacement   = "systemd-journal"
     }
-    // if systemd_user_unit exists
     rule {
         source_labels = ["__journal__systemd_user_unit"]
         regex         = "(.+)\\.service"
@@ -160,139 +216,292 @@ loki.relabel "journal_relabel" {
         replacement   = "$1"
     }
     rule {
-        source_labels = ["__journal_priority_keyword"]
+        source_labels = ["__journal__systemd_user_unit"]
-        target_label  = "level"
+        regex         = "(.+)\\.timer"
+        target_label  = "job"
+        replacement   = "$1.timer"
     }
-}
-{% if node['name'] == "fw" %}
-loki.relabel "suricata_relabel" {
-    forward_to = [loki.process.suricata_json.receiver]
     rule {
-        target_label = "instance"
+        source_labels = ["__journal__systemd_user_unit"]
-        replacement  = "{{ node['name'] }}"
+        regex         = "(.+)\\.socket"
+        target_label  = "job"
+        replacement   = "$1.socket"
+    }
+    rule {
+        source_labels = ["job"]
+        regex         = "(.+)"
+        target_label  = "service_name"
+        replacement   = "$1"
+    }
+    rule {
+        source_labels = ["__journal_priority_keyword"]
+        regex         = "(.+)"
+        target_label  = "journal_level"
+        replacement   = "$1"
     }
     rule {
         target_label = "level"
         replacement  = "info"
     }
+    rule {
+        source_labels = ["__journal_priority_keyword"]
+        regex         = "debug"
+        target_label  = "level"
+        replacement   = "debug"
+    }
+    rule {
+        source_labels = ["__journal_priority_keyword"]
+        regex         = "info|notice"
+        target_label  = "level"
+        replacement   = "info"
+    }
+    rule {
+        source_labels = ["__journal_priority_keyword"]
+        regex         = "warning"
+        target_label  = "level"
+        replacement   = "warn"
+    }
+    rule {
+        source_labels = ["__journal_priority_keyword"]
+        regex         = "err"
+        target_label  = "level"
+        replacement   = "error"
+    }
+    rule {
+        source_labels = ["__journal_priority_keyword"]
+        regex         = "crit|alert|emerg"
+        target_label  = "level"
+        replacement   = "fatal"
+    }
+}
+
+// Journald logs parser
+
+loki.process "journal_parser" {
+    forward_to = [loki.write.loki.receiver]
+    stage.match {
+        selector = "{level!=\"fatal\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(fatal|critical|panic|uncaught|unhandled|assertion failed|out of memory|\\bboom\\b|killed process)\""
+        stage.static_labels {
+            values = {
+                level = "fatal",
+            }
+        }
+    }
+    stage.match {
+        selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\berror\\b|\\berr\\b|\\bfailed\\b|\\bfailure\\b|\\bexception\\b|traceback|segfault|exit-code|container died|permission denied|access denied|connection refused|connection reset|timed out|timeout exceeded|no such file|authentication failed|sqlstate|duplicate key|relation .* does not exist|column .* does not exist|database .* does not exist|relation .* already exists|column .* already exists|x509|unknown authority|certificate .*expired|tls handshake|bad gateway|upstream .*failed|upstream .*error)\""
+        stage.static_labels {
+            values = {
+                level = "error",
+            }
+        }
+    }
+    stage.match {
+        selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(Main process exited|Failed with result|start request repeated too quickly|Dependency failed|Unit .* entered failed state)\""
+
+        stage.static_labels {
+            values = {
+                level = "error",
+            }
+        }
+    }
+    stage.match {
+        selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\bwarn\\b|warning|deprecated|retrying|rate.?limit|too many requests|slow query|slow request)\""
+
+        stage.static_labels {
+            values = {
+                level = "warn",
+            }
+        }
+    }
+    stage.match {
+        selector = "{job=~\"ssh|sshd\"}"
+
+        stage.regex {
+            expression = "Accepted \\w+ for (?P<user>[^ ]+) from (?P<ip>[0-9a-fA-F:.]+)"
+        }
+
+        stage.labels {
+            values = {
+                user = "",
+            }
+        }
+    }
+}
+{% if node['name'] == "fw" %}
+
+// Suricata logs relabel
+
+loki.relabel "suricata_logs" {
+    forward_to = [loki.process.suricata_parser.receiver]
+    
+    rule {
+        target_label = "instance"
+        replacement  = "{{ node['name'] }}"
+    }
     rule {
         target_label = "job"
         replacement  = "suricata_eve"
     }
-}
-{% endif %}
-{% if node['name'] == "auth" %}
-loki.relabel "caddy_relabel" {
-    forward_to = [loki.process.caddy_json.receiver]
     rule {
-        target_label = "instance"
+        target_label = "service_name"
-        replacement  = "{{ node['name'] }}"
+        replacement  = "suricata_eve"
     }
     rule {
         target_label = "level"
         replacement  = "info"
     }
-    rule {
-        target_label  = "job"
-        replacement   = "caddy_access"
-    }
 }
-{% endif %}
-//// Log parser
-///// journal
-loki.process "journal_parser" {
-        forward_to = [loki.write.loki.receiver]
-        // Severity parsing
-        stage.regex {
-            // Regex to extract the log level from the content.
-            expression = "(?i)(?:level[\"\\s:=]+|\\[|\\s|^)(?P<content_level>info|warn|warning|error|debug|fatal|critical|trace)(?:[\"\\]\\s]|$)"
-        }
 
-        stage.labels {
+// Suricata logs parser
-            values = {
+
-                "level" = "content_level",
+loki.process "suricata_parser" {
-            }
-        }
-        // Add this section as parser for each service
-        // common
-        stage.match {
-            selector = "{job=\"sshd\"}"
-            stage.regex {
-                expression = "Accepted \\w+ for (?P<user>\\w+) from (?P<ip>[\\d\\.]+)"
-            }
-            stage.labels {
-                values = { "user" = "" }
-            }
-        }
-        // infra
-        {% if node['name'] == 'infra' %}
-        // auth
-        {% elif node['name'] == 'auth' %}
-        // app
-        {% elif node['name'] == 'app' %}
-        {% endif %}
-}
-{% if node['name'] == "fw" %}
-////// suricata
-loki.process "suricata_json" {
     forward_to = [loki.write.loki.receiver]
+
     stage.json {
         expressions = {
             event_type = "event_type",
-            src_ip     = "src_ip",
+        }
+    }
+    stage.labels {
+        values = {
+            event_type = "",
+        }
+    }
+    stage.match {
+        selector = "{event_type=\"alert\"}"
+        stage.json {
+            expressions = {
                 severity = "alert.severity",
             }
         }
         stage.labels {
-        values = { event_type = "", severity = "" }
+            values = {
+                severity = "",
             }
+        }
+        stage.static_labels {
+            values = {
+                level = "warn",
+            }
+        }
+    }
+    stage.match {
+        selector = "{event_type=\"alert\", severity=~\"1|2\"}"
+        stage.static_labels {
+            values = {
+                level = "error",
+            }
+        }
+    }
+}
+
+// Suricata eve.json input
+
+local.file_match "suricata_logs" {
+    path_targets = [
+        { "__path__" = "/var/log/suricata/eve.json" },
+    ]
+}
+
+loki.source.file "suricata" {
+    targets    = local.file_match.suricata_logs.targets
+    forward_to = [loki.relabel.suricata_logs.receiver]
 }
 {% endif %}
 {% if node['name'] == "auth" %}
-////// caddy
+
-loki.process "caddy_json" {
+// Caddy access logs relabel
+
+loki.relabel "caddy_logs" {
+    forward_to = [loki.process.caddy_parser.receiver]
+
+    rule {
+        target_label = "instance"
+        replacement  = "{{ node['name'] }}"
+    }
+
+    rule {
+        target_label = "job"
+        replacement  = "caddy_access"
+    }
+
+    rule {
+        target_label = "service_name"
+        replacement  = "caddy_access"
+    }
+
+    rule {
+        target_label = "level"
+        replacement  = "info"
+    }
+}
+
+// Caddy access logs parser
+
+loki.process "caddy_parser" {
     forward_to = [loki.write.loki.receiver]
+
     stage.json {
         expressions = {
             status    = "status",
-            method      = "method",
-            remote_ip   = "remote_ip",
             duration  = "duration",
+            method    = "request.method",
+            host      = "request.host",
+            uri       = "request.uri",
+            remote_ip = "request.remote_ip",
         }
     }
     stage.labels {
-        values = { status = "", method = "" }
+        values = {
+            status = "",
+            method = "",
+            host   = "",
+        }
+    }
+    stage.structured_metadata {
+        values = {
+            uri       = "",
+            remote_ip = "",
+            duration  = "",
+        }
+    }
+    stage.match {
+        selector = "{status=~\"401|403|429\"}"
+
+        stage.static_labels {
+            values = {
+                level = "warn",
+            }
+        }
+    }
+    stage.match {
+        selector = "{status=~\"5..\"}"
+
+        stage.static_labels {
+            values = {
+                level = "error",
+            }
+        }
     }
 }
-{% endif %}
-//// Logs input
-////// journald
-loki.source.journal "systemd" {
-    forward_to = [loki.process.journal_parser.receiver]
-    // Temporary tags like "__journal__systemd_unit" is automatically removed when logs is passing "forward_to"
-    // To relabel tags with temporary tags, relabel_rules command is necessary.
-    relabel_rules = loki.relabel.journal_relabel.rules
-}
 
-{% if node['name'] == 'fw' %}
+// Caddy access log input
-////// suricata
-local.file_match "suricata_logs" {
-    path_targets = [{ "__path__" = "/var/log/suricata/eve.json", "instance" = "{{ node['name'] }}" }]
-}
-loki.source.file "suricata" {
-    targets    = local.file_match.suricata_logs.targets
-    forward_to = [loki.relabel.suricata_relabel.receiver]
-}
-{% endif %}
 
-{% if node['name'] == 'auth' %}
-////// caddy
 local.file_match "caddy_logs" {
-    path_targets = [{ "__path__" = "/var/log/caddy/access.log", "instance" = "{{ node['name'] }}" }]
+    path_targets = [
+        { "__path__" = "/var/log/caddy/access.log" },
+    ]
 }
 
 loki.source.file "caddy" {
     targets    = local.file_match.caddy_logs.targets
-    forward_to = [loki.relabel.caddy_relabel.receiver]
+    forward_to = [loki.relabel.caddy_logs.receiver]
 }
 {% endif %}
+
+// Journald logs input
+
+loki.source.journal "systemd" {
+    forward_to    = [loki.process.journal_parser.receiver]
+    relabel_rules = loki.relabel.journal_logs.rules
+}