508 lines
13 KiB
Django/Jinja
508 lines
13 KiB
Django/Jinja
// prometheus
|
|
|
|
// Metrics output
|
|
|
|
prometheus.remote_write "prometheus" {
|
|
endpoint {
|
|
url = "https://{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}/api/v1/write"
|
|
}
|
|
}
|
|
|
|
// System metrics relabel
|
|
|
|
prometheus.relabel "system_metrics" {
|
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
|
|
|
rule {
|
|
target_label = "instance"
|
|
replacement = "{{ node['name'] }}"
|
|
}
|
|
rule {
|
|
target_label = "job"
|
|
replacement = "system"
|
|
}
|
|
rule {
|
|
source_labels = ["name"]
|
|
regex = "(.+)\\.service"
|
|
target_label = "name"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["name"]
|
|
regex = "(.+)\\.timer"
|
|
target_label = "name"
|
|
replacement = "$1.timer"
|
|
}
|
|
rule {
|
|
source_labels = ["name"]
|
|
regex = "(.+)\\.socket"
|
|
target_label = "name"
|
|
replacement = "$1.socket"
|
|
}
|
|
}
|
|
|
|
// Generic service metrics relabel
|
|
|
|
prometheus.relabel "service_metrics" {
|
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
|
|
|
rule {
|
|
target_label = "instance"
|
|
replacement = "{{ node['name'] }}"
|
|
}
|
|
rule {
|
|
source_labels = ["job"]
|
|
regex = "prometheus\\.scrape\\.(.+)"
|
|
target_label = "job"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["job"]
|
|
regex = "integrations\\/(.+)"
|
|
target_label = "job"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["name", "job"]
|
|
separator = ";"
|
|
regex = ";(.+)"
|
|
target_label = "name"
|
|
replacement = "$1"
|
|
}
|
|
}
|
|
|
|
// Metrics input
|
|
|
|
// System metrics input
|
|
|
|
prometheus.exporter.unix "system" {
|
|
enable_collectors = [
|
|
"systemd",
|
|
"cgroup",
|
|
"processes",
|
|
"cpu",
|
|
"meminfo",
|
|
"filesystem",
|
|
"netdev",
|
|
]
|
|
filesystem {
|
|
mount_points_exclude = "^/(boot|sys|proc|dev|run|mnt|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
|
|
fs_types_exclude = "^(tmpfs|devtmpfs|devfs|iso9660|overlay|aufs|squashfs)$"
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "system" {
|
|
targets = prometheus.exporter.unix.system.targets
|
|
forward_to = [prometheus.relabel.system_metrics.receiver]
|
|
}
|
|
{% if node['name'] == "fw" %}
|
|
|
|
// CrowdSec metrics input
|
|
prometheus.scrape "crowdsec" {
|
|
targets = [
|
|
{
|
|
"__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:6060",
|
|
"job" = "crowdsec",
|
|
"name" = "crowdsec",
|
|
},
|
|
{
|
|
"__address__" = "{{ services['crowdsec']['domain']['internal'] }}.{{ domain['internal'] }}:60601",
|
|
"job" = "crowdsec",
|
|
"name" = "crowdsec-bouncer",
|
|
},
|
|
]
|
|
|
|
honor_labels = true
|
|
forward_to = [prometheus.relabel.service_metrics.receiver]
|
|
}
|
|
{% endif %}
|
|
{% if node['name'] == "infra" %}
|
|
|
|
// PostgreSQL metrics
|
|
prometheus.exporter.postgres "postgresql" {
|
|
data_source_names = [
|
|
"postgres://alloy@{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}/postgres?sslmode=verify-full",
|
|
]
|
|
}
|
|
|
|
prometheus.scrape "postgresql" {
|
|
targets = prometheus.exporter.postgres.postgresql.targets
|
|
forward_to = [prometheus.relabel.service_metrics.receiver]
|
|
}
|
|
|
|
// x509-exporter metrics
|
|
prometheus.scrape "x509" {
|
|
targets = [
|
|
{
|
|
"__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:{{ services['x509-exporter']['ports']['http'] }}",
|
|
},
|
|
]
|
|
|
|
forward_to = [prometheus.relabel.service_metrics.receiver]
|
|
}
|
|
{% endif %}
|
|
{% if node['name'] in ["infra", "auth", "app"] %}
|
|
|
|
// Caddy metrics
|
|
prometheus.scrape "caddy" {
|
|
targets = [
|
|
{
|
|
"__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:443",
|
|
},
|
|
]
|
|
scheme = "https"
|
|
metrics_path = "/metrics"
|
|
forward_to = [prometheus.relabel.service_metrics.receiver]
|
|
}
|
|
{% endif %}
|
|
|
|
// Loki
|
|
|
|
// Logs output
|
|
|
|
loki.write "loki" {
|
|
endpoint {
|
|
url = "https://{{ services['loki']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['loki']['ports']['https'] }}/loki/api/v1/push"
|
|
tenant_id = "{{ domain['internal'] }}"
|
|
}
|
|
}
|
|
|
|
// Journald logs relabel
|
|
|
|
loki.relabel "journal_logs" {
|
|
forward_to = []
|
|
rule {
|
|
target_label = "instance"
|
|
replacement = "{{ node['name'] }}"
|
|
}
|
|
rule {
|
|
target_label = "job"
|
|
replacement = "systemd-journal"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
regex = "(.+)\\.service"
|
|
target_label = "job"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
regex = "(.+)\\.timer"
|
|
target_label = "job"
|
|
replacement = "$1.timer"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
regex = "(.+)\\.socket"
|
|
target_label = "job"
|
|
replacement = "$1.socket"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_syslog_identifier"]
|
|
regex = "(.+)"
|
|
target_label = "job"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["job"]
|
|
regex = "user@\\d+"
|
|
target_label = "job"
|
|
replacement = "systemd-journal"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal__systemd_user_unit"]
|
|
regex = "(.+)\\.service"
|
|
target_label = "job"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal__systemd_user_unit"]
|
|
regex = "(.+)\\.timer"
|
|
target_label = "job"
|
|
replacement = "$1.timer"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal__systemd_user_unit"]
|
|
regex = "(.+)\\.socket"
|
|
target_label = "job"
|
|
replacement = "$1.socket"
|
|
}
|
|
rule {
|
|
source_labels = ["job"]
|
|
regex = "(.+)"
|
|
target_label = "service_name"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
regex = "(.+)"
|
|
target_label = "journal_level"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
target_label = "level"
|
|
replacement = "info"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
regex = "debug"
|
|
target_label = "level"
|
|
replacement = "debug"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
regex = "info|notice"
|
|
target_label = "level"
|
|
replacement = "info"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
regex = "warning"
|
|
target_label = "level"
|
|
replacement = "warn"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
regex = "err"
|
|
target_label = "level"
|
|
replacement = "error"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
regex = "crit|alert|emerg"
|
|
target_label = "level"
|
|
replacement = "fatal"
|
|
}
|
|
}
|
|
|
|
// Journald logs parser
|
|
|
|
loki.process "journal_parser" {
|
|
forward_to = [loki.write.loki.receiver]
|
|
stage.match {
|
|
selector = "{level!=\"fatal\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(fatal|critical|panic|uncaught|unhandled|assertion failed|out of memory|\\bboom\\b|killed process)\""
|
|
stage.static_labels {
|
|
values = {
|
|
level = "fatal",
|
|
}
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\berror\\b|\\berr\\b|\\bfailed\\b|\\bfailure\\b|\\bexception\\b|traceback|segfault|exit-code|container died|permission denied|access denied|connection refused|connection reset|timed out|timeout exceeded|no such file|authentication failed|sqlstate|duplicate key|relation .* does not exist|column .* does not exist|database .* does not exist|relation .* already exists|column .* already exists|x509|unknown authority|certificate .*expired|tls handshake|bad gateway|upstream .*failed|upstream .*error)\""
|
|
stage.static_labels {
|
|
values = {
|
|
level = "error",
|
|
}
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(Main process exited|Failed with result|start request repeated too quickly|Dependency failed|Unit .* entered failed state)\""
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
level = "error",
|
|
}
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\bwarn\\b|warning|deprecated|retrying|rate.?limit|too many requests|slow query|slow request)\""
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
level = "warn",
|
|
}
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{job=~\"ssh|sshd\"}"
|
|
|
|
stage.regex {
|
|
expression = "Accepted \\w+ for (?P<user>[^ ]+) from (?P<ip>[0-9a-fA-F:.]+)"
|
|
}
|
|
|
|
stage.labels {
|
|
values = {
|
|
user = "",
|
|
}
|
|
}
|
|
}
|
|
}
|
|
{% if node['name'] == "fw" %}
|
|
|
|
// Suricata logs relabel
|
|
|
|
loki.relabel "suricata_logs" {
|
|
forward_to = [loki.process.suricata_parser.receiver]
|
|
|
|
rule {
|
|
target_label = "instance"
|
|
replacement = "{{ node['name'] }}"
|
|
}
|
|
rule {
|
|
target_label = "job"
|
|
replacement = "suricata_eve"
|
|
}
|
|
rule {
|
|
target_label = "service_name"
|
|
replacement = "suricata_eve"
|
|
}
|
|
rule {
|
|
target_label = "level"
|
|
replacement = "info"
|
|
}
|
|
}
|
|
|
|
// Suricata logs parser
|
|
|
|
loki.process "suricata_parser" {
|
|
forward_to = [loki.write.loki.receiver]
|
|
|
|
stage.json {
|
|
expressions = {
|
|
event_type = "event_type",
|
|
}
|
|
}
|
|
stage.labels {
|
|
values = {
|
|
event_type = "",
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{event_type=\"alert\"}"
|
|
stage.json {
|
|
expressions = {
|
|
severity = "alert.severity",
|
|
}
|
|
}
|
|
stage.labels {
|
|
values = {
|
|
severity = "",
|
|
}
|
|
}
|
|
stage.static_labels {
|
|
values = {
|
|
level = "warn",
|
|
}
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{event_type=\"alert\", severity=~\"1|2\"}"
|
|
stage.static_labels {
|
|
values = {
|
|
level = "error",
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Suricata eve.json input
|
|
|
|
local.file_match "suricata_logs" {
|
|
path_targets = [
|
|
{ "__path__" = "/var/log/suricata/eve.json" },
|
|
]
|
|
}
|
|
|
|
loki.source.file "suricata" {
|
|
targets = local.file_match.suricata_logs.targets
|
|
forward_to = [loki.relabel.suricata_logs.receiver]
|
|
}
|
|
{% endif %}
|
|
{% if node['name'] == "auth" %}
|
|
|
|
// Caddy access logs relabel
|
|
|
|
loki.relabel "caddy_logs" {
|
|
forward_to = [loki.process.caddy_parser.receiver]
|
|
|
|
rule {
|
|
target_label = "instance"
|
|
replacement = "{{ node['name'] }}"
|
|
}
|
|
|
|
rule {
|
|
target_label = "job"
|
|
replacement = "caddy_access"
|
|
}
|
|
|
|
rule {
|
|
target_label = "service_name"
|
|
replacement = "caddy_access"
|
|
}
|
|
|
|
rule {
|
|
target_label = "level"
|
|
replacement = "info"
|
|
}
|
|
}
|
|
|
|
// Caddy access logs parser
|
|
|
|
loki.process "caddy_parser" {
|
|
forward_to = [loki.write.loki.receiver]
|
|
|
|
stage.json {
|
|
expressions = {
|
|
status = "status",
|
|
duration = "duration",
|
|
method = "request.method",
|
|
host = "request.host",
|
|
uri = "request.uri",
|
|
remote_ip = "request.remote_ip",
|
|
}
|
|
}
|
|
stage.labels {
|
|
values = {
|
|
status = "",
|
|
method = "",
|
|
host = "",
|
|
}
|
|
}
|
|
stage.structured_metadata {
|
|
values = {
|
|
uri = "",
|
|
remote_ip = "",
|
|
duration = "",
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{status=~\"401|403|429\"}"
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
level = "warn",
|
|
}
|
|
}
|
|
}
|
|
stage.match {
|
|
selector = "{status=~\"5..\"}"
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
level = "error",
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Caddy access log input
|
|
|
|
local.file_match "caddy_logs" {
|
|
path_targets = [
|
|
{ "__path__" = "/var/log/caddy/access.log" },
|
|
]
|
|
}
|
|
|
|
loki.source.file "caddy" {
|
|
targets = local.file_match.caddy_logs.targets
|
|
forward_to = [loki.relabel.caddy_logs.receiver]
|
|
}
|
|
{% endif %}
|
|
|
|
// Journald logs input
|
|
|
|
loki.source.journal "systemd" {
|
|
forward_to = [loki.process.journal_parser.receiver]
|
|
relabel_rules = loki.relabel.journal_logs.rules
|
|
}
|