refactor(dns): set source of dns domain on fw.yaml

config/services/containers/infra/loki/etc/loki.yaml.j2

@@ -11,6 +11,14 @@ server:
 #  bind_addr: ['::']
 #  bind_port: 7946
 
+ruler:
+  enable_api: true
+  rule_path: /loki/rules
+  storage:
+    type: local
+    local:
+      directory: /etc/loki/rules
+
 schema_config:
   configs:
     - from: "2023-01-01"

config/services/containers/infra/loki/etc/rules.yaml.j2

@@ -0,0 +1,63 @@
+groups:
+  - name: backup_logs
+    interval: 1m
+    rules:
+      - alert: KopiaBackupFailure
+        expr: |
+          sum by (instance, job, service_name) (
+            count_over_time({job="kopia-backup", level=~"error|fatal"}[20m])
+          ) > 0
+        for: 1m
+        labels:
+          severity: critical
+          source: loki
+          category: backup
+          service: kopia
+        annotations:
+          summary: {% raw %}"Kopia backup failed on {{ $labels.instance }}"{% endraw %}{{ '' }}
+          description: "kopia-backup emitted error/fatal logs."
+      - alert: RcloneBackupFailure
+        expr: |
+          sum by (instance, job, service_name) (
+            count_over_time({job="rclone", level=~"error|fatal"}[20m])
+          ) > 0
+        for: 1m
+        labels:
+          severity: critical
+          source: loki
+          category: backup
+          service: rclone
+        annotations:
+          summary: {% raw %}"rclone backup failed on {{ $labels.instance }}"{% endraw %}{{ '' }}
+          description: "rclone emitted error/fatal logs."
+  - name: caddy_logs
+    interval: 1m
+    rules:
+      - alert: Caddy5xxBurst
+        expr: |
+          sum by (instance, host) (
+            count_over_time({job="caddy_access", status=~"5.."}[5m])
+          ) > 5
+        for: 2m
+        labels:
+          severity: warning
+          source: loki
+          category: http
+          service: caddy
+        annotations:
+          summary: {% raw %}"Caddy 5xx burst on {{ $labels.host }}"{% endraw %}{{ '' }}
+          description: {% raw %}"Caddy returned more than 5 server errors in 5 minutes on {{ $labels.instance }}."{% endraw %}{{ '' }}
+      - alert: CaddyRateLimitBurst
+        expr: |
+          sum by (instance, host) (
+            count_over_time({job="caddy_access", status="429"}[10m])
+          ) > 20
+        for: 2m
+        labels:
+          severity: warning
+          source: loki
+          category: http
+          service: caddy
+        annotations:
+          summary: {% raw %}"Caddy 429 burst on {{ $labels.host }}"{% endraw %}{{ '' }}
+          description: "Caddy returned more than 20 rate-limit responses in 10 minutes."

config/services/containers/infra/postgresql/config/pg_hba.conf.j2

@@ -4,25 +4,25 @@
 local all all trust
 
 # Local monitoring connection (host - infra VM) `trust`
-hostssl postgres alloy {{ hostvars['fw']['network4']['infra']['server'] }}/32 trust
-hostssl postgres alloy {{ hostvars['fw']['network6']['infra']['server'] }}/128 trust
-hostssl postgres alloy {{ hostvars['fw']['network4']['subnet']['lla'] }} trust
-hostssl postgres alloy {{ hostvars['fw']['network6']['subnet']['lla'] }} trust
+hostssl postgres alloy {{ hostvars['fw']['network']['infra']['ipv4']['server'] }}/32 trust
+hostssl postgres alloy {{ hostvars['fw']['network']['infra']['ipv6']['server'] }}/128 trust
+hostssl postgres alloy {{ hostvars['fw']['network']['subnet']['ipv4']['lla'] }} trust
+hostssl postgres alloy {{ hostvars['fw']['network']['subnet']['ipv6']['lla'] }} trust
 
 # Local connection (in postgresql container) needs password (127.0.0.1 - container loopback)
 host all all 127.0.0.1/32 scram-sha-256
 host all all ::1/128 scram-sha-256
 
 # Local connection (host - infra VM) needs password (169.254.1.0/24 - link_local subnet for containers in pasta mode)
-hostssl all all {{ hostvars['fw']['network4']['infra']['server'] }}/32 scram-sha-256
-hostssl all all {{ hostvars['fw']['network6']['infra']['server'] }}/128 scram-sha-256
-hostssl all all {{ hostvars['fw']['network4']['subnet']['lla'] }} scram-sha-256
-hostssl all all {{ hostvars['fw']['network6']['subnet']['lla'] }} scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['infra']['ipv4']['server'] }}/32 scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['infra']['ipv6']['server'] }}/128 scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['subnet']['ipv4']['lla'] }} scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['subnet']['ipv6']['lla'] }} scram-sha-256
 
 # auth VM
-hostssl all all {{ hostvars['fw']['network4']['auth']['server'] }}/32 scram-sha-256
-hostssl all all {{ hostvars['fw']['network6']['auth']['server'] }}/128 scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['auth']['ipv4']['server'] }}/32 scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['auth']['ipv6']['server'] }}/128 scram-sha-256
 
 # app VM (Applications, 192.168.10.13)
-hostssl all all {{ hostvars['fw']['network4']['app']['server'] }}/32 scram-sha-256
-hostssl all all {{ hostvars['fw']['network6']['app']['server'] }}/128 scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['app']['ipv4']['server'] }}/32 scram-sha-256
+hostssl all all {{ hostvars['fw']['network']['app']['ipv6']['server'] }}/128 scram-sha-256

config/services/containers/infra/prometheus/etc/rules.yaml.j2

@@ -4,7 +4,7 @@ groups:
 {% for instance in (((groups['hypervisor'] | default([])) + (groups['vms'] | default([]))) | difference(['vmm_init']) | sort) %}
     - alert: {{ instance }}_node_exporter_down
       expr: |
-        (present_over_time(up{instance="{{ instance }}"}[5m]) or on() vector(0)) == 0
+        (max_over_time(up{instance="{{ instance }}"}[5m]) or on() vector(0)) == 0
       for: 30s
       labels:
         severity: critical
@@ -16,7 +16,7 @@ groups:
     rules:
       - alert: Postgresql_Down
         expr: |
-          (present_over_time(pg_up{instance="infra", job="postgres"}[5m]) or on() vector(0)) == 0
+          (max_over_time(pg_up{instance="infra", job="postgres"}[5m]) or on() vector(0)) == 0
         for: 30s
         labels:
           severity: critical