refactor(dns): set source of dns domain on fw.yaml

config/services/containers/infra/loki/etc/loki.yaml.j2

@@ -11,6 +11,14 @@ server:
 #  bind_addr: ['::']
 #  bind_port: 7946
 
+ruler:
+  enable_api: true
+  rule_path: /loki/rules
+  storage:
+    type: local
+    local:
+      directory: /etc/loki/rules
+
 schema_config:
   configs:
     - from: "2023-01-01"

config/services/containers/infra/loki/etc/rules.yaml.j2

@@ -0,0 +1,63 @@
+groups:
+  - name: backup_logs
+    interval: 1m
+    rules:
+      - alert: KopiaBackupFailure
+        expr: |
+          sum by (instance, job, service_name) (
+            count_over_time({job="kopia-backup", level=~"error|fatal"}[20m])
+          ) > 0
+        for: 1m
+        labels:
+          severity: critical
+          source: loki
+          category: backup
+          service: kopia
+        annotations:
+          summary: {% raw %}"Kopia backup failed on {{ $labels.instance }}"{% endraw %}{{ '' }}
+          description: "kopia-backup emitted error/fatal logs."
+      - alert: RcloneBackupFailure
+        expr: |
+          sum by (instance, job, service_name) (
+            count_over_time({job="rclone", level=~"error|fatal"}[20m])
+          ) > 0
+        for: 1m
+        labels:
+          severity: critical
+          source: loki
+          category: backup
+          service: rclone
+        annotations:
+          summary: {% raw %}"rclone backup failed on {{ $labels.instance }}"{% endraw %}{{ '' }}
+          description: "rclone emitted error/fatal logs."
+  - name: caddy_logs
+    interval: 1m
+    rules:
+      - alert: Caddy5xxBurst
+        expr: |
+          sum by (instance, host) (
+            count_over_time({job="caddy_access", status=~"5.."}[5m])
+          ) > 5
+        for: 2m
+        labels:
+          severity: warning
+          source: loki
+          category: http
+          service: caddy
+        annotations:
+          summary: {% raw %}"Caddy 5xx burst on {{ $labels.host }}"{% endraw %}{{ '' }}
+          description: {% raw %}"Caddy returned more than 5 server errors in 5 minutes on {{ $labels.instance }}."{% endraw %}{{ '' }}
+      - alert: CaddyRateLimitBurst
+        expr: |
+          sum by (instance, host) (
+            count_over_time({job="caddy_access", status="429"}[10m])
+          ) > 20
+        for: 2m
+        labels:
+          severity: warning
+          source: loki
+          category: http
+          service: caddy
+        annotations:
+          summary: {% raw %}"Caddy 429 burst on {{ $labels.host }}"{% endraw %}{{ '' }}
+          description: "Caddy returned more than 20 rate-limit responses in 10 minutes."