groups:
  - name: node_exporters_heartbeat
    rules:
{% for instance in (((groups['hypervisor'] | default([])) + (groups['vms'] | default([]))) | difference(['vmm_init']) | sort) %}
    - alert: {{ instance }}_node_exporter_down
      expr: |
        (max_over_time(up{instance="{{ instance }}"}[5m]) or on() vector(0)) == 0
      for: 30s
      labels:
        severity: critical
      annotations:
        summary: "Exporter heartbeat is down: {{ instance }}"
        description: "{{ instance }} exporter is down for 5 mins"
{% endfor %}
  - name: postgresql_heartbeat
    rules:
      - alert: Postgresql_Down
        expr: |
          (max_over_time(pg_up{instance="infra", job="postgres"}[5m]) or on() vector(0)) == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "Postgresql Heartbeat Lost: postgresql"
          description: "postgresql node is down for 5 mins."
  - name: certificate_expiry_check
    rules:
{% for filename in ['root', 'intermediate'] + (services | dict2items | selectattr('value.dependency.cert', 'defined') | selectattr('value.dependency.cert', 'equalto', true) | map(attribute='key') | list) %}
      - alert: {{ filename | replace('.', '_') }}_is_expired_soon
        expr: |
          max(x509_cert_not_after{filename="{{ filename }}.crt"}) - time() < 2592000
        for: 1d
        labels:
          severity: critical
        annotations:
          summary: "{{ filename }}.crt is expired in 30 days"
          description: "{{ filename }}.crt is expired in 30 days."
{% endfor %}
  - name: smartctl
    rules:
      - alert: SmartctlMetricsMissing
        expr: |
          absent_over_time(smartctl_devices{job="smartctl", instance="vmm"}[15m])
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "SMART metrics are missing on vmm"
          description: "No smartctl_devices metric has been received from vmm for 15 minutes. Check smartctl-generator.timer, Alloy textfile collector, and /etc/smartctl-exporter/smartctl.prom."

      - alert: SmartctlNoDevices
        expr: |
          smartctl_devices{job="smartctl", instance="vmm"} < 1
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "No SMART devices discovered on vmm"
          description: "smartctl-generator is running, but it reports fewer than one SMART device. Check USB filtering, /sys/class/block discovery, and smartctl output."

      - alert: SmartctlDeviceHealthFailed
        expr: |
          (
            smartctl_device_smart_status{job="smartctl"}
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "SMART health failed"
          description: "SMART overall-health failed. Check the alert labels for instance, device, model_name, serial_number, and protocol."

      - alert: SmartctlCommandFailed
        expr: |
          (
            (floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 4) % 2)
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl", protocol!~"NVMe|nvme"}
          ) == 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "smartctl command or log read failed"
          description: "smartctl exit status has bit 2 set on a non-NVMe device. Check smartctl output, cable/backplane, and device logs. NVMe devices are intentionally excluded because bit 2 is often caused by unsupported optional log pages."

      - alert: SmartctlDiskFailing
        expr: |
          (
            (floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 8) % 2)
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) == 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "smartctl reports disk failing"
          description: "smartctl exit status has bit 3 set. Check the alert labels and replace or evacuate the device if confirmed."

      - alert: SmartctlPrefailAttributeThreshold
        expr: |
          (
            (floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 16) % 2)
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) == 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "SMART prefail attribute threshold hit"
          description: "smartctl exit status has bit 4 set. Check the alert labels and the SMART attribute table."

      - alert: SmartctlDiskTemperatureHigh
        expr: |
          (
            smartctl_device_temperature{job="smartctl", temperature_type="current"}
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) > 50
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "SMART disk temperature high"
          description: "Disk temperature is above 50C for 15 minutes. Check the alert labels for the affected disk."

      - alert: SmartctlDiskTemperatureCritical
        expr: |
          (
            smartctl_device_temperature{job="smartctl", temperature_type="current"}
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) > 55
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "SMART disk temperature critical"
          description: "Disk temperature is above 55C. Check cooling and disk workload immediately."

      - alert: SmartctlCriticalAttributeNonZero
        expr: |
          (
            smartctl_device_attribute{
              job="smartctl",
              attribute_value_type="raw",
              attribute_name=~"Reallocated_Sector_Ct|Current_Pending_Sector|Offline_Uncorrectable|Spin_Retry_Count"
            }
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) > 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "SMART critical attribute is non-zero"
          description: "A critical SMART raw attribute is non-zero. Check the alert labels for attribute_name, instance, device, model_name, and serial_number."

      - alert: SmartctlCrcErrorIncreased
        expr: |
          (
            delta(smartctl_device_attribute{
              job="smartctl",
              attribute_value_type="raw",
              attribute_name="UDMA_CRC_Error_Count"
            }[24h])
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "SMART CRC error count increased"
          description: "UDMA_CRC_Error_Count increased in the last 24 hours. Check SATA cable, backplane, and power path."

      - alert: SmartctlErrorLogIncreased
        expr: |
          (
            delta(smartctl_device_error_log_count{job="smartctl"}[24h])
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "SMART error log increased"
          description: "SMART error log count increased in the last 24 hours. Check smartctl output and the alert labels."

      - alert: SmartctlSelfTestErrorsPresent
        expr: |
          (
            smartctl_device_self_test_log_error_count{job="smartctl"}
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl"}
          ) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "SMART self-test errors present"
          description: "SMART self-test error count is non-zero. Check self-test history and rerun a long self-test if needed."

      - alert: SmartctlNvmeCriticalWarning
        expr: |
          (
            smartctl_device_critical_warning{job="smartctl"}
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
          ) > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "NVMe SMART critical warning"
          description: "NVMe critical_warning is non-zero. This is a real NVMe health signal; check smartctl output and alert labels."

      - alert: SmartctlNvmeMediaErrorsIncreased
        expr: |
          (
            delta(smartctl_device_media_errors{job="smartctl"}[24h])
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
          ) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "NVMe media errors increased"
          description: "NVMe media error count increased in the last 24 hours. Check smartctl output and alert labels."

      - alert: SmartctlNvmePercentageUsedHigh
        expr: |
          (
            smartctl_device_percentage_used{job="smartctl"}
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
          ) > 90
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "NVMe percentage used high"
          description: "NVMe percentage_used is above 90. Check remaining endurance and plan replacement."

      - alert: SmartctlNvmeAvailableSpareLow
        expr: |
          (
            (
              smartctl_device_available_spare{job="smartctl"}
              <= on(instance, device)
              smartctl_device_available_spare_threshold{job="smartctl"}
            )
            * on(instance, device) group_left(model_name, serial_number, protocol)
              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
          )
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "NVMe available spare below threshold"
          description: "NVMe available_spare is at or below available_spare_threshold. Check smartctl output and alert labels."