282 lines
11 KiB
Django/Jinja
282 lines
11 KiB
Django/Jinja
groups:
|
|
- name: node_exporters_heartbeat
|
|
rules:
|
|
{% for instance in (((groups['hypervisor'] | default([])) + (groups['vms'] | default([]))) | difference(['vmm_init']) | sort) %}
|
|
- alert: {{ instance }}_node_exporter_down
|
|
expr: |
|
|
(max_over_time(up{instance="{{ instance }}"}[5m]) or on() vector(0)) == 0
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Exporter heartbeat is down: {{ instance }}"
|
|
description: "{{ instance }} exporter is down for 5 mins"
|
|
{% endfor %}
|
|
- name: postgresql_heartbeat
|
|
rules:
|
|
- alert: Postgresql_Down
|
|
expr: |
|
|
(max_over_time(pg_up{instance="infra", job="postgres"}[5m]) or on() vector(0)) == 0
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Postgresql Heartbeat Lost: postgresql"
|
|
description: "postgresql node is down for 5 mins."
|
|
- name: certificate_expiry_check
|
|
rules:
|
|
{% for filename in ['root', 'intermediate'] + (services | dict2items | selectattr('value.dependency.cert', 'defined') | selectattr('value.dependency.cert', 'equalto', true) | map(attribute='key') | list) %}
|
|
- alert: {{ filename | replace('.', '_') }}_is_expired_soon
|
|
expr: |
|
|
min(x509_cert_not_after{filename="{{ filename }}.crt"}) - time() < 2592000
|
|
for: 1d
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "{{ filename }}.crt is expired in 30 days"
|
|
description: "{{ filename }}.crt is expired in 30 days."
|
|
{% endfor %}
|
|
- name: smartctl
|
|
rules:
|
|
- alert: SmartctlMetricsMissing
|
|
expr: |
|
|
absent_over_time(smartctl_devices{job="smartctl", instance="vmm"}[15m])
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART metrics are missing on vmm"
|
|
description: "No smartctl_devices metric has been received from vmm for 15 minutes. Check smartctl-generator.timer, Alloy textfile collector, and /etc/smartctl-exporter/smartctl.prom."
|
|
|
|
- alert: SmartctlNoDevices
|
|
expr: |
|
|
smartctl_devices{job="smartctl", instance="vmm"} < 1
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "No SMART devices discovered on vmm"
|
|
description: "smartctl-generator is running, but it reports fewer than one SMART device. Check USB filtering, /sys/class/block discovery, and smartctl output."
|
|
|
|
{% set devices_count =
|
|
(hostvars['vmm']['hardware']['disk']['data']['serials'] | default([]) | length) +
|
|
(hostvars['vmm']['hardware']['disk']['backup']['serials'] | default([]) | length) +
|
|
(hostvars['vmm']['hardware']['disk']['nvme']['serials'] | default([]) | length) %}
|
|
- alert: SmartctlDeviceCountLow
|
|
expr: |
|
|
(max_over_time(smartctl_devices{job="smartctl", instance="vmm"}[5m]) or on() vector(0)) < {{ devices_count }}
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART device count is low on vmm"
|
|
description: "Expected {{ devices_count }} SMART devices on vmm, but fewer than {{ devices_count }} are currently reported. Check missing HDD/NVMe devices, smartctl output, and device discovery."
|
|
|
|
- alert: SmartctlDeviceHealthFailed
|
|
expr: |
|
|
(
|
|
smartctl_device_smart_status{job="smartctl"}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART health failed"
|
|
description: "SMART overall-health failed. Check the alert labels for instance, device, model_name, serial_number, and protocol."
|
|
|
|
- alert: SmartctlCommandFailed
|
|
expr: |
|
|
(
|
|
(floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 4) % 2)
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl", protocol!~"NVMe|nvme"}
|
|
) == 1
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "smartctl command or log read failed"
|
|
description: "smartctl exit status has bit 2 set on a non-NVMe device. Check smartctl output, cable/backplane, and device logs. NVMe devices are intentionally excluded because bit 2 is often caused by unsupported optional log pages."
|
|
|
|
- alert: SmartctlDiskFailing
|
|
expr: |
|
|
(
|
|
(floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 8) % 2)
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) == 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "smartctl reports disk failing"
|
|
description: "smartctl exit status has bit 3 set. Check the alert labels and replace or evacuate the device if confirmed."
|
|
|
|
- alert: SmartctlPrefailAttributeThreshold
|
|
expr: |
|
|
(
|
|
(floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 16) % 2)
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) == 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART prefail attribute threshold hit"
|
|
description: "smartctl exit status has bit 4 set. Check the alert labels and the SMART attribute table."
|
|
|
|
- alert: SmartctlDiskTemperatureHigh
|
|
expr: |
|
|
(
|
|
smartctl_device_temperature{job="smartctl", temperature_type="current"}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) > 50
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "SMART disk temperature high"
|
|
description: "Disk temperature is above 50C for 15 minutes. Check the alert labels for the affected disk."
|
|
|
|
- alert: SmartctlDiskTemperatureCritical
|
|
expr: |
|
|
(
|
|
smartctl_device_temperature{job="smartctl", temperature_type="current"}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) > 55
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART disk temperature critical"
|
|
description: "Disk temperature is above 55C. Check cooling and disk workload immediately."
|
|
|
|
- alert: SmartctlCriticalAttributeNonZero
|
|
expr: |
|
|
(
|
|
smartctl_device_attribute{
|
|
job="smartctl",
|
|
attribute_value_type="raw",
|
|
attribute_name=~"Reallocated_Sector_Ct|Current_Pending_Sector|Offline_Uncorrectable|Spin_Retry_Count"
|
|
}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART critical attribute is non-zero"
|
|
description: "A critical SMART raw attribute is non-zero. Check the alert labels for attribute_name, instance, device, model_name, and serial_number."
|
|
|
|
- alert: SmartctlCrcErrorIncreased
|
|
expr: |
|
|
(
|
|
delta(smartctl_device_attribute{
|
|
job="smartctl",
|
|
attribute_value_type="raw",
|
|
attribute_name="UDMA_CRC_Error_Count"
|
|
}[24h])
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "SMART CRC error count increased"
|
|
description: "UDMA_CRC_Error_Count increased in the last 24 hours. Check SATA cable, backplane, and power path."
|
|
|
|
- alert: SmartctlErrorLogIncreased
|
|
expr: |
|
|
(
|
|
delta(smartctl_device_error_log_count{job="smartctl"}[24h])
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "SMART error log increased"
|
|
description: "SMART error log count increased in the last 24 hours. Check smartctl output and the alert labels."
|
|
|
|
- alert: SmartctlSelfTestErrorsPresent
|
|
expr: |
|
|
(
|
|
smartctl_device_self_test_log_error_count{job="smartctl"}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl"}
|
|
) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "SMART self-test errors present"
|
|
description: "SMART self-test error count is non-zero. Check self-test history and rerun a long self-test if needed."
|
|
|
|
- alert: SmartctlNvmeCriticalWarning
|
|
expr: |
|
|
(
|
|
smartctl_device_critical_warning{job="smartctl"}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
|
) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "NVMe SMART critical warning"
|
|
description: "NVMe critical_warning is non-zero. This is a real NVMe health signal; check smartctl output and alert labels."
|
|
|
|
- alert: SmartctlNvmeMediaErrorsIncreased
|
|
expr: |
|
|
(
|
|
delta(smartctl_device_media_errors{job="smartctl"}[24h])
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
|
) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "NVMe media errors increased"
|
|
description: "NVMe media error count increased in the last 24 hours. Check smartctl output and alert labels."
|
|
|
|
- alert: SmartctlNvmePercentageUsedHigh
|
|
expr: |
|
|
(
|
|
smartctl_device_percentage_used{job="smartctl"}
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
|
) > 90
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "NVMe percentage used high"
|
|
description: "NVMe percentage_used is above 90. Check remaining endurance and plan replacement."
|
|
|
|
- alert: SmartctlNvmeAvailableSpareLow
|
|
expr: |
|
|
(
|
|
(
|
|
smartctl_device_available_spare{job="smartctl"}
|
|
<= on(instance, device)
|
|
smartctl_device_available_spare_threshold{job="smartctl"}
|
|
)
|
|
* on(instance, device) group_left(model_name, serial_number, protocol)
|
|
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
|
)
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "NVMe available spare below threshold"
|
|
description: "NVMe available_spare is at or below available_spare_threshold. Check smartctl output and alert labels."
|