feat(smartctl): add smartctl exporter
This commit is contained in:
@@ -15,10 +15,6 @@ domain:
|
||||
dc: "dc=ilnmors,dc=internal"
|
||||
org: "ilnmors"
|
||||
|
||||
# DNS configuration including bind and blocky should be set manually.
|
||||
# named.conf.j2 is also set manually.
|
||||
# Check the hosts.j2 when cname records are fixed
|
||||
|
||||
services:
|
||||
# common
|
||||
wireguard:
|
||||
@@ -31,6 +27,17 @@ services:
|
||||
internal: "crowdsec"
|
||||
ports:
|
||||
https: "8080"
|
||||
sub:
|
||||
machines:
|
||||
- "fw"
|
||||
- "vmm"
|
||||
- "bk"
|
||||
- "infra"
|
||||
- "auth"
|
||||
- "app"
|
||||
bouncers:
|
||||
- "fw"
|
||||
- "caddy"
|
||||
dependency:
|
||||
cert: true
|
||||
kopia:
|
||||
@@ -272,9 +279,9 @@ version:
|
||||
packages:
|
||||
sops: "3.13.1"
|
||||
step: "0.30.2"
|
||||
kopia: "0.23.0"
|
||||
blocky: "0.29.0"
|
||||
alloy: "1.16.1"
|
||||
kopia: "0.23.1"
|
||||
blocky: "0.32.1"
|
||||
alloy: "1.17.0"
|
||||
containers:
|
||||
# common
|
||||
caddy: "2.11.4"
|
||||
@@ -291,8 +298,8 @@ version:
|
||||
maddy: "0.9.5"
|
||||
x509-exporter: "4.1.0"
|
||||
prometheus: "v3.12.0"
|
||||
loki: "3.7.1"
|
||||
grafana: "13.0.1"
|
||||
loki: "3.7.3"
|
||||
grafana: "13.0.2"
|
||||
# Auth
|
||||
authelia: "4.39.20"
|
||||
# App
|
||||
|
||||
@@ -143,6 +143,15 @@
|
||||
when: inventory_hostname != "vmm_init"
|
||||
tags: ["site", "crowdsec"]
|
||||
|
||||
- name: Set smartctl generator
|
||||
ansible.builtin.include_role:
|
||||
name: "vmm"
|
||||
tasks_from: "services/set_smart_metric_generator"
|
||||
apply:
|
||||
tags: ["site", "smart"]
|
||||
when: inventory_hostname != "vmm_init"
|
||||
tags: ["site", "smart"]
|
||||
|
||||
- name: Set alloy
|
||||
ansible.builtin.include_role:
|
||||
name: "common"
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
---
|
||||
- name: Check rclone installation
|
||||
ansible.builtin.shell: |
|
||||
command -v rclone
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_rclone_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install rclone
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- "rclone"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_rclone_installed.rc != 0
|
||||
|
||||
- name: Create rclone config directory
|
||||
ansible.builtin.file:
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
---
|
||||
- name: Check nftables installation
|
||||
ansible.builtin.shell: |
|
||||
command -v nft
|
||||
become: true # nftables is located in /usr/sbin, which means root permission is needed.
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_nftables_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install nftables
|
||||
ansible.builtin.apt:
|
||||
name: "nftables"
|
||||
name:
|
||||
- "nftables"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_nftables_installed.rc != 0
|
||||
|
||||
- name: Enable nftables.service
|
||||
ansible.builtin.systemd:
|
||||
|
||||
@@ -1,36 +1,18 @@
|
||||
---
|
||||
- name: Check crowdsec installed
|
||||
ansible.builtin.shell: |
|
||||
command -v crowdsec
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_crowdsec_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Check crowdsec bouncer installed
|
||||
ansible.builtin.shell: |
|
||||
command -v crowdsec-firewall-bouncer
|
||||
when: node['name'] == "fw"
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_crowdsec_bouncer_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install crowdsec
|
||||
ansible.builtin.apt:
|
||||
name: "crowdsec"
|
||||
name:
|
||||
- "crowdsec"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_crowdsec_installed.rc != 0
|
||||
|
||||
- name: Install crowdsec bouncers
|
||||
ansible.builtin.apt:
|
||||
name: "crowdsec-firewall-bouncer"
|
||||
name:
|
||||
- "crowdsec-firewall-bouncer"
|
||||
state: "present"
|
||||
become: true
|
||||
when:
|
||||
- node['name'] == "fw"
|
||||
- is_crowdsec_bouncer_installed.rc != 0
|
||||
when: node['name'] == "fw"
|
||||
|
||||
- name: Set acquis.d list for bouncer
|
||||
ansible.builtin.set_fact:
|
||||
@@ -132,7 +114,7 @@
|
||||
|
||||
- name: Set goal machines' name
|
||||
ansible.builtin.set_fact:
|
||||
machines_name: ["fw", "vmm", "bk", "infra", "auth", "app"]
|
||||
machines_name: "{{ services['crowdsec']['sub']['machines'] | list }}"
|
||||
no_log: true
|
||||
|
||||
- name: Prune unknown (random) machines
|
||||
@@ -170,7 +152,7 @@
|
||||
|
||||
- name: Set bouncers' name
|
||||
ansible.builtin.set_fact:
|
||||
bouncers_name: ["fw", "caddy"]
|
||||
bouncers_name: "{{ services['crowdsec']['sub']['bouncers'] | list }}"
|
||||
|
||||
- name: Register Firewall Bouncer to LAPI
|
||||
ansible.builtin.command: |
|
||||
|
||||
@@ -52,6 +52,7 @@
|
||||
deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
|
||||
state: "present"
|
||||
become: true
|
||||
notify: "notification_restart_kopia_server"
|
||||
|
||||
- name: Deploy kopia env
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -1,12 +1,4 @@
|
||||
---
|
||||
- name: Check podman installation
|
||||
ansible.builtin.shell: |
|
||||
command -v podman
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_podman_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Create container directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ node['home_path'] }}/containers"
|
||||
@@ -54,4 +46,3 @@
|
||||
- "podman"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_podman_installed.rc != 0
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
---
|
||||
- name: Check bind9 installation
|
||||
ansible.builtin.shell: |
|
||||
command -v named
|
||||
become: true # named is located in /usr/sbin, which means root permission is needed.
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_bind_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install bind9
|
||||
ansible.builtin.apt:
|
||||
name: "bind9"
|
||||
name:
|
||||
- "bind9"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_bind_installed.rc != 0
|
||||
|
||||
- name: Deploy acme.key
|
||||
ansible.builtin.copy:
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
---
|
||||
- name: Check chrnoy installation
|
||||
ansible.builtin.shell: |
|
||||
command -v chronyc
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_chrony_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install chrony
|
||||
ansible.builtin.apt:
|
||||
name: "chrony"
|
||||
name:
|
||||
- "chrony"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_chrony_installed.rc != 0
|
||||
|
||||
- name: Deploy local acl file
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -1,20 +1,10 @@
|
||||
---
|
||||
- name: Check Kea dhcp4 installation
|
||||
ansible.builtin.shell: |
|
||||
command -v kea-dhcp4
|
||||
become: true # kea-dhcp4 is located in /usr/sbin, which means root permission is needed.
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_kea4_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install kea dhcp 4
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- "kea-dhcp4-server"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_kea4_installed.rc != 0
|
||||
|
||||
- name: Deploy kea dhcp4 conf
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -1,12 +1,4 @@
|
||||
---
|
||||
- name: Check suricata installation
|
||||
ansible.builtin.shell: |
|
||||
command -v suricata
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_suricata_installed"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install suricata
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
@@ -14,7 +6,6 @@
|
||||
- "suricata-update"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_suricata_installed.rc != 0
|
||||
|
||||
- name: Deploy suricata-update service files
|
||||
ansible.builtin.copy:
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
---
|
||||
- name: Check btrfs installation
|
||||
ansible.builtin.shell: |
|
||||
command -v btrfs
|
||||
become: true # btrfs is located in /usr/sbin, which means root permission is needed.
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: "is_btrfs_installed"
|
||||
|
||||
- name: Install btrfs
|
||||
ansible.builtin.apt:
|
||||
name: "btrfs-progs"
|
||||
name:
|
||||
- "btrfs-progs"
|
||||
state: "present"
|
||||
become: true
|
||||
when: is_btrfs_installed.rc != 0
|
||||
|
||||
- name: Mount data disk
|
||||
block:
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
- name: Install smartmontools
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- "smartmontools"
|
||||
- "jq"
|
||||
state: "present"
|
||||
become: true
|
||||
|
||||
- name: Create SMART metric result directory
|
||||
ansible.builtin.file:
|
||||
path: "/etc/smartctl-exporter"
|
||||
state: "directory"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
mode: "0755"
|
||||
become: true
|
||||
|
||||
- name: Deploy smart_metric_generator script
|
||||
ansible.builtin.copy:
|
||||
src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/smart_metric/smart_metric_generator.sh"
|
||||
dest: "/usr/local/sbin"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
mode: "0711"
|
||||
become: true
|
||||
|
||||
- name: Deploy smart_metric_generator service files
|
||||
ansible.builtin.template:
|
||||
src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/smart_metric/{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
mode: "0600"
|
||||
validate: "/usr/bin/systemd-analyze verify %s"
|
||||
loop:
|
||||
- "smart_metric_generator.service"
|
||||
- "smart_metric_generator.timer"
|
||||
become: true
|
||||
|
||||
- name: Register smart_metric_generator timer
|
||||
ansible.builtin.systemd:
|
||||
name: "smart_metric_generator.timer"
|
||||
state: "started"
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
become: true
|
||||
@@ -16,6 +16,9 @@
|
||||
- OJ:
|
||||
- abbr: SJU
|
||||
href: https://oj.sejong.ac.kr/
|
||||
- OPiC:
|
||||
- abbr: ENG
|
||||
href: https://lc.multicampus.com/prodsedu/
|
||||
|
||||
- Entertainment:
|
||||
- YouTube:
|
||||
|
||||
@@ -23,7 +23,7 @@ groups:
|
||||
annotations:
|
||||
summary: "Postgresql Heartbeat Lost: postgresql"
|
||||
description: "postgresql node is down for 5 mins."
|
||||
- name: Certificate_expiry_check
|
||||
- name: certificate_expiry_check
|
||||
rules:
|
||||
{% for filename in ['root', 'intermediate'] + (services | dict2items | selectattr('value.dependency.cert', 'defined') | selectattr('value.dependency.cert', 'equalto', true) | map(attribute='key') | list) %}
|
||||
- alert: {{ filename | replace('.', '_') }}_is_expired_soon
|
||||
@@ -36,3 +36,232 @@ groups:
|
||||
summary: "{{ filename }}.crt is expired in 30 days"
|
||||
description: "{{ filename }}.crt is expired in 30 days."
|
||||
{% endfor %}
|
||||
- name: smartctl
|
||||
rules:
|
||||
- alert: SmartctlMetricsMissing
|
||||
expr: |
|
||||
absent_over_time(smartctl_devices{job="smartctl", instance="vmm"}[15m])
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART metrics are missing on vmm"
|
||||
description: "No smartctl_devices metric has been received from vmm for 15 minutes. Check smartctl-generator.timer, Alloy textfile collector, and /etc/smartctl-exporter/smartctl.prom."
|
||||
|
||||
- alert: SmartctlNoDevices
|
||||
expr: |
|
||||
smartctl_devices{job="smartctl", instance="vmm"} < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No SMART devices discovered on vmm"
|
||||
description: "smartctl-generator is running, but it reports fewer than one SMART device. Check USB filtering, /sys/class/block discovery, and smartctl output."
|
||||
|
||||
- alert: SmartctlDeviceHealthFailed
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_smart_status{job="smartctl"}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART health failed"
|
||||
description: "SMART overall-health failed. Check the alert labels for instance, device, model_name, serial_number, and protocol."
|
||||
|
||||
- alert: SmartctlCommandFailed
|
||||
expr: |
|
||||
(
|
||||
(floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 4) % 2)
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl", protocol!~"NVMe|nvme"}
|
||||
) == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "smartctl command or log read failed"
|
||||
description: "smartctl exit status has bit 2 set on a non-NVMe device. Check smartctl output, cable/backplane, and device logs. NVMe devices are intentionally excluded because bit 2 is often caused by unsupported optional log pages."
|
||||
|
||||
- alert: SmartctlDiskFailing
|
||||
expr: |
|
||||
(
|
||||
(floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 8) % 2)
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "smartctl reports disk failing"
|
||||
description: "smartctl exit status has bit 3 set. Check the alert labels and replace or evacuate the device if confirmed."
|
||||
|
||||
- alert: SmartctlPrefailAttributeThreshold
|
||||
expr: |
|
||||
(
|
||||
(floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 16) % 2)
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART prefail attribute threshold hit"
|
||||
description: "smartctl exit status has bit 4 set. Check the alert labels and the SMART attribute table."
|
||||
|
||||
- alert: SmartctlDiskTemperatureHigh
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) > 50
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SMART disk temperature high"
|
||||
description: "Disk temperature is above 50C for 15 minutes. Check the alert labels for the affected disk."
|
||||
|
||||
- alert: SmartctlDiskTemperatureCritical
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) > 55
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART disk temperature critical"
|
||||
description: "Disk temperature is above 55C. Check cooling and disk workload immediately."
|
||||
|
||||
- alert: SmartctlCriticalAttributeNonZero
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_attribute{
|
||||
job="smartctl",
|
||||
attribute_value_type="raw",
|
||||
attribute_name=~"Reallocated_Sector_Ct|Current_Pending_Sector|Offline_Uncorrectable|Spin_Retry_Count"
|
||||
}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART critical attribute is non-zero"
|
||||
description: "A critical SMART raw attribute is non-zero. Check the alert labels for attribute_name, instance, device, model_name, and serial_number."
|
||||
|
||||
- alert: SmartctlCrcErrorIncreased
|
||||
expr: |
|
||||
(
|
||||
delta(smartctl_device_attribute{
|
||||
job="smartctl",
|
||||
attribute_value_type="raw",
|
||||
attribute_name="UDMA_CRC_Error_Count"
|
||||
}[24h])
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SMART CRC error count increased"
|
||||
description: "UDMA_CRC_Error_Count increased in the last 24 hours. Check SATA cable, backplane, and power path."
|
||||
|
||||
- alert: SmartctlErrorLogIncreased
|
||||
expr: |
|
||||
(
|
||||
delta(smartctl_device_error_log_count{job="smartctl"}[24h])
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SMART error log increased"
|
||||
description: "SMART error log count increased in the last 24 hours. Check smartctl output and the alert labels."
|
||||
|
||||
- alert: SmartctlSelfTestErrorsPresent
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_self_test_log_error_count{job="smartctl"}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl"}
|
||||
) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SMART self-test errors present"
|
||||
description: "SMART self-test error count is non-zero. Check self-test history and rerun a long self-test if needed."
|
||||
|
||||
- alert: SmartctlNvmeCriticalWarning
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_critical_warning{job="smartctl"}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
||||
) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe SMART critical warning"
|
||||
description: "NVMe critical_warning is non-zero. This is a real NVMe health signal; check smartctl output and alert labels."
|
||||
|
||||
- alert: SmartctlNvmeMediaErrorsIncreased
|
||||
expr: |
|
||||
(
|
||||
delta(smartctl_device_media_errors{job="smartctl"}[24h])
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
||||
) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe media errors increased"
|
||||
description: "NVMe media error count increased in the last 24 hours. Check smartctl output and alert labels."
|
||||
|
||||
- alert: SmartctlNvmePercentageUsedHigh
|
||||
expr: |
|
||||
(
|
||||
smartctl_device_percentage_used{job="smartctl"}
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
||||
) > 90
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe percentage used high"
|
||||
description: "NVMe percentage_used is above 90. Check remaining endurance and plan replacement."
|
||||
|
||||
- alert: SmartctlNvmeAvailableSpareLow
|
||||
expr: |
|
||||
(
|
||||
(
|
||||
smartctl_device_available_spare{job="smartctl"}
|
||||
<= on(instance, device)
|
||||
smartctl_device_available_spare_threshold{job="smartctl"}
|
||||
)
|
||||
* on(instance, device) group_left(model_name, serial_number, protocol)
|
||||
smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
|
||||
)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe available spare below threshold"
|
||||
description: "NVMe available_spare is at or below available_spare_threshold. Check smartctl output and alert labels."
|
||||
|
||||
@@ -39,6 +39,14 @@ prometheus.relabel "system_metrics" {
|
||||
target_label = "name"
|
||||
replacement = "$1.socket"
|
||||
}
|
||||
{% if node['name'] == "vmm" %}
|
||||
rule {
|
||||
source_labels = ["__name__"]
|
||||
regex = "smartctl_.+"
|
||||
target_label = "job"
|
||||
replacement = "smartctl"
|
||||
}
|
||||
{% endif %}
|
||||
}
|
||||
|
||||
// Generic service metrics relabel
|
||||
@@ -89,6 +97,11 @@ prometheus.exporter.unix "system" {
|
||||
mount_points_exclude = "^/(boot|sys|proc|dev|run|mnt|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
|
||||
fs_types_exclude = "^(tmpfs|devtmpfs|devfs|iso9660|overlay|aufs|squashfs)$"
|
||||
}
|
||||
{% if node['name'] == "vmm" %}
|
||||
textfile {
|
||||
directory = "/etc/smartctl-exporter"
|
||||
}
|
||||
{% endif %}
|
||||
}
|
||||
|
||||
prometheus.scrape "system" {
|
||||
@@ -228,6 +241,12 @@ loki.relabel "journal_logs" {
|
||||
target_label = "job"
|
||||
replacement = "$1.socket"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_syslog_identifier"]
|
||||
regex = "podman"
|
||||
target_label = "job"
|
||||
replacement = "podman"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["job"]
|
||||
regex = "(.+)"
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=SMART Metric generator
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
# Run the script
|
||||
ExecStart=/usr/local/sbin/smart_metric_generator.sh
|
||||
@@ -0,0 +1,352 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
out_dir="${OUT_DIR:-/etc/smartctl-exporter}"
|
||||
smartctl_bin="${SMARTCTL:-/usr/sbin/smartctl}"
|
||||
|
||||
tmp="$(mktemp "${out_dir}/smartctl.prom.XXXXXX")"
|
||||
out="${out_dir}/smartctl.prom"
|
||||
|
||||
cleanup() {
|
||||
rm -f "$tmp"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
discover_devices() {
|
||||
for sys in /sys/class/block/*; do
|
||||
dev="${sys##*/}"
|
||||
|
||||
[ -e "$sys/device" ] || continue
|
||||
[ ! -e "$sys/partition" ] || continue
|
||||
|
||||
case "$dev" in
|
||||
loop*|ram*|zram*|dm-*|md*|nbd*|sr*|fd*)
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
sys_path="$(readlink -f "$sys" 2>/dev/null || true)"
|
||||
|
||||
case "$sys_path" in
|
||||
*usb*)
|
||||
continue
|
||||
;;
|
||||
*/virtual/*)
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
[ -b "/dev/$dev" ] || continue
|
||||
printf '%s\n' "/dev/$dev"
|
||||
done | sort -V
|
||||
}
|
||||
|
||||
write_header() {
|
||||
cat <<'HDR'
|
||||
# HELP smartctl_version smartctl version
|
||||
# TYPE smartctl_version gauge
|
||||
# HELP smartctl_device Device info
|
||||
# TYPE smartctl_device gauge
|
||||
# HELP smartctl_devices Number of devices configured or dynamically discovered
|
||||
# TYPE smartctl_devices gauge
|
||||
# HELP smartctl_device_capacity_blocks Device capacity in blocks
|
||||
# TYPE smartctl_device_capacity_blocks gauge
|
||||
# HELP smartctl_device_capacity_bytes Device capacity in bytes
|
||||
# TYPE smartctl_device_capacity_bytes gauge
|
||||
# HELP smartctl_device_nvme_capacity_bytes NVMe device total capacity bytes
|
||||
# TYPE smartctl_device_nvme_capacity_bytes gauge
|
||||
# HELP smartctl_device_block_size Device block size
|
||||
# TYPE smartctl_device_block_size gauge
|
||||
# HELP smartctl_device_interface_speed Device interface speed, bits per second
|
||||
# TYPE smartctl_device_interface_speed gauge
|
||||
# HELP smartctl_device_attribute Device attributes
|
||||
# TYPE smartctl_device_attribute gauge
|
||||
# HELP smartctl_device_power_on_seconds Device power on seconds
|
||||
# TYPE smartctl_device_power_on_seconds counter
|
||||
# HELP smartctl_device_rotation_rate Device rotation rate
|
||||
# TYPE smartctl_device_rotation_rate gauge
|
||||
# HELP smartctl_device_temperature Device temperature celsius
|
||||
# TYPE smartctl_device_temperature gauge
|
||||
# HELP smartctl_device_power_cycle_count Device power cycle count
|
||||
# TYPE smartctl_device_power_cycle_count counter
|
||||
# HELP smartctl_device_percentage_used Device write percentage used
|
||||
# TYPE smartctl_device_percentage_used counter
|
||||
# HELP smartctl_device_available_spare Normalized percentage of the remaining spare capacity available
|
||||
# TYPE smartctl_device_available_spare counter
|
||||
# HELP smartctl_device_available_spare_threshold Available spare threshold
|
||||
# TYPE smartctl_device_available_spare_threshold counter
|
||||
# HELP smartctl_device_critical_warning Critical warnings for the state of the controller
|
||||
# TYPE smartctl_device_critical_warning counter
|
||||
# HELP smartctl_device_media_errors Media/data integrity errors
|
||||
# TYPE smartctl_device_media_errors counter
|
||||
# HELP smartctl_device_num_err_log_entries Number of error information log entries
|
||||
# TYPE smartctl_device_num_err_log_entries counter
|
||||
# HELP smartctl_device_bytes_read Device bytes read
|
||||
# TYPE smartctl_device_bytes_read counter
|
||||
# HELP smartctl_device_bytes_written Device bytes written
|
||||
# TYPE smartctl_device_bytes_written counter
|
||||
# HELP smartctl_device_smart_status General smart status
|
||||
# TYPE smartctl_device_smart_status gauge
|
||||
# HELP smartctl_device_smartctl_exit_status Exit status of smartctl on device
|
||||
# TYPE smartctl_device_smartctl_exit_status gauge
|
||||
# HELP smartctl_device_state Device state
|
||||
# TYPE smartctl_device_state gauge
|
||||
# HELP smartctl_device_statistics Device statistics
|
||||
# TYPE smartctl_device_statistics gauge
|
||||
# HELP smartctl_device_error_log_count Device SMART error log count
|
||||
# TYPE smartctl_device_error_log_count gauge
|
||||
# HELP smartctl_device_self_test_log_count Device SMART self test log count
|
||||
# TYPE smartctl_device_self_test_log_count gauge
|
||||
# HELP smartctl_device_self_test_log_error_count Device SMART self test log error count
|
||||
# TYPE smartctl_device_self_test_log_error_count gauge
|
||||
# HELP smartctl_device_erc_seconds Device SMART Error Recovery Control Seconds
|
||||
# TYPE smartctl_device_erc_seconds gauge
|
||||
# HELP smartctl_scsi_grown_defect_list Device SCSI grown defect list counter
|
||||
# TYPE smartctl_scsi_grown_defect_list gauge
|
||||
# HELP smartctl_read_errors_corrected_by_rereads_rewrites Read Errors Corrected by ReReads/ReWrites
|
||||
# TYPE smartctl_read_errors_corrected_by_rereads_rewrites gauge
|
||||
# HELP smartctl_read_errors_corrected_by_eccfast Read Errors Corrected by ECC Fast
|
||||
# TYPE smartctl_read_errors_corrected_by_eccfast gauge
|
||||
# HELP smartctl_read_errors_corrected_by_eccdelayed Read Errors Corrected by ECC Delayed
|
||||
# TYPE smartctl_read_errors_corrected_by_eccdelayed gauge
|
||||
# HELP smartctl_read_total_uncorrected_errors Read Total Uncorrected Errors
|
||||
# TYPE smartctl_read_total_uncorrected_errors gauge
|
||||
# HELP smartctl_write_errors_corrected_by_rereads_rewrites Write Errors Corrected by ReReads/ReWrites
|
||||
# TYPE smartctl_write_errors_corrected_by_rereads_rewrites gauge
|
||||
# HELP smartctl_write_errors_corrected_by_eccfast Write Errors Corrected by ECC Fast
|
||||
# TYPE smartctl_write_errors_corrected_by_eccfast gauge
|
||||
# HELP smartctl_write_errors_corrected_by_eccdelayed Write Errors Corrected by ECC Delayed
|
||||
# TYPE smartctl_write_errors_corrected_by_eccdelayed gauge
|
||||
# HELP smartctl_write_total_uncorrected_errors Write Total Uncorrected Errors
|
||||
# TYPE smartctl_write_total_uncorrected_errors gauge
|
||||
HDR
|
||||
}
|
||||
|
||||
emit_version() {
|
||||
jq -r '
|
||||
def esc: tostring | gsub("\\\\";"\\\\") | gsub("\"";"\\\"");
|
||||
(.json_format_version // [0,0]) as $jv
|
||||
| (.smartctl.version // [0,0]) as $sv
|
||||
| "smartctl_version{json_format_version=\"\($jv[0]).\($jv[1])\",smartctl_version=\"\($sv[0]).\($sv[1])\",svn_revision=\"\((.smartctl.svn_revision // "")|esc)\",build_info=\"\((.smartctl.build_info // "")|esc)\"} 1"
|
||||
'
|
||||
}
|
||||
|
||||
emit_metrics() {
|
||||
jq -r '
|
||||
def esc: tostring | gsub("\\\\";"\\\\") | gsub("\"";"\\\"");
|
||||
def lbl($pairs): $pairs | map("\(.[0])=\"\(.[1]|esc)\"") | join(",");
|
||||
def n($x):
|
||||
if $x == true then 1
|
||||
elif $x == false then 0
|
||||
elif $x == null then 0
|
||||
else $x
|
||||
end;
|
||||
def emit($name; $pairs; $value):
|
||||
"\($name){\(lbl($pairs))} \(n($value))";
|
||||
def emit_if_exists($name; $pairs; $value):
|
||||
if $value == null then empty else emit($name; $pairs; $value) end;
|
||||
def trimstr: tostring | gsub("^\\s+|\\s+$";"");
|
||||
def build_device_label($name; $type):
|
||||
($name | sub("^/dev/(disk/by-id/|disk/by-path/|)"; "") | gsub("/"; "_")) as $base
|
||||
| if ($type | contains(",")) then
|
||||
"\($base)_\($type | gsub(","; "_"))"
|
||||
else
|
||||
$base
|
||||
end;
|
||||
def longflags($obj; $names):
|
||||
[$names[] as $k | select(($obj[$k] // false) == true) | $k] | join(",");
|
||||
|
||||
. as $j
|
||||
| build_device_label(($j.device.name // ""); ($j.device.type // "")) as $device
|
||||
| ($j.device.type // "") as $interface
|
||||
| ($j.device.protocol // "") as $protocol
|
||||
| [
|
||||
emit("smartctl_device_smartctl_exit_status"; [["device", $device]]; $j.smartctl.exit_status),
|
||||
|
||||
emit("smartctl_device"; [
|
||||
["device", $device],
|
||||
["interface", $interface],
|
||||
["protocol", $protocol],
|
||||
["model_family", (($j.model_family // "unknown") | trimstr)],
|
||||
["model_name", (($j.model_name // $j.scsi_model_name // "unknown") | if . == "" then "unknown" else . end | trimstr)],
|
||||
["serial_number", (($j.serial_number // "") | trimstr)],
|
||||
["ata_additional_product_id", ($j.ata_additional_product_id // "unknown")],
|
||||
["firmware_version", ($j.firmware_version // "")],
|
||||
["ata_version", ($j.ata_version.string // "")],
|
||||
["sata_version", ($j.sata_version.string // "")],
|
||||
["form_factor", ($j.form_factor.name // "")],
|
||||
["scsi_vendor", ($j.scsi_vendor // "")],
|
||||
["scsi_product", ($j.scsi_product // "")],
|
||||
["scsi_revision", ($j.scsi_revision // "")],
|
||||
["scsi_version", ($j.scsi_version // "")]
|
||||
]; 1),
|
||||
|
||||
emit("smartctl_device_capacity_blocks"; [["device", $device]]; $j.user_capacity.blocks),
|
||||
emit("smartctl_device_capacity_bytes"; [["device", $device]]; $j.user_capacity.bytes),
|
||||
emit_if_exists("smartctl_device_nvme_capacity_bytes"; [["device", $device]]; $j.nvme_total_capacity),
|
||||
|
||||
emit("smartctl_device_block_size"; [["device", $device], ["blocks_type", "logical"]]; $j.logical_block_size),
|
||||
emit("smartctl_device_block_size"; [["device", $device], ["blocks_type", "physical"]]; $j.physical_block_size),
|
||||
|
||||
(if $j.interface_speed != null then
|
||||
($j.interface_speed
|
||||
| to_entries[]
|
||||
| select(.key == "max" or .key == "current")
|
||||
| emit("smartctl_device_interface_speed";
|
||||
[["device", $device], ["speed_type", .key]];
|
||||
n(.value.units_per_second) * n(.value.bits_per_unit)))
|
||||
else empty end),
|
||||
|
||||
($j.ata_smart_attributes.table[]?
|
||||
| . as $a
|
||||
| longflags(($a.flags // {}); ["prefailure", "updated_online", "performance", "error_rate", "event_count", "auto_keep"]) as $lf
|
||||
| [
|
||||
["value", $a.value],
|
||||
["worst", $a.worst],
|
||||
["thresh", $a.thresh],
|
||||
["raw", $a.raw.value]
|
||||
][]
|
||||
| emit("smartctl_device_attribute"; [
|
||||
["device", $device],
|
||||
["attribute_name", (($a.name // "") | trimstr)],
|
||||
["attribute_flags_short", (($a.flags.string // "") | trimstr)],
|
||||
["attribute_flags_long", $lf],
|
||||
["attribute_value_type", .[0]],
|
||||
["attribute_id", (($a.id // "") | tostring)]
|
||||
]; .[1])
|
||||
),
|
||||
|
||||
(if $j.power_on_time != null then
|
||||
emit("smartctl_device_power_on_seconds"; [["device", $device]];
|
||||
n($j.power_on_time.hours) * 3600 + n($j.power_on_time.minutes) * 60)
|
||||
else empty end),
|
||||
|
||||
(if n($j.rotation_rate) > 0 then
|
||||
emit("smartctl_device_rotation_rate"; [["device", $device]]; $j.rotation_rate)
|
||||
else empty end),
|
||||
|
||||
($j.temperature? // {}
|
||||
| to_entries[]?
|
||||
| emit("smartctl_device_temperature"; [["device", $device], ["temperature_type", .key]]; .value)
|
||||
),
|
||||
|
||||
(if $j.power_cycle_count != null then
|
||||
emit("smartctl_device_power_cycle_count"; [["device", $device]]; $j.power_cycle_count)
|
||||
elif $j.scsi_start_stop_cycle_counter.accumulated_start_stop_cycles != null then
|
||||
emit("smartctl_device_power_cycle_count"; [["device", $device]]; $j.scsi_start_stop_cycle_counter.accumulated_start_stop_cycles)
|
||||
else empty end),
|
||||
|
||||
(if $j.ata_sct_status != null then
|
||||
emit("smartctl_device_state"; [["device", $device]]; $j.ata_sct_status.device_state)
|
||||
else empty end),
|
||||
|
||||
(if $j.smart_status != null then
|
||||
emit("smartctl_device_smart_status"; [["device", $device]];
|
||||
if $j.smart_status.passed == true then 1
|
||||
elif $j.smart_status.passed == false then 0
|
||||
else $j.smart_status.passed
|
||||
end)
|
||||
else empty end),
|
||||
|
||||
($j.ata_device_statistics.pages[]?
|
||||
| select(((.name // "") | trimstr) != "Vendor Specific Statistics") as $page
|
||||
| $page.table[]?
|
||||
| emit("smartctl_device_statistics"; [
|
||||
["device", $device],
|
||||
["statistic_table", (($page.name // "") | trimstr)],
|
||||
["statistic_name", ((.name // "") | trimstr)],
|
||||
["statistic_flags_short", ((.flags.string // "") | trimstr)],
|
||||
["statistic_flags_long", longflags((.flags // {}); ["valid", "normalized", "supports_dsn", "monitored_condition_met"])]
|
||||
]; .value)
|
||||
),
|
||||
|
||||
($j.sata_phy_event_counters.table[]?
|
||||
| emit("smartctl_device_statistics"; [
|
||||
["device", $device],
|
||||
["statistic_table", "SATA PHY Event Counters"],
|
||||
["statistic_name", ((.name // "") | trimstr)],
|
||||
["statistic_flags_short", "V---"],
|
||||
["statistic_flags_long", "valid"]
|
||||
]; .value)
|
||||
),
|
||||
|
||||
($j.ata_smart_error_log? // {}
|
||||
| to_entries[]?
|
||||
| emit("smartctl_device_error_log_count"; [["device", $device], ["error_log_type", .key]]; .value.count)
|
||||
),
|
||||
|
||||
($j.ata_smart_self_test_log? // {}
|
||||
| to_entries[]?
|
||||
| emit("smartctl_device_self_test_log_count"; [["device", $device], ["self_test_log_type", .key]]; .value.count)
|
||||
),
|
||||
|
||||
($j.ata_smart_self_test_log? // {}
|
||||
| to_entries[]?
|
||||
| emit("smartctl_device_self_test_log_error_count"; [["device", $device], ["self_test_log_type", .key]]; .value.error_count_total)
|
||||
),
|
||||
|
||||
($j.ata_sct_erc? // {}
|
||||
| to_entries[]?
|
||||
| emit("smartctl_device_erc_seconds"; [["device", $device], ["op_type", .key]]; n(.value.deciseconds) / 10.0)
|
||||
),
|
||||
|
||||
(if $interface == "nvme" then
|
||||
emit("smartctl_device_percentage_used"; [["device", $device]]; $j.nvme_smart_health_information_log.percentage_used),
|
||||
emit("smartctl_device_available_spare"; [["device", $device]]; $j.nvme_smart_health_information_log.available_spare),
|
||||
emit("smartctl_device_available_spare_threshold"; [["device", $device]]; $j.nvme_smart_health_information_log.available_spare_threshold),
|
||||
emit("smartctl_device_critical_warning"; [["device", $device]]; $j.nvme_smart_health_information_log.critical_warning),
|
||||
emit("smartctl_device_media_errors"; [["device", $device]]; $j.nvme_smart_health_information_log.media_errors),
|
||||
emit("smartctl_device_num_err_log_entries"; [["device", $device]]; $j.nvme_smart_health_information_log.num_err_log_entries),
|
||||
(if $j.nvme_smart_health_information_log.data_units_read != null and n($j.nvme_smart_health_information_log.data_units_read) != 0 then
|
||||
emit("smartctl_device_bytes_read"; [["device", $device]]; n($j.nvme_smart_health_information_log.data_units_read) * 1000 * 512)
|
||||
else empty end),
|
||||
(if $j.nvme_smart_health_information_log.data_units_written != null and n($j.nvme_smart_health_information_log.data_units_written) != 0 then
|
||||
emit("smartctl_device_bytes_written"; [["device", $device]]; n($j.nvme_smart_health_information_log.data_units_written) * 1000 * 512)
|
||||
else empty end)
|
||||
else empty end),
|
||||
|
||||
(if $interface == "scsi" then
|
||||
emit_if_exists("smartctl_scsi_grown_defect_list"; [["device", $device]]; $j.scsi_grown_defect_list),
|
||||
(if $j.scsi_error_counter_log != null then
|
||||
emit("smartctl_device_bytes_read"; [["device", $device]]; n($j.scsi_error_counter_log.read.gigabytes_processed) * 1000000000),
|
||||
emit("smartctl_device_bytes_written"; [["device", $device]]; n($j.scsi_error_counter_log.write.gigabytes_processed) * 1000000000),
|
||||
emit("smartctl_read_errors_corrected_by_rereads_rewrites"; [["device", $device]]; $j.scsi_error_counter_log.read.errors_corrected_by_rereads_rewrites),
|
||||
emit("smartctl_read_errors_corrected_by_eccfast"; [["device", $device]]; $j.scsi_error_counter_log.read.errors_corrected_by_eccfast),
|
||||
emit("smartctl_read_errors_corrected_by_eccdelayed"; [["device", $device]]; $j.scsi_error_counter_log.read.errors_corrected_by_eccdelayed),
|
||||
emit("smartctl_read_total_uncorrected_errors"; [["device", $device]]; $j.scsi_error_counter_log.read.total_uncorrected_errors),
|
||||
emit("smartctl_write_errors_corrected_by_rereads_rewrites"; [["device", $device]]; $j.scsi_error_counter_log.write.errors_corrected_by_rereads_rewrites),
|
||||
emit("smartctl_write_errors_corrected_by_eccfast"; [["device", $device]]; $j.scsi_error_counter_log.write.errors_corrected_by_eccfast),
|
||||
emit("smartctl_write_errors_corrected_by_eccdelayed"; [["device", $device]]; $j.scsi_error_counter_log.write.errors_corrected_by_eccdelayed),
|
||||
emit("smartctl_write_total_uncorrected_errors"; [["device", $device]]; $j.scsi_error_counter_log.write.total_uncorrected_errors)
|
||||
else empty end)
|
||||
else empty end)
|
||||
]
|
||||
| .[]
|
||||
'
|
||||
}
|
||||
|
||||
write_header > "$tmp"
|
||||
|
||||
version_emitted=0
|
||||
device_count=0
|
||||
|
||||
while IFS= read -r dev; do
|
||||
json="$("$smartctl_bin" -a -j "$dev" 2>/dev/null || true)"
|
||||
|
||||
printf '%s' "$json" | jq -e . >/dev/null 2>&1 || continue
|
||||
|
||||
if [ "$version_emitted" -eq 0 ]; then
|
||||
printf '%s' "$json" | emit_version >> "$tmp"
|
||||
version_emitted=1
|
||||
fi
|
||||
|
||||
if printf '%s' "$json" | emit_metrics >> "$tmp"; then
|
||||
device_count=$((device_count + 1))
|
||||
fi
|
||||
done < <(discover_devices)
|
||||
|
||||
printf 'smartctl_devices %s\n' "$device_count" >> "$tmp"
|
||||
|
||||
chmod 0644 "$tmp"
|
||||
mv "$tmp" "$out"
|
||||
trap - EXIT
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run SMART Metric generator service every 5 minutes
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitInactiveSec=5min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user