feat(smartctl): add smartctl exporter

ansible/inventory/group_vars/all.yaml

@@ -15,10 +15,6 @@ domain:
   dc: "dc=ilnmors,dc=internal"
   org: "ilnmors"
 
-# DNS configuration including bind and blocky should be set manually.
-# named.conf.j2 is also set manually.
-# Check the hosts.j2 when cname records are fixed
-
 services:
   # common
   wireguard:
@@ -31,6 +27,17 @@ services:
       internal: "crowdsec"
     ports:
       https: "8080"
+    sub:
+      machines:
+        - "fw"
+        - "vmm"
+        - "bk"
+        - "infra"
+        - "auth"
+        - "app"
+      bouncers:
+        - "fw"
+        - "caddy"
     dependency:
       cert: true
   kopia:
@@ -272,9 +279,9 @@ version:
   packages:
     sops: "3.13.1"
     step: "0.30.2"
-    kopia: "0.23.0"
+    kopia: "0.23.1"
-    blocky: "0.29.0"
+    blocky: "0.32.1"
-    alloy: "1.16.1"
+    alloy: "1.17.0"
   containers:
     # common
     caddy: "2.11.4"
@@ -291,8 +298,8 @@ version:
     maddy: "0.9.5"
     x509-exporter: "4.1.0"
     prometheus: "v3.12.0"
-    loki: "3.7.1"
+    loki: "3.7.3"
-    grafana: "13.0.1"
+    grafana: "13.0.2"
     # Auth
     authelia: "4.39.20"
     # App

ansible/playbooks/vmm/site.yaml

@@ -143,6 +143,15 @@
       when: inventory_hostname != "vmm_init"
       tags: ["site", "crowdsec"]
 
+    - name: Set smartctl generator
+      ansible.builtin.include_role:
+        name: "vmm"
+        tasks_from: "services/set_smart_metric_generator"
+        apply:
+          tags: ["site", "smart"]
+      when: inventory_hostname != "vmm_init"
+      tags: ["site", "smart"]
+
     - name: Set alloy
       ansible.builtin.include_role:
         name: "common"

ansible/roles/bk/tasks/services/set_rclone.yaml

@@ -1,19 +1,10 @@
 ---
-- name: Check rclone installation
-  ansible.builtin.shell: |
-    command -v rclone
-  changed_when: false
-  failed_when: false
-  register: "is_rclone_installed"
-  ignore_errors: true
-
 - name: Install rclone
   ansible.builtin.apt:
     name:
       - "rclone"
     state: "present"
   become: true
-  when: is_rclone_installed.rc != 0
 
 - name: Create rclone config directory
   ansible.builtin.file:

ansible/roles/common/tasks/node/set_nftables.yaml

@@ -1,19 +1,10 @@
 ---
-- name: Check nftables installation
-  ansible.builtin.shell: |
-    command -v nft
-  become: true # nftables is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_nftables_installed"
-  ignore_errors: true
-
 - name: Install nftables
   ansible.builtin.apt:
-    name: "nftables"
+    name:
+      - "nftables"
     state: "present"
   become: true
-  when: is_nftables_installed.rc != 0
 
 - name: Enable nftables.service
   ansible.builtin.systemd:

ansible/roles/common/tasks/services/set_crowdsec.yaml

@@ -1,36 +1,18 @@
 ---
-- name: Check crowdsec installed
-  ansible.builtin.shell: |
-    command -v crowdsec
-  changed_when: false
-  failed_when: false
-  register: "is_crowdsec_installed"
-  ignore_errors: true
-
-- name: Check crowdsec bouncer installed
-  ansible.builtin.shell: |
-    command -v crowdsec-firewall-bouncer
-  when: node['name'] == "fw"
-  changed_when: false
-  failed_when: false
-  register: "is_crowdsec_bouncer_installed"
-  ignore_errors: true
-
 - name: Install crowdsec
   ansible.builtin.apt:
-    name: "crowdsec"
+    name:
+      - "crowdsec"
     state: "present"
   become: true
-  when: is_crowdsec_installed.rc != 0
 
 - name: Install crowdsec bouncers
   ansible.builtin.apt:
-    name: "crowdsec-firewall-bouncer"
+    name:
+      - "crowdsec-firewall-bouncer"
     state: "present"
   become: true
-  when:
+  when: node['name'] == "fw"
-    - node['name'] == "fw"
-    - is_crowdsec_bouncer_installed.rc != 0
 
 - name: Set acquis.d list for bouncer
   ansible.builtin.set_fact:
@@ -132,7 +114,7 @@
 
     - name: Set goal machines' name
       ansible.builtin.set_fact:
-        machines_name: ["fw", "vmm", "bk", "infra", "auth", "app"]
+        machines_name: "{{ services['crowdsec']['sub']['machines'] | list }}"
       no_log: true
 
     - name: Prune unknown (random) machines
@@ -170,7 +152,7 @@
 
     - name: Set bouncers' name
       ansible.builtin.set_fact:
-        bouncers_name: ["fw", "caddy"]
+        bouncers_name: "{{ services['crowdsec']['sub']['bouncers'] | list }}"
 
     - name: Register Firewall Bouncer to LAPI
       ansible.builtin.command: |

ansible/roles/common/tasks/services/set_kopia.yaml

@@ -52,6 +52,7 @@
         deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         state: "present"
       become: true
+      notify: "notification_restart_kopia_server"
 
     - name: Deploy kopia env
       ansible.builtin.template:

ansible/roles/common/tasks/services/set_podman.yaml

@@ -1,12 +1,4 @@
 ---
-- name: Check podman installation
-  ansible.builtin.shell: |
-    command -v podman
-  changed_when: false
-  failed_when: false
-  register: "is_podman_installed"
-  ignore_errors: true
-
 - name: Create container directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers"
@@ -54,4 +46,3 @@
       - "podman"
     state: "present"
   become: true
-  when: is_podman_installed.rc != 0

ansible/roles/fw/tasks/services/set_bind.yaml

@@ -1,19 +1,10 @@
 ---
-- name: Check bind9 installation
-  ansible.builtin.shell: |
-    command -v named
-  become: true # named is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_bind_installed"
-  ignore_errors: true
-
 - name: Install bind9
   ansible.builtin.apt:
-    name: "bind9"
+    name:
+      - "bind9"
     state: "present"
   become: true
-  when: is_bind_installed.rc != 0
 
 - name: Deploy acme.key
   ansible.builtin.copy:

ansible/roles/fw/tasks/services/set_chrony.yaml

@@ -1,18 +1,10 @@
 ---
-- name: Check chrnoy installation
-  ansible.builtin.shell: |
-    command -v chronyc
-  changed_when: false
-  failed_when: false
-  register: "is_chrony_installed"
-  ignore_errors: true
-
 - name: Install chrony
   ansible.builtin.apt:
-    name: "chrony"
+    name:
+      - "chrony"
     state: "present"
   become: true
-  when: is_chrony_installed.rc != 0
 
 - name: Deploy local acl file
   ansible.builtin.template:

ansible/roles/fw/tasks/services/set_kea.yaml

@@ -1,20 +1,10 @@
 ---
-- name: Check Kea dhcp4 installation
-  ansible.builtin.shell: |
-    command -v kea-dhcp4
-  become: true # kea-dhcp4 is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_kea4_installed"
-  ignore_errors: true
-
 - name: Install kea dhcp 4
   ansible.builtin.apt:
     name:
       - "kea-dhcp4-server"
     state: "present"
   become: true
-  when: is_kea4_installed.rc != 0
 
 - name: Deploy kea dhcp4 conf
   ansible.builtin.template:

ansible/roles/fw/tasks/services/set_suricata.yaml

@@ -1,12 +1,4 @@
 ---
-- name: Check suricata installation
-  ansible.builtin.shell: |
-    command -v suricata
-  changed_when: false
-  failed_when: false
-  register: "is_suricata_installed"
-  ignore_errors: true
-
 - name: Install suricata
   ansible.builtin.apt:
     name:
@@ -14,7 +6,6 @@
       - "suricata-update"
     state: "present"
   become: true
-  when: is_suricata_installed.rc != 0
 
 - name: Deploy suricata-update service files
   ansible.builtin.copy:

ansible/roles/vmm/tasks/node/set_disk.yaml

@@ -1,18 +1,10 @@
 ---
-- name: Check btrfs installation
-  ansible.builtin.shell: |
-    command -v btrfs
-  become: true # btrfs is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_btrfs_installed"
-
 - name: Install btrfs
   ansible.builtin.apt:
-    name: "btrfs-progs"
+    name:
+      - "btrfs-progs"
     state: "present"
   become: true
-  when: is_btrfs_installed.rc != 0
 
 - name: Mount data disk
   block:

ansible/roles/vmm/tasks/services/set_smart_metric_generator.yaml

@@ -0,0 +1,47 @@
+---
+- name: Install smartmontools
+  ansible.builtin.apt:
+    name:
+      - "smartmontools"
+      - "jq"
+    state: "present"
+  become: true
+
+- name: Create SMART metric result directory
+  ansible.builtin.file:
+    path: "/etc/smartctl-exporter"
+    state: "directory"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+  become: true
+
+- name: Deploy smart_metric_generator script
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/smart_metric/smart_metric_generator.sh"
+    dest: "/usr/local/sbin"
+    owner: "root"
+    group: "root"
+    mode: "0711"
+  become: true
+
+- name: Deploy smart_metric_generator service files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/smart_metric/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0600"
+    validate: "/usr/bin/systemd-analyze verify %s"
+  loop:
+    - "smart_metric_generator.service"
+    - "smart_metric_generator.timer"
+  become: true
+
+- name: Register smart_metric_generator timer
+  ansible.builtin.systemd:
+    name: "smart_metric_generator.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+  become: true

config/services/containers/app/homepage/config/bookmarks.yaml.j2

@@ -16,6 +16,9 @@
     - OJ:
         - abbr: SJU
           href: https://oj.sejong.ac.kr/
+    - OPiC:
+        - abbr: ENG
+          href: https://lc.multicampus.com/prodsedu/
 
 - Entertainment:
     - YouTube:

config/services/containers/infra/prometheus/etc/rules.yaml.j2

@@ -23,7 +23,7 @@ groups:
         annotations:
           summary: "Postgresql Heartbeat Lost: postgresql"
           description: "postgresql node is down for 5 mins."
-  - name: Certificate_expiry_check
+  - name: certificate_expiry_check
     rules:
 {% for filename in ['root', 'intermediate'] + (services | dict2items | selectattr('value.dependency.cert', 'defined') | selectattr('value.dependency.cert', 'equalto', true) | map(attribute='key') | list) %}
       - alert: {{ filename | replace('.', '_') }}_is_expired_soon
@@ -36,3 +36,232 @@ groups:
           summary: "{{ filename }}.crt is expired in 30 days"
           description: "{{ filename }}.crt is expired in 30 days."
 {% endfor %}
+  - name: smartctl
+    rules:
+      - alert: SmartctlMetricsMissing
+        expr: |
+          absent_over_time(smartctl_devices{job="smartctl", instance="vmm"}[15m])
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "SMART metrics are missing on vmm"
+          description: "No smartctl_devices metric has been received from vmm for 15 minutes. Check smartctl-generator.timer, Alloy textfile collector, and /etc/smartctl-exporter/smartctl.prom."
+
+      - alert: SmartctlNoDevices
+        expr: |
+          smartctl_devices{job="smartctl", instance="vmm"} < 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "No SMART devices discovered on vmm"
+          description: "smartctl-generator is running, but it reports fewer than one SMART device. Check USB filtering, /sys/class/block discovery, and smartctl output."
+
+      - alert: SmartctlDeviceHealthFailed
+        expr: |
+          (
+            smartctl_device_smart_status{job="smartctl"}
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "SMART health failed"
+          description: "SMART overall-health failed. Check the alert labels for instance, device, model_name, serial_number, and protocol."
+
+      - alert: SmartctlCommandFailed
+        expr: |
+          (
+            (floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 4) % 2)
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl", protocol!~"NVMe|nvme"}
+          ) == 1
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "smartctl command or log read failed"
+          description: "smartctl exit status has bit 2 set on a non-NVMe device. Check smartctl output, cable/backplane, and device logs. NVMe devices are intentionally excluded because bit 2 is often caused by unsupported optional log pages."
+
+      - alert: SmartctlDiskFailing
+        expr: |
+          (
+            (floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 8) % 2)
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) == 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "smartctl reports disk failing"
+          description: "smartctl exit status has bit 3 set. Check the alert labels and replace or evacuate the device if confirmed."
+
+      - alert: SmartctlPrefailAttributeThreshold
+        expr: |
+          (
+            (floor(smartctl_device_smartctl_exit_status{job="smartctl"} / 16) % 2)
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) == 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "SMART prefail attribute threshold hit"
+          description: "smartctl exit status has bit 4 set. Check the alert labels and the SMART attribute table."
+
+      - alert: SmartctlDiskTemperatureHigh
+        expr: |
+          (
+            smartctl_device_temperature{job="smartctl", temperature_type="current"}
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) > 50
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "SMART disk temperature high"
+          description: "Disk temperature is above 50C for 15 minutes. Check the alert labels for the affected disk."
+
+      - alert: SmartctlDiskTemperatureCritical
+        expr: |
+          (
+            smartctl_device_temperature{job="smartctl", temperature_type="current"}
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) > 55
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "SMART disk temperature critical"
+          description: "Disk temperature is above 55C. Check cooling and disk workload immediately."
+
+      - alert: SmartctlCriticalAttributeNonZero
+        expr: |
+          (
+            smartctl_device_attribute{
+              job="smartctl",
+              attribute_value_type="raw",
+              attribute_name=~"Reallocated_Sector_Ct|Current_Pending_Sector|Offline_Uncorrectable|Spin_Retry_Count"
+            }
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) > 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "SMART critical attribute is non-zero"
+          description: "A critical SMART raw attribute is non-zero. Check the alert labels for attribute_name, instance, device, model_name, and serial_number."
+
+      - alert: SmartctlCrcErrorIncreased
+        expr: |
+          (
+            delta(smartctl_device_attribute{
+              job="smartctl",
+              attribute_value_type="raw",
+              attribute_name="UDMA_CRC_Error_Count"
+            }[24h])
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "SMART CRC error count increased"
+          description: "UDMA_CRC_Error_Count increased in the last 24 hours. Check SATA cable, backplane, and power path."
+
+      - alert: SmartctlErrorLogIncreased
+        expr: |
+          (
+            delta(smartctl_device_error_log_count{job="smartctl"}[24h])
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "SMART error log increased"
+          description: "SMART error log count increased in the last 24 hours. Check smartctl output and the alert labels."
+
+      - alert: SmartctlSelfTestErrorsPresent
+        expr: |
+          (
+            smartctl_device_self_test_log_error_count{job="smartctl"}
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl"}
+          ) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "SMART self-test errors present"
+          description: "SMART self-test error count is non-zero. Check self-test history and rerun a long self-test if needed."
+
+      - alert: SmartctlNvmeCriticalWarning
+        expr: |
+          (
+            smartctl_device_critical_warning{job="smartctl"}
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
+          ) > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "NVMe SMART critical warning"
+          description: "NVMe critical_warning is non-zero. This is a real NVMe health signal; check smartctl output and alert labels."
+
+      - alert: SmartctlNvmeMediaErrorsIncreased
+        expr: |
+          (
+            delta(smartctl_device_media_errors{job="smartctl"}[24h])
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
+          ) > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "NVMe media errors increased"
+          description: "NVMe media error count increased in the last 24 hours. Check smartctl output and alert labels."
+
+      - alert: SmartctlNvmePercentageUsedHigh
+        expr: |
+          (
+            smartctl_device_percentage_used{job="smartctl"}
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
+          ) > 90
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "NVMe percentage used high"
+          description: "NVMe percentage_used is above 90. Check remaining endurance and plan replacement."
+
+      - alert: SmartctlNvmeAvailableSpareLow
+        expr: |
+          (
+            (
+              smartctl_device_available_spare{job="smartctl"}
+              <= on(instance, device)
+              smartctl_device_available_spare_threshold{job="smartctl"}
+            )
+            * on(instance, device) group_left(model_name, serial_number, protocol)
+              smartctl_device{job="smartctl", protocol=~"NVMe|nvme"}
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "NVMe available spare below threshold"
+          description: "NVMe available_spare is at or below available_spare_threshold. Check smartctl output and alert labels."

config/services/systemd/common/alloy/config.alloy.j2

@@ -39,6 +39,14 @@ prometheus.relabel "system_metrics" {
         target_label  = "name"
         replacement   = "$1.socket"
     }
+{% if node['name'] == "vmm" %}
+    rule {
+        source_labels = ["__name__"]
+        regex         = "smartctl_.+"
+        target_label  = "job"
+        replacement   = "smartctl"
+    }
+{% endif %}
 }
 
 // Generic service metrics relabel
@@ -89,6 +97,11 @@ prometheus.exporter.unix "system" {
         mount_points_exclude = "^/(boot|sys|proc|dev|run|mnt|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
         fs_types_exclude     = "^(tmpfs|devtmpfs|devfs|iso9660|overlay|aufs|squashfs)$"
     }
+{% if node['name'] == "vmm" %}
+    textfile {
+        directory = "/etc/smartctl-exporter"
+    }
+{% endif %}
 }
 
 prometheus.scrape "system" {
@@ -228,6 +241,12 @@ loki.relabel "journal_logs" {
         target_label  = "job"
         replacement   = "$1.socket"
     }
+    rule {
+        source_labels = ["__journal_syslog_identifier"]
+        regex         = "podman"
+        target_label  = "job"
+        replacement   = "podman"
+    }
     rule {
         source_labels = ["job"]
         regex         = "(.+)"

config/services/systemd/vmm/smart_metric/smart_metric_generator.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=SMART Metric generator
+
+[Service]
+Type=oneshot
+
+StandardOutput=journal
+StandardError=journal
+
+# Run the script
+ExecStart=/usr/local/sbin/smart_metric_generator.sh

config/services/systemd/vmm/smart_metric/smart_metric_generator.sh

@@ -0,0 +1,352 @@
+#!/bin/bash
+set -euo pipefail
+
+out_dir="${OUT_DIR:-/etc/smartctl-exporter}"
+smartctl_bin="${SMARTCTL:-/usr/sbin/smartctl}"
+
+tmp="$(mktemp "${out_dir}/smartctl.prom.XXXXXX")"
+out="${out_dir}/smartctl.prom"
+
+cleanup() {
+  rm -f "$tmp"
+}
+trap cleanup EXIT
+
+discover_devices() {
+  for sys in /sys/class/block/*; do
+    dev="${sys##*/}"
+
+    [ -e "$sys/device" ] || continue
+    [ ! -e "$sys/partition" ] || continue
+
+    case "$dev" in
+      loop*|ram*|zram*|dm-*|md*|nbd*|sr*|fd*)
+        continue
+        ;;
+    esac
+
+    sys_path="$(readlink -f "$sys" 2>/dev/null || true)"
+
+    case "$sys_path" in
+      *usb*)
+        continue
+        ;;
+      */virtual/*)
+        continue
+        ;;
+    esac
+
+    [ -b "/dev/$dev" ] || continue
+    printf '%s\n' "/dev/$dev"
+  done | sort -V
+}
+
+write_header() {
+  cat <<'HDR'
+# HELP smartctl_version smartctl version
+# TYPE smartctl_version gauge
+# HELP smartctl_device Device info
+# TYPE smartctl_device gauge
+# HELP smartctl_devices Number of devices configured or dynamically discovered
+# TYPE smartctl_devices gauge
+# HELP smartctl_device_capacity_blocks Device capacity in blocks
+# TYPE smartctl_device_capacity_blocks gauge
+# HELP smartctl_device_capacity_bytes Device capacity in bytes
+# TYPE smartctl_device_capacity_bytes gauge
+# HELP smartctl_device_nvme_capacity_bytes NVMe device total capacity bytes
+# TYPE smartctl_device_nvme_capacity_bytes gauge
+# HELP smartctl_device_block_size Device block size
+# TYPE smartctl_device_block_size gauge
+# HELP smartctl_device_interface_speed Device interface speed, bits per second
+# TYPE smartctl_device_interface_speed gauge
+# HELP smartctl_device_attribute Device attributes
+# TYPE smartctl_device_attribute gauge
+# HELP smartctl_device_power_on_seconds Device power on seconds
+# TYPE smartctl_device_power_on_seconds counter
+# HELP smartctl_device_rotation_rate Device rotation rate
+# TYPE smartctl_device_rotation_rate gauge
+# HELP smartctl_device_temperature Device temperature celsius
+# TYPE smartctl_device_temperature gauge
+# HELP smartctl_device_power_cycle_count Device power cycle count
+# TYPE smartctl_device_power_cycle_count counter
+# HELP smartctl_device_percentage_used Device write percentage used
+# TYPE smartctl_device_percentage_used counter
+# HELP smartctl_device_available_spare Normalized percentage of the remaining spare capacity available
+# TYPE smartctl_device_available_spare counter
+# HELP smartctl_device_available_spare_threshold Available spare threshold
+# TYPE smartctl_device_available_spare_threshold counter
+# HELP smartctl_device_critical_warning Critical warnings for the state of the controller
+# TYPE smartctl_device_critical_warning counter
+# HELP smartctl_device_media_errors Media/data integrity errors
+# TYPE smartctl_device_media_errors counter
+# HELP smartctl_device_num_err_log_entries Number of error information log entries
+# TYPE smartctl_device_num_err_log_entries counter
+# HELP smartctl_device_bytes_read Device bytes read
+# TYPE smartctl_device_bytes_read counter
+# HELP smartctl_device_bytes_written Device bytes written
+# TYPE smartctl_device_bytes_written counter
+# HELP smartctl_device_smart_status General smart status
+# TYPE smartctl_device_smart_status gauge
+# HELP smartctl_device_smartctl_exit_status Exit status of smartctl on device
+# TYPE smartctl_device_smartctl_exit_status gauge
+# HELP smartctl_device_state Device state
+# TYPE smartctl_device_state gauge
+# HELP smartctl_device_statistics Device statistics
+# TYPE smartctl_device_statistics gauge
+# HELP smartctl_device_error_log_count Device SMART error log count
+# TYPE smartctl_device_error_log_count gauge
+# HELP smartctl_device_self_test_log_count Device SMART self test log count
+# TYPE smartctl_device_self_test_log_count gauge
+# HELP smartctl_device_self_test_log_error_count Device SMART self test log error count
+# TYPE smartctl_device_self_test_log_error_count gauge
+# HELP smartctl_device_erc_seconds Device SMART Error Recovery Control Seconds
+# TYPE smartctl_device_erc_seconds gauge
+# HELP smartctl_scsi_grown_defect_list Device SCSI grown defect list counter
+# TYPE smartctl_scsi_grown_defect_list gauge
+# HELP smartctl_read_errors_corrected_by_rereads_rewrites Read Errors Corrected by ReReads/ReWrites
+# TYPE smartctl_read_errors_corrected_by_rereads_rewrites gauge
+# HELP smartctl_read_errors_corrected_by_eccfast Read Errors Corrected by ECC Fast
+# TYPE smartctl_read_errors_corrected_by_eccfast gauge
+# HELP smartctl_read_errors_corrected_by_eccdelayed Read Errors Corrected by ECC Delayed
+# TYPE smartctl_read_errors_corrected_by_eccdelayed gauge
+# HELP smartctl_read_total_uncorrected_errors Read Total Uncorrected Errors
+# TYPE smartctl_read_total_uncorrected_errors gauge
+# HELP smartctl_write_errors_corrected_by_rereads_rewrites Write Errors Corrected by ReReads/ReWrites
+# TYPE smartctl_write_errors_corrected_by_rereads_rewrites gauge
+# HELP smartctl_write_errors_corrected_by_eccfast Write Errors Corrected by ECC Fast
+# TYPE smartctl_write_errors_corrected_by_eccfast gauge
+# HELP smartctl_write_errors_corrected_by_eccdelayed Write Errors Corrected by ECC Delayed
+# TYPE smartctl_write_errors_corrected_by_eccdelayed gauge
+# HELP smartctl_write_total_uncorrected_errors Write Total Uncorrected Errors
+# TYPE smartctl_write_total_uncorrected_errors gauge
+HDR
+}
+
+emit_version() {
+  jq -r '
+    def esc: tostring | gsub("\\\\";"\\\\") | gsub("\"";"\\\"");
+    (.json_format_version // [0,0]) as $jv
+    | (.smartctl.version // [0,0]) as $sv
+    | "smartctl_version{json_format_version=\"\($jv[0]).\($jv[1])\",smartctl_version=\"\($sv[0]).\($sv[1])\",svn_revision=\"\((.smartctl.svn_revision // "")|esc)\",build_info=\"\((.smartctl.build_info // "")|esc)\"} 1"
+  '
+}
+
+emit_metrics() {
+  jq -r '
+    def esc: tostring | gsub("\\\\";"\\\\") | gsub("\"";"\\\"");
+    def lbl($pairs): $pairs | map("\(.[0])=\"\(.[1]|esc)\"") | join(",");
+    def n($x):
+      if $x == true then 1
+      elif $x == false then 0
+      elif $x == null then 0
+      else $x
+      end;
+    def emit($name; $pairs; $value):
+      "\($name){\(lbl($pairs))} \(n($value))";
+    def emit_if_exists($name; $pairs; $value):
+      if $value == null then empty else emit($name; $pairs; $value) end;
+    def trimstr: tostring | gsub("^\\s+|\\s+$";"");
+    def build_device_label($name; $type):
+      ($name | sub("^/dev/(disk/by-id/|disk/by-path/|)"; "") | gsub("/"; "_")) as $base
+      | if ($type | contains(",")) then
+          "\($base)_\($type | gsub(","; "_"))"
+        else
+          $base
+        end;
+    def longflags($obj; $names):
+      [$names[] as $k | select(($obj[$k] // false) == true) | $k] | join(",");
+
+    . as $j
+    | build_device_label(($j.device.name // ""); ($j.device.type // "")) as $device
+    | ($j.device.type // "") as $interface
+    | ($j.device.protocol // "") as $protocol
+    | [
+        emit("smartctl_device_smartctl_exit_status"; [["device", $device]]; $j.smartctl.exit_status),
+
+        emit("smartctl_device"; [
+          ["device", $device],
+          ["interface", $interface],
+          ["protocol", $protocol],
+          ["model_family", (($j.model_family // "unknown") | trimstr)],
+          ["model_name", (($j.model_name // $j.scsi_model_name // "unknown") | if . == "" then "unknown" else . end | trimstr)],
+          ["serial_number", (($j.serial_number // "") | trimstr)],
+          ["ata_additional_product_id", ($j.ata_additional_product_id // "unknown")],
+          ["firmware_version", ($j.firmware_version // "")],
+          ["ata_version", ($j.ata_version.string // "")],
+          ["sata_version", ($j.sata_version.string // "")],
+          ["form_factor", ($j.form_factor.name // "")],
+          ["scsi_vendor", ($j.scsi_vendor // "")],
+          ["scsi_product", ($j.scsi_product // "")],
+          ["scsi_revision", ($j.scsi_revision // "")],
+          ["scsi_version", ($j.scsi_version // "")]
+        ]; 1),
+
+        emit("smartctl_device_capacity_blocks"; [["device", $device]]; $j.user_capacity.blocks),
+        emit("smartctl_device_capacity_bytes"; [["device", $device]]; $j.user_capacity.bytes),
+        emit_if_exists("smartctl_device_nvme_capacity_bytes"; [["device", $device]]; $j.nvme_total_capacity),
+
+        emit("smartctl_device_block_size"; [["device", $device], ["blocks_type", "logical"]]; $j.logical_block_size),
+        emit("smartctl_device_block_size"; [["device", $device], ["blocks_type", "physical"]]; $j.physical_block_size),
+
+        (if $j.interface_speed != null then
+          ($j.interface_speed
+            | to_entries[]
+            | select(.key == "max" or .key == "current")
+            | emit("smartctl_device_interface_speed";
+                [["device", $device], ["speed_type", .key]];
+                n(.value.units_per_second) * n(.value.bits_per_unit)))
+         else empty end),
+
+        ($j.ata_smart_attributes.table[]?
+          | . as $a
+          | longflags(($a.flags // {}); ["prefailure", "updated_online", "performance", "error_rate", "event_count", "auto_keep"]) as $lf
+          | [
+              ["value",  $a.value],
+              ["worst",  $a.worst],
+              ["thresh", $a.thresh],
+              ["raw",    $a.raw.value]
+            ][]
+          | emit("smartctl_device_attribute"; [
+              ["device", $device],
+              ["attribute_name", (($a.name // "") | trimstr)],
+              ["attribute_flags_short", (($a.flags.string // "") | trimstr)],
+              ["attribute_flags_long", $lf],
+              ["attribute_value_type", .[0]],
+              ["attribute_id", (($a.id // "") | tostring)]
+            ]; .[1])
+        ),
+
+        (if $j.power_on_time != null then
+          emit("smartctl_device_power_on_seconds"; [["device", $device]];
+            n($j.power_on_time.hours) * 3600 + n($j.power_on_time.minutes) * 60)
+         else empty end),
+
+        (if n($j.rotation_rate) > 0 then
+          emit("smartctl_device_rotation_rate"; [["device", $device]]; $j.rotation_rate)
+         else empty end),
+
+        ($j.temperature? // {}
+          | to_entries[]?
+          | emit("smartctl_device_temperature"; [["device", $device], ["temperature_type", .key]]; .value)
+        ),
+
+        (if $j.power_cycle_count != null then
+          emit("smartctl_device_power_cycle_count"; [["device", $device]]; $j.power_cycle_count)
+         elif $j.scsi_start_stop_cycle_counter.accumulated_start_stop_cycles != null then
+          emit("smartctl_device_power_cycle_count"; [["device", $device]]; $j.scsi_start_stop_cycle_counter.accumulated_start_stop_cycles)
+         else empty end),
+
+        (if $j.ata_sct_status != null then
+          emit("smartctl_device_state"; [["device", $device]]; $j.ata_sct_status.device_state)
+         else empty end),
+
+        (if $j.smart_status != null then
+          emit("smartctl_device_smart_status"; [["device", $device]];
+            if $j.smart_status.passed == true then 1
+            elif $j.smart_status.passed == false then 0
+            else $j.smart_status.passed
+            end)
+         else empty end),
+
+        ($j.ata_device_statistics.pages[]?
+          | select(((.name // "") | trimstr) != "Vendor Specific Statistics") as $page
+          | $page.table[]?
+          | emit("smartctl_device_statistics"; [
+              ["device", $device],
+              ["statistic_table", (($page.name // "") | trimstr)],
+              ["statistic_name", ((.name // "") | trimstr)],
+              ["statistic_flags_short", ((.flags.string // "") | trimstr)],
+              ["statistic_flags_long", longflags((.flags // {}); ["valid", "normalized", "supports_dsn", "monitored_condition_met"])]
+            ]; .value)
+        ),
+
+        ($j.sata_phy_event_counters.table[]?
+          | emit("smartctl_device_statistics"; [
+              ["device", $device],
+              ["statistic_table", "SATA PHY Event Counters"],
+              ["statistic_name", ((.name // "") | trimstr)],
+              ["statistic_flags_short", "V---"],
+              ["statistic_flags_long", "valid"]
+            ]; .value)
+        ),
+
+        ($j.ata_smart_error_log? // {}
+          | to_entries[]?
+          | emit("smartctl_device_error_log_count"; [["device", $device], ["error_log_type", .key]]; .value.count)
+        ),
+
+        ($j.ata_smart_self_test_log? // {}
+          | to_entries[]?
+          | emit("smartctl_device_self_test_log_count"; [["device", $device], ["self_test_log_type", .key]]; .value.count)
+        ),
+
+        ($j.ata_smart_self_test_log? // {}
+          | to_entries[]?
+          | emit("smartctl_device_self_test_log_error_count"; [["device", $device], ["self_test_log_type", .key]]; .value.error_count_total)
+        ),
+
+        ($j.ata_sct_erc? // {}
+          | to_entries[]?
+          | emit("smartctl_device_erc_seconds"; [["device", $device], ["op_type", .key]]; n(.value.deciseconds) / 10.0)
+        ),
+
+        (if $interface == "nvme" then
+          emit("smartctl_device_percentage_used"; [["device", $device]]; $j.nvme_smart_health_information_log.percentage_used),
+          emit("smartctl_device_available_spare"; [["device", $device]]; $j.nvme_smart_health_information_log.available_spare),
+          emit("smartctl_device_available_spare_threshold"; [["device", $device]]; $j.nvme_smart_health_information_log.available_spare_threshold),
+          emit("smartctl_device_critical_warning"; [["device", $device]]; $j.nvme_smart_health_information_log.critical_warning),
+          emit("smartctl_device_media_errors"; [["device", $device]]; $j.nvme_smart_health_information_log.media_errors),
+          emit("smartctl_device_num_err_log_entries"; [["device", $device]]; $j.nvme_smart_health_information_log.num_err_log_entries),
+          (if $j.nvme_smart_health_information_log.data_units_read != null and n($j.nvme_smart_health_information_log.data_units_read) != 0 then
+            emit("smartctl_device_bytes_read"; [["device", $device]]; n($j.nvme_smart_health_information_log.data_units_read) * 1000 * 512)
+           else empty end),
+          (if $j.nvme_smart_health_information_log.data_units_written != null and n($j.nvme_smart_health_information_log.data_units_written) != 0 then
+            emit("smartctl_device_bytes_written"; [["device", $device]]; n($j.nvme_smart_health_information_log.data_units_written) * 1000 * 512)
+           else empty end)
+         else empty end),
+
+        (if $interface == "scsi" then
+          emit_if_exists("smartctl_scsi_grown_defect_list"; [["device", $device]]; $j.scsi_grown_defect_list),
+          (if $j.scsi_error_counter_log != null then
+            emit("smartctl_device_bytes_read"; [["device", $device]]; n($j.scsi_error_counter_log.read.gigabytes_processed) * 1000000000),
+            emit("smartctl_device_bytes_written"; [["device", $device]]; n($j.scsi_error_counter_log.write.gigabytes_processed) * 1000000000),
+            emit("smartctl_read_errors_corrected_by_rereads_rewrites"; [["device", $device]]; $j.scsi_error_counter_log.read.errors_corrected_by_rereads_rewrites),
+            emit("smartctl_read_errors_corrected_by_eccfast"; [["device", $device]]; $j.scsi_error_counter_log.read.errors_corrected_by_eccfast),
+            emit("smartctl_read_errors_corrected_by_eccdelayed"; [["device", $device]]; $j.scsi_error_counter_log.read.errors_corrected_by_eccdelayed),
+            emit("smartctl_read_total_uncorrected_errors"; [["device", $device]]; $j.scsi_error_counter_log.read.total_uncorrected_errors),
+            emit("smartctl_write_errors_corrected_by_rereads_rewrites"; [["device", $device]]; $j.scsi_error_counter_log.write.errors_corrected_by_rereads_rewrites),
+            emit("smartctl_write_errors_corrected_by_eccfast"; [["device", $device]]; $j.scsi_error_counter_log.write.errors_corrected_by_eccfast),
+            emit("smartctl_write_errors_corrected_by_eccdelayed"; [["device", $device]]; $j.scsi_error_counter_log.write.errors_corrected_by_eccdelayed),
+            emit("smartctl_write_total_uncorrected_errors"; [["device", $device]]; $j.scsi_error_counter_log.write.total_uncorrected_errors)
+           else empty end)
+         else empty end)
+      ]
+    | .[]
+  '
+}
+
+write_header > "$tmp"
+
+version_emitted=0
+device_count=0
+
+while IFS= read -r dev; do
+  json="$("$smartctl_bin" -a -j "$dev" 2>/dev/null || true)"
+
+  printf '%s' "$json" | jq -e . >/dev/null 2>&1 || continue
+
+  if [ "$version_emitted" -eq 0 ]; then
+    printf '%s' "$json" | emit_version >> "$tmp"
+    version_emitted=1
+  fi
+
+  if printf '%s' "$json" | emit_metrics >> "$tmp"; then
+    device_count=$((device_count + 1))
+  fi
+done < <(discover_devices)
+
+printf 'smartctl_devices %s\n' "$device_count" >> "$tmp"
+
+chmod 0644 "$tmp"
+mv "$tmp" "$out"
+trap - EXIT

config/services/systemd/vmm/smart_metric/smart_metric_generator.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run SMART Metric generator service every 5 minutes
+
+[Timer]
+OnBootSec=1min
+OnUnitInactiveSec=5min
+Persistent=true
+
+[Install]
+WantedBy=timers.target