refactor(observation): update alloy, grafana, prometheus

config/services/containers/infra/prometheus/etc/prometheus.yaml.j2

@@ -24,9 +24,9 @@ scrape_configs:
     scheme: "https"
     tls_config:
       ca_file: "/etc/ssl/prometheus/{{ root_cert_filename }}"
-      server_name: "{{ ['internal'] }}.{{ domain['internal'] }}"
+      server_name: "{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}"
     static_configs:
-      - targets: ["localhost:9090"]
+      - targets: ["{{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}"]
        # The label name is added as a label `label_name=<label_value>` to any timeseries scraped from this config.
         labels:
           instance: "{{ node['name'] }}"

config/services/containers/infra/prometheus/etc/rules.yaml.j2

@@ -1,7 +1,7 @@
 groups:
   - name: node_exporters_heartbeat
     rules:
-{% for instance in ["vmm", "fw", "infra", "auth", "app"] %}
+{% for instance in (((groups['hypervisor'] | default([])) + (groups['vms'] | default([]))) | difference(['vmm_init']) | sort) %}
     - alert: {{ instance }}_node_exporter_down
       expr: |
         (present_over_time(up{instance="{{ instance }}"}[5m]) or on() vector(0)) == 0
@@ -25,14 +25,14 @@ groups:
           description: "postgresql node is down for 5 mins."
   - name: Certificate_expiry_check
     rules:
-{% for filename in ["root.crt", "intermediate.crt", "crowdsec.crt", "blocky.crt", "kopia.crt", "postgresql.crt", "ldap.crt", "prometheus.crt", "loki.crt"] %}
+{% for filename in ['root', 'intermediate'] + (services | dict2items | selectattr('value.dependency.cert', 'defined') | selectattr('value.dependency.cert', 'equalto', true) | map(attribute='key') | list) %}
       - alert: {{ filename | replace('.', '_') }}_is_expired_soon
         expr: |
-          max(x509_cert_not_after{filename="{{ filename }}"}) - time() < 2592000
+          max(x509_cert_not_after{filename="{{ filename }}.crt"}) - time() < 2592000
         for: 1d
         labels:
           severity: critical
         annotations:
-          summary: "{{ filename }} is expired in 30 days"
-          description: "{{ filename }} is expired in 30 days."
+          summary: "{{ filename }}.crt is expired in 30 days"
+          description: "{{ filename }}.crt is expired in 30 days."
 {% endfor %}

config/services/containers/infra/prometheus/prometheus.container.j2

@@ -13,6 +13,8 @@ Image=docker.io/prom/prometheus:{{ version['containers']['prometheus'] }}
 ContainerName=prometheus
 HostName=prometheus
 
+AddHost={{ services['prometheus']['domain']['internal'] }}.{{ domain['internal'] }}:host-gateway
+
 PublishPort={{ services['prometheus']['ports']['https'] }}:9090/tcp
 
 Volume=%h/containers/prometheus/data:/prometheus:rw