feat(redis): update redis from 8.6.1 to 8.6.3

update notes:
- run 'ansible-playbook playbooks/app/site.yaml --tags "site"' so that update all redis at onece

ansible/inventory/group_vars/all.yaml

@@ -185,28 +185,28 @@ version:
     step: "0.30.2"
     kopia: "0.22.3"
     blocky: "0.29.0"
-    alloy: "1.13.0"
+    alloy: "1.16.1"
   containers:
     # common
     caddy: "2.11.2"
     # infra
     step: "0.30.2"
-    ldap: "v0.6.2"
-    x509-exporter: "3.21.0"
-    prometheus: "v3.9.1"
-    loki: "3.6.5"
-    grafana: "12.3.3"
+    ldap: "v0.6.3"
+    x509-exporter: "4.1.0"
+    prometheus: "v3.11.3"
+    loki: "3.7.1"
+    grafana: "13.0.1"
     ## Postgresql
-    postgresql: "18.2"
+    postgresql: "18.3"
     # For immich - https://github.com/immich-app/base-images/blob/main/postgres/versions.yaml
 #    pgvector: "v0.8.1"
-    vectorchord: "0.5.3"
+    vectorchord: "1.1.1"
     # Auth
     authelia: "4.39.19"
     # App
-    vaultwarden: "1.35.8"
+    vaultwarden: "1.36.0"
     gitea: "1.26.1"
-    redis: "8.6.1"
+    redis: "8.6.3"
     immich: "v2.7.5"
     actualbudget: "26.3.0"
     paperless: "2.20.15"

ansible/playbooks/console/site.yaml

@@ -122,3 +122,11 @@
         apply:
           tags: ["init", "site", "tools"]
       tags: ["init", "site", "tools"]
+
+    - name: Set kopia
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_kopia"
+        apply:
+          tags: ["init", "site", "kopia"]
+      tags: ["init", "site", "kopia"]

ansible/roles/common/tasks/services/set_alloy.yaml

@@ -5,9 +5,10 @@
       - hardware
   become: true
 
-- name: Deploy alloy deb file (x86_64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-amd64.deb"
+- name: Download alloy deb file (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.amd64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -15,9 +16,10 @@
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
-- name: Deploy alloy deb file (aarch64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-arm64.deb"
+- name: Download alloy deb file (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.arm64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -30,6 +32,7 @@
     deb: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     state: "present"
   become: true
+  notify: "notification_restart_alloy"
 
 - name: Deploy alloy config
   ansible.builtin.template:

ansible/roles/common/tasks/services/set_kopia.yaml

@@ -5,34 +5,36 @@
       - hardware
   become: true
 
-- name: Check kopia installation
-  ansible.builtin.shell: |
-    command -v kopia
-  changed_when: false
-  failed_when: false
-  register: "is_kopia_installed"
-  ignore_errors: true
-
 - name: Set console kopia
   when: node['name'] == 'console'
   block:
-    - name: Apply cli tools (x86_64)
+    - name: Download kopia
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
+        dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
+        mode: "0600"
+      loop:
+        - "amd64"
+        - "arm64"
+
+    - name: Install kopia (x86_64)
       ansible.builtin.apt:
         deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
         state: "present"
       become: true
-      when:
-        - ansible_facts['architecture'] == "x86_64"
-        - is_kopia_installed.rc != 0
-    - name: Apply cli tools (aarch64)
+      when: ansible_facts['architecture'] == "x86_64"
+
+    - name: Install kopia (aarch64)
       ansible.builtin.apt:
         deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
         state: "present"
       become: true
-      when:
-        - ansible_facts['architecture'] == "aarch64"
-        - is_kopia_installed.rc != 0
-    - name: Connect kopia server
+      when: ansible_facts['architecture'] == "aarch64"
+
+    - name: Connect console kopia server
       environment:
         KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['user']['console'] }}"
       ansible.builtin.shell: |
@@ -51,30 +53,36 @@
     - name: Set kopia uid
       ansible.builtin.set_fact:
         kopia_uid: 951
-    - name: Deploy kopia deb file (x86_64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
+
+    - name: Download kopia deb file (x86_64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_amd64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "x86_64"
-    - name: Deploy kopia deb file (aarch64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
+
+    - name: Download kopia deb file (aarch64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_arm64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "aarch64"
+
     - name: Create kopia group
       ansible.builtin.group:
         name: "kopia"
         gid: "{{ kopia_uid }}"
         state: "present"
       become: true
+
     - name: Create kopia user
       ansible.builtin.user:
         name: "kopia"
@@ -85,6 +93,7 @@
         comment: "Kopia backup User"
         state: "present"
       become: true
+
     - name: Create kopia directory
       ansible.builtin.file:
         path: "{{ item.name }}"
@@ -101,12 +110,13 @@
           mode: "0700"
       become: true
       no_log: true
+
     - name: Install kopia
       ansible.builtin.apt:
         deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         state: "present"
       become: true
-      when: is_kopia_installed.rc != 0
+
     - name: Deploy kopia env
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2"
@@ -116,6 +126,7 @@
         mode: "0400"
       become: true
       no_log: true
+
     - name: Deploy kopia service files
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/{{ item }}.j2"
@@ -128,6 +139,7 @@
         - "kopia-backup.service"
         - "kopia-backup.timer"
       become: true
+
     - name: Enable auto kopia rules update
       ansible.builtin.systemd:
         name: "kopia-backup.timer"

ansible/roles/console/tasks/services/set_cli_tools.yaml

@@ -49,42 +49,6 @@
     - "amd64"
     - "arm64"
 
-- name: Download kopia
-  ansible.builtin.get_url:
-    url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
-      kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
-- name: Download blocky
-  ansible.builtin.get_url:
-    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
-      blocky_v{{ version['packages']['blocky'] }}_Linux_{{ item }}.tar.gz"
-    dest: "{{ node['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-{{ item }}.tar.gz"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "x86_64"
-    - "arm64"
-
-- name: Download alloy
-  ansible.builtin.get_url:
-    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
-      alloy-{{ version['packages']['alloy'] }}-1.{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
 - name: Apply cli tools (x86_64)
   ansible.builtin.apt:
     deb: "{{ node['data_path'] }}/bin/{{ item }}"
@@ -92,7 +56,6 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-amd64.deb"
     - "step-{{ version['packages']['step'] }}-amd64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-amd64.deb"
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
@@ -103,6 +66,5 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-arm64.deb"
     - "step-{{ version['packages']['step'] }}-arm64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-arm64.deb"
   become: true
   when: ansible_facts['architecture'] == "aarch64"

ansible/roles/fw/tasks/services/set_blocky.yaml

@@ -23,7 +23,7 @@
     state: "present"
   become: true
 
-- name: Create blocky etc directory
+- name: Create blocky directory
   ansible.builtin.file:
     path: "{{ item }}"
     owner: "blocky"
@@ -31,13 +31,38 @@
     mode: "0750"
     state: "directory"
   loop:
+    - "/home/blocky"
+    - "/home/blocky/bin"
     - "/etc/blocky"
     - "/etc/blocky/ssl"
   become: true
 
+- name: Download blocky (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_x86_64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "x86_64"
+
+- name: Download blocky (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_arm64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "aarch64"
+
 - name: Deploy blocky binary file (x86_64)
   ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    remote_src: true
     dest: "/usr/local/bin/"
     owner: "root"
     group: "root"
@@ -52,7 +77,8 @@
 
 - name: Deploy blocky binary file (aarch64)
   ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    remote_src: true
     dest: "/usr/local/bin/"
     owner: "root"
     group: "root"

ansible/roles/infra/tasks/services/set_x509-exporter.yaml

@@ -8,9 +8,20 @@
     mode: "0770"
   loop:
     - "x509-exporter"
+    - "x509-exporter/config"
     - "x509-exporter/certs"
   become: true
 
+- name: Deploy config.yaml
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/x509-exporter/config/config.yaml"
+    dest: "{{ node['home_path'] }}/containers/x509-exporter/config/config.yaml"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  no_log: true
+
 - name: Deploy certificates
   ansible.builtin.copy:
     content: |

config/node/fw/nftables.conf.j2

@@ -82,6 +82,8 @@ table inet filter {
         chain global {
                 # invalid packets
                 ct state invalid drop comment "deny invalid connection"
+                # VPN connection exception handling
+                udp dport $PORTS_VPN return comment "return vpn connection to input and forward chain"
                 # crowdsec
                 ip saddr @crowdsec-blacklists counter drop comment "deny all crowdsec blacklist"
                 ip6 saddr @crowdsec6-blacklists counter drop comment "deny all ipv6 crowdsec blacklist"

config/services/containers/infra/x509-exporter/config/config.yaml

@@ -0,0 +1,11 @@
+server:
+  listen: :9793
+
+sources:
+  - kind: file
+    name: homelab-certs
+    paths:
+      - /certs/*.crt
+      - /certs/*.pem
+      - /certs/*.cer
+    refreshInterval: 1m

config/services/containers/infra/x509-exporter/x509-exporter.container.j2

@@ -11,11 +11,12 @@ Image=docker.io/enix/x509-certificate-exporter:{{ version['containers']['x509-ex
 ContainerName=x509-exporter
 HostName=X509-exporter
 
+Volume=%h/containers/x509-exporter/config/config.yaml:/etc/config.yaml:ro
 Volume=%h/containers/x509-exporter/certs:/certs:ro
 
 PublishPort={{ services['x509-exporter']['ports']['http'] }}:9793
 
-Exec=--listen-address :9793 --watch-dir=/certs
+Exec=--config /etc/config.yaml
 
 [Service]
 Restart=always

config/services/systemd/common/crowdsec/bouncers/whitelists.yaml.j2

@@ -13,9 +13,11 @@ whitelist:
 {% if node['name'] == 'auth' %}
     expression:
         # budget local-first sql scrap rule
-        - "evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'"
+        - "evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/data/migrations/'"
         # immich thumbnail request 404 error false positive
-        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
+        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
         # opencloud chunk request false positive
-        - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
+        - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'"
+        # nextcloud thumbnail/preview request error false positive
+        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'"
 {% endif %}

docs/issues/crowdsec/260321_actual_budget.md

@@ -21,13 +21,14 @@
 ## Timeline
 - 2026-03-21: Release actual budget
 - 2026-03-21: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression
 
 ## Solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'
+  - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/data/migrations/'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260321_immich.md

@@ -20,13 +20,14 @@
 ## Timeline
 - 2026-03-21: Release Immich
 - 2026-03-21: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression
 
 ## Solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260404_opencloud.md

@@ -20,13 +20,14 @@
 ## Timeline
 - 2026-04-04: Release OpenCloud
 - 2026-04-04: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression
 
 ## Solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'
+  - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260502_nextcloud.md

@@ -21,9 +21,13 @@
 - 2026-05-02: Find the false positive case, and add whitelist
 - 2026-05-03: Install crowdsecurity/nextcloud-whitelist parser
 - 2026-05-03: Make previous expressions annotation
+- 2026-05-07: Find the false positive case, which is not on `crowdsecurity/nextcloud-whitelist`
+- 2026-05-07: Set whitelist expression
 
 ## Solution
 - Install crowdsecurity/nextcloud-whitelist on auth node
+- Add expression on whitelist
+  - evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'
 
 ### Deprecated solution
 - Access to fw

docs/services/common/crowdsec.md

@@ -234,9 +234,9 @@ fw@fw:~$ sudo cscli alerts inspect 230 -d
 
 - check the log and analyze and make expression
   - e.g. immich
-    - evt.Meta.target_fqdn == 'immich.ilnmors.com' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+    - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
   - e.g. opencloud
-    - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
+    - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'"
 - free false positive decision
 
 fw@fw:~$ sudo cscli decision list

docs/services/infra/postgresql.md

@@ -37,14 +37,14 @@ podman exec -it -u postgres postgresql "psql -U postgres"
 > \l
 > \q
 
-# Restor database (manually)
+# Restore database (manually)
 podman exec -u postgres postgresql "psql -U postgres -f $POSTGRESQL_BACKUP_PATH_IN_CONTAINER/script.sql"
 
 # Backup service executes
 systemctl --user start postgresql-cluster-backup.service
 
 # Stop and remove all data
-systemctl --stop postgresql
+systemctl --user stop postgresql
 sudo find "/home/infra/data/containers/postgresql/data" -mindepth 1 -delete
 
 # Restore database
@@ -62,3 +62,39 @@ postgres=# SHOW shared_preload_libraries;
  vchord.so
 (1 row)
 ```
+
+## Update and upgrade version
+
+### Update version
+
+#### Prerequisite
+
+- Shutdown all related services on [infra, auth, app] vms.
+    - [service list](../../../ansible/roles/infra/tasks/services/set_postgresql.yaml)
+    - `systemctl --user stop $SERVICE`
+
+- Run backup service unit on infra vm.
+    - `systemctl --user start postgresql-cluster-backup.service`
+    - `systemctl --user start postgresql-data-backup@$SERVICE.service`
+
+- Modify postgresql and extension version and run ansible playbook
+    - [version info](../../../ansible/inventory/group_vars/all.yaml)
+    - `ansible-playbook playbooks/infra/site.yaml --tags "postgresql"`
+
+- Check postgresql container and update extension
+
+```postgresql
+# immich example
+# extension should be checked on each database which needs the extension
+\c immich_db
+\dx
+# check the installed_version and default_version
+ALTER EXTENSION vchord UPDATE;
+REINDEX INDEX face_index;
+REINDEX INDEX clip_index;
+```
+
+- Run playbook to start all services
+    - `ansible-playbook playbooks/[infra, auth, app]/site.yaml --tags "site"`
+
+- Check all services