2026-SEP-16

false positive:
- outline session problem (LePresidente/http-generic-401-bf)

.gitignore

@@ -1,8 +1,11 @@
-.ansible
 data/bin/*
-data/volumes/*
-data/images/*
-docs/archives/textfiles/
-docs/notes/*
+data/images/containers/*
+data/images/seeds/*
+data/vmm_init/*
+*.raw
+*.tar
+*.qcow2
 *.sql
+*.iso
+*.git_bak
 !.gitkeep

README.md

@@ -1,26 +0,0 @@
-# ilnmors homelab README
-
-This homelab project implements single-node On-premise IaaS system. The homelab contains virtual machines which are divided by their roles, such as private firewall, DNS, PKI, LDAP and database, SSO\(OIDC\). The standard domain is used to implement this system without specific vendors. All components are defined as code and initiated by IaC \(Ansible\) except hypervisor initial configuration.
-
-## RTO times
-- Feb/25/2026 - Reprovisioning Hypervisor and vms
-    - RTO: 1 hour 30 min - verified
-        - Manual install and set vmm: 20 min
-        - Create and reprovision fw including services: 15 min
-        - Create and reprovision infra including services: 20 min
-        - Create and reprovision auth including services: 10 min
-        - Create and reprovision app except services: 10 min
-        - Intermediate tasks (ACME issuance, DNS propagation, etc.): 15 min
-
-- Mar/5/2026 - Reprovisioning Hardware and Hypervisor and vms
-    - RTO: 2 hour 20 min
-        - console: 15min - verified
-        - certificate: 0 min \(When it needs to be created, RTO will be 20 min) - not verified
-        - wireguard: 0 min \(When it needs to be created, RTO will be 1 min) - not verified
-        - hypervisor\(+fw\): 45 min - verified
-        - switch: 1 min - verified
-        - dsm: 30 min - verified
-        - kopia: 0 min \(When it needs to be created, RTO will be 10 min) - verified
-        - Extra vms: 30 min - verified
-        - Etc: 30 min
-

ansible/convention.yaml

@@ -1,27 +1,28 @@
 ---
-# `""` should be used for every string values
-# except name, boolean, varible(environment) name, number (only for decimal)
+# String values need `""`
+# Task names, boolean values, environment names, decimal numbers don't need `""`
 - name: Convention task
   delegate_to: "host"
   delegate_facts: true
   environment:
-    env: "environment"
+    env: "value"
   ansible.builtin.file:
-    # Identifier: name:, src, dest, url
-    path: "/path/of/example"
-    # State
+    # Identifier: name, src, dest, url, etc.
+    path: "/example/path"
+    # state parameters
     state: "directory"
     owner: "root"
     group: "root"
     mode: "0640"
-    # optional data
+    # optional parameters
     recurse: true
   loop:
     - "list"
   loop_control:
     label: "{{ item }}"
   become: true
-  # become_user: "root"
+  # default become user is root
+  become_user: "user"
   when: condition.stat.exists
   changed_when: condition.rc != 0
   failed_when: condition.rc == 0
@@ -30,10 +31,11 @@
   listen: "notification_example"
   no_log: true
   run_once: true
+  # Ensure execution even if it fails (it prints errors on the screen, but it ignores them)
+  ignore_errors: true
   tags:
     - "always"
     - "init"
-    - "upgrade"
-    - "update"
-
+    - "site"
+    - "[service_name]"
 # when: "'tags' is not in ansible_run_tags"

ansible/inventory/group_vars/all.yaml

@@ -1,5 +1,6 @@
 ---
 # Global vars
+timezone: "Asia/Seoul"
 ansible_ssh_private_key_file: "/etc/secrets/{{ hostvars['console']['node']['uid'] }}/id_console"
 
 # CA
@@ -7,45 +8,114 @@ root_cert_filename: "ilnmors_root_ca.crt"
 intermediate_cert_filename: "ilnmors_intermediate_ca.crt"
 intermediate_key_filename: "ilnmors_intermediate_ca.key"
 
-
-# local SAN and SSH SAN should be updated manually on host_vars
+# Domains
 domain:
   public: "ilnmors.com"
   internal: "ilnmors.internal"
   dc: "dc=ilnmors,dc=internal"
   org: "ilnmors"
 
-# DNS configuration including bind and blocky should be set manually.
-# named.conf.j2 is also set manually.
-# Check the hosts.j2 when cname records are fixed
-
 services:
+  # common
+  wireguard:
+    domain:
+      public: "vpn"
+    ports:
+      udp: "11290"
   crowdsec:
-    domain: "crowdsec"
+    domain:
+      internal: "crowdsec"
     ports:
       https: "8080"
+    sub:
+      machines:
+        - "fw"
+        - "vmm"
+        - "bk"
+        - "infra"
+        - "auth"
+        - "app"
+      bouncers:
+        - "fw"
+        - "caddy"
+    dependency:
+      cert: true
+  kopia:
+    domain:
+      internal: "kopia"
+    ports:
+      https: "51515"
+    uid: "951"
+    dependency:
+      cert: true
+  caddy:
+    ports:
+      http: "2080"
+      https: "2443"
+  # fw
   bind:
-    domain: "bind"
+    domain:
+      internal: "bind"
     ports:
       dns: "53"
   blocky:
-    domain: "blocky"
+    domain:
+      internal: "blocky"
     ports:
       https: "443"
       dns: "53"
+      dot: "853"
+    dependency:
+      cert: true
+  # infra
+  rustfs:
+    domain:
+      internal: "rustfs"
+      public: "rustfs"
+    ports:
+      http: "9001"
+    subuid: "110000"
+    dependency:
+      web: "infra-public"
   postgresql:
-    domain: "postgresql"
+    domain:
+      internal: "postgresql"
     ports:
       tcp: "5432" # postgresql db connection port
     subuid: "100998"
+    dependency:
+      cert: true
+  mariadb:
+    domain:
+      internal: "mariadb"
+    ports:
+      tcp: "3306" # mariadb db connection port
+    subuid: "100998"
+    dependency:
+      cert: true
   ldap:
-    domain: "ldap"
+    domain:
+      internal: "ldap"
     ports:
       http: "17170"
       ldaps: "6360"
     subuid: "100999"
+    dependency:
+      cert: true
+      web: "infra-internal"
+      db: "postgresql"
+  maddy:
+    domain:
+      internal: "maddy"
+    ports:
+      smtp: "2587"
+      imaps: "2993"
+    dependency:
+      cert: true
+      db: "postgresql"
   ca:
-    domain: "ca"
+    domain:
+      internal: "ca"
     ports:
       https: "9000"
     subuid: "100999"
@@ -54,133 +124,348 @@ services:
       http: "9793"
     subuid: "165533"
   prometheus:
-    domain: "prometheus"
+    domain:
+      internal: "prometheus"
     ports:
       https: "9090"
     subuid: "165533"
+    dependency:
+      cert: true
+      web: "infra-internal"
   loki:
-    domain: "loki"
+    domain:
+      internal: "loki"
     ports:
       https: "3100"
     subuid: "110000"
+    dependency:
+      cert: true
   grafana:
-    domain: "grafana"
+    domain:
+      internal: "grafana"
     ports:
-      http: "3000"
+      http: "3000" # infra
     subuid: "100471"
-  caddy:
-    ports:
-      http: "2080"
-      https: "2443"
-  nas:
-    domain: "nas"
-    ports:
-      https: "5001"
-  kopia:
-    domain: "nas"
-    ports:
-      https: "51515"
+    dependency:
+      web: "infra-internal"
+      db: "postgresql"
+  # auth
   authelia:
-    domain: "authelia"
+    domain:
+      public: "authelia"
     ports:
       http: "9091"
+    dependency:
+      web: "auth-public"
+      db: "postgresql"
+  # app
   redis:
     subuid: "100998"
+  homepage:
+    domain:
+      public: "@"
+      internal: "homepage.app"
+    ports:
+      http: "3000" # app
+    dependency:
+      web: "app-public"
+  openwebui:
+    domain:
+      public: "openwebui"
+      internal: "openwebui.app"
+    ports:
+      http: "8002"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  llama:
+    ports:
+      http: "10000"
+    dependency:
+      models:
+        default: |
+          n-gpu-layers = all
+          kv-unified = true
+          cache-prompt = true
+          cache-type-k = q8_0
+          cache-type-v = q8_0
+          flash-attn = auto
+          reasoning = off
+          jinja = true
+          lazy-mode = off
+        Nemotron3.5-Lightning-30B-A3B: |
+          model = /models/Nemotron3.5-Lightning-30B-A3B/Nemotron-3.5-Lightning-30B-A3B-Heretic-Uncensored-Q4_K_M-GGUF.gguf
+          c = 65536
+          parallel = 2
+          load-on-startup = true
+        # version: mradermacher/LFM2-24B-A2B-heretic-ara-GGUF
+        # LFM2-24B-A2B: |
+        #   model = /models/LFM2-24B-A2B/LFM2-24B-A2B-heretic-ara.Q4_K_M.gguf
+        #   c = 65536
+        #   parallel = 2
+        #   load-on-startup = true
+        # # version: SC117/LFM2.5-VL-3B-Uncensored-GGUF
+        # LFM2-VL-3B: |
+        #   model = /models/LFM2-VL-3B/LFM2.5-VL-3B-Uncensored-Q4_K_M.gguf
+        #   mmproj = /models/LFM2-VL-3B/mmproj-LFM2.5-VL-3B-BF16.gguf
+        #   c = 65536
+        #   parallel = 1
+        #   load-on-startup = true
+        # version: HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M
+        # Qwen3.6: |
+        #   model = /models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-Q4_K_M.gguf
+        #   mmproj = /models/Qwen3.6-35B-A3B/mmproj-Qwen3.6-35B-BF16.gguf
+        #   c = 65536
+        #   parallel = 2
+        #   load-on-startup = true
+        #   image-min-tokens = 1024
+        # # version: HauhauCS/Qwen3.5-2B-Uncensored-HauhauCS-Aggressiv-Q8_0
+        # Qwen3.5-image: |
+        #   model = /models/Qwen3.5-2B/Qwen3.5-2B-Q8_0.gguf
+        #   c = 32768
+        #   parallel = 1
+        #   load-on-startup = true
+        # version: LiquidAI/LFM2.5-2.6B-GGUF
+        # LFM2.5: |
+        #   model = /models/LFM2.5-2.6B/LFM2.5-2.6B-Q8_0.gguf
+        #   c = 32768
+        #   parallel = 2
+        #   load-on-startup = true
+  stablediffusion:
+    ports:
+      http: "10001"
+    dependency:
+      models:
+        active: "zImage"
+        zImage: |
+          --diffusion-model /models/zImage/moodyRealMix_zitV7Global-Q8_0.gguf \
+          --llm /models/zImage/Qwen3-4B-Instruct-2507-Q4_K_M.gguf \
+          --vae /models/zImage/ae.safetensors \
+          --backend gpu \
+          --cfg-scale 1.0 \
+          --steps 9 \
+          --vae-tiling \
+          --vae-tile-size 32x32 \
+          --vae-tile-overlap 0.5
+  snappymail:
+    domain:
+      public: "webmail"
+      internal: "webmail.app"
+    ports:
+      http: "8888"
+    subuid: "100081"
+    dependency:
+      web: "app-public"
   vaultwarden:
     domain:
       public: "vault"
       internal: "vault.app"
     ports:
       http: "8000"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  linkding:
+    domain:
+      public: "linkding"
+      internal: "linkding.app"
+    ports:
+      http: "9090"
+    subuid: "100032"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
   gitea:
     domain:
       public: "gitea"
       internal: "gitea.app"
     ports:
-      http: "3000"
+      http: "3001"
     subuid: "100999"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  mastodon:
+    domain:
+      public: "mastodon"
+      internal: "mastodon.app"
+    ports:
+      http: "3002"
+      api: "4000"
+      redis: "6379"
+    subuid: "100990"
+    route:
+      api: "/api/v1/streaming*"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  groupoffice:
+    domain:
+      public: "groupoffice"
+      internal: "groupoffice.app"
+    ports:
+      http: "8005"
+    subuid: "100032"
+    route:
+      dav: true
+    dependency:
+      web: "app-public"
+      db: "mariadb"
+  kaneo:
+    domain:
+      public: "kaneo"
+      internal: "kaneo.app"
+    ports:
+      http: "8006"
+    subuid: "101000"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  nextcloud:
+    domain:
+      public: "nextcloud"
+      internal: "nextcloud.app"
+    ports:
+      http: "8001"
+      redis: "6380"
+    subuid: "100032"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  collabora:
+    domain:
+      public: "collabora"
+      internal: "collabora.app"
+    ports:
+      http: "9980"
+    subuid: "101000"
+    dependency:
+      web: "app-public"
   immich:
     domain:
       public: "immich"
       internal: "immich.app"
     ports:
       http: "2283"
-      redis: "6379"
+      redis: "6381"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
   immich-ml:
     ports:
       http: "3003"
-  actualbudget:
+  outline:
     domain:
-      public: "budget"
-      internal: "budget.app"
+      public: "outline"
+      internal: "outline.app"
     ports:
-      http: "5006"
+      http: "3004"
+      redis: "6382"
     subuid: "101000"
-  paperless:
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  bookstack:
     domain:
-      public: "paperless"
-      internal: "paperless.app"
+      public: "bookstack"
+      internal: "bookstack.app"
     ports:
-      http: "8001"
-      redis: "6380"
-    subuid: "100999"
-  vikunja:
+      http: "8004"
+    subuid: "100032"
+    dependency:
+      web: "app-public"
+      db: "mariadb"
+  sure:
     domain:
-      public: "vikunja"
-      internal: "vikunja.app"
+      public: "sure"
+      internal: "sure.app"
     ports:
-      http: "3456"
+      http: "3006"
+      redis: "6383"
     subuid: "100999"
-  opencloud:
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  papra:
     domain:
-      public: "opencloud"
-      internal: "opencloud.app"
+      public: "papra"
+      internal: "papra.app"
     ports:
-      http: "9200"
-    subuid: "100999"
-  manticore:
+      http: "1221"
     subuid: "100998"
-  affine:
+    dependency:
+      web: "app-public"
+  tandoor:
     domain:
-      public: "affine"
-      internal: "affine.app"
+      public: "tandoor"
+      internal: "tandoor.app"
     ports:
-      http: "3010"
-      redis: "6381"
-      manticore: "9308"
+      http: "8003"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
+  homebox:
+    domain:
+      public: "homebox"
+      internal: "homebox.app"
+    ports:
+      http: "7745"
+    dependency:
+      web: "app-public"
+      db: "postgresql"
 
 version:
   packages:
-    sops: "3.12.1"
-    step: "0.29.0"
-    kopia: "0.22.3"
-    blocky: "0.28.2"
-    alloy: "1.13.0"
+    sops: "3.13.2"
+    step: "0.30.2"
+    kopia: "0.23.1"
+    blocky: "0.33.0"
+    alloy: "1.17.1"
   containers:
     # common
-    caddy: "2.10.2"
+    caddy: "2.11.4"
     # infra
-    step: "0.29.0"
-    ldap: "v0.6.2"
-    x509-exporter: "3.19.1"
-    prometheus: "v3.9.1"
-    loki: "3.6.5"
-    grafana: "12.3.3"
+    rustfs: "1.0.0-rc.6"
+    ca: "0.30.2"
     ## Postgresql
-    postgresql: "18.2"
-    # For immich - https://github.com/immich-app/base-images/blob/main/postgres/versions.yaml
-#    pgvector: "v0.8.1"
-    vectorchord: "0.5.3"
+    postgresql: "18.6"
+    ## Mariadb
+    mariadb: "12.3.3"
+    ### For immich - https://github.com/immich-app/base-images/blob/main/postgres/versions.yaml
+    vectorchord: "1.1.1"
+    ## lldap
+    ldap: "v0.6.3"
+    ## maddy
+    maddy: "0.9.5"
+    x509-exporter: "4.1.0"
+    prometheus: "v3.14.0"
+    loki: "3.7.7"
+    grafana: "13.2.1"
     # Auth
-    authelia: "4.39.15"
+    authelia: "4.39.20"
     # App
-    vaultwarden: "1.35.4"
-    gitea: "1.25.5"
-    redis: "8.6.1"
-    immich: "v2.7.4"
-    actualbudget: "26.3.0"
-    paperless: "2.20.13"
-    vikunja: "2.2.2"
-    opencloud: "4.0.4"
-    manticore: "25.0.0"
-    affine: "0.26.3"
+    vaultwarden: "1.37.2"
+    gitea: "1.27.3"
+    redis: "8.10.1"
+    immich: "v3.2.2"
+    nextcloud: "34.0.3"
+    collabora: "26.04.3.2.1"
+    sure: "0.7.4"
+    outline: "1.10.1"
+    homebox: "0.26.2"
+    homepage: "v2.0.0"
+    snappymail: "v2.38.2"
+    mastodon: "v4.6.7"
+    papra: "26.6.1-rootless"
+    tandoor: "2.6.13"
+    bookstack: "26.5.4"
+    linkding: "1.46.2"
+    openwebui: "v0.11.3"
+    # https://github.com/ggml-org/llama.cpp/pkgs/container/llama.cpp/versions
+    llama: "server-vulkan-b10830"
+    # https://github.com/leejet/stable-diffusion.cpp/pkgs/container/stable-diffusion.cpp
+    # master-sycl@sha256:f27ebe3fea7dcb5978241425d3a115ba6736cfc33c42e75233f22f7411449e9c has problem in it, it cannot get vRAM from UMA properly
+    # Pin due to a SYCL/UMA false-OOM regression in newer stable-diffusion.cpp ModelManager memory accounting.
+    stablediffusion: "master-sycl@sha256:67ccb5d11bf55578f7938daeb7c9fee1d0dd1344c089c090b9065714d7960fd5"
+    groupoffice: "26.0.43"
+    kaneo: "2.23.2"

ansible/inventory/group_vars/hypervisor.yaml

@@ -1,7 +1,71 @@
 ---
 node:
   name: "vmm"
+  group: "svadmins"
   uid: 2000
   home_path: "/home/vmm"
-  ssh_san: "vmm,vmm_init,vmm.ilnmors.internal,init.vmm.ilnmors.internal"
-  local_san: "localhost vmm.ilnmors.internal"
+  libvirt_path: "/var/lib/libvirt"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}_init,{{ node['name'] }}.{{ domain['internal'] }},init.{{ node['name'] }}.{{ domain['internal'] }}"
+  local_san: "localhost {{ node['name'] }}.{{ domain['internal'] }}"
+
+hardware:
+  mac:
+    nic0: "c8:ff:bf:07:02:34"
+    nic1: "c8:ff:bf:07:02:33"
+    sfpp0: "c8:ff:bf:07:02:31"
+    sfpp1: "c8:ff:bf:07:02:32"
+  igpu:
+    id: "8086:4628"
+    address: "0000:00:02.0"
+    domain: "0x0000"
+    bus: "0x00"
+    slot: "0x02"
+    function: "0x0"
+  disk:
+    # NVMe is set when the hypervisor is initiated manually.
+    nvme:
+      serials:
+        - "S5GXNX0T228635E"
+      fs:
+        type: "ext4"
+        label: ""
+        mount_point:
+          - "/"
+          - "/var/lib/libvirt"
+        mount_opts: ""
+    data:
+      serials:
+        - "PN1138P2GH354H"
+        - "PN2131P6GJKSMP"
+      fs:
+        type: "btrfs"
+        label: "DATA"
+        mount_point: "{{ node['libvirt_path'] }}/disk/data"
+        mount_opts:
+          - "defaults"
+          - "noatime"
+          - "nodatacow"
+          - "degraded"
+          - "nofail"
+        btrfs:
+          data_profile: "raid1"
+          metadata_profile: "raid1"
+          min_devices: 2
+    backup:
+      serials:
+        - "PN1134P6HVTL0W"
+      # Spare
+      # - "WMC6N0E617S8"
+      fs:
+        type: "btrfs"
+        label: "BACKUP"
+        mount_point: "{{ node['libvirt_path'] }}/disk/backup"
+        mount_opts:
+          - "defaults"
+          - "noatime"
+          - "nodatacow"
+          - "nofail"
+        btrfs:
+          data_profile: "single"
+          metadata_profile: "dup"
+          min_devices: 1

ansible/inventory/host_vars/app.yaml

@@ -2,40 +2,30 @@
 # Node Factors
 node:
   name: "app"
+  group: "svadmins"
   uid: 2004
   home_path: "/home/app"
-  ssh_san: "app,app.ilnmors.internal"
-  local_san: "localhost app.ilnmors.internal"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}.{{ domain['internal'] }}"
+  local_san: "localhost {{ node['name'] }}.{{ domain['internal'] }}"
 # VM Factors
 vm:
   name: "app"
-  cpu: 4
+  cpu: 6
+  cpu_pin: |
+    <vcpupin vcpu='0' cpuset='0'/>
+    <vcpupin vcpu='1' cpuset='1'/>
+    <vcpupin vcpu='2' cpuset='2'/>
+    <vcpupin vcpu='3' cpuset='3'/>
+    <vcpupin vcpu='4' cpuset='10'/>
+    <vcpupin vcpu='5' cpuset='11'/>
+    <emulatorpin cpuset='10-11'/>
   shares: 1024
-  memory: 16
+  memory: 48
   storage: 256
+  data_storage:
+    name: "vdb"
+    pool: "data"
+    size: 1.75
   lan_mac: "0a:49:6e:4d:03:00"
   lan_net: "lan-net"
   lan_group: "vlan10-access"
-  # PCIe passthrough address
-  # result of `lspci | grep -i -e "sata controller" -e "vga"` and parse it.
-  # Ex) 04:00.0 > domain: "0x0000", bus: "0x04", slot: "0x00", function: "0x0"
-  pass_through:
-    igpu:
-      address: "0000:00:02.0"
-      domain: "0x0000"
-      bus: "0x00"
-      slot: "0x02"
-      function: "0x0"
-    sata_controller: # Additional SATA Controller
-      address: "0000:04:00.0"
-      domain: "0x0000"
-      bus: "0x04"
-      slot: "0x00"
-      function: "0x0"
-
-# BTRFS configuration for hdd which is passthroughed
-storage:
-  btrfs:
-    label: "APP_DATA"
-    level: "raid10"
-    mount_point: "/home/app/data"

ansible/inventory/host_vars/auth.yaml

@@ -2,17 +2,22 @@
 # Node Factors
 node:
   name: "auth"
+  group: "svadmins"
   uid: 2003
   home_path: "/home/auth"
-  ssh_san: "auth,auth.ilnmors.internal"
-  local_san: "localhost auth.ilnmors.internal"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}.{{ domain['internal'] }}"
+  local_san: "localhost {{ node['name'] }}.{{ domain['internal'] }}"
 # VM Factors
 vm:
   name: "auth"
   cpu: 2
-  shares: 512
-  memory: 2
-  storage: 64
+  cpu_pin: |
+    <vcpupin vcpu='0' cpuset='6'/>
+    <vcpupin vcpu='1' cpuset='7'/>
+    <emulatorpin cpuset='6-7'/>
+  shares: 1024
+  memory: 1
+  storage: 32
   lan_mac: "0a:49:6e:4d:02:00"
   lan_net: "lan-net"
   lan_group: "vlan10-access"

ansible/inventory/host_vars/bk.yaml

@@ -0,0 +1,27 @@
+---
+# Node Factors
+node:
+  name: "bk"
+  group: "svadmins"
+  uid: 2009
+  home_path: "/home/bk"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}.{{ domain['internal'] }}"
+  local_san: "localhost {{ node['name'] }}.{{ domain['internal'] }}"
+# VM Factors
+vm:
+  name: "bk"
+  cpu: 2
+  cpu_pin: |
+    <vcpupin vcpu='0' cpuset='6'/>
+    <vcpupin vcpu='1' cpuset='7'/>
+    <emulatorpin cpuset='6-7'/>
+  shares: 512
+  memory: 2
+  storage: 64
+  data_storage:
+    name: "vdb"
+    pool: "backup"
+    size: 1.75
+  lan_mac: "0a:49:6e:4d:09:00"
+  lan_net: "lan-net"
+  lan_group: "vlan10-access"

ansible/inventory/host_vars/console.yaml

@@ -10,17 +10,17 @@ age_key: !vault |
           33653638373537323363316363646534323362353836373665636265663939353862663532313230
           30643038313138626464626161373835646665326334393834616234643931656536343130316238
           61656264643830616639
-# Node Factors
+
 node:
   name: "console"
+  group: "svadmins"
   uid: 2999
   home_path: "/home/console"
-  workspace_path: "{{ node.home_path }}/workspace"
-  homelab_path: "{{ node.home_path }}/workspace/homelab"
-  data_path: "{{ node.homelab_path }}/data"
-  config_path: "{{ node.homelab_path }}/config"
-  ssh_san: "console,console.ilnmors.internal"
-  ssh_users: "vmm,fw,infra,auth,app"
+  workspace_path: "{{ node['home_path'] }}/workspace"
+  homelab_path: "{{ node['home_path'] }}/workspace/homelab"
+  data_path: "{{ node['homelab_path'] }}/data"
+  config_path: "{{ node['homelab_path'] }}/config"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}.{{ domain['internal'] }}"
+  ssh_users: "vmm,fw,bk,infra,auth,app"
   # add the hostname of wsl, it is needed to improve the sudo problem
-  local_san: "localhost console.ilnmors.internal surface"
-# ansible_python_interpreter: "{{ ansible_playbook_python }}"
+  local_san: "localhost {{ node['name'] }} {{ node['name'] }}.{{ domain['internal'] }}"

ansible/inventory/host_vars/fw.yaml

@@ -2,15 +2,22 @@
 # Node Factors
 node:
   name: "fw"
+  group: "svadmins"
   uid: 2001
   home_path: "/home/fw"
-  ssh_san: "fw,fw.ilnmors.internal"
-  local_san: "localhost fw.ilnmors.internal"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}.{{ domain['internal'] }}"
+  local_san: "localhost {{ node['name'] }}.{{ domain['internal'] }}"
 
 # VM Factors
 vm:
   name: "fw"
-  cpu: 2
+  cpu: 4
+  cpu_pin: |
+    <vcpupin vcpu='0' cpuset='0'/>
+    <vcpupin vcpu='1' cpuset='1'/>
+    <vcpupin vcpu='2' cpuset='4'/>
+    <vcpupin vcpu='3' cpuset='5'/>
+    <emulatorpin cpuset='4-5'/>
   shares: 2048
   memory: 4
   storage: 64
@@ -21,75 +28,131 @@ vm:
   lan_group: "vlan-trunk"
 
 # Network Factors
-# LLA is like MAC address for L3 (Network layer). Usually, subnet is used to seperate network.
-network4:
+network:
+  dns_serial: "{{ now().strftime('%Y%m%d') }}00"
   subnet:
-    client: "192.168.1.0/24"
-    server: "192.168.10.0/24"
-    user: "192.168.20.0/24"
-    wg: "192.168.99.0/24"
-    lla: "169.254.0.0/16"
-  # You can use "{{ hostvars['fw']['network4']['firewall'].values() | join(', ') }}" for all
-  firewall:
-    client: "192.168.1.1"
-    server: "192.168.10.1"
-    user: "192.168.20.1"
-    wg: "192.168.99.1"
+    ipv4:
+      client: "192.168.1.0/24"
+      server: "192.168.10.0/24"
+      user: "192.168.20.0/24"
+      wg: "192.168.99.0/24"
+      lla: "169.254.0.0/16"
+    ipv6:
+      client: "fd00:1::/64"
+      server: "fd00:10::/64"
+      wg: "fd00:99::/64"
+      lla: "fe80::/10"
+  podman:
+    ipv4:
+      lla: "169.254.1.2"
+  reverse_zone:
+    ipv4:
+      client: "1.168.192.in-addr.arpa"
+      server: "10.168.192.in-addr.arpa"
+    ipv6:
+      client: "0.0.0.0.0.0.0.0.1.0.0.0.0.0.d.f.ip6.arpa"
+      server: "0.0.0.0.0.0.0.0.0.1.0.0.0.0.d.f.ip6.arpa"
+  fw:
+    ipv4:
+      client: "192.168.1.1"
+      server: "192.168.10.1"
+      user: "192.168.20.1"
+      wg: "192.168.99.1"
+      reverse: "1"
+    ipv6:
+      client: "fd00:1::1"
+      server: "fd00:10::1"
+      wg: "fd00:99::1"
+      reverse: "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
+    alias:
+      - "ntp"
+      - "crowdsec"
   blocky:
-    server: "192.168.10.2"
+    ipv4:
+      server: "192.168.10.2"
+      reverse: "2"
+    ipv6:
+      server: "fd00:10::2"
+      reverse: "2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
   bind:
-    server: "192.168.10.3"
+    ipv4:
+      server: "192.168.10.3"
+      reverse: "3"
+    ipv6:
+      server: "fd00:10::3"
+      reverse: "3.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
   console:
-    client: "192.168.1.20"
-    wg: "192.168.99.20"
+    ipv4:
+      client: "192.168.1.20"
+      wg: "192.168.99.20"
+      reverse: "20"
+    ipv6:
+      client: "fd00:1::20"
+      wg: "fd00:99::20"
+      reverse: "0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
   vmm:
-    client: "192.168.1.10"
-    server: "192.168.10.10"
+    ipv4:
+      client: "192.168.1.10"
+      server: "192.168.10.10"
+      reverse: "10"
+    ipv6:
+      client: "fd00:1::10"
+      server: "fd00:10::10"
+      reverse: "0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
+    alias:
+      - "init.vmm"
+  bk:
+    ipv4:
+      server: "192.168.10.9"
+      reverse: "9"
+    ipv6:
+      server: "fd00:10::9"
+      reverse: "9.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
+    alias:
+      - "kopia"
   infra:
-    server: "192.168.10.11"
+    ipv4:
+      server: "192.168.10.11"
+      reverse: "11"
+    ipv6:
+      server: "fd00:10::11"
+      reverse: "1.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
+    alias:
+      - "rustfs"
+      - "ca"
+      - "postgresql"
+      - "mariadb"
+      - "ldap"
+      - "maddy"
+      - "prometheus"
+      - "loki"
+      - "grafana"
   auth:
-    server: "192.168.10.12"
+    ipv4:
+      server: "192.168.10.12"
+      reverse: "12"
+    ipv6:
+      server: "fd00:10::12"
+      reverse: "2.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
+    alias:
+      - "authelia"
   app:
-    server: "192.168.10.13"
+    ipv4:
+      server: "192.168.10.13"
+      reverse: "13"
+    ipv6:
+      server: "fd00:10::13"
+      reverse: "3.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0"
+    alias:
+      - "*.app"
   switch:
-    client: "192.168.1.2"
-  nas:
-    client: "192.168.1.11"
+    ipv4:
+      client: "192.168.1.2"
+      reverse: "2"
   printer:
-    client: "192.168.1.101"
-
-network6:
-  subnet:
-    client: "fd00:1::/64"
-    server: "fd00:10::/64"
-    wg: "fd00:99::/64"
-    lla: "fe80::/10"
-  firewall:
-    client: "fd00:1::1"
-    server: "fd00:10::1"
-    wg: "fd00:99::1"
-  blocky:
-    server: "fd00:10::2"
-  bind:
-    server: "fd00:10::3"
-  console:
-    client: "fd00:1::20"
-    wg: "fd00:99::20"
-  vmm:
-    client: "fd00:1::10"
-    server: "fd00:10::10"
-  infra:
-    server: "fd00:10::11"
-  auth:
-    server: "fd00:10::12"
-  app:
-    server: "fd00:10::13"
-  switch:
-    client: "fd00:1::2"
-  nas:
-    client: "fd00:1::11"
-  printer:
-    client: "fd00:1::101"
+    ipv4:
+      client: "192.168.1.101"
+      reverse: "101"
 
 # Suricata Factors
 # suricata_home_net: '[10.0.0.0/8,172.16.0.0/12,192.168.0.0/16]'

ansible/inventory/host_vars/infra.yaml

@@ -2,16 +2,23 @@
 # Node Factors
 node:
   name: "infra"
+  group: "svadmins"
   uid: 2002
   home_path: "/home/infra"
-  ssh_san: "infra,infra.ilnmors.internal"
-  local_san: "localhost infra.ilnmors.internal"
+  ssh_san: "{{ node['name'] }},{{ node['name'] }}.{{ domain['internal'] }}"
+  local_san: "localhost {{ node['name'] }}.{{ domain['internal'] }}"
 
 # VM Factors
 vm:
   name: "infra"
-  cpu: 2
-  shares: 1024
+  cpu: 4
+  cpu_pin: |
+    <vcpupin vcpu='0' cpuset='2'/>
+    <vcpupin vcpu='1' cpuset='3'/>
+    <vcpupin vcpu='2' cpuset='8'/>
+    <vcpupin vcpu='3' cpuset='9'/>
+    <emulatorpin cpuset='8-9'/>
+  shares: 2048
   memory: 6
   storage: 256
   lan_mac: "0a:49:6e:4d:01:00"

ansible/inventory/hosts.ini

@@ -9,6 +9,7 @@ vmm ansible_host=vmm.ilnmors.internal ansible_user=vmm
 # --- Virtual Machines ---
 [vms]
 fw ansible_host=fw.ilnmors.internal ansible_user=fw
+bk ansible_host=bk.ilnmors.internal ansible_user=bk
 infra ansible_host=infra.ilnmors.internal ansible_user=infra
 auth  ansible_host=auth.ilnmors.internal ansible_user=auth
-app   ansible_host=app.ilnmors.internal ansible_user=app
+app   ansible_host=app.ilnmors.internal ansible_user=app

ansible/playbooks/app/site.yaml

@@ -20,12 +20,13 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['app'] }}"
+      no_log: true
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -77,6 +78,14 @@
           tags: ["init", "resolved"]
       tags: ["init", "resolved"]
 
+    - name: Set timesyncd
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_timesyncd"
+        apply:
+          tags: ["init", "timesyncd"]
+      tags: ["init", "timesyncd"]
+
     - name: Update and upgrade apt
       ansible.builtin.apt:
         upgrade: "dist"
@@ -93,14 +102,32 @@
           - "jq"
           - "netcat-openbsd"
           - "dbus-user-session"
+          - "ksmtuned"
+          - "rsync"
         state: "present"
       become: true
-      tags: ["init", "install-packages"]
+      tags: ["init", "site", "install-packages"]
 
-    - name: Set raid
+    - name: Enable ksm.service
+      ansible.builtin.systemd:
+        name: "ksm.service"
+        state: "started"
+        enabled: true
+      become: true
+      tags: ["init", "site", "install-packages"]
+
+    - name: Disable ksmtuned.service
+      ansible.builtin.systemd:
+        name: "ksmtuned.service"
+        state: "stopped"
+        enabled: false
+      become: true
+      tags: ["init", "site", "install-packages"]
+
+    - name: Set data volume
       ansible.builtin.include_role:
-        name: "app"
-        tasks_from: "node/set_raid"
+        name: "common"
+        tasks_from: "node/set_data_volume"
         apply:
           tags: ["init", "raid"]
       tags: ["init", "raid"]
@@ -113,14 +140,6 @@
           tags: ["init", "linger"]
       tags: ["init", "linger"]
 
-    - name: Set podman
-      ansible.builtin.include_role:
-        name: "common"
-        tasks_from: "services/set_podman"
-        apply:
-          tags: ["init", "podman"]
-      tags: ["init", "podman"]
-
     - name: Set nftables
       ansible.builtin.include_role:
         name: "common"
@@ -129,6 +148,14 @@
           tags: ["init", "nftables"]
       tags: ["init", "nftables"]
 
+    - name: Set alloy
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_alloy"
+        apply:
+          tags: ["site", "alloy"]
+      tags: ["site", "alloy"]
+
     - name: Set crowdsec
       ansible.builtin.include_role:
         name: "common"
@@ -137,14 +164,6 @@
           tags: ["site", "crowdsec"]
       tags: ["site", "crowdsec"]
 
-    - name: Set alloy
-      ansible.builtin.include_role:
-        name: "common"
-        tasks_from: "services/set_alloy"
-        apply:
-          tags: ["init", "update", "alloy"]
-      tags: ["init", "update", "alloy"]
-
     - name: Set kopia
       ansible.builtin.include_role:
         name: "common"
@@ -153,6 +172,14 @@
           tags: ["site", "kopia"]
       tags: ["site", "kopia"]
 
+    - name: Set podman
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_podman"
+        apply:
+          tags: ["site", "podman"]
+      tags: ["site", "podman"]
+
     - name: Set caddy
       ansible.builtin.include_role:
         name: "common"
@@ -169,6 +196,14 @@
           tags: ["site", "vaultwarden"]
       tags: ["site", "vaultwarden"]
 
+    - name: Set snappymail
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_snappymail"
+        apply:
+          tags: ["site", "snappymail"]
+      tags: ["site", "snappymail"]
+
     - name: Set gitea
       ansible.builtin.include_role:
         name: "app"
@@ -182,49 +217,136 @@
         name: "app"
         tasks_from: "services/set_immich"
         apply:
-          tags: ["site", "immich"]
-      tags: ["site", "immich"]
+          tags: ["site", "immich", "redis"]
+      tags: ["site", "immich", "redis"]
 
-    - name: Set actual budget
+    - name: Set nextcloud
       ansible.builtin.include_role:
         name: "app"
-        tasks_from: "services/set_actual-budget"
+        tasks_from: "services/set_nextcloud"
         apply:
-          tags: ["site", "actual-budget"]
-      tags: ["site", "actual-budget"]
+          tags: ["site", "nextcloud", "redis"]
+      tags: ["site", "nextcloud", "redis"]
 
-    - name: Set paperless
+    - name: Set groupoffice
       ansible.builtin.include_role:
         name: "app"
-        tasks_from: "services/set_paperless"
+        tasks_from: "services/set_groupoffice"
         apply:
-          tags: ["site", "paperless"]
-      tags: ["site", "paperless"]
+          tags: ["site", "groupoffice"]
+      tags: ["site", "groupoffice"]
 
-    - name: Set vikunja
+    - name: Set collabora
       ansible.builtin.include_role:
         name: "app"
-        tasks_from: "services/set_vikunja"
+        tasks_from: "services/set_collabora"
         apply:
-          tags: ["site", "vikunja"]
-      tags: ["site", "vikunja"]
+          tags: ["site", "collabora"]
+      tags: ["site", "collabora"]
 
-    - name: Set opencloud
+    - name: Set kaneo
       ansible.builtin.include_role:
         name: "app"
-        tasks_from: "services/set_opencloud"
+        tasks_from: "services/set_kaneo"
         apply:
-          tags: ["site", "opencloud"]
-      tags: ["site", "opencloud"]
+          tags: ["site", "kaneo"]
+      tags: ["site", "kaneo"]
 
-    - name: Set affine
+    - name: Set sure
       ansible.builtin.include_role:
         name: "app"
-        tasks_from: "services/set_affine"
+        tasks_from: "services/set_sure"
         apply:
-          tags: ["site", "affine"]
-      tags: ["site", "affine"]
+          tags: ["site", "sure", "redis"]
+      tags: ["site", "sure", "redis"]
 
+    - name: Set outline
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_outline"
+        apply:
+          tags: ["site", "outline", "redis"]
+      tags: ["site", "outline", "redis"]
+
+    - name: Set homebox
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_homebox"
+        apply:
+          tags: ["site", "homebox"]
+      tags: ["site", "homebox"]
+
+    - name: Set mastodon
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_mastodon"
+        apply:
+          tags: ["site", "mastodon", "redis"]
+      tags: ["site", "mastodon", "redis"]
+
+    - name: Set papra
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_papra"
+        apply:
+          tags: ["site", "papra"]
+      tags: ["site", "papra"]
+
+    - name: Set tandoor
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_tandoor"
+        apply:
+          tags: ["site", "tandoor"]
+      tags: ["site", "tandoor"]
+
+    - name: Set bookstack
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_bookstack"
+        apply:
+          tags: ["site", "bookstack"]
+      tags: ["site", "bookstack"]
+
+    - name: Set linkding
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_linkding"
+        apply:
+          tags: ["site", "linkding"]
+      tags: ["site", "linkding"]
+
+    - name: Set llama
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_llama"
+        apply:
+          tags: ["site", "llama"]
+      tags: ["site", "llama"]
+
+    - name: Set stablediffusion
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_stablediffusion"
+        apply:
+          tags: ["site", "stablediffusion"]
+      tags: ["site", "stablediffusion"]
+
+    - name: Set openwebui
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_openwebui"
+        apply:
+          tags: ["site", "openwebui"]
+      tags: ["site", "openwebui"]
+
+    - name: Set homepage
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_homepage"
+        apply:
+          tags: ["site", "homepage"]
+      tags: ["site", "homepage"]
 
     - name: Flush handlers right now
       ansible.builtin.meta: "flush_handlers"
@@ -239,10 +361,19 @@
         state: "present"
       become: true
       notify:
+        - "notification_add_user_in_group"
         - "notification_update_initramfs"
         - "notification_reboot_app"
       tags: ["init"]
   handlers:
+    - name: Add user in video, render group
+      ansible.builtin.user:
+        name: "{{ ansible_user }}"
+        state: "present"
+        groups: "video, render"
+        append: true
+      become: true
+      listen: "notification_add_user_in_group"
     - name: Update initramfs
       ansible.builtin.command:
         update-initramfs -u

ansible/playbooks/auth/site.yaml

@@ -20,12 +20,13 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['auth'] }}"
+      no_log: true
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -77,6 +78,14 @@
           tags: ["init", "resolved"]
       tags: ["init", "resolved"]
 
+    - name: Set timesyncd
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_timesyncd"
+        apply:
+          tags: ["init", "timesyncd"]
+      tags: ["init", "timesyncd"]
+
     - name: Update and upgrade apt
       ansible.builtin.apt:
         upgrade: "dist"
@@ -105,14 +114,6 @@
           tags: ["init", "linger"]
       tags: ["init", "linger"]
 
-    - name: Set podman
-      ansible.builtin.include_role:
-        name: "common"
-        tasks_from: "services/set_podman"
-        apply:
-          tags: ["init", "podman"]
-      tags: ["init", "podman"]
-
     - name: Set nftables
       ansible.builtin.include_role:
         name: "common"
@@ -129,6 +130,14 @@
           tags: ["site", "crowdsec"]
       tags: ["site", "crowdsec"]
 
+    - name: Set podman
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_podman"
+        apply:
+          tags: ["site", "podman"]
+      tags: ["site", "podman"]
+
     - name: Set caddy
       ansible.builtin.include_role:
         name: "common"

ansible/playbooks/bk/site.yaml

@@ -0,0 +1,163 @@
+---
+- name: Load secret values
+  hosts: "console"
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Load secret from secrets.yaml
+      ansible.builtin.include_role:
+        name: "console"
+        tasks_from: "node/load_secret_vars"
+        apply:
+          tags: ["always"]
+      tags: ["always"]
+
+- name: Site bk
+  hosts: "bk"
+  gather_facts: false
+  become: false
+  pre_tasks:
+    - name: Set become password
+      ansible.builtin.set_fact:
+        ansible_become_pass: "{{ hostvars['console']['sudo']['password']['bk'] }}"
+      no_log: true
+      tags: ["always"]
+
+  tasks:
+    - name: Set timezone
+      community.general.timezone:
+        name: "{{ timezone }}"
+      become: true
+      tags: ["init", "timezone"]
+
+    - name: Deploy root_ca certificate
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/deploy_root_ca"
+        apply:
+          tags: ["init", "root_crt"]
+      tags: ["init", "root_crt"]
+
+    - name: Deploy hosts file
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/deploy_hosts"
+        apply:
+          tags: ["init", "hosts"]
+      tags: ["init", "hosts"]
+
+    - name: Create default directory
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/create_default_dir"
+        apply:
+          tags: ["init", "default_dir"]
+      tags: ["init", "default_dir"]
+
+    - name: Set ssh host
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_ssh_host"
+        apply:
+          tags: ["init", "ssh_host"]
+      tags: ["init", "ssh_host"]
+
+    - name: Set networkd
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_networkd"
+        apply:
+          tags: ["init", "networkd"]
+      tags: ["init", "networkd"]
+
+    - name: Set resolved
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_resolved"
+        apply:
+          tags: ["init", "resolved"]
+      tags: ["init", "resolved"]
+
+    - name: Set timesyncd
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_timesyncd"
+        apply:
+          tags: ["init", "timesyncd"]
+      tags: ["init", "timesyncd"]
+
+    - name: Update and upgrade apt
+      ansible.builtin.apt:
+        upgrade: "dist"
+        update_cache: true
+        cache_valid_time: 3600
+      become: true
+      tags: ["init", "site", "upgrade-packages"]
+
+    - name: Install common packages
+      ansible.builtin.apt:
+        name:
+          - "acl"
+          - "curl"
+          - "jq"
+          - "netcat-openbsd"
+          - "dbus-user-session"
+        state: "present"
+      become: true
+      tags: ["init", "site", "install-packages"]
+
+    - name: Set data volume
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_data_volume"
+        apply:
+          tags: ["init", "raid"]
+      tags: ["init", "raid"]
+
+    - name: Set linger
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_linger"
+        apply:
+          tags: ["init", "linger"]
+      tags: ["init", "linger"]
+
+    - name: Set nftables
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_nftables"
+        apply:
+          tags: ["init", "nftables"]
+      tags: ["init", "nftables"]
+
+    - name: Set crowdsec
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_crowdsec"
+        apply:
+          tags: ["site", "crowdsec"]
+      tags: ["site", "crowdsec"]
+
+    - name: Set kopia
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_kopia"
+        apply:
+          tags: ["site", "kopia"]
+      tags: ["site", "kopia"]
+
+    - name: Set rclone
+      ansible.builtin.include_role:
+        name: "bk"
+        tasks_from: "services/set_rclone"
+        apply:
+          tags: ["site", "rclone"]
+      tags: ["site", "rclone"]
+
+    - name: Set alloy
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_alloy"
+        apply:
+          tags: ["site", "alloy"]
+      tags: ["site", "alloy"]

ansible/playbooks/console/create_cert.yaml

@@ -0,0 +1,223 @@
+---
+- name: Load secret values
+  hosts: "console"
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Load secret from secrets.yaml
+      ansible.builtin.include_role:
+        name: "console"
+        tasks_from: "node/load_secret_vars"
+        apply:
+          tags: ["always"]
+      tags: ["always"]
+
+- name: Site console
+  hosts: "console"
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Create root CA
+      tags: ["never", "root"]
+      block:
+        - name: Clean previous CA temporary directory
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/ca"
+            state: "absent"
+          no_log: true
+        - name: Create ca directory
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/ca"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            state: "directory"
+            mode: "0700"
+        - name: Get root CA password
+          ansible.builtin.copy:
+            content: |
+              {{ hostvars['console']['ca']['root']['password'] }}
+            dest: "/run/user/{{ node['uid'] }}/ca/root_ca.password"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            mode: "0400"
+          no_log: true
+        - name: Create root CA
+          ansible.builtin.command: |
+            step certificate create \
+            "{{ domain['internal'] }} Root CA" \
+            /run/user/{{ node['uid'] }}/ca/root_ca.crt \
+            /run/user/{{ node['uid'] }}/ca/root_ca.key \
+            --password-file /run/user/{{ node['uid'] }}/ca/root_ca.password \
+            --profile root-ca \
+            --not-after 87600h
+          changed_when: true
+          no_log: true
+        - name: Update root CA
+          environment:
+            SOPS_AGE_KEY: "{{ hostvars['console']['age_key'] }}"
+          ansible.builtin.shell: |
+            set -euo pipefail
+            jq -Rs . "/run/user/{{ node['uid'] }}/ca/root_ca.crt" \
+            | sops set --value-stdin "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml" \
+            '["ca"]["root"]["crt"]'
+            jq -Rs . "/run/user/{{ node['uid'] }}/ca/root_ca.key" \
+            | sops set --value-stdin "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml" \
+            '["ca"]["root"]["key"]'
+          changed_when: true
+          no_log: true
+      always:
+        - name: Clean temporary files
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/ca"
+            state: "absent"
+          no_log: true
+
+    - name: Get root CA and create intermediate CA
+      tags: ["never", "intermediate"]
+      block:
+        - name: Clean previous CA temporary directory
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/ca"
+            state: "absent"
+          no_log: true
+        - name: Create ca certificates directory
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/ca"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            state: "directory"
+            mode: "0700"
+        - name: Get root key and cert and password
+          ansible.builtin.copy:
+            content: |
+              {{ hostvars['console']['ca']['root'][item] }}
+            dest: "/run/user/{{ node['uid'] }}/ca/root_ca.{{ item }}"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            mode: "0400"
+          loop:
+            - "key"
+            - "crt"
+            - "password"
+          no_log: true
+        - name: Get intermediate CA password
+          ansible.builtin.copy:
+            content: |
+              {{ hostvars['console']['ca']['intermediate']['password'] }}
+            dest: "/run/user/{{ node['uid'] }}/ca/intermediate_ca.password"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            mode: "0400"
+          no_log: true
+        - name: Create intermediate CA
+          ansible.builtin.command: |
+            step certificate create \
+            "{{ domain['internal'] }} intermediate CA" \
+            /run/user/{{ node['uid'] }}/ca/intermediate_ca.crt \
+            /run/user/{{ node['uid'] }}/ca/intermediate_ca.key \
+            --password-file /run/user/{{ node['uid'] }}/ca/intermediate_ca.password \
+            --profile intermediate-ca \
+            --ca /run/user/{{ node['uid'] }}/ca/root_ca.crt \
+            --ca-key /run/user/{{ node['uid'] }}/ca/root_ca.key \
+            --ca-password-file /run/user/{{ node['uid'] }}/ca/root_ca.password \
+            --not-after 43800h
+          changed_when: true
+          no_log: true
+        - name: Update intermediate CA
+          environment:
+            SOPS_AGE_KEY: "{{ hostvars['console']['age_key'] }}"
+          ansible.builtin.shell: |
+            set -euo pipefail
+            jq -Rs . "/run/user/{{ node['uid'] }}/ca/intermediate_ca.crt" \
+            | sops set --value-stdin "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml" \
+            '["ca"]["intermediate"]["crt"]'
+            jq -Rs . "/run/user/{{ node['uid'] }}/ca/intermediate_ca.key" \
+            | sops set --value-stdin "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml" \
+            '["ca"]["intermediate"]["key"]'
+          changed_when: true
+          no_log: true
+      always:
+        - name: Clean temporary files
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/ca"
+            state: "absent"
+          no_log: true
+
+    - name: Get intermediate CA and create leaf cert
+      tags: ["never", "leaf"]
+      block:
+        - name: Initialize self-cert services list
+          ansible.builtin.set_fact:
+            cert_services: []
+        - name: Set self-cert services list
+          ansible.builtin.set_fact:
+            cert_services: >-
+              {{
+                services
+                | dict2items
+                | selectattr('value.dependency.cert', 'defined')
+                | selectattr('value.dependency.cert', 'equalto', true)
+                | map(attribute='key')
+                | list
+              }}
+        - name: Clean previous leaf temporary directory
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/leaf"
+            state: "absent"
+          no_log: true
+        - name: Create leaf certificates directory
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/leaf"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            state: "directory"
+            mode: "0700"
+        - name: Get intermediate key and cert and password
+          ansible.builtin.copy:
+            content: |
+              {{ hostvars['console']['ca']['intermediate'][item] }}
+            dest: "/run/user/{{ node['uid'] }}/leaf/intermediate_ca.{{ item }}"
+            owner: "{{ ansible_user }}"
+            group: "{{ node['group'] }}"
+            mode: "0400"
+          loop:
+            - "key"
+            - "crt"
+            - "password"
+          no_log: true
+        - name: Create leaf certificates
+          ansible.builtin.command: |
+            step certificate create \
+            "{{ services[item]['domain']['internal'] }}.{{ domain['internal'] }}" \
+            /run/user/{{ node['uid'] }}/leaf/{{ item }}.crt \
+            /run/user/{{ node['uid'] }}/leaf/{{ item }}.key \
+            --profile leaf \
+            --san {{ services[item]['domain']['internal'] }}.{{ domain['internal'] }} \
+            --ca /run/user/{{ node['uid'] }}/leaf/intermediate_ca.crt \
+            --ca-key /run/user/{{ node['uid'] }}/leaf/intermediate_ca.key \
+            --ca-password-file /run/user/{{ node['uid'] }}/leaf/intermediate_ca.password \
+            --not-after 21900h \
+            --insecure --no-password
+          loop: "{{ cert_services }}"
+          changed_when: true
+          no_log: true
+        - name: Update leaf keys and certificates
+          environment:
+            SOPS_AGE_KEY: "{{ hostvars['console']['age_key'] }}"
+          ansible.builtin.shell: |
+            set -euo pipefail
+            jq -Rs . "/run/user/{{ node['uid'] }}/leaf/{{ item }}.crt" \
+            | sops set --value-stdin "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml" \
+            '["{{ item }}"]["crt"]'
+            jq -Rs . "/run/user/{{ node['uid'] }}/leaf/{{ item }}.key" \
+            | sops set --value-stdin "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml" \
+            '["{{ item }}"]["key"]'
+          loop: "{{ cert_services }}"
+          changed_when: true
+          no_log: true
+      always:
+        - name: Clean temporary files
+          ansible.builtin.file:
+            path: "/run/user/{{ node['uid'] }}/leaf"
+            state: "absent"
+          no_log: true

ansible/playbooks/console/site.yaml

@@ -20,13 +20,14 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['console'] }}"
+      no_log: true
       tags: ["always"]
 
   tasks:
     # init
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -70,32 +71,6 @@
           tags: ["init", "ssh_client"]
       tags: ["init", "ssh_client"]
 
-    - name: Check file permissions
-      ansible.builtin.file:
-        path: "{{ node['workspace_path'] }}/{{ item }}"
-        state: "directory"
-        owner: "{{ ansible_user }}"
-        group: "svadmins"
-        mode: "u=rwX,g=,o="
-        recurse: true
-      loop:
-        - "homelab/ansible"
-        - "homelab/config"
-        - "homelab/docs"
-        - "university"
-      tags: ["init", "site", "file_permission"]
-      # kopia snashot is mounted on homelab/data/volumes.
-      # NEVER CHANGE permission and owners
-
-    - name: Download vm cloud-init
-      ansible.builtin.get_url:
-        url: "https://cdimage.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2"
-        dest: "{{ node['data_path'] }}/images/debian-13-generic-amd64.qcow2"
-        owner: "console"
-        group: "svadmins"
-        mode: "0600"
-      tags: ["init", "site", "cloud-init-image"]
-
     - name: Install packages
       ansible.builtin.apt:
         name:
@@ -108,8 +83,6 @@
           - "logrotate"
           - "nftables"
           - "build-essential"
-          - "g++"
-          - "gcc"
           - "fuse3"
         state: "present"
       become: true
@@ -122,3 +95,28 @@
         apply:
           tags: ["init", "site", "tools"]
       tags: ["init", "site", "tools"]
+
+    - name: Set vmm_init config
+      ansible.builtin.include_role:
+        name: "console"
+        tasks_from: "node/create_vmm_init"
+        apply:
+          tags: ["site", "vmm_init"]
+      tags: ["site", "vmm_init"]
+
+    - name: Download vm cloud-init
+      ansible.builtin.get_url:
+        url: "https://cdimage.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2"
+        dest: "{{ node['data_path'] }}/images/debian-13-generic-amd64.qcow2"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+        mode: "0600"
+      tags: ["init", "site", "cloud-init-image"]
+
+    - name: Set kopia
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_kopia"
+        apply:
+          tags: ["init", "site", "kopia"]
+      tags: ["init", "site", "kopia"]

ansible/playbooks/fw/site.yaml

@@ -20,12 +20,13 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['fw'] }}"
+      no_log: true
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -162,8 +163,8 @@
         name: "fw"
         tasks_from: "services/set_bind"
         apply:
-          tags: ["init", "update", "bind"]
-      tags: ["init", "update", "bind"]
+          tags: ["init", "bind"]
+      tags: ["init", "bind"]
 
     - name: Set blocky
       ansible.builtin.include_role:

ansible/playbooks/infra/site.yaml

@@ -20,12 +20,13 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['infra'] }}"
+      no_log: true
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -77,6 +78,14 @@
           tags: ["init", "resolved"]
       tags: ["init", "resolved"]
 
+    - name: Set timesyncd
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "node/set_timesyncd"
+        apply:
+          tags: ["init", "timesyncd"]
+      tags: ["init", "timesyncd"]
+
     - name: Update and upgrade apt
       ansible.builtin.apt:
         upgrade: "dist"
@@ -105,14 +114,6 @@
           tags: ["init", "linger"]
       tags: ["init", "linger"]
 
-    - name: Set podman
-      ansible.builtin.include_role:
-        name: "common"
-        tasks_from: "services/set_podman"
-        apply:
-          tags: ["init", "podman"]
-      tags: ["init", "podman"]
-
     - name: Set nftables
       ansible.builtin.include_role:
         name: "common"
@@ -129,6 +130,22 @@
           tags: ["site", "crowdsec"]
       tags: ["site", "crowdsec"]
 
+    - name: Set kopia
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_kopia"
+        apply:
+          tags: ["site", "kopia"]
+      tags: ["site", "kopia"]
+
+    - name: Set podman
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_podman"
+        apply:
+          tags: ["site", "podman"]
+      tags: ["site", "podman"]
+
     - name: Set ca
       ansible.builtin.include_role:
         name: "infra"
@@ -137,6 +154,14 @@
           tags: ["site", "ca"]
       tags: ["site", "ca"]
 
+    - name: Set rustfs
+      ansible.builtin.include_role:
+        name: "infra"
+        tasks_from: "services/set_rustfs"
+        apply:
+          tags: ["site", "rustfs"]
+      tags: ["site", "rustfs"]
+
     - name: Set postgresql
       ansible.builtin.include_role:
         name: "infra"
@@ -145,6 +170,14 @@
           tags: ["site", "postgresql"]
       tags: ["site", "postgresql"]
 
+    - name: Set mariadb
+      ansible.builtin.include_role:
+        name: "infra"
+        tasks_from: "services/set_mariadb"
+        apply:
+          tags: ["site", "mariadb"]
+      tags: ["site", "mariadb"]
+
     - name: Set caddy
       ansible.builtin.include_role:
         name: "common"
@@ -161,6 +194,14 @@
           tags: ["site", "ldap"]
       tags: ["site", "ldap"]
 
+    - name: Set maddy
+      ansible.builtin.include_role:
+        name: "infra"
+        tasks_from: "services/set_maddy"
+        apply:
+          tags: ["site", "maddy"]
+      tags: ["site", "maddy"]
+
     - name: Set x509 exporter
       ansible.builtin.include_role:
         name: "infra"
@@ -200,11 +241,3 @@
         apply:
           tags: ["site", "grafana"]
       tags: ["site", "grafana"]
-
-    - name: Set kopia
-      ansible.builtin.include_role:
-        name: "common"
-        tasks_from: "services/set_kopia"
-        apply:
-          tags: ["site", "kopia"]
-      tags: ["site", "kopia"]

ansible/playbooks/vmm/create_vm.yaml

@@ -19,6 +19,7 @@
   vars:
     valid_vm_names:
       - "fw"
+      - "bk"
       - "infra"
       - "auth"
       - "app"
@@ -36,6 +37,7 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['vmm'] }}"
+      no_log: true
 
     - name: Create seed file
       ansible.builtin.include_role:

ansible/playbooks/vmm/site.yaml

@@ -27,12 +27,13 @@
     - name: Set become password
       ansible.builtin.set_fact:
         ansible_become_pass: "{{ hostvars['console']['sudo']['password']['vmm'] }}"
+      no_log: true
       tags: ["always"]
   tasks:
     # init
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -100,6 +101,14 @@
           tags: ["init", "linger"]
       tags: ["init", "linger"]
 
+    - name: Set raid
+      ansible.builtin.include_role:
+        name: "vmm"
+        tasks_from: "node/set_disk"
+        apply:
+          tags: ["init", "raid"]
+      tags: ["init", "raid"]
+
     - name: Set libvirt
       ansible.builtin.include_role:
         name: "vmm"
@@ -134,6 +143,15 @@
       when: inventory_hostname != "vmm_init"
       tags: ["site", "crowdsec"]
 
+    - name: Set smartctl generator
+      ansible.builtin.include_role:
+        name: "vmm"
+        tasks_from: "services/set_smart_metric_generator"
+        apply:
+          tags: ["site", "smart"]
+      when: inventory_hostname != "vmm_init"
+      tags: ["site", "smart"]
+
     - name: Set alloy
       ansible.builtin.include_role:
         name: "common"
@@ -155,10 +173,12 @@
           - ksmtuned
           - libvirt-daemon-system
           - virt-top
+          - btrfs-progs
           - python3
           - python3-apt
           - python3-libvirt
           - python3-lxml
+          - rsync
         state: "present"
       become: true
       when: "'init' is not in ansible_run_tags"

ansible/roles/app/handlers/main.yaml

@@ -43,59 +43,198 @@
   listen: "notification_restart_immich-ml"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Restart actual-budget
+- name: Restart nextcloud
   ansible.builtin.systemd:
-    name: "actual-budget.service"
+    name: "nextcloud.service"
     state: "restarted"
     enabled: true
-    scope: "user"
     daemon_reload: true
+    scope: "user"
+  when: is_nextcloud_init.stat.exists
   changed_when: false
-  listen: "notification_restart_actual-budget"
+  listen: "notification_restart_nextcloud"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Restart paperless
+- name: Restart collabora
   ansible.builtin.systemd:
-    name: "paperless.service"
+    name: "collabora.service"
     state: "restarted"
     enabled: true
     scope: "user"
     daemon_reload: true
   changed_when: false
-  listen: "notification_restart_paperless"
+  listen: "notification_restart_collabora"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Restart vikunja
+- name: Restart sure
   ansible.builtin.systemd:
-    name: "vikunja.service"
+    name: "{{ item }}"
     state: "restarted"
     enabled: true
     scope: "user"
     daemon_reload: true
+  loop:
+    - "sure-web.service"
+    - "sure-worker.service"
   changed_when: false
-  listen: "notification_restart_vikunja"
+  listen: "notification_restart_sure"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Restart opencloud
+- name: Restart outline
   ansible.builtin.systemd:
-    name: "opencloud.service"
+    name: "outline.service"
     state: "restarted"
     enabled: true
-    daemon_reload: true
     scope: "user"
-  when: is_opencloud_init.stat.exists
+    daemon_reload: true
+  when: is_outline_init.stat.exists
   changed_when: false
-  listen: "notification_restart_opencloud"
+  listen: "notification_restart_outline"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Restart affine
+- name: Restart homebox
   ansible.builtin.systemd:
-    name: "affine.service"
+    name: "homebox.service"
     state: "restarted"
     enabled: true
-    daemon_reload: true
     scope: "user"
-  when: is_affine_init.stat.exists
+    daemon_reload: true
   changed_when: false
-  listen: "notification_restart_affine"
+  listen: "notification_restart_homebox"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart homepage
+  ansible.builtin.systemd:
+    name: "homepage.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_homepage"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart snappymail
+  ansible.builtin.systemd:
+    name: "snappymail.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_snappymail"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart mastodon
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  loop:
+    - "mastodon-web.service"
+    - "mastodon-worker.service"
+    - "mastodon-streaming.service"
+  changed_when: false
+  listen: "notification_restart_mastodon"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart papra
+  ansible.builtin.systemd:
+    name: "papra.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_papra"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart tandoor
+  ansible.builtin.systemd:
+    name: "tandoor.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_tandoor"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart bookstack
+  ansible.builtin.systemd:
+    name: "bookstack.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_bookstack"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart linkding
+  ansible.builtin.systemd:
+    name: "linkding.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_linkding"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart openwebui
+  ansible.builtin.systemd:
+    name: "openwebui.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_openwebui"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart llama
+  ansible.builtin.systemd:
+    name: "llama.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_llama"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart stablediffusion
+  ansible.builtin.systemd:
+    name: "stablediffusion.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_stablediffusion"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart groupoffice
+  ansible.builtin.systemd:
+    name: "groupoffice.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_groupoffice"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart kaneo
+  ansible.builtin.systemd:
+    name: "kaneo.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_kaneo"
   ignore_errors: true # noqa: ignore-errors

ansible/roles/app/tasks/node/set_raid.yaml

@@ -1,70 +0,0 @@
----
-- name: Check btrfs installation
-  ansible.builtin.shell: |
-    command -v btrfs
-  become: true # btrfs is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_btrfs_installed"
-  ignore_errors: true
-
-- name: Install btrfs
-  ansible.builtin.apt:
-    name: "btrfs-progs"
-    state: "present"
-  become: true
-  when: is_btrfs_installed.rc != 0
-
-- name: Set hard disk path
-  ansible.builtin.shell: |
-    set -o pipefail
-    ls -1 /dev/disk/by-path/*{{ vm['pass_through']['sata_controller']['address'] }}* | \
-    grep -v '\.0$' | \
-    sort
-  changed_when: false
-  register: "hdd_path_list"
-
-- name: Check app_hdd filesystem already exists
-  ansible.builtin.command: |
-    blkid -L {{ storage['btrfs']['label'] }}
-  register: is_app_data
-  changed_when: false
-  failed_when: false
-  become: true
-
-- name: Check disk number
-  ansible.builtin.fail:
-    msg: "Below 4 disks for RAID10, found {{ hdd_path_list.stdout_lines | length }}"
-  when: (hdd_path_list.stdout_lines | length) < 4
-
-- name: Set btrfs raid10 volume
-  ansible.builtin.shell: |
-    mkfs.btrfs -f \
-    -L {{ storage['btrfs']['label'] }} \
-    -d {{ storage['btrfs']['level'] }} \
-    -m {{ storage['btrfs']['level'] }} \
-    {{ hdd_path_list.stdout_lines | join(' ') }}
-  become: true
-  when:
-    - is_app_data.rc != 0
-    - (hdd_path_list.stdout_lines | length) >= 4
-  changed_when: is_mkfs.rc == 0
-  register: "is_mkfs"
-
-- name: Mount btrfs raid10 volume
-  ansible.posix.mount:
-    path: "{{ storage['btrfs']['mount_point'] }}"
-    src: "LABEL={{ storage['btrfs']['label'] }}"
-    state: "mounted"
-    fstype: "btrfs"
-    opts: "defaults,noatime,compress=zstd:3,autodefrag,degraded,nofail"
-  become: true
-
-- name: Set hard disk path permissions
-  ansible.builtin.file:
-    path: "{{ storage['btrfs']['mount_point'] }}"
-    state: "directory"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0770"
-  become: true

ansible/roles/app/tasks/services/set_bookstack.yaml

@@ -0,0 +1,87 @@
+---
+- name: Create bookstack directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['bookstack']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/bookstack"
+    - "data/containers/bookstack/public"
+    - "data/containers/bookstack/storage"
+    - "containers/bookstack"
+    - "containers/bookstack/ssl"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/bookstack/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['bookstack']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_bookstack"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "BOOKSTACK_SECRET_KEY"
+      value: "{{ hostvars['console']['bookstack']['secret_key'] }}"
+    - name: "BOOKSTACK_DB_PASSWORD"
+      value: "{{ hostvars['console']['mariadb']['password']['bookstack'] }}"
+    - name: "BOOKSTACK_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['bookstack']['oidc']['secret'] }}"
+  notify: "notification_restart_bookstack"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/solidnerd/bookstack"
+    tag: "{{ version['containers']['bookstack'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy bookstack.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/bookstack/bookstack.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/bookstack.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_bookstack"
+
+- name: Enable bookstack.service
+  ansible.builtin.systemd:
+    name: "bookstack.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_solidnerd_bookstack_{{ version['containers']['bookstack'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/solidnerd/bookstack:{{ version['containers']['bookstack'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_solidnerd_bookstack_{{ version['containers']['bookstack'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_solidnerd_bookstack_{{ version['containers']['bookstack'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_collabora.yaml

@@ -0,0 +1,44 @@
+---
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/collabora/code"
+    tag: "{{ version['containers']['collabora'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/collabora/collabora.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/collabora.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_collabora"
+
+- name: Enable collabora.service
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_collabora_code_{{ version['containers']['collabora'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/collabora/code:{{ version['containers']['collabora'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_collabora_code_{{ version['containers']['collabora'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_collabora_code_{{ version['containers']['collabora'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_gitea.yaml

@@ -4,10 +4,11 @@
     path: "{{ node['home_path'] }}/{{ item }}"
     state: "directory"
     owner: "{{ services['gitea']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "data/containers/gitea"
+    - "data/containers/gitea/data"
     - "containers/gitea"
     - "containers/gitea/ssl"
   become: true
@@ -18,7 +19,7 @@
       {{ hostvars['console']['ca']['root']['crt'] }}
     dest: "{{ node['home_path'] }}/containers/gitea/ssl/{{ root_cert_filename }}"
     owner: "{{ services['gitea']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0440"
   become: true
   notify: "notification_restart_gitea"
@@ -33,12 +34,19 @@
   notify: "notification_restart_gitea"
   no_log: true
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/gitea/gitea"
+    tag: "{{ version['containers']['gitea'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/gitea/gitea.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/gitea.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_gitea"
 
@@ -49,3 +57,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_gitea_gitea_{{ version['containers']['gitea'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/gitea/gitea:{{ version['containers']['gitea'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_gitea_gitea_{{ version['containers']['gitea'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_gitea_gitea_{{ version['containers']['gitea'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_groupoffice.yaml

@@ -0,0 +1,118 @@
+---
+- name: Create groupoffice directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['groupoffice']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/groupoffice"
+    - "data/containers/groupoffice/data"
+    - "containers/groupoffice"
+    - "containers/groupoffice/ssl"
+    - "containers/groupoffice/config"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/groupoffice/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['groupoffice']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_groupoffice"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "GROUPOFFICE_DB_PASSWORD"
+      value: "{{ hostvars['console']['mariadb']['password']['groupoffice'] }}"
+  notify: "notification_restart_groupoffice"
+  no_log: true
+
+- name: Deploy config files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/groupoffice/config/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/containers/groupoffice/config/{{ item }}"
+    owner: "{{ services['groupoffice']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+  loop:
+    - "config.php"
+    - "groupoffice-ca.ini"
+    - "Connection.php"
+    - "Authenticator.php"
+    - "configfile.php"
+  become: true
+  notify: "notification_restart_groupoffice"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/intermesh/groupoffice"
+    tag: "{{ version['containers']['groupoffice'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy groupoffice.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/groupoffice/groupoffice.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/groupoffice.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_groupoffice"
+
+- name: Deploy groupoffice-cron service
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/groupoffice/services/{{ item }}"
+    dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "groupoffice-cron.service"
+    - "groupoffice-cron.timer"
+
+- name: Enable groupoffice.service
+  ansible.builtin.systemd:
+    name: "groupoffice.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Enable groupoffice-cron.timer
+  ansible.builtin.systemd:
+    name: "groupoffice-cron.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_intermesh_groupoffice_{{ version['containers']['groupoffice'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/intermesh/groupoffice:{{ version['containers']['groupoffice'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_intermesh_groupoffice_{{ version['containers']['groupoffice'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_intermesh_groupoffice_{{ version['containers']['groupoffice'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_homebox.yaml

@@ -0,0 +1,84 @@
+---
+- name: Create homebox directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/homebox"
+    - "data/containers/homebox/data"
+    - "containers/homebox"
+    - "containers/homebox/ssl"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/homebox/ssl/{{ root_cert_filename }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  notify: "notification_restart_homebox"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "HBOX_AUTH_API_KEY_PEPPER"
+      value: "{{ hostvars['console']['homebox']['api_secret'] }}"
+    - name: "HBOX_DATABASE_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['homebox'] }}"
+    - name: "HBOX_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['homebox']['oidc']['secret'] }}"
+  notify: "notification_restart_homebox"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/sysadminsmedia/homebox"
+    tag: "{{ version['containers']['homebox'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy homebox.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/homebox/homebox.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/homebox.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_homebox"
+
+- name: Enable homebox.service
+  ansible.builtin.systemd:
+    name: "homebox.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_sysadminsmedia_homebox_{{ version['containers']['homebox'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/sysadminsmedia/homebox:{{ version['containers']['homebox'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_sysadminsmedia_homebox_{{ version['containers']['homebox'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_sysadminsmedia_homebox_{{ version['containers']['homebox'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_homepage.yaml

@@ -0,0 +1,99 @@
+---
+- name: Set homepage services list
+  ansible.builtin.set_fact:
+    homepage_public_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.web', 'defined')
+        | selectattr('value.dependency.web', 'in', ['infra-public', 'auth-public', 'app-public'])
+        | map(attribute='key')
+        | list
+      }}
+    homepage_internal_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.web', 'defined')
+        | selectattr('value.dependency.web', 'in', ['infra-internal', 'auth-internal', 'app-internal'])
+        | map(attribute='key')
+        | list
+      }}
+
+- name: Create homepage directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/homepage"
+    - "containers/homepage/config"
+    - "containers/homepage/images"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/gethomepage/homepage"
+    tag: "{{ version['containers']['homepage'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy config files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/homepage/config/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/containers/homepage/config/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "settings.yaml"
+    - "widgets.yaml"
+    - "services.yaml"
+    - "bookmarks.yaml"
+  notify: "notification_restart_homepage"
+
+- name: Deploy image directory
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/homepage/images/"
+    dest: "{{ node['home_path'] }}/containers/homepage/images/"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/homepage/homepage.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/homepage.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_homepage"
+
+- name: Enable homepage.service
+  ansible.builtin.systemd:
+    name: "homepage.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_gethomepage_homepage_{{ version['containers']['homepage'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/gethomepage/homepage:{{ version['containers']['homepage'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_gethomepage_homepage_{{ version['containers']['homepage'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_gethomepage_homepage_{{ version['containers']['homepage'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_immich.yaml

@@ -8,7 +8,7 @@
     path: "{{ node['home_path'] }}/{{ item }}"
     state: "directory"
     owner: "{{ services['redis']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "containers/redis"
@@ -21,7 +21,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
     dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   register: "is_redis_conf"
 
@@ -30,7 +30,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   register: "is_redis_containerfile"
 
@@ -43,23 +43,16 @@
     scope: "user"
   when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
 
-- name: Add user in video, render group
-  ansible.builtin.user:
-    name: "{{ ansible_user }}"
-    state: "present"
-    groups: "video, render"
-    append: true
-  become: true
-
 - name: Create immich directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/{{ item }}"
     state: "directory"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "data/containers/immich"
+    - "data/containers/immich/data"
     - "containers/immich"
     - "containers/immich/ssl"
     - "containers/immich/ml"
@@ -71,7 +64,7 @@
       {{ hostvars['console']['ca']['root']['crt'] }}
     dest: "{{ node['home_path'] }}/containers/immich/ssl/{{ root_cert_filename }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0440"
   notify: "notification_restart_immich"
   no_log: true
@@ -85,12 +78,28 @@
   notify: "notification_restart_immich"
   no_log: true
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "{{ item.image }}"
+    tag: "{{ item.tag }}"
+    state: "present"
+    pull: true
+  loop:
+    - name: "immich_ml"
+      image: "ghcr.io/immich-app/immich-machine-learning"
+      tag: "{{ version['containers']['immich'] }}-openvino"
+    - name: "immich"
+      image: "ghcr.io/immich-app/immich-server"
+      tag: "{{ version['containers']['immich'] }}"
+  loop_control:
+    label: "{{ item.name }}"
+
 - name: Deploy immich.container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/immich/immich.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/immich.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_immich"
 
@@ -99,7 +108,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/immich/immich-ml.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/immich-ml.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_immich-ml"
 
@@ -118,3 +127,41 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - name: "redis"
+      image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - name: "immich_ml"
+      image: "ghcr.io/immich-app/immich-machine-learning:{{ version['containers']['immich'] }}-openvino"
+      file: "ghcr.io_immich-app_immich-machine-learning_{{ version['containers']['immich'] }}-openvino"
+    - name: "immich"
+      image: "ghcr.io/immich-app/immich-server:{{ version['containers']['immich'] }}"
+      file: "ghcr.io_immich-app_immich-server_{{ version['containers']['immich'] }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_kaneo.yaml

@@ -0,0 +1,86 @@
+---
+- name: Create kaneo directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['kaneo']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/kaneo"
+    - "containers/kaneo/ssl"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/kaneo/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['kaneo']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_kaneo"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "KANEO_AUTH_SECRET"
+      value: "{{ hostvars['console']['kaneo']['secret_key'] }}"
+    - name: "KANEO_DB_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['kaneo'] }}"
+    - name: "KANEO_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['kaneo']['oidc']['secret'] }}"
+    - name: "KANEO_S3_SECRET_KEY"
+      value: "{{ hostvars['console']['rustfs']['password']['kaneo'] }}"
+  notify: "notification_restart_kaneo"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/usekaneo/kaneo"
+    tag: "{{ version['containers']['kaneo'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy kaneo.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/kaneo/kaneo.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/kaneo.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_kaneo"
+
+- name: Enable kaneo.service
+  ansible.builtin.systemd:
+    name: "kaneo.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_usekaneo_kane_{{ version['containers']['kaneo'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/usekaneo/kaneo:{{ version['containers']['kaneo'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_usekaneo_kane_{{ version['containers']['kaneo'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_usekaneo_kane_{{ version['containers']['kaneo'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_linkding.yaml

@@ -0,0 +1,84 @@
+---
+- name: Create linkding directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['linkding']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/linkding"
+    - "data/containers/linkding/data"
+    - "containers/linkding"
+    - "containers/linkding/ssl"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/linkding/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['linkding']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_linkding"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "LINKDING_DB_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['linkding'] }}"
+    - name: "LINKDING_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['linkding']['oidc']['secret'] }}"
+  notify: "notification_restart_linkding"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/sissbruecker/linkding"
+    tag: "{{ version['containers']['linkding'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy linkding.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/linkding/linkding.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/linkding.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_linkding"
+
+- name: Enable linkding.service
+  ansible.builtin.systemd:
+    name: "linkding.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_sissbruecker_linkding_{{ version['containers']['linkding'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/sissbruecker/linkding:{{ version['containers']['linkding'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_sissbruecker_linkding_{{ version['containers']['linkding'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_sissbruecker_linkding_{{ version['containers']['linkding'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_llama.yaml

@@ -0,0 +1,90 @@
+---
+- name: Check model directory in data volume
+  ansible.builtin.find:
+    path: "{{ node['home_path'] }}/data/containers/llama/models"
+    file_type: "file"
+    patterns: "*.gguf"
+    hidden: true
+    recurse: true
+  register: llama_models
+
+- name: Failure when models directory in data volume
+  ansible.builtin.fail:
+    msg: "There are no models in data directory, please ensure download some model files for llama.cpp"
+  when: llama_models.matched == 0
+
+- name: Create llama directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/llama"
+    - "containers/llama/models"
+    - "containers/llama/config"
+
+- name: Sync model data (--delete=true)
+  delegate_to: "{{ inventory_hostname }}"
+  ansible.posix.synchronize:
+    src: "{{ node['home_path'] }}/data/containers/llama/models/"
+    dest: "{{ node['home_path'] }}/containers/llama/models/"
+    owner: true
+    group: true
+    perms: true
+    delete: true
+  notify: "notification_restart_llama"
+
+- name: Deploy configuration file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/llama/config/models.ini.j2"
+    dest: "{{ node['home_path'] }}/containers/llama/config/models.ini"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+  notify: "notification_restart_llama"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/ggml-org/llama.cpp"
+    tag: "{{ version['containers']['llama'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/llama/llama.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/llama.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_llama"
+
+- name: Enable llama.service
+  ansible.builtin.systemd:
+    name: "llama.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_ggml-org_llama.cpp_{{ version['containers']['llama'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/ggml-org/llama.cpp:{{ version['containers']['llama'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_ggml-org_llama.cpp_{{ version['containers']['llama'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_ggml-org_llama.cpp_{{ version['containers']['llama'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_mastodon.yaml

@@ -0,0 +1,276 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "mastodon"
+
+- name: Create redis_mastodon directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create mastodon directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['mastodon']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/mastodon"
+    - "data/containers/mastodon/system"
+    - "containers/mastodon"
+    - "containers/mastodon/ssl"
+  become: true
+
+- name: Check data directory empty
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/mastodon/.init"
+  register: "is_mastodon_init"
+
+- name: Check migrate file
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/mastodon/.migrated_{{ version['containers']['mastodon'] }}"
+  register: "is_mastodon_migrated"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/mastodon/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['mastodon']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_mastodon"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "MASTODON_SECRET_KEY_BASE"
+      value: "{{ hostvars['console']['mastodon']['secret_key'] }}"
+    - name: "MASTODON_ENCRYPTION_PRIMARY_KEY"
+      value: "{{ hostvars['console']['mastodon']['encryption']['primary_key'] }}"
+    - name: "MASTODON_ENCRYPTION_DETERMINISTIC_KEY"
+      value: "{{ hostvars['console']['mastodon']['encryption']['deterministic_key'] }}"
+    - name: "MASTODON_ENCRYPTION_SALT"
+      value: "{{ hostvars['console']['mastodon']['encryption']['salt'] }}"
+    - name: "MASTODON_POSTGRES_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['mastodon'] }}"
+    - name: "MASTODON_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['mastodon']['oidc']['secret'] }}"
+    - name: "MASTODON_VAPID_PRIVATE_KEY"
+      value: "{{ hostvars['console']['mastodon']['vapid']['private_key'] }}"
+    - name: "MASTODON_VAPID_PUBLIC_KEY"
+      value: "{{ hostvars['console']['mastodon']['vapid']['public_key'] }}"
+  notify: "notification_restart_mastodon"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "{{ item }}"
+    tag: "{{ version['containers']['mastodon'] }}"
+    state: "present"
+    pull: true
+  loop:
+    - "ghcr.io/mastodon/mastodon"
+    - "ghcr.io/mastodon/mastodon-streaming"
+
+- name: Initialize mastodon
+  when: not is_mastodon_init.stat.exists
+  block:
+    - name: Execute init command
+      containers.podman.podman_container:
+        name: "mastodon_init"
+        image: "ghcr.io/mastodon/mastodon:{{ version['containers']['mastodon'] }}"
+        command: "bundle exec rails db:prepare"
+        state: "started"
+        rm: true
+        detach: false
+        env:
+          LOCAL_DOMAIN: "{{ services['mastodon']['domain']['public'] }}.{{ domain['public'] }}"
+          SAFETY_ASSURED: "1"
+          DB_HOST: "{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}"
+          DB_PORT: "{{ services['postgresql']['ports']['tcp'] }}"
+          DB_NAME: "mastodon_db"
+          DB_USER: "mastodon"
+          DB_PASS: "{{ hostvars['console']['postgresql']['password']['mastodon'] }}"
+          DB_SSLMODE: "verify-full"
+          PGSSLROOTCERT: "/etc/ssl/mastodon/{{ root_cert_filename }}"
+          REDIS_URL: "redis://host.containers.internal:{{ services['mastodon']['ports']['redis'] }}"
+          SECRET_KEY_BASE: "{{ hostvars['console']['mastodon']['secret_key'] }}"
+          ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: "{{ hostvars['console']['mastodon']['encryption']['primary_key'] }}"
+          ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: "{{ hostvars['console']['mastodon']['encryption']['deterministic_key'] }}"
+          ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: "{{ hostvars['console']['mastodon']['encryption']['salt'] }}"
+        volume:
+          - "{{ node['home_path'] }}/containers/mastodon/ssl:/etc/ssl/mastodon:ro"
+          - "{{ node['home_path'] }}/data/containers/mastodon/system:/mastodon/public/system:rw"
+      no_log: true
+
+    - name: Create .init file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/containers/mastodon/.init"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+
+    - name: Create .migrated file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/containers/mastodon/.migrated_{{ version['containers']['mastodon'] }}"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+
+- name: Migrate DB of mastodon
+  when: not is_mastodon_migrated.stat.exists and is_mastodon_init.stat.exists
+  block:
+    - name: Stop the services
+      ansible.builtin.systemd:
+        name: "{{ item }}"
+        state: "stopped"
+        enabled: true
+        daemon_reload: true
+        scope: "user"
+      loop:
+        - "mastodon-web.service"
+        - "mastodon-worker.service"
+        - "mastodon-streaming.service"
+
+    - name: Execute migrate command
+      containers.podman.podman_container:
+        name: "mastodon_migrate"
+        image: "ghcr.io/mastodon/mastodon:{{ version['containers']['mastodon'] }}"
+        command: "bundle exec rails db:migrate"
+        state: "started"
+        rm: true
+        detach: false
+        env:
+          LOCAL_DOMAIN: "{{ services['mastodon']['domain']['public'] }}.{{ domain['public'] }}"
+          DB_HOST: "{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}"
+          DB_PORT: "{{ services['postgresql']['ports']['tcp'] }}"
+          DB_NAME: "mastodon_db"
+          DB_USER: "mastodon"
+          DB_PASS: "{{ hostvars['console']['postgresql']['password']['mastodon'] }}"
+          DB_SSLMODE: "verify-full"
+          PGSSLROOTCERT: "/etc/ssl/mastodon/{{ root_cert_filename }}"
+          REDIS_URL: "redis://host.containers.internal:{{ services['mastodon']['ports']['redis'] }}"
+          SECRET_KEY_BASE: "{{ hostvars['console']['mastodon']['secret_key'] }}"
+          ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: "{{ hostvars['console']['mastodon']['encryption']['primary_key'] }}"
+          ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: "{{ hostvars['console']['mastodon']['encryption']['deterministic_key'] }}"
+          ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: "{{ hostvars['console']['mastodon']['encryption']['salt'] }}"
+        volume:
+          - "{{ node['home_path'] }}/containers/mastodon/ssl:/etc/ssl/mastodon:ro"
+          - "{{ node['home_path'] }}/data/containers/mastodon/system:/mastodon/public/system:rw"
+      no_log: true
+
+    - name: Create .migrated file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/containers/mastodon/.migrated_{{ version['containers']['mastodon'] }}"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+
+- name: Deploy mastodon.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/mastodon/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "mastodon-web.container"
+    - "mastodon-worker.container"
+    - "mastodon-streaming.container"
+  notify: "notification_restart_mastodon"
+
+- name: Enable mastodon.service
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  loop:
+    - "mastodon-web.service"
+    - "mastodon-worker.service"
+    - "mastodon-streaming.service"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - name: "redis"
+      image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - name: "mastodon"
+      image: "ghcr.io/mastodon/mastodon:{{ version['containers']['mastodon'] }}"
+      file: "ghcr.io_mastodon_mastodon_{{ version['containers']['mastodon'] }}"
+    - name: "mastodon-streaming"
+      image: "ghcr.io/mastodon/mastodon-streaming:{{ version['containers']['mastodon'] }}"
+      file: "ghcr.io_mastodon_mastodon-streaming_{{ version['containers']['mastodon'] }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_nextcloud.yaml

@@ -0,0 +1,218 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "nextcloud"
+
+- name: Create redis_nextcloud directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create nextcloud directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['nextcloud']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/nextcloud"
+    - "data/containers/nextcloud/html"
+    - "containers/nextcloud"
+    - "containers/nextcloud/ssl"
+    - "containers/nextcloud/ini"
+  become: true
+
+- name: Check .init file
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/nextcloud/.init"
+  register: "is_nextcloud_init"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/nextcloud/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['nextcloud']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_nextcloud"
+  no_log: true
+
+- name: Initialize nextcloud
+  when: not is_nextcloud_init.stat.exists
+  block:
+    - name: Execute init command
+      containers.podman.podman_container:
+        name: "nextcloud_init"
+        image: "docker.io/library/nextcloud:{{ version['containers']['nextcloud'] }}"
+        command: "/bin/true"
+        state: "started"
+        rm: true
+        detach: false
+        env:
+          NEXTCLOUD_UPDATE: "1"
+          NEXTCLOUD_ADMIN_USER: "{{ hostvars['console']['nextcloud']['admin']['id'] }}"
+          NEXTCLOUD_ADMIN_PASSWORD: "{{ hostvars['console']['nextcloud']['admin']['password'] }}"
+          POSTGRES_HOST: "{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}"
+          POSTGRES_DB: "nextcloud_db"
+          POSTGRES_USER: "nextcloud"
+          POSTGRES_PASSWORD: "{{ hostvars['console']['postgresql']['password']['nextcloud'] }}"
+          PGSSLMODE: "verify-full"
+          PGSSLROOTCERT: "/etc/ssl/nextcloud/{{ root_cert_filename }}"
+          PGSSLCERTMODE: "disable"
+          REDIS_HOST: "host.containers.internal"
+          REDIS_HOST_PORT: "{{ services['nextcloud']['ports']['redis'] }}"
+        volume:
+          - "{{ node['home_path'] }}/containers/nextcloud/ssl:/etc/ssl/nextcloud:ro"
+          - "{{ node['home_path'] }}/data/containers/nextcloud/html:/var/www/html:rw"
+      no_log: true
+
+    - name: Create .init file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/containers/nextcloud/.init"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+
+- name: Deploy config files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/config/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/data/containers/nextcloud/html/config/{{ item }}"
+    owner: "{{ services['nextcloud']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+  loop:
+    - "background.config.php"
+    - "cache.config.php"
+    - "domain.config.php"
+    - "local_remote.config.php"
+    - "user_oidc.config.php"
+  become: true
+  notify: "notification_restart_nextcloud"
+
+- name: Deploy opcache.ini file
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/ini/{{ item }}"
+    dest: "{{ node['home_path'] }}/containers/nextcloud/ini/{{ item }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "opcache.ini"
+    - "upload.ini"
+  notify: "notification_restart_nextcloud"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/library/nextcloud"
+    tag: "{{ version['containers']['nextcloud'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy nextcloud.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/nextcloud.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/nextcloud.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_nextcloud"
+
+- name: Deploy nextcloud-cron service
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/services/{{ item }}"
+    dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "nextcloud-cron.service"
+    - "nextcloud-cron.timer"
+
+- name: Enable nextcloud.service
+  ansible.builtin.systemd:
+    name: "nextcloud.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Enable nextcloud-cron.timer
+  ansible.builtin.systemd:
+    name: "nextcloud-cron.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - name: "redis"
+      image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - name: "nextcloud"
+      image: "docker.io/library/nextcloud:{{ version['containers']['nextcloud'] }}"
+      file: "docker.io_library_nextcloud_{{ version['containers']['nextcloud'] }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_openwebui.yaml

@@ -0,0 +1,84 @@
+---
+- name: Create openwebui directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/openwebui"
+    - "data/containers/openwebui/data"
+    - "containers/openwebui"
+    - "containers/openwebui/ssl"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/openwebui/ssl/{{ root_cert_filename }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  notify: "notification_restart_openwebui"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "OPEN_WEBUI_SECRET_KEY"
+      value: "{{ hostvars['console']['openwebui']['secret_key'] }}"
+    - name: "OPEN_WEBUI_POSTGRES_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['openwebui'] }}"
+    - name: "OPEN_WEBUI_OAUTH_CLIENT_SECRET"
+      value: "{{ hostvars['console']['openwebui']['oidc']['secret'] }}"
+  notify: "notification_restart_openwebui"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/open-webui/open-webui"
+    tag: "{{ version['containers']['openwebui'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/openwebui/openwebui.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/openwebui.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_openwebui"
+
+- name: Enable openwebui.service
+  ansible.builtin.systemd:
+    name: "openwebui.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_open-webui_open-webui_{{ version['containers']['openwebui'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/open-webui/open-webui:{{ version['containers']['openwebui'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_open-webui_open-webui_{{ version['containers']['openwebui'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_open-webui_open-webui_{{ version['containers']['openwebui'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_outline.yaml

@@ -0,0 +1,161 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "outline"
+
+- name: Create redis_outline directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create outline directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['outline']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/outline"
+    - "data/containers/outline/data"
+    - "containers/outline"
+    - "containers/outline/ssl"
+  become: true
+
+- name: Check data directory empty
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/outline/.init"
+  register: "is_outline_init"
+
+- name: Create .init file
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/data/containers/outline/.init"
+    state: "touch"
+    mode: "0644"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+  when: not is_outline_init.stat.exists
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/outline/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['outline']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_outline"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "OUTLINE_SECRET_KEY"
+      value: "{{ hostvars['console']['outline']['secret_key'] }}"
+    - name: "OUTLINE_UTILS_SECRET"
+      value: "{{ hostvars['console']['outline']['utils_secret'] }}"
+    - name: "OUTLINE_DATABASE_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['outline'] }}"
+    - name: "OUTLINE_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['outline']['oidc']['secret'] }}"
+  notify: "notification_restart_outline"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/outlinewiki/outline"
+    tag: "{{ version['containers']['outline'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy outline.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/outline/outline.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/outline.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_outline"
+
+- name: Enable outline.service
+  ansible.builtin.systemd:
+    name: "outline.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - name: "redis"
+      image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - name: "outline"
+      image: "docker.io/outlinewiki/outline:{{ version['containers']['outline'] }}"
+      file: "docker.io_outlinewiki_outline_{{ version['containers']['outline'] }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_papra.yaml

@@ -0,0 +1,137 @@
+---
+- name: Create papra directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['papra']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/papra"
+    - "data/containers/papra/data"
+    - "data/containers/papra/backup"
+    - "containers/papra"
+    - "containers/papra/script"
+    - "containers/papra/db"
+  become: true
+
+- name: Check backup file
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/papra/backup/db.sqlite"
+  register: "is_db_backup"
+
+- name: Check database file
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/containers/papra/db/db.sqlite"
+  register: "is_db"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/papra-hq/papra"
+    tag: "{{ version['containers']['papra'] }}"
+    state: "present"
+    pull: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "PAPRA_AUTH_SECRET"
+      value: "{{ hostvars['console']['papra']['secret_key'] }}"
+    - name: "PAPRA_AUTH_PROVIDERS_CUSTOMS"
+      value: |
+        [
+          {
+            "providerId": "authelia",
+            "providerName": "Authelia",
+            "providerIconUrl": "https://www.authelia.com/images/branding/logo-cropped.png",
+            "clientId": "papra",
+            "clientSecret": "{{ hostvars['console']['papra']['oidc']['secret'] }}",
+            "type": "oidc",
+            "discoveryUrl": "https://{{ services['authelia']['domain']['public'] }}.{{ domain['public'] }}/.well-known/openid-configuration",
+            "scopes": ["openid", "profile", "email"]
+          }
+        ]
+  notify: "notification_restart_papra"
+  no_log: true
+
+- name: Restore db file
+  ansible.builtin.copy:
+    src: "{{ node['home_path'] }}/data/containers/papra/backup/db.sqlite"
+    remote_src: true
+    dest: "{{ node['home_path'] }}/containers/papra/db/db.sqlite"
+    owner: "{{ services['papra']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0660"
+  become: true
+  when:
+    - is_db_backup.stat.exists
+    - not is_db.stat.exists
+
+- name: Deploy papra.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/papra/papra.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/papra.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  become: true
+  notify: "notification_restart_papra"
+
+- name: Deploy papra-backup service
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/papra/services/{{ item }}"
+    dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "papra-backup.service"
+    - "papra-backup.timer"
+
+- name: Deploy papra script
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/papra/script/"
+    dest: "{{ node['home_path'] }}/containers/papra/script/"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0755"
+
+- name: Enable papra.service
+  ansible.builtin.systemd:
+    name: "papra.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Enable papra-backup.timer
+  ansible.builtin.systemd:
+    name: "papra-backup.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_papra-hq_papra_{{ version['containers']['papra'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/papra-hq/papra:{{ version['containers']['papra'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_papra-hq_papra_{{ version['containers']['papra'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_papra-hq_papra_{{ version['containers']['papra'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_snappymail.yaml

@@ -0,0 +1,90 @@
+---
+- name: Create snappymail directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['snappymail']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/snappymail"
+    - "data/containers/snappymail/data"
+    - "containers/snappymail"
+    - "containers/snappymail/ssl"
+    - "containers/snappymail/configs"
+    - "containers/snappymail/domains"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/snappymail/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['snappymail']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_snappymail"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/the-djmaze/snappymail"
+    tag: "{{ version['containers']['snappymail'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy config files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/snappymail/{{ item.src }}.j2"
+    dest: "{{ node['home_path'] }}/containers/snappymail/{{ item.dest }}"
+    owner: "{{ services['snappymail']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - src: "configs/application.ini"
+      dest: "configs/application.ini"
+    - src: "domains/domain.ini"
+      dest: "domains/{{ domain['internal'] }}.ini"
+  loop_control:
+    label: "{{ item.src | basename }}"
+  become: true
+  notify: "notification_restart_snappymail"
+
+- name: Deploy snappymail.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/snappymail/snappymail.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/snappymail.container"
+    owner: "{{ services['snappymail']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  become: true
+  notify: "notification_restart_snappymail"
+
+- name: Enable snappymail.service
+  ansible.builtin.systemd:
+    name: "snappymail.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_the-djmaze_snappymail_{{ version['containers']['snappymail'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/the-djmaze/snappymail:{{ version['containers']['snappymail'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_the-djmaze_snappymail_{{ version['containers']['snappymail'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_the-djmaze_snappymail_{{ version['containers']['snappymail'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_stablediffusion.yaml

@@ -0,0 +1,91 @@
+---
+- name: Check model directory in data volume
+  ansible.builtin.find:
+    path: "{{ node['home_path'] }}/data/containers/stablediffusion/models"
+    file_type: "file"
+    patterns:
+      - "*.gguf"
+      - "*.safetensors"
+    hidden: true
+    recurse: true
+  register: stablediffusion_models
+
+- name: Failure when models directory in data volume
+  ansible.builtin.fail:
+    msg: "There are no models in data directory, please ensure download some model files for stable-diffusion.cpp"
+  when: stablediffusion_models.matched == 0
+
+- name: Create stablediffusion directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/stablediffusion"
+    - "containers/stablediffusion/models"
+
+- name: Sync model data (--delete=true)
+  delegate_to: "{{ inventory_hostname }}"
+  ansible.posix.synchronize:
+    src: "{{ node['home_path'] }}/data/containers/stablediffusion/models/"
+    dest: "{{ node['home_path'] }}/containers/stablediffusion/models/"
+    owner: true
+    group: true
+    perms: true
+    delete: true
+  notify: "notification_restart_stablediffusion"
+
+- name: Set active model
+  ansible.builtin.set_fact:
+    stabeldiffusion_active_model: "{{ services['stablediffusion']['dependency']['models']['active'] }}"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/leejet/stable-diffusion.cpp"
+    tag: "{{ version['containers']['stablediffusion'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/stablediffusion/stablediffusion.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/stablediffusion.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_stablediffusion"
+
+- name: Enable stablediffusion.service
+  ansible.builtin.systemd:
+    name: "stablediffusion.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Set stable-diffusion archive version
+  ansible.builtin.set_fact:
+    stablediffusion_archive_version: >-
+      {{ version['containers']['stablediffusion'].split('@sha256:')[0] }}@{{ version['containers']['stablediffusion'].split('@sha256:')[1][0:7] }}
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/ghcr.io_leejet_stable-diffusion.cpp_{{ stablediffusion_archive_version }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "ghcr.io/leejet/stable-diffusion.cpp:{{ version['containers']['stablediffusion'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/ghcr.io_leejet_stable-diffusion.cpp_{{ stablediffusion_archive_version }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/ghcr.io_leejet_stable-diffusion.cpp_{{ stablediffusion_archive_version }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_sure.yaml

@@ -0,0 +1,152 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "sure"
+
+- name: Create redis_sure directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create sure directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['sure']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/sure"
+    - "data/containers/sure/storage"
+    - "containers/sure"
+    - "containers/sure/ssl"
+  become: true
+
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/sure/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['sure']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_sure"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "SURE_SECRET_KEY_BASE"
+      value: "{{ hostvars['console']['sure']['secret_key'] }}"
+    - name: "SURE_POSTGRES_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['sure'] }}"
+    - name: "SURE_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['sure']['oidc']['secret'] }}"
+  notify: "notification_restart_sure"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "ghcr.io/we-promise/sure"
+    tag: "{{ version['containers']['sure'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy sure.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/sure/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "sure-web.container"
+    - "sure-worker.container"
+  notify: "notification_restart_sure"
+
+- name: Enable sure.service
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  loop:
+    - "sure-web.service"
+    - "sure-worker.service"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - name: "redis"
+      image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - name: "sure"
+      image: "ghcr.io/we-promise/sure:{{ version['containers']['sure'] }}"
+      file: "ghcr.io_we-promise_sure_{{ version['containers']['sure'] }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_tandoor.yaml

@@ -0,0 +1,102 @@
+---
+- name: Create tandoor directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/tandoor"
+    - "data/containers/tandoor/mediafiles"
+    - "containers/tandoor"
+    - "containers/tandoor/ssl"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/tandoor/ssl/{{ root_cert_filename }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  notify: "notification_restart_tandoor"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "TANDOOR_SECRET_KEY"
+      value: "{{ hostvars['console']['tandoor']['secret_key'] }}"
+    - name: "TANDOOR_POSTGRES_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['tandoor'] }}"
+    - name: "TANDOOR_SOCIALACCOUNT_PROVIDERS"
+      value: |
+        {
+          "openid_connect": {
+            "SCOPE": ["openid", "profile", "email"],
+            "OAUTH_PKCE_ENABLED": true,
+            "APPS": [
+              {
+                "provider_id": "authelia",
+                "name": "authelia",
+                "client_id": "tandoor",
+                "secret": "{{ hostvars['console']['tandoor']['oidc']['secret'] }}",
+                "settings": {
+                  "server_url": "https://{{ services['authelia']['domain']['public'] }}.{{ domain['public'] }}/.well-known/openid-configuration",
+                  "token_auth_method": "client_secret_basic"
+                }
+              }
+            ]
+          }
+        }
+  notify: "notification_restart_tandoor"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/vabene1111/recipes"
+    tag: "{{ version['containers']['tandoor'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy tandoor.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/tandoor/tandoor.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/tandoor.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_tandoor"
+
+- name: Enable tandoor.service
+  ansible.builtin.systemd:
+    name: "tandoor.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_vabene1111_recipes_{{ version['containers']['tandoor'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/vabene1111/recipes:{{ version['containers']['tandoor'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_vabene1111_recipes_{{ version['containers']['tandoor'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_vabene1111_recipes_{{ version['containers']['tandoor'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_vaultwarden.yaml

@@ -4,10 +4,11 @@
     path: "{{ node['home_path'] }}/{{ item }}"
     state: "directory"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "data/containers/vaultwarden"
+    - "data/containers/vaultwarden/data"
     - "containers/vaultwarden"
     - "containers/vaultwarden/ssl"
 
@@ -17,7 +18,7 @@
       {{ hostvars['console']['ca']['root']['crt'] }}
     dest: "{{ node['home_path'] }}/containers/vaultwarden/ssl/{{ root_cert_filename }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0440"
   become: true
   notify: "notification_restart_vaultwarden"
@@ -34,17 +35,24 @@
       value: "{{ hostvars['console']['vaultwarden']['admin']['hash'] }}"
     - name: "VW_DATABASE_URL"
       value: "postgresql://vaultwarden:{{ hostvars['console']['postgresql']['password']['vaultwarden'] | urlencode | replace('/', '%2F') }}\
-        @{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}/vaultwarden_db?sslmode=verify-full&\
+        @{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}/vaultwarden_db?sslmode=verify-full&\
         sslrootcert=/etc/ssl/vaultwarden/{{ root_cert_filename }}"
   notify: "notification_restart_vaultwarden"
   no_log: true
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/vaultwarden/server"
+    tag: "{{ version['containers']['vaultwarden'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/vaultwarden/vaultwarden.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/vaultwarden.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_vaultwarden"
 
@@ -55,3 +63,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_vaultwarden_server_{{ version['containers']['vaultwarden'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/vaultwarden/server:{{ version['containers']['vaultwarden'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_vaultwarden_server_{{ version['containers']['vaultwarden'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_vaultwarden_server_{{ version['containers']['vaultwarden'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/auth/tasks/services/set_authelia.yaml

@@ -4,7 +4,7 @@
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "authelia"
@@ -17,7 +17,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/auth/authelia/config/authelia.yaml.j2"
     dest: "{{ node['home_path'] }}/containers/authelia/config/authelia.yaml"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
   become: true
   notify: "notification_restart_authelia"
@@ -29,7 +29,7 @@
       {{ hostvars['console']['ca']['root']['crt'] }}
     dest: "{{ node['home_path'] }}/containers/authelia/certs/{{ root_cert_filename }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0440"
   become: true
   no_log: true
@@ -54,18 +54,25 @@
     - name: "AUTHELIA_JWKS_ES256"
       value: "{{ hostvars['console']['authelia']['jwk_es256'] }}"
     - name: "AUTHELIA_LDAP_PASSWORD"
-      value: "{{ hostvars['console']['ldap']['password']['authelia'] }}"
+      value: "{{ hostvars['console']['ldap']['password']['services']['authelia'] }}"
     - name: "POSTGRES_AUTHELIA_PASSWORD"
       value: "{{ hostvars['console']['postgresql']['password']['authelia'] }}"
   notify: "notification_restart_authelia"
   no_log: true
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/authelia/authelia"
+    tag: "{{ version['containers']['authelia'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/auth/authelia/authelia.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/authelia.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_authelia"
 
@@ -76,3 +83,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_authelia_authelia_{{ version['containers']['authelia'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/authelia/authelia:{{ version['containers']['authelia'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_authelia_authelia_{{ version['containers']['authelia'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_authelia_authelia_{{ version['containers']['authelia'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/bk/tasks/services/set_rclone.yaml

@@ -0,0 +1,45 @@
+---
+- name: Install rclone
+  ansible.builtin.apt:
+    name:
+      - "rclone"
+    state: "present"
+  become: true
+
+- name: Create rclone config directory
+  ansible.builtin.file:
+    path: "/etc/secrets/{{ node['uid'] }}/rclone"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "root"
+    mode: "0700"
+  become: true
+
+- name: Deploy rclone config
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/bk/rclone/config/rclone.config.j2"
+    dest: "/etc/secrets/{{ node['uid'] }}/rclone/rclone.config"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0600"
+  no_log: true
+
+- name: Deploy rclone backup service files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/bk/rclone/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+    validate: "/usr/bin/systemd-analyze verify %s"
+  loop:
+    - "rclone.service"
+    - "rclone.timer"
+
+- name: Enable rclone timer
+  ansible.builtin.systemd:
+    name: "rclone.timer"
+    state: "started"
+    enabled: true
+    scope: "user"
+    daemon_reload: true

ansible/roles/common/handlers/main.yaml

@@ -99,3 +99,14 @@
   changed_when: false
   listen: "notification_restart_alloy"
   ignore_errors: true # noqa: ignore-errors
+
+- name: Restart kopia server
+  ansible.builtin.systemd:
+    name: "kopia-server.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_kopia_server"
+  ignore_errors: true # noqa: ignore-errors

ansible/roles/common/tasks/node/create_default_dir.yaml

@@ -22,7 +22,7 @@
     path: "{{ node['home_path'] }}/.config/systemd/user"
     state: "directory"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0700"
 
 - name: Create quadlet directory
@@ -30,5 +30,14 @@
     path: "{{ node['home_path'] }}/.config/containers/systemd"
     state: "directory"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0700"
+
+- name: Set data volume path
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/data"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  when: node['name'] in ["infra", "app"]

ansible/roles/common/tasks/node/set_data_volume.yaml

@@ -0,0 +1,41 @@
+---
+- name: Check data volume filesystem already exists
+  ansible.builtin.command: |
+    blkid /dev/{{ vm['data_storage']['name'] }}
+  register: is_volume
+  changed_when: false
+  failed_when: false
+  become: true
+
+- name: Create ext4 filesystem on data raw
+  ansible.builtin.command: |
+    mkfs.ext4 -F /dev/{{ vm['data_storage']['name'] }}
+  when: is_volume.rc != 0
+  changed_when: true
+  become: true
+
+- name: Set data volume path
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/data"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+
+- name: Mount data volume
+  ansible.posix.mount:
+    path: "{{ node['home_path'] }}/data"
+    src: "/dev/{{ vm['data_storage']['name'] }}"
+    opts: "noatime"
+    state: "mounted"
+    fstype: "ext4"
+  become: true
+
+- name: Set data volume path permissions
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/data"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  become: true

ansible/roles/common/tasks/node/set_networkd.yaml

@@ -11,7 +11,7 @@
 - name: Deploy networkd files
   ansible.builtin.template:
     src: "{{ item }}"
-    dest: "/etc/systemd/network/{{ item | basename }}"
+    dest: "/etc/systemd/network/{{ item | basename | replace('.j2', '') }}"
     owner: "root"
     group: "systemd-network"
     mode: "0640"

ansible/roles/common/tasks/node/set_nftables.yaml

@@ -1,19 +1,10 @@
 ---
-- name: Check nftables installation
-  ansible.builtin.shell: |
-    command -v nft
-  become: true # nftables is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_nftables_installed"
-  ignore_errors: true
-
 - name: Install nftables
   ansible.builtin.apt:
-    name: "nftables"
+    name:
+      - "nftables"
     state: "present"
   become: true
-  when: is_nftables_installed.rc != 0
 
 - name: Enable nftables.service
   ansible.builtin.systemd:

ansible/roles/common/tasks/node/set_timesyncd.yaml

@@ -9,8 +9,8 @@
   become: true
 
 - name: Deploy timesyncd.conf.d/local-ntp.conf
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['config_path'] }}/node/common/timesyncd/local-ntp.conf"
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/node/common/timesyncd/local-ntp.conf.j2"
     dest: "/etc/systemd/timesyncd.conf.d/local-ntp.conf"
     owner: "root"
     group: "systemd-timesync"

ansible/roles/common/tasks/services/set_alloy.yaml

@@ -5,9 +5,10 @@
       - hardware
   become: true
 
-- name: Deploy alloy deb file (x86_64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-amd64.deb"
+- name: Download alloy deb file (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.amd64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -15,9 +16,10 @@
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
-- name: Deploy alloy deb file (aarch64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-arm64.deb"
+- name: Download alloy deb file (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.arm64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -30,6 +32,7 @@
     deb: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     state: "present"
   become: true
+  notify: "notification_restart_alloy"
 
 - name: Deploy alloy config
   ansible.builtin.template:
@@ -64,6 +67,19 @@
   become: true
   notify: "notification_restart_alloy"
 
+- name: Set ACL for data directory
+  ansible.posix.acl:
+    path: "{{ item }}"
+    entity: "alloy"
+    etype: "user"
+    permissions: "x"
+    state: "present"
+  loop:
+    - "{{ node['home_path'] }}"
+    - "{{ node['home_path'] }}/data"
+  become: true
+  when: node['name'] in ["bk", "app"]
+
 - name: Enable alloy service
   ansible.builtin.systemd:
     name: "alloy.service"
@@ -71,3 +87,17 @@
     enabled: true
     daemon_reload: true
   become: true
+
+- name: Check alloy.deb on console
+  delegate_to: "console"
+  ansible.builtin.stat:
+    path: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}.deb"
+  register: is_console_alloy_deb
+
+- name: Fetch deb bin file
+  ansible.builtin.fetch:
+    src: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/bin/"
+    flat: true
+  when: not is_console_alloy_deb.stat.exists
+  become: true

ansible/roles/common/tasks/services/set_caddy.yaml

@@ -1,10 +1,53 @@
 ---
+- name: Initialize Caddy services list
+  ansible.builtin.set_fact:
+    caddy_services: []
+
 # infra, auth, app (vmm, fw has no podman in it)
+- name: Set Caddy infra services list
+  ansible.builtin.set_fact:
+    caddy_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.web', 'defined')
+        | selectattr('value.dependency.web', 'in', ['infra-internal', 'infra-public'])
+        | map(attribute='key')
+        | list
+      }}
+  when: node['name'] == 'infra'
+
+- name: Set Caddy auth services list
+  ansible.builtin.set_fact:
+    caddy_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.web', 'defined')
+        | selectattr('value.dependency.web', 'in', ['auth-public', 'infra-public', 'app-public'])
+        | map(attribute='key')
+        | list
+      }}
+  when: node['name'] == 'auth'
+
+- name: Set Caddy app services list
+  ansible.builtin.set_fact:
+    caddy_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.web', 'defined')
+        | selectattr('value.dependency.web', 'in', ['app-internal', 'app-public'])
+        | map(attribute='key')
+        | list
+      }}
+  when: node['name'] == 'app'
+
 - name: Create caddy directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0770"
   loop:
@@ -18,7 +61,7 @@
   ansible.builtin.file:
     path: /var/log/caddy
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0755"
   become: true
@@ -48,16 +91,20 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/common/caddy/build/caddy.containerfile.j2"
     dest: "{{ node['home_path'] }}/containers/caddy/build/Containerfile"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0640"
+  register: "is_caddy_containerfile"
+  notify: "notification_restart_caddy"
 
 - name: Deploy root crt for build
   ansible.builtin.copy:
     content: "{{ hostvars['console']['ca']['root']['crt'] }}"
     dest: "{{ node['home_path'] }}/containers/caddy/build/{{ root_cert_filename }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0640"
+  register: "is_caddy_root_crt"
+  notify: "notification_restart_caddy"
   no_log: true
 
 - name: Build caddy container image
@@ -67,6 +114,7 @@
     tag: "{{ version['containers']['caddy'] }}"
     state: "build"
     path: "{{ node['home_path'] }}/containers/caddy/build"
+    force: "{{ is_caddy_root_crt.changed or is_caddy_containerfile.changed }}"
 
 - name: Prune caddy dangling images
   containers.podman.podman_prune:
@@ -74,10 +122,10 @@
 
 - name: Deploy caddyfile
   ansible.builtin.template:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/common/caddy/etc/{{ node['name'] }}/Caddyfile.j2"
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/common/caddy/etc/Caddyfile.j2"
     dest: "{{ node['home_path'] }}/containers/caddy/etc/Caddyfile"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
   notify: "notification_restart_caddy"
 
@@ -86,7 +134,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/common/caddy/caddy.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/caddy.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_caddy"
 
@@ -97,3 +145,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ domain['internal'] }}_{{ node['name'] }}_caddy_{{ version['containers']['caddy'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ domain['internal'] }}/{{ node['name'] }}/caddy:{{ version['containers']['caddy'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ domain['internal'] }}_{{ node['name'] }}_caddy_{{ version['containers']['caddy'] }}.tar"
+    format: "oci-archive"
+    force: "{{ is_caddy_root_crt.changed or is_caddy_containerfile.changed }}"
+  when: not container_archive_images.stat.exists or is_caddy_root_crt.changed or is_caddy_containerfile.changed
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ domain['internal'] }}_{{ node['name'] }}_caddy_{{ version['containers']['caddy'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/common/tasks/services/set_crowdsec.yaml

@@ -1,45 +1,32 @@
 ---
-- name: Check crowdsec installed
-  ansible.builtin.shell: |
-    command -v crowdsec
-  changed_when: false
-  failed_when: false
-  register: "is_crowdsec_installed"
-  ignore_errors: true
-
-- name: Check crowdsec bouncer installed
-  ansible.builtin.shell: |
-    command -v crowdsec-firewall-bouncer
-  when: node['name'] == "fw"
-  changed_when: false
-  failed_when: false
-  register: "is_crowdsec_bouncer_installed"
-  ignore_errors: true
-
 - name: Install crowdsec
   ansible.builtin.apt:
-    name: "crowdsec"
+    name:
+      - "crowdsec"
     state: "present"
   become: true
-  when: is_crowdsec_installed.rc != 0
 
 - name: Install crowdsec bouncers
   ansible.builtin.apt:
-    name: "crowdsec-firewall-bouncer"
+    name:
+      - "crowdsec-firewall-bouncer"
     state: "present"
   become: true
-  when:
-    - node['name'] == "fw"
-    - is_crowdsec_bouncer_installed.rc != 0
+  when: node['name'] == "fw"
 
 - name: Set acquis.d list for bouncer
   ansible.builtin.set_fact:
     acquisd_list:
       fw:
-        collection: "crowdsecurity/suricata"
+        collection:
+          - "crowdsecurity/suricata"
+        parser: []
         config: "suricata.yaml"
       auth:
-        collection: "crowdsecurity/caddy"
+        collection:
+          - "crowdsecurity/caddy"
+        parser:
+          - "crowdsecurity/nextcloud-whitelist"
         config: "caddy.yaml"
 
 - name: Deploy crowdsec-update service files
@@ -115,8 +102,8 @@
       no_log: true
 
     - name: Get existing machines list
-      ansible.builtin.command:
-        cmd: "cscli machines list -o json"
+      ansible.builtin.command: |
+        cscli machines list -o json
       become: true
       changed_when: false
       register: "existing_crowdsec_machines_list"
@@ -127,27 +114,27 @@
 
     - name: Set goal machines' name
       ansible.builtin.set_fact:
-        machines_name: ["fw", "vmm", "infra", "auth", "app"]
+        machines_name: "{{ services['crowdsec']['sub']['machines'] | list }}"
       no_log: true
 
     - name: Prune unknown (random) machines
-      ansible.builtin.command:
-        cmd: "cscli machines delete {{ item }}"
+      ansible.builtin.command: |
+        cscli machines delete {{ item }}
       loop: "{{ existing_machines_name | difference(machines_name) }}"
       become: true
       changed_when: true
 
     - name: Register crowdsec machines to LAPI server
-      ansible.builtin.command:
-        cmd: "cscli machines add {{ item }} --password {{ hostvars['console']['crowdsec']['machine'][item] }} --force -f /dev/null"
+      ansible.builtin.command: |
+        cscli machines add {{ item }} --password {{ hostvars['console']['crowdsec']['machine'][item] }} --force -f /dev/null
       loop: "{{ machines_name }}"
       become: true
       changed_when: false
       no_log: true
 
     - name: Get existing bouncers list
-      ansible.builtin.command:
-        cmd: "cscli bouncers list -o json"
+      ansible.builtin.command: |
+        cscli bouncers list -o json
       become: true
       register: "existing_crowdsec_bouncers_list"
       changed_when: false
@@ -157,19 +144,19 @@
         existing_bouncers_name: "{{ existing_crowdsec_bouncers_list.stdout | from_json | map(attribute='name') | list }}"
 
     - name: Flush bouncers
-      ansible.builtin.command:
-        cmd: "cscli bouncers delete {{ item }}"
+      ansible.builtin.command: |
+        cscli bouncers delete {{ item }}
       loop: "{{ existing_bouncers_name }}"
       become: true
       changed_when: true
 
     - name: Set bouncers' name
       ansible.builtin.set_fact:
-        bouncers_name: ["fw", "caddy"]
+        bouncers_name: "{{ services['crowdsec']['sub']['bouncers'] | list }}"
 
     - name: Register Firewall Bouncer to LAPI
-      ansible.builtin.command:
-        cmd: "cscli bouncers add {{ item }}-bouncer -k {{ hostvars['console']['crowdsec']['bouncer'][item] }}"
+      ansible.builtin.command: |
+        cscli bouncers add {{ item }}-bouncer -k {{ hostvars['console']['crowdsec']['bouncer'][item] }}
       loop: "{{ bouncers_name }}"
       become: true
       changed_when: true
@@ -179,9 +166,16 @@
 - name: Set crowdsec bouncer
   when: node['name'] in acquisd_list
   block:
+    - name: Update and upgrade crowdsec hub
+      ansible.builtin.shell: |
+        cscli hub update && cscli hub upgrade
+      become: true
+      changed_when: false
+
     - name: Install crowdsec collection
-      ansible.builtin.command:
-        cmd: "cscli collections install {{ acquisd_list[node['name']]['collection'] }}"
+      ansible.builtin.command: |
+        cscli collections install {{ item }}
+      loop: "{{ acquisd_list[node['name']]['collection'] }}"
       become: true
       changed_when: "'overwrite' not in is_collection_installed.stderr"
       failed_when:
@@ -189,6 +183,17 @@
         - "'already installed' not in is_collection_installed.stderr"
       register: "is_collection_installed"
 
+    - name: Install crowdsec parser
+      ansible.builtin.command: |
+        cscli parsers install {{ item }}
+      loop: "{{ acquisd_list[node['name']]['parser'] }}"
+      become: true
+      changed_when: "'overwrite' not in is_parser_installed.stderr"
+      failed_when:
+        - is_parser_installed.rc != 0
+        - "'already installed' not in is_parser_installed.stderr"
+      register: "is_parser_installed"
+
     - name: Create crowdsec acquis.d directory
       ansible.builtin.file:
         path: "/etc/crowdsec/acquis.d"
@@ -298,7 +303,3 @@
     enabled: true
     daemon_reload: true
   become: true
-
-# cscli bouncers list
-# cscli machines list
-# cscli metrics

ansible/roles/common/tasks/services/set_kopia.yaml

@@ -5,86 +5,336 @@
       - hardware
   become: true
 
-- name: Check kopia installation
-  ansible.builtin.shell: |
-    command -v kopia
-  changed_when: false
-  failed_when: false
-  register: "is_kopia_installed"
-  ignore_errors: true
-
-- name: Set console kopia
-  when: node['name'] == 'console'
+- name: Set backup kopia repository server
+  when: node['name'] == "bk"
   block:
-    - name: Apply cli tools (x86_64)
-      ansible.builtin.apt:
-        deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
-        state: "present"
-      become: true
-      when:
-        - ansible_facts['architecture'] == "x86_64"
-        - is_kopia_installed.rc != 0
-    - name: Apply cli tools (aarch64)
-      ansible.builtin.apt:
-        deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
-        state: "present"
-      become: true
-      when:
-        - ansible_facts['architecture'] == "aarch64"
-        - is_kopia_installed.rc != 0
-    - name: Connect kopia server
-      environment:
-        KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['user']['console'] }}"
-      ansible.builtin.shell: |
-        /usr/bin/kopia repository connect server \
-        --url=https://{{ services['kopia']['domain'] }}.{{ domain['internal'] }}:{{ services['kopia']['ports']['https'] }} \
-        --override-username=console \
-        --override-hostname=console.{{ domain['internal'] }}
-      changed_when: false
-      failed_when: is_kopia_connected.rc != 0
-      register: "is_kopia_connected"
-      no_log: true
-
-- name: Set infra/app kopia
-  when: node['name'] in ['infra', 'app']
-  block:
-    - name: Set kopia uid
-      ansible.builtin.set_fact:
-        kopia_uid: 951
-    - name: Deploy kopia deb file (x86_64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
+    - name: Download kopia deb file (x86_64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_amd64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "x86_64"
-    - name: Deploy kopia deb file (aarch64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
+
+    - name: Download kopia deb file (aarch64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_arm64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "aarch64"
+
+    - name: Create kopia directory
+      ansible.builtin.file:
+        path: "{{ item.name }}"
+        state: "directory"
+        owner: "{{ ansible_user }}"
+        group: "root"
+        mode: "{{ item.mode }}"
+      loop:
+        - name: "/etc/kopia"
+          mode: "0700"
+        - name: "/etc/kopia/ssl"
+          mode: "0700"
+        - name: "/var/cache/kopia"
+          mode: "0700"
+      become: true
+      no_log: true
+
+    - name: Install kopia
+      ansible.builtin.apt:
+        deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
+        state: "present"
+      become: true
+      notify: "notification_restart_kopia_server"
+
+    - name: Deploy kopia env
+      ansible.builtin.template:
+        src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2"
+        dest: "/etc/secrets/{{ node['uid'] }}/kopia.env"
+        owner: "{{ ansible_user }}"
+        group: "root"
+        mode: "0400"
+      become: true
+      notify: "notification_restart_kopia_server"
+      no_log: true
+
+    - name: Deploy kopia certificate and key
+      ansible.builtin.copy:
+        content: |
+          {{ item.value }}
+        dest: "/etc/kopia/ssl/{{ item.name }}"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+        mode: "{{ item.mode }}"
+      loop:
+        - name: "kopia.crt"
+          value: |
+            {{ hostvars['console']['kopia']['crt'] | trim }}
+            {{ hostvars['console']['ca']['intermediate']['crt'] }}
+          mode: "0440"
+        - name: "kopia.key"
+          value: "{{ hostvars['console']['kopia']['key'] }}"
+          mode: "0400"
+      notify: "notification_restart_kopia_server"
+      no_log: true
+
+    - name: Create repository directory
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/kopia/repository"
+        state: "directory"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+        mode: "0770"
+
+    - name: Check repository directory empty
+      ansible.builtin.find:
+        path: "{{ node['home_path'] }}/data/kopia/repository/"
+        file_type: "any"
+        hidden: true
+        recurse: false
+      register: "is_kopia_repository_empty"
+
+    - name: Create kopia repository
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia repository create filesystem \
+        --path="{{ node['home_path'] }}/data/kopia/repository" \
+        --description="{{ domain['internal'] }} kopia repository"
+      when: is_kopia_repository_empty.matched == 0
+      changed_when: true
+      no_log: true
+
+    - name: Connect to kopia repository
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia repository connect filesystem \
+        --path="{{ node['home_path'] }}/data/kopia/repository" \
+        --override-username="il" \
+        --override-hostname="{{ node['name'] }}.{{ domain['internal'] }}"
+      when: is_kopia_repository_empty.matched > 0
+      changed_when: false
+      failed_when:
+        - is_kopia_repository_empty.matched > 0
+        - is_repository_connected.rc != 0
+      register: "is_repository_connected"
+      no_log: true
+
+    - name: Create kopia user
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia server users add \
+        --user-password {{ item.password | quote }} \
+        {{ item.name }}
+      loop:
+        - name: "console@console.{{ domain['internal'] }}"
+          password: "{{ hostvars['console']['kopia']['users']['console'] }}"
+        - name: "infra@infra.{{ domain['internal'] }}"
+          password: "{{ hostvars['console']['kopia']['users']['infra'] }}"
+        - name: "app@app.{{ domain['internal'] }}"
+          password: "{{ hostvars['console']['kopia']['users']['app'] }}"
+      when: is_kopia_repository_empty.matched == 0
+      changed_when: false
+      no_log: true
+
+    - name: Update kopia user
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia server users set \
+        --user-password {{ item.password | quote }} \
+        {{ item.name }}
+      loop:
+        - name: "console@console.{{ domain['internal'] }}"
+          password: "{{ hostvars['console']['kopia']['users']['console'] }}"
+        - name: "infra@infra.{{ domain['internal'] }}"
+          password: "{{ hostvars['console']['kopia']['users']['infra'] }}"
+        - name: "app@app.{{ domain['internal'] }}"
+          password: "{{ hostvars['console']['kopia']['users']['app'] }}"
+      when: is_kopia_repository_empty.matched > 0
+      changed_when: false
+      no_log: true
+
+    - name: Set kopia console ACL
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia server acl add \
+        --user console@console.{{ domain['internal'] }} \
+        --target {{ item }} \
+        --access FULL \
+        --overwrite
+      loop:
+        - "type=snapshot"
+        - "type=policy"
+        - "type=user"
+        - "type=acl"
+        - "type=content"
+      changed_when: false
+      no_log: true
+
+    - name: Set kopia infra ACL
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia server acl add \
+        --user infra@infra.{{ domain['internal'] }} \
+        --target {{ item.target }} \
+        --access {{ item.access }} \
+        --overwrite
+      loop:
+        - target: "type=snapshot"
+          access: "FULL"
+        - target: "type=policy"
+          access: "READ"
+        - target: "type=policy,username=infra,hostname=infra.{{ domain['internal'] }}"
+          access: "FULL"
+        - target: "type=content"
+          access: "APPEND"
+      changed_when: false
+      no_log: true
+
+    - name: Set kopia app ACL
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia server acl add \
+        --user app@app.{{ domain['internal'] }} \
+        --target {{ item.target }} \
+        --access {{ item.access }} \
+        --overwrite
+      loop:
+        - target: "type=snapshot"
+          access: "FULL"
+        - target: "type=policy"
+          access: "READ"
+        - target: "type=policy,username=app,hostname=app.{{ domain['internal'] }}"
+          access: "FULL"
+        - target: "type=content"
+          access: "APPEND"
+      changed_when: false
+      no_log: true
+
+    - name: Set kopia policy
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia policy set \
+        {{ item.content }}
+      loop:
+        - content: |
+            --global \
+            --compression=zstd
+        - content: |
+            infra@infra.{{ domain['internal'] }} \
+            --compression=zstd \
+            --keep-latest=10 \
+            --keep-hourly=0 \
+            --keep-daily=14 \
+            --keep-weekly=4 \
+            --keep-monthly=6 \
+            --keep-annual=1
+        - content: |
+            app@app.{{ domain['internal'] }} \
+            --compression=none \
+            --keep-latest=10 \
+            --keep-hourly=0 \
+            --keep-daily=14 \
+            --keep-weekly=4 \
+            --keep-monthly=6 \
+            --keep-annual=1
+      changed_when: false
+      no_log: true
+
+    - name: Set kopia parameters
+      ansible.builtin.shell: |
+        set -a
+        . /etc/secrets/{{ node['uid'] }}/kopia.env
+        set +a
+        /usr/bin/kopia repository set-parameters \
+        --max-pack-size-mb=100
+      changed_when: false
+      no_log: true
+
+    - name: Deploy kopia server service file
+      ansible.builtin.template:
+        src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia-server.service.j2"
+        dest: "{{ node['home_path'] }}/.config/systemd/user/kopia-server.service"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+        mode: "0644"
+        validate: "/usr/bin/systemd-analyze verify %s"
+      become: true
+      notify: "notification_restart_kopia_server"
+
+    - name: Enable kopia server service
+      ansible.builtin.systemd:
+        name: "kopia-server.service"
+        state: "started"
+        enabled: true
+        scope: "user"
+        daemon_reload: true
+
+- name: Set infra/app backup kopia
+  when: node['name'] in ["infra", "app"]
+  block:
+    - name: Download kopia deb file (x86_64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_amd64.deb"
+        dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      become: true
+      when: ansible_facts['architecture'] == "x86_64"
+
+    - name: Download kopia deb file (aarch64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_arm64.deb"
+        dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      become: true
+      when: ansible_facts['architecture'] == "aarch64"
+
     - name: Create kopia group
       ansible.builtin.group:
         name: "kopia"
-        gid: "{{ kopia_uid }}"
+        gid: "{{ services['kopia']['uid'] }}"
         state: "present"
       become: true
+
     - name: Create kopia user
       ansible.builtin.user:
         name: "kopia"
-        uid: "{{ kopia_uid }}"
+        uid: "{{ services['kopia']['uid'] }}"
         group: "kopia"
         shell: "/usr/sbin/nologin"
         password_lock: true
         comment: "Kopia backup User"
         state: "present"
       become: true
+
     - name: Create kopia directory
       ansible.builtin.file:
         path: "{{ item.name }}"
@@ -101,22 +351,109 @@
           mode: "0700"
       become: true
       no_log: true
+
     - name: Install kopia
       ansible.builtin.apt:
         deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         state: "present"
       become: true
-      when: is_kopia_installed.rc != 0
+
     - name: Deploy kopia env
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2"
-        dest: "/etc/secrets/{{ kopia_uid }}/kopia.env"
-        owner: "{{ kopia_uid }}"
+        dest: "/etc/secrets/{{ services['kopia']['uid'] }}/kopia.env"
+        owner: "{{ services['kopia']['uid'] }}"
         group: "root"
         mode: "0400"
       become: true
       no_log: true
-    - name: Deploy kopia service files
+
+    - name: Check data directory existance
+      ansible.builtin.stat:
+        path: "{{ node['home_path'] }}/data/"
+      register: "is_data_dir"
+
+    - name: Delete lost+found directory on data directory
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/lost+found/"
+        state: "absent"
+      become: true
+      when: is_data_dir.stat.exists
+
+    - name: Check data directory empty
+      ansible.builtin.find:
+        path: "{{ node['home_path'] }}/data/"
+        file_type: "any"
+        hidden: true
+        recurse: false
+      register: "is_data_dir_empty"
+      when: is_data_dir.stat.exists
+
+    - name: Connect kopia server
+      environment:
+        KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['users'][node['name']] }}"
+      ansible.builtin.shell: |
+        /usr/bin/kopia repository connect server \
+        --url=https://{{ services['kopia']['domain']['internal'] }}.{{ domain['internal'] }}:443 \
+        --override-username={{ node['name'] }} \
+        --override-hostname={{ node['name'] }}.{{ domain['internal'] }}
+      become: true
+      when: (is_data_dir_empty.matched | default(1)) == 0
+      changed_when: false
+      failed_when: is_kopia_connected.rc != 0
+      register: "is_kopia_connected"
+      no_log: true
+
+    - name: Check whether Kopia snapshot exists
+      environment:
+        KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['users'][node['name']] }}"
+      ansible.builtin.shell: |
+        /usr/bin/kopia snapshot list \
+        {{ node['name'] }}@{{ node['name'] }}.{{ domain['internal'] }}:{{ node['home_path'] }}/data \
+        --json \
+        --max-results=1
+      become: true
+      when: (is_kopia_connected.rc | default(1)) == 0
+      changed_when: false
+      failed_when: is_snapshot_exist.rc != 0
+      register: "is_snapshot_exist"
+      no_log: true
+
+    - name: Restore data directory
+      environment:
+        KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['users'][node['name']] }}"
+      ansible.builtin.shell: |
+        /usr/bin/kopia snapshot restore \
+        --snapshot-time=latest \
+        {{ node['name'] }}@{{ node['name'] }}.{{ domain['internal'] }}:{{ node['home_path'] }}/data \
+        {{ node['home_path'] }}/data
+      become: true
+      when:
+        - (is_data_dir_empty.matched | default(1)) == 0
+        - (is_kopia_connected.rc | default(1)) == 0
+        - (is_snapshot_exist.stdout |
+           default('[]', true) |
+           from_json |
+           list |
+           length
+          ) > 0
+      changed_when: is_kopia_restored.rc == 0
+      failed_when: is_kopia_restored.rc != 0
+      register: "is_kopia_restored"
+      no_log: true
+
+    - name: Disconnect kopia server
+      ansible.builtin.shell: |
+        /usr/bin/kopia repository disconnect
+      become: true
+      when:
+        - (is_data_dir_empty.matched | default(1)) == 0
+        - (is_kopia_connected.rc | default(1)) == 0
+      changed_when: false
+      failed_when: false
+      no_log: true
+
+    - name: Deploy kopia backup service files
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/{{ item }}.j2"
         dest: "/etc/systemd/system/{{ item }}"
@@ -128,10 +465,53 @@
         - "kopia-backup.service"
         - "kopia-backup.timer"
       become: true
-    - name: Enable auto kopia rules update
+
+    - name: Enable Kopia backup timer
       ansible.builtin.systemd:
         name: "kopia-backup.timer"
         state: "started"
         enabled: true
         daemon_reload: true
       become: true
+
+- name: Set console kopia
+  when: node['name'] == "console"
+  block:
+    - name: Download kopia
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
+        dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
+        mode: "0600"
+      loop:
+        - "amd64"
+        - "arm64"
+
+    - name: Install kopia (x86_64)
+      ansible.builtin.apt:
+        deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
+        state: "present"
+      become: true
+      when: ansible_facts['architecture'] == "x86_64"
+
+    - name: Install kopia (aarch64)
+      ansible.builtin.apt:
+        deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
+        state: "present"
+      become: true
+      when: ansible_facts['architecture'] == "aarch64"
+
+    - name: Connect console kopia server
+      environment:
+        KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['users']['console'] }}"
+      ansible.builtin.shell: |
+        /usr/bin/kopia repository connect server \
+        --url=https://{{ services['kopia']['domain']['internal'] }}.{{ domain['internal'] }}:443 \
+        --override-username={{ node['name'] }} \
+        --override-hostname={{ node['name'] }}.{{ domain['internal'] }}
+      changed_when: false
+      failed_when: is_kopia_connected.rc != 0
+      register: "is_kopia_connected"
+      no_log: true

ansible/roles/common/tasks/services/set_podman.yaml

@@ -1,46 +1,48 @@
 ---
-- name: Check podman installation
-  ansible.builtin.shell: |
-    command -v podman
-  changed_when: false
-  failed_when: false
-  register: "is_podman_installed"
-  ignore_errors: true
-
 - name: Create container directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0700"
 
-- name: Create container data directory for app
+- name: Create container data directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/data/containers"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0770"
-  when: node['name'] == "app"
+  when: node['name'] in ["infra", "app"]
 
-- name: Install podman and reset ssh connection for initiating
-  when: is_podman_installed.rc != 0
+- name: Create container image archive directory
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    state: "directory"
+    mode: "0700"
+  loop:
+    - "{{ node['home_path'] }}/archives"
+    - "{{ node['home_path'] }}/archives/containers"
+
+- name: Set subid scope (Overwrite)
+  ansible.builtin.copy:
+    content: |
+      {{ ansible_user }}:100000:65536
+    dest: "/etc/sub{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  loop:
+    - "uid"
+    - "gid"
+  become: true
+
+- name: Install podman
+  ansible.builtin.apt:
+    name:
+      - "podman"
+    state: "present"
   become: true
-  block:
-    - name: Set subid scope (Overwrite)
-      ansible.builtin.copy:
-        content: |
-          {{ ansible_user }}:100000:65536
-        dest: "/etc/sub{{ item }}"
-        owner: "root"
-        group: "root"
-        mode: "0644"
-      loop:
-        - "uid"
-        - "gid"
-    - name: Install podman
-      ansible.builtin.apt:
-        name:
-          - "podman"
-        state: "present"

ansible/roles/console/handlers/main.yaml

@@ -1,8 +1 @@
 ---
-- name: Register font
-  ansible.builtin.shell: |
-    fc-cache -f -v
-  become: true
-  changed_when: false
-  listen: "notification_update_font"
-  ignore_errors: true # noqa: ignore-errors

ansible/roles/console/tasks/node/create_vmm_init.yaml

@@ -0,0 +1,90 @@
+---
+- name: Create vmm_init directory
+  ansible.builtin.file:
+    path: "{{ node['data_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0750"
+  loop:
+    - "vmm_init"
+    - "vmm_init/grub.d"
+    - "vmm_init/modprobe.d"
+    - "vmm_init/networkd"
+    - "vmm_init/ssh"
+    - "vmm_init/ssh/sshd_config.d"
+    - "vmm_init/sysctl.d"
+
+- name: Create nftables.conf
+  ansible.builtin.template:
+    src: "{{ node['config_path'] }}/node/vmm/nftables.conf.j2"
+    dest: "{{ node['data_path'] }}/vmm_init/nftables.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+
+- name: Create grub.d iommu config
+  ansible.builtin.copy:
+    content: |
+      GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt"
+    dest: "{{ node['data_path'] }}/vmm_init/grub.d/iommu.cfg"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+
+- name: Create modprobe.d vfio config
+  ansible.builtin.copy:
+    content: |
+      options vfio-pci ids={{ hostvars['vmm']['hardware']['igpu']['id'] }}
+      softdep i915 pre: vfio-pci
+      # softdep ahci pre: vfio-pci # sata controller
+    dest: "{{ node['data_path'] }}/vmm_init/modprobe.d/vfio.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+
+- name: Create networkd network config
+  ansible.builtin.template:
+    src: "{{ item }}"
+    dest: "{{ node['data_path'] }}/vmm_init/networkd/{{ item | basename | replace('.j2', '') }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+  loop: "{{ query('fileglob', hostvars['console']['node']['config_path'] + '/node/vmm/networkd/*') | sort }}"
+
+- name: Create ssh CA
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ssh']['ca']['pub'] }}
+    dest: "{{ node['data_path'] }}/vmm_init/ssh/local_ssh_ca.pub"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+
+- name: Create ssh config
+  ansible.builtin.copy:
+    content: "{{ item.value }}"
+    dest: "{{ node['data_path'] }}/vmm_init/ssh/sshd_config.d/{{ item.name }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"
+  loop:
+    - name: "prohibit_root.conf"
+      value: |
+        PermitRootLogin no
+    - name: "ssh_ca.conf"
+      value: |
+        TrustedUserCAKeys /etc/ssh/local_ssh_ca.pub
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Create sysctl.d bridge config
+  ansible.builtin.copy:
+    content: |
+      net.bridge.bridge-nf-call-ip6tables = 0
+      net.bridge.bridge-nf-call-iptables = 0
+      net.bridge.bridge-nf-call-arptables = 0
+    dest: "{{ node['data_path'] }}/vmm_init/sysctl.d/bridge.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0640"

ansible/roles/console/tasks/node/load_secret_vars.yaml

@@ -16,13 +16,13 @@
   environment:
     SOPS_AGE_KEY: "{{ hostvars['console']['age_key'] }}"
   ansible.builtin.command: |
-    sops -d --output-type yaml {{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml
+    sops -d --output-type yaml "{{ hostvars['console']['node']['config_path'] }}/secrets/secrets.yaml"
   changed_when: false
   register: "decrypted_secrets"
   run_once: true
   no_log: true
 
-- name: Load decrypted secret vaules in console
+- name: Load decrypted secret values in console
   ansible.builtin.set_fact:
     "{{ item.key }}": "{{ item.value }}"
   loop: "{{ decrypted_secrets.stdout | from_yaml | dict2items }}"

ansible/roles/console/tasks/node/set_ssh_client.yaml

@@ -66,8 +66,8 @@
         content: |
           {{ hostvars['console']['ssh']['ca']['key'] }}
         dest: "/run/user/{{ node['uid'] }}/local_ssh_ca_private_key"
-        owner: "console"
-        group: "svadmins"
+        owner: "{{ ansible_user }}"
+        group: "{{ node['group'] }}"
         mode: "0400"
       no_log: true
     - name: Sign on ssh console key (pub file)
@@ -91,7 +91,7 @@
     path: "{{ node['home_path'] }}/.ssh"
     state: "directory"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0700"
 
 - name: Create ssh config file
@@ -105,5 +105,5 @@
       {% endfor %}
     dest: "{{ node['home_path'] }}/.ssh/config"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"

ansible/roles/console/tasks/services/set_cli_tools.yaml

@@ -31,7 +31,7 @@
       sops_{{ version['packages']['sops'] }}_{{ item }}.deb"
     dest: "{{ node['data_path'] }}/bin/sops-{{ version['packages']['sops'] }}-{{ item }}.deb"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
   loop:
     - "amd64"
@@ -43,43 +43,7 @@
       step-cli_{{ version['packages']['step'] }}-1_{{ item }}.deb"
     dest: "{{ node['data_path'] }}/bin/step-{{ version['packages']['step'] }}-{{ item }}.deb"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
-- name: Download kopia
-  ansible.builtin.get_url:
-    url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
-      kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
-- name: Download blocky
-  ansible.builtin.get_url:
-    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
-      blocky_v{{ version['packages']['blocky'] }}_Linux_{{ item }}.tar.gz"
-    dest: "{{ node['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-{{ item }}.tar.gz"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "x86_64"
-    - "arm64"
-
-- name: Download alloy
-  ansible.builtin.get_url:
-    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
-      alloy-{{ version['packages']['alloy'] }}-1.{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
   loop:
     - "amd64"
@@ -92,7 +56,6 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-amd64.deb"
     - "step-{{ version['packages']['step'] }}-amd64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-amd64.deb"
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
@@ -103,6 +66,5 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-arm64.deb"
     - "step-{{ version['packages']['step'] }}-arm64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-arm64.deb"
   become: true
   when: ansible_facts['architecture'] == "aarch64"

ansible/roles/fw/handlers/main.yaml

@@ -44,7 +44,7 @@
   ansible.builtin.systemd:
     name: "blocky.service"
     state: "restarted"
-    enabled: "true"
+    enabled: true
     daemon_reload: true
   become: true
   changed_when: false

ansible/roles/fw/tasks/services/set_bind.yaml

@@ -1,31 +1,12 @@
 ---
-- name: Check bind9 installation
-  ansible.builtin.shell: |
-    command -v named
-  become: true # named is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_bind_installed"
-  ignore_errors: true
-
-- name: Set bind9 zone files
-  ansible.builtin.set_fact:
-    bind_zone_files:
-      - "db.ilnmors.internal"
-      - "db.ilnmors.com"
-      - "db.1.168.192.in-addr.arpa"
-      - "db.10.168.192.in-addr.arpa"
-      - "db.1.00df.ip6.arpa"
-      - "db.10.00df.ip6.arpa"
-
 - name: Install bind9
   ansible.builtin.apt:
-    name: "bind9"
+    name:
+      - "bind9"
     state: "present"
   become: true
-  when: is_bind_installed.rc != 0
 
-- name: Deploy acem.key
+- name: Deploy acme.key
   ansible.builtin.copy:
     content: "{{ hostvars['console']['bind']['acme_key'] }}"
     dest: "/etc/bind/acme.key"
@@ -36,23 +17,64 @@
   notify: "notification_restart_bind"
   no_log: true
 
-- name: Deploy db files
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/bind/lib/{{ item }}"
-    dest: "/var/lib/bind/{{ item }}"
+- name: Deploy forward zone files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/bind/lib/{{ item.src }}.j2"
+    dest: "/var/lib/bind/{{ item.dest }}"
     owner: "bind"
     group: "bind"
     mode: "0640"
-  loop: "{{ bind_zone_files }}"
+  loop:
+    - src: "db.public"
+      dest: "db.{{ domain['public'] }}"
+    - src: "db.internal"
+      dest: "db.{{ domain['internal'] }}"
   become: true
   notify: "notification_restart_bind"
   no_log: true
 
+- name: Deploy IPv4 reverse zone files
+  vars:
+    reverse_net: "{{ item.key }}"
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/bind/lib/db.ipv4.rev.j2"
+    dest: "/var/lib/bind/db.{{ item.key }}.ipv4.rev"
+    owner: bind
+    group: bind
+    mode: "0644"
+  loop: "{{ network['reverse_zone']['ipv4'] | dict2items }}"
+  become: true
+  notify: "notification_restart_bind"
+  no_log: true
+
+- name: Deploy IPv6 reverse zones
+  vars:
+    reverse_net: "{{ item.key }}"
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/bind/lib/db.ipv6.rev.j2"
+    dest: "/var/lib/bind/db.{{ item.key }}.ipv6.rev"
+    owner: bind
+    group: bind
+    mode: "0644"
+  loop: "{{ network['reverse_zone']['ipv6'] | dict2items }}"
+  become: true
+  notify: "notification_restart_bind"
+  no_log: true
+
+- name: Find BIND journal files
+  ansible.builtin.find:
+    paths: "/var/lib/bind"
+    patterns: "*.jnl"
+    file_type: file
+  become: true
+  register: bind_journal_files
+  no_log: true
+
 - name: Clean BIND journal files
   ansible.builtin.file:
-    path: "/var/lib/bind/{{ item }}.jnl"
+    path: "{{ item.path }}"
     state: absent
-  loop: "{{ bind_zone_files }}"
+  loop: "{{ bind_journal_files.files }}"
   become: true
   notify: "notification_restart_bind"
   no_log: true
@@ -67,7 +89,7 @@
     validate: "/usr/bin/named-checkconf -z %s"
   become: true
   notify: "notification_restart_bind"
-  no_log: true
+
 
 - name: Create named.service.d
   ansible.builtin.file:

ansible/roles/fw/tasks/services/set_blocky.yaml

@@ -23,7 +23,7 @@
     state: "present"
   become: true
 
-- name: Create blocky etc directory
+- name: Create blocky directory
   ansible.builtin.file:
     path: "{{ item }}"
     owner: "blocky"
@@ -31,28 +31,38 @@
     mode: "0750"
     state: "directory"
   loop:
+    - "/home/blocky"
+    - "/home/blocky/bin"
     - "/etc/blocky"
     - "/etc/blocky/ssl"
   become: true
 
-- name: Deploy blocky binary file (x86_64)
-  ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
-    dest: "/usr/local/bin/"
-    owner: "root"
-    group: "root"
-    mode: "0755"
-    extra_opts:
-      - "--strip-components=0"
-      - "--wildcards"
-      - "blocky"
+- name: Download blocky (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_x86_64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
   become: true
   when: ansible_facts['architecture'] == "x86_64"
-  notify: "notification_restart_blocky"
 
-- name: Deploy blocky binary file (aarch64)
+- name: Download blocky (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_arm64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "aarch64"
+
+- name: Deploy blocky binary file
   ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    remote_src: true
     dest: "/usr/local/bin/"
     owner: "root"
     group: "root"
@@ -62,7 +72,6 @@
       - "--wildcards"
       - "blocky"
   become: true
-  when: ansible_facts['architecture'] == "aarch64"
   notify: "notification_restart_blocky"
 
 - name: Deploy blocky config
@@ -115,3 +124,10 @@
     enabled: true
     daemon_reload: true
   become: true
+
+- name: Fetch deb bin file
+  ansible.builtin.fetch:
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/bin/"
+    flat: true
+  become: true

ansible/roles/fw/tasks/services/set_chrony.yaml

@@ -1,18 +1,10 @@
 ---
-- name: Check chrnoy installation
-  ansible.builtin.shell: |
-    command -v chronyc
-  changed_when: false
-  failed_when: false
-  register: "is_chrony_installed"
-  ignore_errors: true
-
 - name: Install chrony
   ansible.builtin.apt:
-    name: "chrony"
+    name:
+      - "chrony"
     state: "present"
   become: true
-  when: is_chrony_installed.rc != 0
 
 - name: Deploy local acl file
   ansible.builtin.template:

ansible/roles/fw/tasks/services/set_ddns.yaml

@@ -6,7 +6,7 @@
       API_KEY={{ hostvars['console']['ddns']['api_key'] }}
     dest: "/etc/secrets/{{ node['uid'] }}/ddns.env"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0400"
   become: true
   no_log: true
@@ -16,7 +16,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/ddns/ddns.sh"
     dest: "/usr/local/bin"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0711"
   become: true
 
@@ -25,7 +25,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/ddns/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
     validate: "/usr/bin/systemd-analyze verify %s"
   loop:

ansible/roles/fw/tasks/services/set_kea.yaml

@@ -1,20 +1,10 @@
 ---
-- name: Check Kea dhcp4 installation
-  ansible.builtin.shell: |
-    command -v kea-dhcp4
-  become: true # kea-dhcp4 is located in /usr/sbin, which means root permission is needed.
-  changed_when: false
-  failed_when: false
-  register: "is_kea4_installed"
-  ignore_errors: true
-
 - name: Install kea dhcp 4
   ansible.builtin.apt:
     name:
       - "kea-dhcp4-server"
     state: "present"
   become: true
-  when: is_kea4_installed.rc != 0
 
 - name: Deploy kea dhcp4 conf
   ansible.builtin.template:
@@ -26,7 +16,7 @@
   become: true
   notify: "notification_restart_kea4"
 
-- name: Create kea-dhcp-server.service.d
+- name: Create kea-dhcp4-server.service.d
   ansible.builtin.file:
     path: "/etc/systemd/system/kea-dhcp4-server.service.d"
     state: "directory"
@@ -35,7 +25,7 @@
     mode: "0755"
   become: true
 
-- name: Set kea-dhcp-server.service.d/override.conf
+- name: Set kea-dhcp4-server.service.d/override.conf
   ansible.builtin.copy:
     dest: "/etc/systemd/system/kea-dhcp4-server.service.d/override.conf"
     content: |

ansible/roles/fw/tasks/services/set_suricata.yaml

@@ -1,12 +1,4 @@
 ---
-- name: Check suricata installation
-  ansible.builtin.shell: |
-    command -v suricata
-  changed_when: false
-  failed_when: false
-  register: "is_suricata_installed"
-  ignore_errors: true
-
 - name: Install suricata
   ansible.builtin.apt:
     name:
@@ -14,7 +6,6 @@
       - "suricata-update"
     state: "present"
   become: true
-  when: is_suricata_installed.rc != 0
 
 - name: Deploy suricata-update service files
   ansible.builtin.copy:

ansible/roles/infra/handlers/main.yaml

@@ -1,8 +1,20 @@
+---
+- name: Restart rustfs
+  ansible.builtin.systemd:
+    name: "rustfs.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  changed_when: false
+  listen: "notification_restart_rustfs"
+  ignore_errors: true # noqa: ignore-errors
+
 - name: Restart ca
   ansible.builtin.systemd:
     name: "ca.service"
     state: "restarted"
-    enabled: "true"
+    enabled: true
     daemon_reload: true
     scope: "user"
   changed_when: false
@@ -12,7 +24,7 @@
 - name: Reload postgresql
   ansible.builtin.command:
     /usr/bin/podman exec -u postgres postgresql sh -c "pg_ctl reload"
-  when: is_postgresql_init.stat.exists
+  when: is_postgresql_init_marker.stat.exists
   changed_when: false
   listen: "notification_reload_postgresql"
   ignore_errors: true # noqa: ignore-errors
@@ -24,11 +36,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
-  when: is_postgresql_init.stat.exists
+  when: is_postgresql_init_marker.stat.exists
   changed_when: false
   listen: "notification_restart_postgresql"
   ignore_errors: true # noqa: ignore-errors
 
+- name: Restart mariadb
+  ansible.builtin.systemd:
+    name: "mariadb.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_mariadb_init_marker.stat.exists
+  changed_when: false
+  listen: "notification_restart_mariadb"
+  ignore_errors: true # noqa: ignore-errors
+
 - name: Restart ldap
   ansible.builtin.systemd:
     name: "ldap.service"
@@ -73,13 +97,24 @@
   listen: "notification_restart_grafana"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Enable x509-exporter.service
+- name: Restart x509-exporter
   ansible.builtin.systemd:
     name: "x509-exporter.service"
-    state: "started"
+    state: "restarted"
     enabled: true
     daemon_reload: true
     scope: "user"
   changed_when: false
   listen: "notification_restart_x509-exporter"
   ignore_errors: true # noqa: ignore-errors
+
+- name: Restart maddy
+  ansible.builtin.systemd:
+    name: "maddy.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  changed_when: false
+  listen: "notification_restart_maddy"
+  ignore_errors: true # noqa: ignore-errors

ansible/roles/infra/tasks/services/set_ca_server.yaml

@@ -3,7 +3,7 @@
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     owner: "{{ services['ca']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0770"
   loop:
@@ -24,16 +24,20 @@
   notify: "notification_restart_ca"
   no_log: true
 
-- name: Deploy ca config files
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/smallstep/step-ca"
+    tag: "{{ version['containers']['ca'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy ca config file
   ansible.builtin.template:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/ca/config/{{ item }}.j2"
-    dest: "{{ node['home_path'] }}/containers/ca/config/{{ item }}"
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/ca/config/ca.json.j2"
+    dest: "{{ node['home_path'] }}/containers/ca/config/ca.json"
     owner: "{{ services['ca']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0400"
-  loop:
-    - "ca.json"
-    - "defaults.json"
   become: true
   notify: "notification_restart_ca"
 
@@ -43,7 +47,7 @@
       {{ item.value }}
     dest: "{{ item.path }}/{{ item.name }}"
     owner: "{{ services['ca']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "{{ item.mode }}"
   loop:
     - name: "{{ root_cert_filename }}"
@@ -67,7 +71,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/ca/ca.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/ca.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_ca"
 
@@ -78,3 +82,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_smallstep_step-ca_{{ version['containers']['ca'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/smallstep/step-ca:{{ version['containers']['ca'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_smallstep_step-ca_{{ version['containers']['ca'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_smallstep_step-ca_{{ version['containers']['ca'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_grafana.yaml

@@ -3,7 +3,7 @@
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     owner: "{{ services['grafana']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0770"
   loop:
@@ -11,6 +11,8 @@
     - "grafana/data"
     - "grafana/etc"
     - "grafana/etc/provisioning"
+    - "grafana/etc/provisioning/datasources"
+    - "grafana/etc/provisioning/dashboards"
     - "grafana/etc/dashboards"
     - "grafana/ssl"
   become: true
@@ -21,7 +23,7 @@
       {{ hostvars['console']['ca']['root']['crt'] }}
     dest: "{{ node['home_path'] }}/containers/grafana/ssl/{{ root_cert_filename }}"
     owner: "{{ services['grafana']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0400"
   become: true
   notify: "notification_restart_grafana"
@@ -34,21 +36,21 @@
     state: "present"
     force: true
   loop:
+    - name: "GF_LDAP_BIND_PASSWORD"
+      value: "{{ hostvars['console']['ldap']['password']['services']['grafana'] }}"
     - name: "GF_DB_PASSWORD"
       value: "{{ hostvars['console']['postgresql']['password']['grafana'] }}"
-    - name: "LDAP_BIND_PASSWORD"
-      value: "{{ hostvars['console']['ldap']['password']['grafana'] }}"
     - name: "GF_ADMIN_PASSWORD"
-      value: "{{ hostvars['console']['grafana']['user']['password'] }}"
+      value: "{{ hostvars['console']['grafana']['admin']['password'] }}"
   notify: "notification_restart_grafana"
   no_log: true
 
-- name: Deploy configruation files
+- name: Deploy configuration files
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/grafana/etc/{{ item }}"
     owner: "{{ services['grafana']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0400"
   loop:
     - "grafana.ini"
@@ -57,22 +59,48 @@
   notify: "notification_restart_grafana"
   no_log: true
 
-- name: Deploy provisioing file
+- name: Deploy provisioing files
   ansible.builtin.template:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/provisioning/datasources/datasources.yaml.j2"
-    dest: "{{ node['home_path'] }}/containers/grafana/etc/provisioning/datasources/datasources.yaml"
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/provisioning/{{ item.file }}"
+    dest: "{{ node['home_path'] }}/containers/grafana/etc/provisioning/{{ item.dest }}"
     owner: "{{ services['grafana']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
+    mode: "0400"
+  loop:
+    - name: "datasource"
+      file: "datasources/datasources.yaml.j2"
+      dest: "datasources/datasources.yaml"
+    - name: "dashboard"
+      file: "dashboards/dashboards.yaml"
+      dest: "dashboards/dashboards.yaml"
+  loop_control:
+    label: "{{ item.name }}"
+  become: true
+  notify: "notification_restart_grafana"
+
+- name: Deploy dashboard files
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/dashboards/"
+    dest: "{{ node['home_path'] }}/containers/grafana/etc/dashboards/"
+    owner: "{{ services['grafana']['subuid'] }}"
+    group: "{{ node['group'] }}"
     mode: "0400"
   become: true
   notify: "notification_restart_grafana"
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/grafana/grafana"
+    tag: "{{ version['containers']['grafana'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/grafana.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/grafana.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_grafana"
 
@@ -83,3 +111,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_grafana_{{ version['containers']['grafana'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/grafana/grafana:{{ version['containers']['grafana'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_grafana_{{ version['containers']['grafana'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_grafana_{{ version['containers']['grafana'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_ldap.yaml

@@ -3,7 +3,7 @@
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     owner: "{{ services['ldap']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     state: "directory"
     mode: "0770"
   loop:
@@ -18,7 +18,7 @@
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/ldap/ssl/{{ item.name }}"
     owner: "{{ services['ldap']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "{{ item.mode }}"
   loop:
     - name: "{{ root_cert_filename }}"
@@ -33,8 +33,8 @@
       value: "{{ hostvars['console']['ldap']['key'] }}"
       mode: "0400"
   become: true
-  notify: "notification_restart_ldap"
   no_log: true
+  notify: "notification_restart_ldap"
 
 - name: Register secret value to podman secret
   containers.podman.podman_secret:
@@ -46,22 +46,25 @@
     # urlencode doesn't fix `/` as `%2F`. It needs replace
     - name: "LLDAP_DATABASE_URL"
       value: "postgres://ldap:{{ hostvars['console']['postgresql']['password']['ldap'] | urlencode | replace('/', '%2F') }}\
-        @{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}/ldap_db?sslmode=verify-full&sslrootcert=/etc/ssl/ldap/{{ root_cert_filename }}"
+        @{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}/\
+        ldap_db?sslmode=verify-full&sslrootcert=/etc/ssl/ldap/{{ root_cert_filename }}"
     - name: "LLDAP_KEY_SEED"
       value: "{{ hostvars['console']['ldap']['seed_key'] }}"
     - name: "LLDAP_JWT_SECRET"
       value: "{{ hostvars['console']['ldap']['jwt_secret'] }}"
-  notify: "notification_restart_ldap"
   no_log: true
+  notify: "notification_restart_ldap"
 
 - name: Initiate ldap (When = false, If DB data does not exist in postgresql, activate this block)
+# The reason why this task doesn't use the way to check ".init" file is this tasks can override original database.
+# Absent of ".init" file cannot guarantee DB is empty.
   when: false
-  become: true
+  no_log: true
   block:
     - name: Register extra secret value to podman secret
       containers.podman.podman_secret:
-        name: "LLDAP_LDAP_USER_PASSWORD"
-        data: "{{ hostvars['console']['ldap']['password']['user'] }}"
+        name: "LLDAP_LDAP_USER_PASS"
+        data: "{{ hostvars['console']['ldap']['password']['services']['admin'] }}"
         state: "present"
         force: true
     # You must check the image version first (following container file on data/config/containers/infra/ldap/ldap.container)
@@ -73,29 +76,38 @@
         rm: true
         detach: false
         env:
-          TZ: "Asia/Seoul"
+          TZ: "{{ timezone }}"
           LLDAP_LDAP_BASE_DN: "{{ domain['dc'] }}"
         secrets:
           - "LLDAP_DATABASE_URL,type=env"
           - "LLDAP_KEY_SEED,type=env"
           - "LLDAP_JWT_SECRET,type=env"
-          - "LLDAP_LDAP_USER_PASSWORD,type=env"
+          - "LLDAP_LDAP_USER_PASS,type=env"
         volumes:
           - "{{ node['home_path'] }}/containers/ldap/data:/data:rw"
           - "{{ node['home_path'] }}/containers/ldap/ssl:/etc/ssl/ldap:ro"
+        etc_hosts:
+          "{{ {} | combine({ services['postgresql']['domain']['internal'] ~ '.' ~ domain['internal']: 'host-gateway' }) }}"
 
   always:
     - name: Clean extra secret value from podman secret
       containers.podman.podman_secret:
-        name: "LLDAP_LDAP_USER_PASSWORD"
+        name: "LLDAP_LDAP_USER_PASS"
         state: "absent"
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/lldap/lldap"
+    tag: "{{ version['containers']['ldap'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/ldap/ldap.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/ldap.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_ldap"
 
@@ -106,3 +118,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_lldap_lldap_{{ version['containers']['ldap'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/lldap/lldap:{{ version['containers']['ldap'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_lldap_lldap_{{ version['containers']['ldap'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_lldap_lldap_{{ version['containers']['ldap'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_loki.yaml

@@ -4,11 +4,13 @@
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
     owner: "{{ services['loki']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "loki"
     - "loki/etc"
+    - "loki/etc/rules"
+    - "loki/etc/rules/{{ domain['internal'] }}"
     - "loki/data"
     - "loki/ssl"
   become: true
@@ -18,7 +20,18 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/loki/etc/loki.yaml.j2"
     dest: "{{ node['home_path'] }}/containers/loki/etc/loki.yaml"
     owner: "{{ services['loki']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
+    mode: "0600"
+  become: true
+  notify: "notification_restart_loki"
+  no_log: true
+
+- name: Deploy loki rules file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/loki/etc/rules.yaml.j2"
+    dest: "{{ node['home_path'] }}/containers/loki/etc/rules/{{ domain['internal'] }}/rules.yaml"
+    owner: "{{ services['loki']['subuid'] }}"
+    group: "{{ node['group'] }}"
     mode: "0600"
   become: true
   notify: "notification_restart_loki"
@@ -30,7 +43,7 @@
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/loki/ssl/{{ item.name }}"
     owner: "{{ services['loki']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "{{ item.mode }}"
   loop:
     - name: "{{ root_cert_filename }}"
@@ -48,12 +61,19 @@
   notify: "notification_restart_loki"
   no_log: true
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/grafana/loki"
+    tag: "{{ version['containers']['loki'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/loki/loki.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/loki.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_loki"
 
@@ -64,3 +84,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_loki_{{ version['containers']['loki'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/grafana/loki:{{ version['containers']['loki'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_loki_{{ version['containers']['loki'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_loki_{{ version['containers']['loki'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_maddy.yaml

@@ -0,0 +1,107 @@
+---
+- name: Create maddy directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    state: "directory"
+    mode: "0770"
+  loop:
+    - "containers/maddy"
+    - "containers/maddy/ssl"
+    - "data/containers/maddy"
+    - "data/containers/maddy/data"
+
+- name: Deploy maddy certificate and key
+  ansible.builtin.copy:
+    content: |
+      {{ item.value }}
+    dest: "{{ node['home_path'] }}/containers/maddy/ssl/{{ item.name }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - name: "{{ root_cert_filename }}"
+      value: "{{ hostvars['console']['ca']['root']['crt'] }}"
+      mode: "0440"
+    - name: "maddy.crt"
+      value: |
+        {{ hostvars['console']['maddy']['crt'] | trim }}
+        {{ hostvars['console']['ca']['intermediate']['crt'] }}
+      mode: "0440"
+    - name: "maddy.key"
+      value: "{{ hostvars['console']['maddy']['key'] }}"
+      mode: "0400"
+  become: true
+  notify: "notification_restart_maddy"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "MADDY_LDAP_BIND_PASSWORD"
+      value: "{{ hostvars['console']['ldap']['password']['services']['maddy'] }}"
+    - name: "MADDY_POSTGRES_URL"
+      value: "postgresql://maddy:{{ hostvars['console']['postgresql']['password']['maddy'] | urlencode | replace('/', '%2F') }}\
+        @{{ services['postgresql']['domain']['internal'] }}.{{ domain['internal'] }}/maddy_db?sslmode=verify-full&\
+        sslrootcert=/etc/ssl/maddy/{{ root_cert_filename }}"
+  notify: "notification_restart_maddy"
+  no_log: true
+
+- name: Deploy configuration files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/maddy/data/maddy.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/maddy/maddy.conf"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0400"
+  notify: "notification_restart_maddy"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/foxcpp/maddy"
+    tag: "{{ version['containers']['maddy'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/maddy/maddy.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/maddy.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_maddy"
+
+- name: Enable maddy
+  ansible.builtin.systemd:
+    name: "maddy.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_foxcpp_maddy_{{ version['containers']['maddy'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/foxcpp/maddy:{{ version['containers']['maddy'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_foxcpp_maddy_{{ version['containers']['maddy'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_foxcpp_maddy_{{ version['containers']['maddy'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_mariadb.yaml

@@ -0,0 +1,197 @@
+---
+- name: Initialize connected services list
+  ansible.builtin.set_fact:
+    connected_services: []
+
+- name: Set connected services list
+  ansible.builtin.set_fact:
+    connected_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.db', 'defined')
+        | selectattr('value.dependency.db', 'equalto', 'mariadb')
+        | map(attribute='key')
+        | list
+      }}
+
+- name: Create mariadb directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['mariadb']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/mariadb/"
+    - "data/containers/mariadb/backups"
+    - "data/containers/mariadb/backups/sql"
+    - "containers/mariadb"
+    - "containers/mariadb/data"
+    - "containers/mariadb/config"
+    - "containers/mariadb/ssl"
+    - "containers/mariadb/init"
+  become: true
+
+- name: Check mariadb .init marker
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/containers/mariadb/.init"
+  register: "is_mariadb_init_marker"
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/library/mariadb"
+    tag: "{{ version['containers']['mariadb'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy mariadb configuration files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/mariadb/config/mariadb.cnf.j2"
+    dest: "{{ node['home_path'] }}/containers/mariadb/config/mariadb.cnf"
+    owner: "{{ services['mariadb']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0600"
+  become: true
+  notify: "notification_restart_mariadb"
+  no_log: true
+
+- name: Deploy mariadb certificate and key
+  ansible.builtin.copy:
+    content: |
+      {{ item.value }}
+    dest: "{{ node['home_path'] }}/containers/mariadb/ssl/{{ item.name }}"
+    owner: "{{ services['mariadb']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - name: "{{ root_cert_filename }}"
+      value: "{{ hostvars['console']['ca']['root']['crt'] }}"
+      mode: "0440"
+    - name: "mariadb.crt"
+      value: |
+        {{ hostvars['console']['mariadb']['crt'] | trim }}
+        {{ hostvars['console']['ca']['intermediate']['crt'] }}
+      mode: "0440"
+    - name: "mariadb.key"
+      value: "{{ hostvars['console']['mariadb']['key'] }}"
+      mode: "0400"
+  become: true
+  notify: "notification_restart_mariadb"
+  no_log: true
+
+- name: Initiating mariadb
+  when: not is_mariadb_init_marker.stat.exists
+  block:
+    - name: Check initiating SQL files
+      ansible.builtin.find:
+        path: "{{ node['home_path'] }}/data/containers/mariadb/backups/sql/"
+        file_type: "file"
+        patterns:
+          - "mariadb_*.sql"
+        hidden: true
+        recurse: false
+      become: true
+      register: "mariadb_init_sql_files"
+
+    - name: Set initiating SQL file lists
+      ansible.builtin.set_fact:
+        mariadb_init_sql_file_list: >-
+          {{
+            mariadb_init_sql_files.files
+            | map(attribute='path')
+            | map('basename')
+            | map('regex_replace', '^mariadb_', '')
+            | map('regex_replace', '[.]sql$', '')
+            | list
+          }}
+
+    - name: Fail on invalid initiating SQL file set
+      ansible.builtin.fail:
+      when:
+        - mariadb_init_sql_files.matched > 0
+        - ((connected_services)
+          | symmetric_difference(mariadb_init_sql_file_list)
+          | length) > 0
+
+    - name: Remote copy the initiating SQL
+      ansible.builtin.copy:
+        src: "{{ node['home_path'] }}/data/containers/mariadb/backups/sql/mariadb_{{ item }}.sql"
+        remote_src: true
+        dest: "{{ node['home_path'] }}/containers/mariadb/init/{{ '%02d' | format(index_num) }}_mariadb_{{ item }}.sql"
+        owner: "{{ services['mariadb']['subuid'] }}"
+        group: "{{ node['group'] }}"
+        mode: "0600"
+      loop: "{{ connected_services }}"
+      loop_control:
+        index_var: index_num
+      become: true
+      when:
+        - ((connected_services)
+          | symmetric_difference(mariadb_init_sql_file_list)
+          | length) == 0
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/mariadb/mariadb.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/mariadb.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_mariadb"
+
+- name: Deploy backup service files
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/mariadb/services/{{ item }}"
+    dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  loop:
+    - "mariadb-data-backup@.service"
+    - "mariadb-data-backup@.timer"
+
+- name: Enable mariadb
+  ansible.builtin.systemd:
+    name: "mariadb.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Enable data backup timer
+  ansible.builtin.systemd:
+    name: "mariadb-data-backup@{{ item }}.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  loop: "{{ connected_services }}"
+
+- name: Create .init file
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/containers/mariadb/.init"
+    state: "touch"
+    mode: "0644"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+  when: not is_mariadb_init_marker.stat.exists
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_library_mariadb_{{ version['containers']['mariadb'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/library/mariadb:{{ version['containers']['mariadb'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_library_mariadb_{{ version['containers']['mariadb'] }}.tar"
+    format: "oci-archive"
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_library_mariadb_{{ version['containers']['mariadb'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_postgresql.yaml

@@ -1,41 +1,53 @@
 ---
+- name: Initialize connected services list
+  ansible.builtin.set_fact:
+    connected_services: []
+
 - name: Set connected services list
   ansible.builtin.set_fact:
-    connected_services:
-      - "ldap"
-      - "authelia"
-      - "grafana"
-      - "vaultwarden"
-      - "gitea"
-      - "immich"
-      - "paperless"
-      - "vikunja"
-      - "affine"
+    connected_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.db', 'defined')
+        | selectattr('value.dependency.db', 'equalto', 'postgresql')
+        | map(attribute='key')
+        | list
+      }}
 
 - name: Create postgresql directory
   ansible.builtin.file:
-    path: "{{ node['home_path'] }}/containers/{{ item }}"
+    path: "{{ node['home_path'] }}/{{ item }}"
     state: "directory"
     owner: "{{ services['postgresql']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
-    - "postgresql"
-    - "postgresql/data"
-    - "postgresql/config"
-    - "postgresql/ssl"
-    - "postgresql/init"
-    - "postgresql/backups"
-    - "postgresql/build"
+    - "data/containers/postgresql/"
+    - "data/containers/postgresql/backups"
+    - "data/containers/postgresql/backups/sql"
+    - "containers/postgresql"
+    - "containers/postgresql/data"
+    - "containers/postgresql/config"
+    - "containers/postgresql/ssl"
+    - "containers/postgresql/init"
+    - "containers/postgresql/build"
   become: true
 
+- name: Check postgresql .init marker
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/containers/postgresql/.init"
+  register: "is_postgresql_init_marker"
+
 - name: Deploy containerfile for build
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/build/postgresql.containerfile.j2"
     dest: "{{ node['home_path'] }}/containers/postgresql/build/Containerfile"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0640"
+  register: "is_postgresql_containerfile"
+  notify: "notification_restart_postgresql"
 
 - name: Build postgresql container image
   containers.podman.podman_image:
@@ -44,6 +56,7 @@
     tag: "pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}"
     state: "build"
     path: "{{ node['home_path'] }}/containers/postgresql/build"
+    force: "{{ is_postgresql_containerfile.changed }}"
 
 - name: Prune postgresql dangling images
   containers.podman.podman_prune:
@@ -54,7 +67,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/config/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/postgresql/config/{{ item }}"
     owner: "{{ services['postgresql']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
   loop:
     - "postgresql.conf"
@@ -69,7 +82,7 @@
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/postgresql/ssl/{{ item.name }}"
     owner: "{{ services['postgresql']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "{{ item.mode }}"
   loop:
     - name: "{{ root_cert_filename }}"
@@ -87,50 +100,63 @@
   notify: "notification_reload_postgresql"
   no_log: true
 
-- name: Check data directory empty
-  ansible.builtin.stat:
-    path: "{{ node['home_path'] }}/containers/postgresql/data/.init"
-  become: true
-  register: "is_postgresql_init"
-
-- name: Prepare initiating DB
-  when: not is_postgresql_init.stat.exists
-  become: true
+- name: Initiating Postgresql
+  when: not is_postgresql_init_marker.stat.exists
   block:
-    # `init/pg_cluster.sql` should be fetched from postgresql's backup directory before running initiating
-    - name: Deploy init cluster sql file
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/init/pg_cluster.sql"
-        dest: "{{ node['home_path'] }}/containers/postgresql/init/0_pg_cluster.sql"
-        owner: "{{ services['postgresql']['subuid'] }}"
-        group: "svadmins"
-        mode: "0600"
+    - name: Check initiating SQL files
+      ansible.builtin.find:
+        path: "{{ node['home_path'] }}/data/containers/postgresql/backups/sql/"
+        file_type: "file"
+        patterns:
+          - "pg_*.sql"
+        hidden: true
+        recurse: false
+      become: true
+      register: "postgresql_init_sql_files"
 
-    - name: Deploy restoring data sql files
+    - name: Set initiating SQL file lists
+      ansible.builtin.set_fact:
+        postgresql_init_sql_file_list: >-
+          {{
+            postgresql_init_sql_files.files
+            | map(attribute='path')
+            | map('basename')
+            | map('regex_replace', '^pg_', '')
+            | map('regex_replace', '[.]sql$', '')
+            | list
+          }}
+
+    - name: Fail on invalid initiating SQL file set
+      ansible.builtin.fail:
+      when:
+        - postgresql_init_sql_files.matched > 0
+        - ((['cluster'] + connected_services)
+          | symmetric_difference(postgresql_init_sql_file_list)
+          | length) > 0
+
+    - name: Remote copy the initiating SQL
       ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/init/pg_{{ item }}.sql"
-        dest: "{{ node['home_path'] }}/containers/postgresql/init/{{ index_num + 1 }}_pg_{{ item }}.sql"
+        src: "{{ node['home_path'] }}/data/containers/postgresql/backups/sql/pg_{{ item }}.sql"
+        remote_src: true
+        dest: "{{ node['home_path'] }}/containers/postgresql/init/{{ '%02d' | format(index_num) }}_pg_{{ item }}.sql"
         owner: "{{ services['postgresql']['subuid'] }}"
-        group: "svadmins"
+        group: "{{ node['group'] }}"
         mode: "0600"
-      loop: "{{ connected_services }}"
+      loop: "{{ ['cluster'] + connected_services }}"
       loop_control:
         index_var: index_num
-
-    - name: Create .init file
-      ansible.builtin.file:
-        path: "{{ node['home_path'] }}/containers/postgresql/data/.init"
-        state: "touch"
-        mode: "0644"
-        owner: "{{ ansible_user }}"
-        group: "svadmins"
+      become: true
+      when:
+        - ((['cluster'] + connected_services)
+          | symmetric_difference(postgresql_init_sql_file_list)
+          | length) == 0
 
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/postgresql.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/postgresql.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_postgresql"
 
@@ -139,7 +165,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/services/{{ item }}"
     dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   loop:
     - "postgresql-cluster-backup.service"
@@ -171,3 +197,38 @@
     daemon_reload: true
     scope: "user"
   loop: "{{ connected_services }}"
+
+- name: Create .init file
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/containers/postgresql/.init"
+    state: "touch"
+    mode: "0644"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+  when: not is_postgresql_init_marker.stat.exists
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/\
+      {{ domain['internal'] }}_{{ node['name'] }}_postgres_\
+      pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ domain['internal'] }}/{{ node['name'] }}/postgres:pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/\
+      {{ domain['internal'] }}_{{ node['name'] }}_postgres_\
+      pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}.tar"
+    format: "oci-archive"
+    force: "{{ is_postgresql_containerfile.changed }}"
+  when: not container_archive_images.stat.exists or is_postgresql_containerfile.changed
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/\
+      {{ domain['internal'] }}_{{ node['name'] }}_postgres_\
+      pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_prometheus.yaml

@@ -4,7 +4,7 @@
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
     owner: "{{ services['prometheus']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "prometheus"
@@ -18,7 +18,7 @@
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/prometheus/etc/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/prometheus/etc/{{ item }}"
     owner: "{{ services['prometheus']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0600"
   loop:
     - "prometheus.yaml"
@@ -34,7 +34,7 @@
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/prometheus/ssl/{{ item.name }}"
     owner: "{{ services['prometheus']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "{{ item.mode }}"
   loop:
     - name: "{{ root_cert_filename }}"
@@ -52,12 +52,19 @@
   notify: "notification_restart_prometheus"
   no_log: true
 
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/prom/prometheus"
+    tag: "{{ version['containers']['prometheus'] }}"
+    state: "present"
+    pull: true
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/prometheus/prometheus.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/prometheus.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_prometheus"
 
@@ -68,3 +75,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_prom_prometheus_{{ version['containers']['prometheus'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/prom/prometheus:{{ version['containers']['prometheus'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_prom_prometheus_{{ version['containers']['prometheus'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_prom_prometheus_{{ version['containers']['prometheus'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_rustfs.yaml

@@ -0,0 +1,65 @@
+---
+- name: Create rustfs directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['rustfs']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0770"
+  loop:
+    - "data/containers/rustfs"
+    - "data/containers/rustfs/data"
+  become: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "RUSTFS_SECRET_KEY"
+    data: "{{ hostvars['console']['rustfs']['password']['rustfs'] }}"
+    state: "present"
+    force: true
+  notify: "notification_restart_rustfs"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/rustfs/rustfs"
+    tag: "{{ version['containers']['rustfs'] }}"
+    state: "present"
+    pull: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/rustfs/rustfs.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/rustfs.container"
+    owner: "{{ ansible_user }}"
+    group: "{{ node['group'] }}"
+    mode: "0644"
+  notify: "notification_restart_rustfs"
+
+- name: Enable rustfs
+  ansible.builtin.systemd:
+    name: "rustfs.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_rustfs_rustfs_{{ version['containers']['rustfs'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/rustfs/rustfs:{{ version['containers']['rustfs'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_rustfs_rustfs_{{ version['containers']['rustfs'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_rustfs_rustfs_{{ version['containers']['rustfs'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_x509-exporter.yaml

@@ -1,52 +1,87 @@
 ---
+- name: Initialize self-cert services list
+  ansible.builtin.set_fact:
+    cert_services: []
+
+- name: Set self-cert services list
+  ansible.builtin.set_fact:
+    cert_services: >-
+      {{
+        services
+        | dict2items
+        | selectattr('value.dependency.cert', 'defined')
+        | selectattr('value.dependency.cert', 'equalto', true)
+        | map(attribute='key')
+        | list
+      }}
+
 - name: Create x509-exporter directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
     owner: "{{ services['x509-exporter']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0770"
   loop:
     - "x509-exporter"
+    - "x509-exporter/config"
     - "x509-exporter/certs"
   become: true
 
-- name: Deploy certificates
+- name: Deploy config.yaml
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/x509-exporter/config/config.yaml"
+    dest: "{{ node['home_path'] }}/containers/x509-exporter/config/config.yaml"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_x509-exporter"
+
+- name: Deploy CA certificates
   ansible.builtin.copy:
     content: |
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/x509-exporter/certs/{{ item.name }}"
     owner: "{{ services['x509-exporter']['subuid'] }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0440"
   loop:
     - name: "root.crt"
       value: "{{ hostvars['console']['ca']['root']['crt'] }}"
     - name: "intermediate.crt"
       value: "{{ hostvars['console']['ca']['intermediate']['crt'] }}"
-    - name: "crowdsec.crt"
-      value: "{{ hostvars['console']['crowdsec']['crt'] }}"
-    - name: "blocky.crt"
-      value: "{{ hostvars['console']['blocky']['crt'] }}"
-    - name: "postgresql.crt"
-      value: "{{ hostvars['console']['postgresql']['crt'] }}"
-    - name: "ldap.crt"
-      value: "{{ hostvars['console']['ldap']['crt'] }}"
-    - name: "prometheus.crt"
-      value: "{{ hostvars['console']['prometheus']['crt'] }}"
-    - name: "loki.crt"
-      value: "{{ hostvars['console']['loki']['crt'] }}"
-    - name: "dsm.crt"
-      value: "{{ hostvars['console']['dsm']['crt'] }}"
   become: true
+  notify: "notification_restart_x509-exporter"
   no_log: true
 
+- name: Deploy leaf certificates
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console'][item]['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/x509-exporter/certs/{{ item }}.crt"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
+    group: "{{ node['group'] }}"
+    mode: "0440"
+  loop: "{{ cert_services }}"
+  become: true
+  notify: "notification_restart_x509-exporter"
+  no_log: true
+
+- name: Pull image
+  containers.podman.podman_image:
+    name: "docker.io/enix/x509-certificate-exporter"
+    tag: "{{ version['containers']['x509-exporter'] }}"
+    state: "present"
+    pull: true
+
+
 - name: Deploy container file
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/x509-exporter/x509-exporter.container.j2"
     dest: "{{ node['home_path'] }}/.config/containers/systemd/x509-exporter.container"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   notify: "notification_restart_x509-exporter"
 
@@ -57,3 +92,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_enix_x509-certificate-exporter_{{ version['containers']['x509-exporter'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/enix/x509-certificate-exporter:{{ version['containers']['x509-exporter'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_enix_x509-certificate-exporter_{{ version['containers']['x509-exporter'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_enix_x509-certificate-exporter_{{ version['containers']['x509-exporter'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/vmm/tasks/node/set_disk.yaml

@@ -0,0 +1,171 @@
+---
+- name: Install btrfs
+  ansible.builtin.apt:
+    name:
+      - "btrfs-progs"
+    state: "present"
+  become: true
+
+- name: Mount data disk
+  block:
+    - name: Set hard data disk path
+      ansible.builtin.shell: |
+        set -o pipefail
+        ls -1 /dev/disk/by-id/ata-* | \
+        grep -F -e "{{ hardware['disk']['data']['serials'] | join('" -e "') }}" | \
+        grep -vE -- '-part[0-9]+$' | \
+        sort
+      changed_when: false
+      register: "data_disk_path"
+
+    - name: Check vmm_hdd filesystem already exists
+      ansible.builtin.command: |
+        blkid -L {{ hardware['disk']['data']['fs']['label'] }}
+      become: true
+      changed_when: false
+      failed_when: false
+      register: "is_data_disk_fs"
+
+    - name: Check disk number
+      ansible.builtin.fail:
+        msg: "Below 2 disks for RAID, found {{ data_disk_path.stdout_lines | length }}"
+      when: (data_disk_path.stdout_lines | length) < hardware['disk']['data']['fs']['btrfs']['min_devices']
+
+    - name: Set data btrfs volume
+      ansible.builtin.shell: |
+        mkfs.btrfs -f \
+        -L {{ hardware['disk']['data']['fs']['label'] }} \
+        -d {{ hardware['disk']['data']['fs']['btrfs']['data_profile'] }} \
+        -m {{ hardware['disk']['data']['fs']['btrfs']['metadata_profile'] }} \
+        {{ data_disk_path.stdout_lines | join(' ') }}
+      become: true
+      when:
+        - is_data_disk_fs.rc != 0
+        - (data_disk_path.stdout_lines | length) >= hardware['disk']['data']['fs']['btrfs']['min_devices']
+      changed_when: is_mkfs.rc == 0
+      register: "is_mkfs"
+
+    - name: Create mount volume
+      ansible.builtin.file:
+        path: "{{ hardware['disk']['data']['fs']['mount_point'] }}"
+        state: "directory"
+        owner: "root"
+        group: "root"
+        mode: "0711"
+      become: true
+
+    - name: Mount data btrfs volume
+      ansible.posix.mount:
+        path: "{{ hardware['disk']['data']['fs']['mount_point'] }}"
+        src: "LABEL={{ hardware['disk']['data']['fs']['label'] }}"
+        state: "mounted"
+        fstype: "{{ hardware['disk']['data']['fs']['type'] }}"
+        opts: "{{ hardware['disk']['data']['fs']['mount_opts'] | join(',') }}"
+      become: true
+
+    - name: Set mounted btrfs volume permissions
+      ansible.builtin.file:
+        path: "{{ hardware['disk']['data']['fs']['mount_point'] }}"
+        state: "directory"
+        owner: "root"
+        group: "root"
+        mode: "0711"
+      become: true
+
+- name: Mount backup disk
+  block:
+    - name: Set backup disk path
+      ansible.builtin.shell: |
+        set -o pipefail
+        ls -1 /dev/disk/by-id/ata-* | \
+        grep -F -e "{{ hardware['disk']['backup']['serials'] | join('" -e "') }}" | \
+        grep -vE -- '-part[0-9]+$' | \
+        sort
+      changed_when: false
+      register: "backup_disk_path"
+
+    - name: Check vmm_hdd filesystem already exists
+      ansible.builtin.command: |
+        blkid -L {{ hardware['disk']['backup']['fs']['label'] }}
+      become: true
+      changed_when: false
+      failed_when: false
+      register: "is_backup_disk_fs"
+
+    - name: Set backup btrfs volume
+      ansible.builtin.shell: |
+        mkfs.btrfs -f \
+        -L {{ hardware['disk']['backup']['fs']['label'] }} \
+        -d {{ hardware['disk']['backup']['fs']['btrfs']['data_profile'] }} \
+        -m {{ hardware['disk']['backup']['fs']['btrfs']['metadata_profile'] }} \
+        {{ backup_disk_path.stdout_lines | join(' ') }}
+      become: true
+      when:
+        - is_backup_disk_fs.rc != 0
+        - (backup_disk_path.stdout_lines | length) >= hardware['disk']['backup']['fs']['btrfs']['min_devices']
+      changed_when: is_mkfs.rc == 0
+      register: "is_mkfs"
+
+    - name: Create mount volume
+      ansible.builtin.file:
+        path: "{{ hardware['disk']['backup']['fs']['mount_point'] }}"
+        state: "directory"
+        owner: "root"
+        group: "root"
+        mode: "0711"
+      become: true
+
+    - name: Mount backup btrfs volume
+      ansible.posix.mount:
+        path: "{{ hardware['disk']['backup']['fs']['mount_point'] }}"
+        src: "LABEL={{ hardware['disk']['backup']['fs']['label'] }}"
+        state: "mounted"
+        fstype: "{{ hardware['disk']['backup']['fs']['type'] }}"
+        opts: "{{ hardware['disk']['backup']['fs']['mount_opts'] | join(',') }}"
+      become: true
+
+    - name: Set mounted btrfs volume permissions
+      ansible.builtin.file:
+        path: "{{ hardware['disk']['backup']['fs']['mount_point'] }}"
+        state: "directory"
+        owner: "root"
+        group: "root"
+        mode: "0711"
+      become: true
+
+- name: Verify data disk is mounted
+  ansible.builtin.command: |
+    mountpoint -q {{ hardware['disk']['data']['fs']['mount_point'] }}
+  become: true
+  changed_when: false
+
+- name: Verify backup disk is mounted
+  ansible.builtin.command: |
+    mountpoint -q {{ hardware['disk']['backup']['fs']['mount_point'] }}
+  become: true
+  changed_when: false
+
+- name: Deploy btrfs scrub service and timer
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/btrfs/btrfs-scrub-{{ item }}.j2"
+    dest: "/etc/systemd/system/btrfs-scrub-{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  loop:
+    - "data.service"
+    - "data.timer"
+    - "backup.service"
+    - "backup.timer"
+  become: true
+
+- name: Enable auto btrfs scrub
+  ansible.builtin.systemd:
+    name: "btrfs-scrub-{{ item }}.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+  loop:
+    - "data"
+    - "backup"
+  become: true

ansible/roles/vmm/tasks/node/set_libvirt.yaml

@@ -9,23 +9,27 @@
 
 - name: Check libvirt directory
   ansible.builtin.stat:
-    path: "/var/lib/libvirt/{{ item }}"
+    path: "{{ node['libvirt_path'] }}/{{ item }}"
   loop:
     - "images"
     - "seeds"
+    - "disk"
+    - "disk/data"
+    - "disk/backup"
   register: "is_libvirt_dir"
 
 - name: Create libvirt directory
   ansible.builtin.file:
-    path: "/var/lib/libvirt/{{ item.item }}"
+    path: "{{ node['libvirt_path'] }}/{{ item.item }}"
     state: "directory"
     owner: "root"
     group: "root"
     mode: "0711"
   loop: "{{ is_libvirt_dir.results }}"
+  loop_control:
+    label: "{{ item.item }}"
   when: not item.stat.exists
   become: true
-  no_log: true
 
 - name: Set LIBVIRT_DEFAULT_URI
   ansible.builtin.lineinfile:
@@ -72,6 +76,8 @@
   loop:
     - "images-pool"
     - "seeds-pool"
+    - "data-pool"
+    - "backup-pool"
 
 - name: Start virtual storage pool
   community.libvirt.virt_pool:
@@ -81,6 +87,8 @@
   loop:
     - "images-pool"
     - "seeds-pool"
+    - "data-pool"
+    - "backup-pool"
 
 - name: Autostart virtual storage pool
   community.libvirt.virt_pool:
@@ -90,3 +98,5 @@
   loop:
     - "images-pool"
     - "seeds-pool"
+    - "data-pool"
+    - "backup-pool"

ansible/roles/vmm/tasks/services/set_smart_metric_generator.yaml

@@ -0,0 +1,47 @@
+---
+- name: Install smartmontools
+  ansible.builtin.apt:
+    name:
+      - "smartmontools"
+      - "jq"
+    state: "present"
+  become: true
+
+- name: Create SMART metric result directory
+  ansible.builtin.file:
+    path: "/etc/smartctl-exporter"
+    state: "directory"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+  become: true
+
+- name: Deploy smart_metric_generator script
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/smart_metric/smart_metric_generator.sh"
+    dest: "/usr/local/sbin"
+    owner: "root"
+    group: "root"
+    mode: "0711"
+  become: true
+
+- name: Deploy smart_metric_generator service files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/smart_metric/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0600"
+    validate: "/usr/bin/systemd-analyze verify %s"
+  loop:
+    - "smart_metric_generator.service"
+    - "smart_metric_generator.timer"
+  become: true
+
+- name: Register smart_metric_generator timer
+  ansible.builtin.systemd:
+    name: "smart_metric_generator.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+  become: true

ansible/roles/vmm/tasks/vm/create_seed.yaml

@@ -1,13 +1,13 @@
 ---
 # This task is located in vmm roles because of its attributes,
 # but all process should be run in "console".
-# At the playbook, `delegate_to: "console"` option is applyed by `apply:`.
+# At the playbook, `delegate_to: "console"` option is applied by `apply:`.
 - name: Create images directory
   ansible.builtin.file:
     path: "{{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}"
     state: "directory"
-    owner: "console"
-    group: "svadmins"
+    owner: "{{ hostvars['console']['node']['name'] }}"
+    group: "{{ hostvars['console']['node']['group'] }}"
     mode: "0700"
 
 - name: Create temp meta-data
@@ -16,8 +16,8 @@
       instance-id: vm-{{ target_vm }}
       local-hostname: {{ target_vm }}
     dest: "{{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/meta-data"
-    owner: "console"
-    group: "svadmins"
+    owner: "{{ hostvars['console']['node']['name'] }}"
+    group: "{{ hostvars['console']['node']['group'] }}"
     mode: "0600"
   register: "vm_meta_data"
   no_log: true
@@ -26,8 +26,8 @@
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/libvirt/seeds/user-data.j2"
     dest: "{{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/user-data"
-    owner: "console"
-    group: "svadmins"
+    owner: "{{ hostvars['console']['node']['name'] }}"
+    group: "{{ hostvars['console']['node']['group'] }}"
     mode: "0600"
   register: "vm_user_data"
   no_log: true
@@ -37,8 +37,8 @@
     content: |
       network: {config: disabled}
     dest: "{{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/network-config"
-    owner: "console"
-    group: "svadmins"
+    owner: "{{ hostvars['console']['node']['name'] }}"
+    group: "{{ hostvars['console']['node']['group'] }}"
     mode: "0600"
   register: "vm_network_config"
   no_log: true
@@ -49,8 +49,7 @@
   register: "is_seediso"
 
 - name: Create seed.iso
-  ansible.builtin.shell:
-    cmd: |
+  ansible.builtin.shell: |
       cloud-localds -N {{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/network-config \
       {{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/seed.iso \
       {{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/user-data \

ansible/roles/vmm/tasks/vm/deploy_vm_init.yaml

@@ -1,16 +1,16 @@
 ---
 - name: Check vm cloud-init
   ansible.builtin.stat:
-    path: "/var/lib/libvirt/images/debian-13.qcow2"
+    path: "{{ node['libvirt_path'] }}/images/debian-13.qcow2"
   become: true
   register: is_cloud_init_file
 
 - name: Deploy vm cloud-init
   ansible.builtin.copy:
     src: "{{ hostvars['console']['node']['data_path'] }}/images/debian-13-generic-amd64.qcow2"
-    dest: "/var/lib/libvirt/images/debian-13.qcow2"
+    dest: "{{ node['libvirt_path'] }}/images/debian-13.qcow2"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
     force: false
   become: true
@@ -18,18 +18,18 @@
 
 - name: Remote copy vm cloud-init file
   ansible.builtin.copy:
-    src: "/var/lib/libvirt/images/debian-13.qcow2"
-    dest: "/var/lib/libvirt/images/{{ target_vm }}.qcow2"
+    src: "{{ node['libvirt_path'] }}/images/debian-13.qcow2"
+    dest: "{{ node['libvirt_path'] }}/images/{{ target_vm }}.qcow2"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
     remote_src: true
     force: false
   become: true
 
 - name: Check deployed cloud-init file info
-  ansible.builtin.command:
-    cmd: "qemu-img info /var/lib/libvirt/images/{{ target_vm }}.qcow2 --output json"
+  ansible.builtin.command: |
+    qemu-img info {{ node['libvirt_path'] }}/images/{{ target_vm }}.qcow2 --output json
   changed_when: false
   failed_when:
     - deployed_cloudfile_info.rc != 0
@@ -37,19 +37,38 @@
   register: "deployed_cloudfile_info"
 
 - name: Resize deployed cloud-init file
-  ansible.builtin.command:
-    cmd: "qemu-img resize /var/lib/libvirt/images/{{ target_vm }}.qcow2 {{ hostvars[target_vm]['vm']['storage'] }}G"
+  ansible.builtin.command: |
+    qemu-img resize {{ node['libvirt_path'] }}/images/{{ target_vm }}.qcow2 {{ hostvars[target_vm]['vm']['storage'] }}G
   when:
     - deployed_cloudfile_info.rc == 0
     - (deployed_cloudfile_info.stdout | from_json)['virtual-size'] < (hostvars[target_vm]['vm']['storage'] | int * 1024 * 1024 * 1024)
   changed_when: true
 
+- name: Check bk/app raw image
+  ansible.builtin.stat:
+    path: "{{ hardware['disk'][hostvars[target_vm]['vm']['data_storage']['pool']]['fs']['mount_point'] }}/{{ target_vm }}.raw"
+    get_checksum: false
+    get_mime: false
+    get_attributes: false
+  when: target_vm in ["bk", "app"]
+  register: "is_app_data"
+
+- name: Create bk/app raw image
+  ansible.builtin.command: |
+    qemu-img create -f raw {{ hardware['disk'][hostvars[target_vm]['vm']['data_storage']['pool']]['fs']['mount_point'] }}/{{ target_vm }}.raw \
+    {{ hostvars[target_vm]['vm']['data_storage']['size'] }}T
+  become: true
+  when:
+    - target_vm in ["bk", "app"]
+    - not is_app_data.stat.exists
+  changed_when: true
+
 - name: Deploy vm seed.iso
   ansible.builtin.copy:
     src: "{{ hostvars['console']['node']['data_path'] }}/images/seeds/{{ target_vm }}/seed.iso"
-    dest: "/var/lib/libvirt/seeds/{{ target_vm }}_seed.iso"
+    dest: "{{ node['libvirt_path'] }}/seeds/{{ target_vm }}_seed.iso"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0644"
   become: true
   when: deployed_cloudfile_info.rc == 0

ansible/roles/vmm/tasks/vm/register_vm.yaml

@@ -8,11 +8,11 @@
     command: define
 
 - name: Deploy VM systemd file
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/libvirt/services/{{ target_vm }}.service"
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/vmm/libvirt/services/{{ target_vm }}.service.j2"
     dest: "{{ node['home_path'] }}/.config/systemd/user/{{ target_vm }}.service"
     owner: "{{ ansible_user }}"
-    group: "svadmins"
+    group: "{{ node['group'] }}"
     mode: "0400"
 
 - name: Register VM service

config/node/app/nftables.conf.j2

@@ -1,17 +1,17 @@
 #!/usr/sbin/nft -f
 flush ruleset
 
-define NET4_SERVER = {{ hostvars['fw']['network4']['subnet']['server'] }}
-define NET6_SERVER = {{ hostvars['fw']['network6']['subnet']['server'] }}
-define HOSTS4_CONSOLE = { {{ hostvars['fw']['network4']['console'].values() | join(', ') }} }
-define HOSTS6_CONSOLE = { {{ hostvars['fw']['network6']['console'].values() | join(', ') }} }
-define HOSTS4_AUTH = {{ hostvars['fw']['network4']['auth']['server'] }}
-define HOSTS6_AUTH = {{ hostvars['fw']['network6']['auth']['server'] }}
+define NET4_SERVER = {{ hostvars['fw']['network']['subnet']['ipv4']['server'] }}
+define NET6_SERVER = {{ hostvars['fw']['network']['subnet']['ipv6']['server'] }}
+define HOSTS4_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS4_AUTH = {{ hostvars['fw']['network']['auth']['ipv4']['server'] }}
+define HOSTS6_AUTH = {{ hostvars['fw']['network']['auth']['ipv6']['server'] }}
 define PORTS_SSH = 22
 define PORTS_HTTP = 80
-define PORTS_HTTP_FORWARD = 2080
+define PORTS_HTTP_FORWARD = {{ services['caddy']['ports']['http'] }}
 define PORTS_HTTPS = 443
-define PORTS_HTTPS_FORWARD = 2443
+define PORTS_HTTPS_FORWARD = {{ services['caddy']['ports']['https'] }}
 
 table inet nat {
     chain prerouting {

config/node/auth/nftables.conf.j2

@@ -1,15 +1,15 @@
 #!/usr/sbin/nft -f
 flush ruleset
 
-define NET4_SERVER = {{ hostvars['fw']['network4']['subnet']['server'] }}
-define NET6_SERVER = {{ hostvars['fw']['network6']['subnet']['server'] }}
-define HOSTS4_CONSOLE = { {{ hostvars['fw']['network4']['console'].values() | join(', ') }} }
-define HOSTS6_CONSOLE = { {{ hostvars['fw']['network6']['console'].values() | join(', ') }} }
+define NET4_SERVER = {{ hostvars['fw']['network']['subnet']['ipv4']['server'] }}
+define NET6_SERVER = {{ hostvars['fw']['network']['subnet']['ipv6']['server'] }}
+define HOSTS4_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
 define PORTS_SSH = 22
 define PORTS_HTTP = 80
-define PORTS_HTTP_FORWARD = 2080
+define PORTS_HTTP_FORWARD = {{ services['caddy']['ports']['http'] }}
 define PORTS_HTTPS = 443
-define PORTS_HTTPS_FORWARD = 2443
+define PORTS_HTTPS_FORWARD = {{ services['caddy']['ports']['https'] }}
 
 table inet nat {
     chain prerouting {

config/node/bk/nftables.conf.j2

@@ -0,0 +1,48 @@
+#!/usr/sbin/nft -f
+flush ruleset
+
+define NET4_SERVER = {{ hostvars['fw']['network']['subnet']['ipv4']['server'] }}
+define NET6_SERVER = {{ hostvars['fw']['network']['subnet']['ipv6']['server'] }}
+define HOSTS4_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS4_INFRA = {{ hostvars['fw']['network']['infra']['ipv4']['server'] }}
+define HOSTS6_INFRA = {{ hostvars['fw']['network']['infra']['ipv6']['server'] }}
+define HOSTS4_APP = {{ hostvars['fw']['network']['app']['ipv4']['server'] }}
+define HOSTS6_APP = {{ hostvars['fw']['network']['app']['ipv6']['server'] }}
+define PORTS_SSH = 22
+define PORTS_HTTPS = 443
+define PORTS_KOPIA = {{ services['kopia']['ports']['https'] }}
+
+table inet nat {
+    chain prerouting {
+        type nat hook prerouting priority dstnat; policy accept;
+        tcp dport $PORTS_HTTPS dnat to :$PORTS_KOPIA comment "dnat https ports to $PORTS_KOPIA"
+    }
+    chain postrouting {
+
+    }
+    chain output {
+        type nat hook output priority dstnat; policy accept;
+        oifname "lo" tcp dport $PORTS_HTTPS dnat to :$PORTS_KOPIA comment "dnat https ports to $PORTS_KOPIA out of LOCALHOST"
+    }
+}
+
+table inet filter {
+    chain input {
+        type filter hook input priority 0; policy drop;
+        ct state invalid drop comment "deny invalid connection"
+        ct state established, related accept comment "allow all connection already existing"
+        iifname "lo" accept comment "allow local connection: BK > BK"
+        meta l4proto { icmp, icmpv6 } accept comment "allow icmp connection: BK"
+        ip saddr $HOSTS4_CONSOLE tcp dport $PORTS_SSH accept comment "allow ipv4 ssh connection: CONSOLE > BK"
+        ip6 saddr $HOSTS6_CONSOLE tcp dport $PORTS_SSH accept comment "allow ipv6 ssh connection: CONSOLE > BK"
+        ip saddr { $HOSTS4_CONSOLE, $HOSTS4_INFRA, $HOSTS4_APP } tcp dport $PORTS_KOPIA ct original proto-dst $PORTS_HTTPS accept comment "allow ipv4 https connection: > BK"
+        ip6 saddr { $HOSTS6_CONSOLE, $HOSTS6_INFRA, $HOSTS6_APP } tcp dport $PORTS_KOPIA ct original proto-dst $PORTS_HTTPS accept comment "allow ipv6 https connection: > BK"
+    }
+    chain forward {
+        type filter hook forward priority 0; policy drop;
+    }
+    chain output {
+        type filter hook output priority 0; policy accept;
+    }
+}

config/node/common/hosts.j2

@@ -1,34 +1,34 @@
-# localhost
-127.0.0.1 {{ node['local_san'] }}
-::1 {{ node['local_san'] }}
-{% if node['name'] == 'console' %}
-# Hosts IPv4
-{{ hostvars['fw']['network4']['firewall']['server'] }} fw.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['vmm']['client'] }} init.vmm.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['vmm']['server'] }} vmm.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['infra']['server'] }} infra.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['auth']['server'] }} auth.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['app']['server'] }} app.{{ domain['internal'] }}
-# Hosts IPv6
-{{ hostvars['fw']['network6']['firewall']['server'] }} fw.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['vmm']['client'] }} init.vmm.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['vmm']['server'] }} vmm.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['infra']['server'] }} infra.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['auth']['server'] }} auth.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['app']['server'] }} app.{{ domain['internal'] }}
-{% else %}
-# IPv4
-# Crowdsec, blocky, bind(fw)
-{{ hostvars['fw']['network4']['firewall']['server'] }} ntp.{{ domain['internal'] }} crowdsec.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['blocky']['server'] }} blocky.{{ domain['internal'] }}
-{{ hostvars['fw']['network4']['bind']['server'] }} bind.{{ domain['internal'] }}
-# DB, LDAP, CA, Prometheus, Loki, mail (infra)
-{{ hostvars['fw']['network4']['infra']['server'] }} postgresql.{{ domain['internal'] }} ldap.{{ domain['internal'] }} prometheus.{{ domain['internal'] }} loki.{{ domain['internal'] }} mail.{{ domain['internal'] }} ca.{{ domain['internal'] }}
-# IPv6
-# Crowdsec, blocky, bind(fw)
-{{ hostvars['fw']['network6']['firewall']['server'] }} ntp.{{ domain['internal'] }} crowdsec.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['blocky']['server'] }} blocky.{{ domain['internal'] }}
-{{ hostvars['fw']['network6']['bind']['server'] }} bind.{{ domain['internal'] }}
-# DB, LDAP, CA, Prometheus, Loki, mail (infra)
-{{ hostvars['fw']['network6']['infra']['server'] }} postgresql.{{ domain['internal'] }} ldap.{{ domain['internal'] }} prometheus.{{ domain['internal'] }} loki.{{ domain['internal'] }} mail.{{ domain['internal'] }} ca.{{ domain['internal'] }}
-{% endif %}
+# Managed by Ansible
+# Bootstrap name resolution only.
+# Full service discovery is handled by BIND.
+
+127.0.0.1       localhost
+::1             localhost
+
+{% set net = hostvars['fw']['network'] %}
+{% 
+    set server_hosts = (
+        net
+        | dict2items
+        | rejectattr('key', 'equalto', 'dns_serial')
+        | rejectattr('key', 'equalto', 'subnet')
+        | rejectattr('key', 'equalto', 'reverse_zone')
+        | selectattr('value.ipv4.server', 'defined')
+        | selectattr('value.ipv6.server', 'defined')
+        | map(attribute='key')
+        | list
+    )
+%}
+# node canonical names
+{% for ip in ['ipv4', 'ipv6'] %}
+# {{ ip }}
+## vmm init
+{{ "%-36s" | format(net['vmm'][ip]['client']) }} init.vmm init.vmm.{{ domain['internal'] }}
+## console
+{{ "%-36s" | format(net['console'][ip]['client']) }} console console.{{ domain['internal'] }}
+## nodes
+{% for service in server_hosts %}
+{{ "%-36s" | format(net[service][ip]['server']) }} {{ service }} {{ service }}.{{ domain['internal'] }}{% for alias in net[service]['alias'] | default([]) %}{% if service in ['vmm', 'fw'] and alias != 'init.vmm' %} {{ alias }}.{{ domain['internal'] }}{% endif %}{% endfor %}{{ '' }}
+{% endfor %}
+
+{% endfor %}

config/node/common/networkd/20-eth0.network

@@ -3,11 +3,11 @@ Name=eth0
 
 [Network]
 # IPv4
-Address={{ hostvars['fw']['network4'][target_vm]['server'] }}/24
-Gateway={{ hostvars['fw']['network4']['firewall']['server'] }}
-DNS={{ hostvars['fw']['network4']['blocky']['server'] }}
+Address={{ hostvars['fw']['network'][target_vm]['ipv4']['server'] }}/24
+Gateway={{ hostvars['fw']['network']['fw']['ipv4']['server'] }}
+DNS={{ hostvars['fw']['network']['blocky']['ipv4']['server'] }}
 # IPv6
 IPv6AcceptRA=false
-Address={{ hostvars['fw']['network6'][target_vm]['server'] }}/64
-Gateway={{ hostvars['fw']['network6']['firewall']['server'] }}
-DNS={{ hostvars['fw']['network6']['blocky']['server'] }}
+Address={{ hostvars['fw']['network'][target_vm]['ipv6']['server'] }}/64
+Gateway={{ hostvars['fw']['network']['fw']['ipv6']['server'] }}
+DNS={{ hostvars['fw']['network']['blocky']['ipv6']['server'] }}

config/node/common/resolved/global.conf.j2

@@ -1,6 +1,6 @@
 [Resolve]
-{% if node['name'] in ['vmm', 'fw'] %}
+{% if node['name'] in ["vmm", "fw"] %}
 DNS=1.1.1.2 1.0.0.2
 DNS=2606:4700:4700::1112 2606:4700:4700::1002
 {% endif %}
-cache=false
+Cache=false

config/node/fw/networkd/21-fw-client.network

@@ -9,8 +9,8 @@ IPv6AcceptRA=false
 VLAN=server
 VLAN=user
 # IPv4
-Address={{ hostvars['fw']['network4']['firewall']['client'] }}/24
-DNS={{ hostvars['fw']['network4']['blocky']['server'] }}
+Address={{ hostvars['fw']['network']['fw']['ipv4']['client'] }}/24
+DNS={{ hostvars['fw']['network']['blocky']['ipv4']['server'] }}
 # IPv6
-Address={{ hostvars['fw']['network6']['firewall']['client'] }}/64
-DNS={{ hostvars['fw']['network6']['blocky']['server'] }}
+Address={{ hostvars['fw']['network']['fw']['ipv6']['client'] }}/64
+DNS={{ hostvars['fw']['network']['blocky']['ipv6']['server'] }}

config/node/fw/networkd/22-fw-server.network

@@ -6,19 +6,19 @@ IPForward=true
 IPv6SendRA=false
 IPv6AcceptRA=false
 # IPv4
-Address={{ hostvars['fw']['network4']['firewall']['server'] }}/24
-DNS={{ hostvars['fw']['network4']['blocky']['server'] }}
+Address={{ hostvars['fw']['network']['fw']['ipv4']['server'] }}/24
+DNS={{ hostvars['fw']['network']['blocky']['ipv4']['server'] }}
 # IPv6
-Address={{ hostvars['fw']['network6']['firewall']['server'] }}/64
-DNS={{ hostvars['fw']['network6']['blocky']['server'] }}
+Address={{ hostvars['fw']['network']['fw']['ipv6']['server'] }}/64
+DNS={{ hostvars['fw']['network']['blocky']['ipv6']['server'] }}
 
 [Address]
-Address={{ hostvars['fw']['network4']['blocky']['server'] }}/24
+Address={{ hostvars['fw']['network']['blocky']['ipv4']['server'] }}/24
 [Address]
-Address={{ hostvars['fw']['network4']['bind']['server'] }}/24
+Address={{ hostvars['fw']['network']['bind']['ipv4']['server'] }}/24
 [Address]
-Address={{ hostvars['fw']['network6']['blocky']['server'] }}/64
+Address={{ hostvars['fw']['network']['blocky']['ipv6']['server'] }}/64
 PreferredLifetime=0
 [Address]
-Address={{ hostvars['fw']['network6']['bind']['server'] }}/64
+Address={{ hostvars['fw']['network']['bind']['ipv6']['server'] }}/64
 PreferredLifetime=0

config/node/fw/networkd/23-fw-user.network

@@ -7,8 +7,8 @@ IPv6PrefixDelegation=true
 IPv6SendRA=true
 IPv6SendRAExtension=false
 # IPv4
-Address={{ hostvars['fw']['network4']['firewall']['user'] }}/24
-DNS={{ hostvars['fw']['network4']['blocky']['server'] }}
+Address={{ hostvars['fw']['network']['fw']['ipv4']['user'] }}/24
+DNS={{ hostvars['fw']['network']['blocky']['ipv4']['server'] }}
 
 [IPv6PrefixDelegation]
 SubnetId=20
@@ -22,4 +22,4 @@ Managed=false
 # O-Flag: Other information form DHCPv6
 OtherInformation=false
 EmitDNS=true
-DNS={{ hostvars['fw']['network6']['blocky']['server'] }}
+DNS={{ hostvars['fw']['network']['blocky']['ipv6']['server'] }}

config/node/fw/nftables.conf.j2

@@ -9,48 +9,47 @@ define IF_SERVER = "server"
 define IF_USER = "user"
 define IF_WG = "wg0"
 
-define NET4_CLIENT = {{ hostvars['fw']['network4']['subnet']['client'] }}
-define NET4_SERVER = {{ hostvars['fw']['network4']['subnet']['server'] }}
-define NET4_USER = {{ hostvars['fw']['network4']['subnet']['user'] }}
-define NET4_WG = {{ hostvars['fw']['network4']['subnet']['wg'] }}
-define NET4_LLA = {{ hostvars['fw']['network4']['subnet']['lla'] }}
+define NET4_CLIENT = {{ hostvars['fw']['network']['subnet']['ipv4']['client'] }}
+define NET4_SERVER = {{ hostvars['fw']['network']['subnet']['ipv4']['server'] }}
+define NET4_USER = {{ hostvars['fw']['network']['subnet']['ipv4']['user'] }}
+define NET4_WG = {{ hostvars['fw']['network']['subnet']['ipv4']['wg'] }}
+define NET4_LLA = {{ hostvars['fw']['network']['subnet']['ipv4']['lla'] }}
 define NET4_RFC1918 = { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }
 
-define NET6_CLIENT = {{ hostvars['fw']['network6']['subnet']['client'] }}
-define NET6_SERVER = {{ hostvars['fw']['network6']['subnet']['server'] }}
-define NET6_WG = {{ hostvars['fw']['network6']['subnet']['wg'] }}
-define NET6_LLA = {{ hostvars['fw']['network6']['subnet']['lla'] }}
+define NET6_CLIENT = {{ hostvars['fw']['network']['subnet']['ipv6']['client'] }}
+define NET6_SERVER = {{ hostvars['fw']['network']['subnet']['ipv6']['server'] }}
+define NET6_WG = {{ hostvars['fw']['network']['subnet']['ipv6']['wg'] }}
+define NET6_LLA = {{ hostvars['fw']['network']['subnet']['ipv6']['lla'] }}
 
-define HOSTS4_FW = { {{ hostvars['fw']['network4']['firewall'].values() | join(', ') }} }
-define HOSTS4_BLOCKY = {{ hostvars['fw']['network4']['blocky']['server'] }}
-define HOSTS4_BIND = {{ hostvars['fw']['network4']['bind']['server'] }}
-define HOSTS4_CONSOLE = { {{ hostvars['fw']['network4']['console'].values() | join(', ') }} }
-define HOSTS4_VMM = { {{ hostvars['fw']['network4']['vmm'].values() | join(', ') }} }
-define HOSTS4_INFRA = {{ hostvars['fw']['network4']['infra']['server'] }}
-define HOSTS4_AUTH = {{ hostvars['fw']['network4']['auth']['server'] }}
-define HOSTS4_APP = {{ hostvars['fw']['network4']['app']['server'] }}
-define HOSTS4_NAS = {{ hostvars['fw']['network4']['nas']['client'] }}
-define HOSTS4_PRINTER = {{ hostvars['fw']['network4']['printer']['client'] }}
+define HOSTS4_FW = { {{ hostvars['fw']['network']['fw']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS4_BLOCKY = {{ hostvars['fw']['network']['blocky']['ipv4']['server'] }}
+define HOSTS4_BIND = {{ hostvars['fw']['network']['bind']['ipv4']['server'] }}
+define HOSTS4_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS4_VMM = { {{ hostvars['fw']['network']['vmm']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS4_BK = {{ hostvars['fw']['network']['bk']['ipv4']['server'] }}
+define HOSTS4_INFRA = {{ hostvars['fw']['network']['infra']['ipv4']['server'] }}
+define HOSTS4_AUTH = {{ hostvars['fw']['network']['auth']['ipv4']['server'] }}
+define HOSTS4_APP = {{ hostvars['fw']['network']['app']['ipv4']['server'] }}
+define HOSTS4_SWITCH = {{ hostvars['fw']['network']['switch']['ipv4']['client'] }}
+define HOSTS4_PRINTER = {{ hostvars['fw']['network']['printer']['ipv4']['client'] }}
 
-define HOSTS6_FW = { {{ hostvars['fw']['network6']['firewall'].values() | join(', ') }} }
-define HOSTS6_BLOCKY = {{ hostvars['fw']['network6']['blocky']['server'] }}
-define HOSTS6_BIND = {{ hostvars['fw']['network6']['bind']['server'] }}
-define HOSTS6_CONSOLE = { {{ hostvars['fw']['network6']['console'].values() | join(', ') }} }
-define HOSTS6_VMM = { {{ hostvars['fw']['network6']['vmm'].values() | join(', ') }} }
-define HOSTS6_INFRA = {{ hostvars['fw']['network6']['infra']['server'] }}
-define HOSTS6_AUTH = {{ hostvars['fw']['network6']['auth']['server'] }}
-define HOSTS6_APP = {{ hostvars['fw']['network6']['app']['server'] }}
-define HOSTS6_NAS = {{ hostvars['fw']['network6']['nas']['client'] }}
+define HOSTS6_FW = { {{ hostvars['fw']['network']['fw']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_BLOCKY = {{ hostvars['fw']['network']['blocky']['ipv6']['server'] }}
+define HOSTS6_BIND = {{ hostvars['fw']['network']['bind']['ipv6']['server'] }}
+define HOSTS6_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_VMM = { {{ hostvars['fw']['network']['vmm']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_BK = {{ hostvars['fw']['network']['bk']['ipv6']['server'] }}
+define HOSTS6_INFRA = {{ hostvars['fw']['network']['infra']['ipv6']['server'] }}
+define HOSTS6_AUTH = {{ hostvars['fw']['network']['auth']['ipv6']['server'] }}
+define HOSTS6_APP = {{ hostvars['fw']['network']['app']['ipv6']['server'] }}
 
 define PORTS_SSH = 22
 define PORTS_WEB = { 80, 443 }
 define PORTS_DHCP = { 67, 68, 546, 547 }
 define PORTS_DNS = 53
 define PORTS_NTP = 123
-define PORTS_VPN = 11290
-define PORTS_CROWDSEC = 8080
-define PORTS_NAS = { 5000, 5001 }
-define PORTS_KOPIA = 51515
+define PORTS_VPN = {{ services['wireguard']['ports']['udp'] }}
+define PORTS_CROWDSEC = {{ services['crowdsec']['ports']['https'] }}
 
 table inet nat {
         chain prerouting {
@@ -78,10 +77,12 @@ table inet filter {
         set crowdsec6-blacklists {
                 type ipv6_addr
                 flags timeout
-        }        
+        }
         chain global {
                 # invalid packets
                 ct state invalid drop comment "deny invalid connection"
+                # VPN connection exception handling
+                udp dport $PORTS_VPN return comment "return vpn connection to input and forward chain"
                 # crowdsec
                 ip saddr @crowdsec-blacklists counter drop comment "deny all crowdsec blacklist"
                 ip6 saddr @crowdsec6-blacklists counter drop comment "deny all ipv6 crowdsec blacklist"
@@ -111,13 +112,14 @@ table inet filter {
                 jump global comment "set global condition"
                 iifname "lo" accept comment "allow local connection: FW > FW"
                 udp dport $PORTS_VPN accept comment "allow vpn connection: > FW"
+                ip6 saddr $NET6_LLA meta l4proto icmpv6 accept comment "allow icmpv6 connection: ipv6 LLA > FW"
                 iifname { $IF_CLIENT, $IF_SERVER, $IF_USER, $IF_WG } meta l4proto { icmp, icmpv6 } accept comment "allow icmp, icmpv6 connection: CLIENT/SERVER/USER/WG > FW"
-                iifname { $IF_CLIENT, $IF_SERVER, $IF_USER, $IF_WG } udp dport $PORTS_NTP accept comment "allow ntp connection: CLIENT/SERVER/USER/WG > FW"
+                iifname { $IF_CLIENT, $IF_SERVER, $IF_WG } udp dport $PORTS_NTP accept comment "allow ntp connection: CLIENT/SERVER/WG > FW"
                 # Global chain contains "WAN !> :SSH_PORT"
                 ip saddr $HOSTS4_CONSOLE tcp dport $PORTS_SSH accept comment "allow ipv4 ssh connection: CONSOLE > FW"
                 ip6 saddr $HOSTS6_CONSOLE tcp dport $PORTS_SSH accept comment "allow ipv6 ssh connection: CONSOLE > FW"
-                ip saddr { $HOSTS4_VMM, $HOSTS4_INFRA, $HOSTS4_AUTH, $HOSTS4_APP } tcp dport $PORTS_CROWDSEC accept comment "allow ipv4 crowdsec lapi connection: SERVER > FW"
-                ip6 saddr { $HOSTS6_VMM, $HOSTS6_INFRA, $HOSTS6_AUTH, $HOSTS6_APP } tcp dport $PORTS_CROWDSEC accept comment "allow ipv6 crowdsec lapi connection: SERVER > FW"
+                ip saddr { $HOSTS4_VMM, $HOSTS4_BK, $HOSTS4_INFRA, $HOSTS4_AUTH, $HOSTS4_APP } tcp dport $PORTS_CROWDSEC accept comment "allow ipv4 crowdsec lapi connection: SERVER > FW"
+                ip6 saddr { $HOSTS6_VMM, $HOSTS6_BK, $HOSTS6_INFRA, $HOSTS6_AUTH, $HOSTS6_APP } tcp dport $PORTS_CROWDSEC accept comment "allow ipv6 crowdsec lapi connection: SERVER > FW"
                 # Global chain contains "WAN !> :DNS_PORT"
                 ip daddr $HOSTS4_BLOCKY udp dport $PORTS_DNS accept comment "allow ipv4 udp dns connection: !WAN > SERVER BLOCKY(FW)"
                 ip daddr $HOSTS4_BLOCKY tcp dport $PORTS_DNS accept comment "allow ipv4 tcp dns connection: !WAN > SERVER BLOCKY(FW)"
@@ -138,17 +140,14 @@ table inet filter {
                 # SSH connection
                 ip saddr $HOSTS4_CONSOLE tcp dport $PORTS_SSH accept comment "allow ipv4 ssh connection: CONSOLE > FW >"
                 ip6 saddr $HOSTS6_CONSOLE tcp dport $PORTS_SSH accept comment "allow ipv6 ssh connection: CONSOLE > FW >"
-                # Reverse proxy (WAN)
+                # Reverse proxy for all
                 oifname $IF_SERVER ip daddr $HOSTS4_AUTH tcp dport $PORTS_WEB accept comment "allow ipv4 web connection: > FW > SERVER AUTH"
                 oifname $IF_SERVER ip6 daddr $HOSTS6_AUTH tcp dport $PORTS_WEB accept comment "allow ipv6 web connection: > FW > SERVER AUTH"
-                # Reverse proxy (SERVER)
-                oifname $IF_SERVER ip saddr $HOSTS4_CONSOLE ip daddr { $HOSTS4_INFRA, $HOSTS4_APP } tcp dport $PORTS_WEB accept comment "allow ipv4 web connection: CONSOLE > FW > SERVER INFRA/APP"
-                oifname $IF_SERVER ip6 saddr $HOSTS6_CONSOLE ip6 daddr { $HOSTS6_INFRA, $HOSTS6_APP } tcp dport $PORTS_WEB accept comment "allow ipv6 web connection: CONSOLE > FW > SERVER INFRA/APP"
-                # Kopia/NAS Console > NAS
-                oifname $IF_CLIENT ip saddr $HOSTS4_CONSOLE ip daddr $HOSTS4_NAS tcp dport { $PORTS_NAS, $PORTS_KOPIA } accept comment "allow ipv4 web connection (DSM, KOPIA): CONSOLE > FW > CLIENT NAS"
-                oifname $IF_CLIENT ip6 saddr $HOSTS6_CONSOLE ip6 daddr $HOSTS6_NAS tcp dport { $PORTS_NAS, $PORTS_KOPIA } accept comment "allow ipv6 web connection (DSM, KOPIA): CONSOLE > FW > CLIENT NAS"
+                # Reverse proxy (SERVER) and Kopia (SERVER) for CONSOLE
+                oifname $IF_SERVER ip saddr $HOSTS4_CONSOLE ip daddr { $HOSTS4_BK, $HOSTS4_INFRA, $HOSTS4_APP } tcp dport $PORTS_WEB accept comment "allow ipv4 web connection: CONSOLE > FW > SERVER BK/INFRA/APP"
+                oifname $IF_SERVER ip6 saddr $HOSTS6_CONSOLE ip6 daddr { $HOSTS6_BK, $HOSTS6_INFRA, $HOSTS6_APP } tcp dport $PORTS_WEB accept comment "allow ipv6 web connection: CONSOLE > FW > SERVER BK/INFRA/APP"
                 # Printer
-                oifname $IF_CLIENT ip saddr $HOSTS4_CONSOLE ip daddr $HOSTS4_PRINTER accept comment "allow ipv4 printer connection: CONSOLE > FW > PRINTER"
+                ip saddr $HOSTS4_CONSOLE ip daddr $HOSTS4_PRINTER accept comment "allow ipv4 printer connection: CONSOLE > FW > PRINTER"
 
                 iifname $IF_WAN jump wan comment "set WAN interface rules"
                 iifname $IF_CLIENT jump client comment "set CLIENT interface rules"
@@ -160,19 +159,13 @@ table inet filter {
                 return
         }
         chain client {
-                oifname $IF_WAN ip saddr { $HOSTS4_CONSOLE, $HOSTS4_NAS } accept comment "allow ipv4 internet connection: CLIENT CONSOLE/NAS > FW > WAN"
-                oifname $IF_WAN ip6 saddr { $HOSTS6_CONSOLE, $HOSTS6_NAS } accept comment "allow ipv6 internet connection: CLIENT CONSOLE/NAS > FW > WAN"
+                oifname $IF_WAN ip saddr { $HOSTS4_PRINTER, $HOSTS4_SWITCH } drop comment "deny ipv4 internet connection: CLIENT PRINTER/SWITCH !> FW > WAN"
+                oifname $IF_WAN accept comment "allow internet connection: CLIENT > FW > WAN"
                 return
         }
         chain server {
-                # reverse proxy AUTH > NAS
-                oifname $IF_CLIENT ip saddr $HOSTS4_AUTH ip daddr $HOSTS4_NAS tcp dport $PORTS_NAS accept comment "allow ipv4 web connection(DSM): SERVER AUTH > FW > CLIENT NAS"
-                oifname $IF_CLIENT ip6 saddr $HOSTS6_AUTH ip6 daddr $HOSTS6_NAS tcp dport $PORTS_NAS accept comment "allow ipv6 web connection(DSM): SERVER AUTH > FW > CLIENT NAS"
-                # Kopia INFRA, APP > NAS
-                oifname $IF_CLIENT ip saddr { $HOSTS4_INFRA, $HOSTS4_APP } ip daddr $HOSTS4_NAS tcp dport $PORTS_KOPIA accept comment "allow ipv4 web connection(kopia): SERVER INFRA/APP > FW > CLIENT NAS"
-                oifname $IF_CLIENT ip6 saddr { $HOSTS6_INFRA, $HOSTS6_APP } ip6 daddr $HOSTS6_NAS tcp dport $PORTS_KOPIA accept comment "allow ipv6 web connection(kopia): SERVER INFRA/APP > FW > CLIENT NAS"
-                oifname $IF_WAN ip saddr { $HOSTS4_VMM, $HOSTS4_INFRA, $HOSTS4_AUTH, $HOSTS4_APP } accept comment "allow ipv4 internet connection: SERVER VMM/INFRA/AUTH/APP > FW > WAN"
-                oifname $IF_WAN ip6 saddr { $HOSTS6_VMM, $HOSTS6_INFRA, $HOSTS6_AUTH, $HOSTS6_APP } accept comment "allow ipv6 internet connection: SERVER VMM/INFRA/AUTH/APP > FW > WAN"
+                oifname $IF_WAN ip saddr { $HOSTS4_VMM, $HOSTS4_BK, $HOSTS4_INFRA, $HOSTS4_AUTH, $HOSTS4_APP } accept comment "allow ipv4 internet connection: SERVER VMM/BK/INFRA/AUTH/APP > FW > WAN"
+                oifname $IF_WAN ip6 saddr { $HOSTS6_VMM, $HOSTS6_BK, $HOSTS6_INFRA, $HOSTS6_AUTH, $HOSTS6_APP } accept comment "allow ipv6 internet connection: SERVER VMM/BK/INFRA/AUTH/APP > FW > WAN"
                 return
         }
         chain user {

config/node/fw/wireguard/30-fw-wg0.netdev

@@ -2,9 +2,9 @@
 Name=wg0
 Kind=wireguard
 [WireGuard]
-ListenPort=11290
+ListenPort={{ services['wireguard']['ports']['udp'] }}
 PrivateKey={{ hostvars['console']['wireguard']['server']['private_key'] }}
 [WireGuardPeer]
 PublicKey={{ hostvars['console']['wireguard']['console']['public_key'] }}
 PresharedKey={{ hostvars['console']['wireguard']['console']['preshared_key'] }}
-AllowedIPs={{ hostvars['fw']["network4"]["console"]["wg"] }}/32, {{ hostvars['fw']["network6"]["console"]["wg"] }}/128
+AllowedIPs={{ hostvars['fw']["network"]["console"]['ipv4']["wg"] }}/32, {{ hostvars['fw']["network"]["console"]['ipv6']["wg"] }}/128

config/node/fw/wireguard/31-fw-wg0.network

@@ -1,6 +1,6 @@
 [Match]
 Name=wg0
 [Network]
-Address={{ hostvars['fw']["network4"]["firewall"]["wg"] }}/24
-Address={{ hostvars['fw']["network6"]["firewall"]["wg"] }}/64
+Address={{ hostvars['fw']["network"]["fw"]['ipv4']["wg"] }}/24
+Address={{ hostvars['fw']["network"]["fw"]['ipv6']["wg"] }}/64
 IPForward=yes

config/node/infra/nftables.conf.j2

@@ -3,21 +3,27 @@
 # iifname oifname saddr daddr proto dport ct state action / Ellipsis if you can something
 flush ruleset
 
-define NET4_SERVER = {{ hostvars['fw']['network4']['subnet']['server'] }}
-define NET6_SERVER = {{ hostvars['fw']['network6']['subnet']['server'] }}
-define HOSTS4_CONSOLE = { {{ hostvars['fw']['network4']['console'].values() | join(', ') }} }
-define HOSTS6_CONSOLE = { {{ hostvars['fw']['network6']['console'].values() | join(', ') }} }
+define NET4_SERVER = {{ hostvars['fw']['network']['subnet']['ipv4']['server'] }}
+define NET6_SERVER = {{ hostvars['fw']['network']['subnet']['ipv6']['server'] }}
+define HOSTS4_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv4'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS6_CONSOLE = { {{ hostvars['fw']['network']['console']['ipv6'] | dict2items | rejectattr('key', 'equalto', 'reverse') | map(attribute='value') | join(', ')}} }
+define HOSTS4_AUTH = {{ hostvars['fw']['network']['auth']['ipv4']['server'] }}
+define HOSTS6_AUTH = {{ hostvars['fw']['network']['auth']['ipv6']['server'] }}
 define PORTS_SSH = 22
-define PORTS_DB = 5432
-define PORTS_CA = 9000
+define PORTS_DB = { 5432, 3306 }
+define PORTS_CA = {{ services['ca']['ports']['https'] }}
 define PORTS_LDAPS = 636
-define PORTS_LDAPS_FORWARD = 6360
+define PORTS_LDAPS_FORWARD = {{ services['ldap']['ports']['ldaps'] }}
 define PORTS_HTTP = 80
-define PORTS_HTTP_FORWARD = 2080
+define PORTS_HTTP_FORWARD = {{ services['caddy']['ports']['http'] }}
 define PORTS_HTTPS = 443
-define PORTS_HTTPS_FORWARD = 2443
-define PORTS_PROMETHEUS = 9090
-define PORTS_LOKI = 3100
+define PORTS_HTTPS_FORWARD = {{ services['caddy']['ports']['https'] }}
+define PORTS_PROMETHEUS = {{ services['prometheus']['ports']['https'] }}
+define PORTS_LOKI = {{ services['loki']['ports']['https'] }}
+define PORTS_SMTP = 587
+define PORTS_SMTP_FORWARD = {{ services['maddy']['ports']['smtp'] }}
+define PORTS_IMAPS = 993
+define PORTS_IMAPS_FORWARD = {{ services['maddy']['ports']['imaps'] }}
 
 table inet nat {
     chain prerouting {
@@ -25,6 +31,8 @@ table inet nat {
         tcp dport $PORTS_HTTP dnat to :$PORTS_HTTP_FORWARD comment "DNAT http ports to $PORTS_HTTP_FORWARD"
         tcp dport $PORTS_HTTPS dnat to :$PORTS_HTTPS_FORWARD comment "DNAT https ports to $PORTS_HTTPS_FORWARD"
         tcp dport $PORTS_LDAPS dnat to :$PORTS_LDAPS_FORWARD comment "DNAT ldaps ports to $PORTS_LDAPS_FORWARD"
+        tcp dport $PORTS_SMTP dnat to :$PORTS_SMTP_FORWARD comment "DNAT smtp ports to $PORTS_SMTP_FORWARD"
+        tcp dport $PORTS_IMAPS dnat to :$PORTS_IMAPS_FORWARD comment "DNAT imaps ports to $PORTS_IMAPS_FORWARD"
     }
     chain postrouting {
 
@@ -34,6 +42,8 @@ table inet nat {
         oifname "lo" tcp dport $PORTS_HTTP dnat to :$PORTS_HTTP_FORWARD comment "DNAT http ports to $PORTS_HTTP_FORWARD out of LOCALHOST"
         oifname "lo" tcp dport $PORTS_HTTPS dnat to :$PORTS_HTTPS_FORWARD comment "DNAT https ports to $PORTS_HTTPS_FORWARD out of LOCALHOST"
         oifname "lo" tcp dport $PORTS_LDAPS dnat to :$PORTS_LDAPS_FORWARD comment "DNAT ldaps ports to $PORTS_LDAPS_FORWARD out of LOCALHOST"
+        oifname "lo" tcp dport $PORTS_SMTP dnat to :$PORTS_SMTP_FORWARD comment "DNAT smtp ports to $PORTS_SMTP_FORWARD"
+        oifname "lo" tcp dport $PORTS_IMAPS dnat to :$PORTS_IMAPS_FORWARD comment "DNAT imaps ports to $PORTS_IMAPS_FORWARD"
     }
 }
 
@@ -50,16 +60,20 @@ table inet filter {
         ip6 saddr $NET6_SERVER tcp dport $PORTS_CA accept comment "allow ipv6 ca connection: SERVER > INFRA"
         ip saddr $NET4_SERVER tcp dport $PORTS_DB accept comment "allow ipv4 db connection: SERVER > INFRA"
         ip6 saddr $NET6_SERVER tcp dport $PORTS_DB accept comment "allow ipv6 db connection: SERVER > INFRA"
-        ip saddr $HOSTS4_CONSOLE tcp dport $PORTS_HTTP_FORWARD ct original proto-dst $PORTS_HTTP accept comment "allow ipv4 http connection: CONSOLE > INFRA"
-        ip6 saddr $HOSTS6_CONSOLE tcp dport $PORTS_HTTP_FORWARD ct original proto-dst $PORTS_HTTP accept comment "allow ipv6 http connection: CONSOLE > INFRA"
-        ip saddr $HOSTS4_CONSOLE tcp dport $PORTS_HTTPS_FORWARD ct original proto-dst $PORTS_HTTPS accept comment "allow ipv4 https connection: CONSOLE > INFRA"
-        ip6 saddr $HOSTS6_CONSOLE tcp dport $PORTS_HTTPS_FORWARD ct original proto-dst $PORTS_HTTPS accept comment "allow ipv6 https connection: CONSOLE > INFRA"
+        ip saddr { $HOSTS4_CONSOLE, $HOSTS4_AUTH } tcp dport $PORTS_HTTP_FORWARD ct original proto-dst $PORTS_HTTP accept comment "allow ipv4 http connection: CONSOLE, AUTH > APP"
+        ip6 saddr { $HOSTS6_CONSOLE, $HOSTS6_AUTH } tcp dport $PORTS_HTTP_FORWARD ct original proto-dst $PORTS_HTTP accept comment "allow ipv6 http connection: CONSOLE, AUTH > APP"
+        ip saddr { $HOSTS4_CONSOLE, $HOSTS4_AUTH } tcp dport $PORTS_HTTPS_FORWARD ct original proto-dst $PORTS_HTTPS accept comment "allow ipv4 https connection: CONSOLE, AUTH > APP"
+        ip6 saddr { $HOSTS6_CONSOLE, $HOSTS6_AUTH } tcp dport $PORTS_HTTPS_FORWARD ct original proto-dst $PORTS_HTTPS accept comment "allow ipv6 https connection: CONSOLE, AUTH > APP"
         ip saddr $NET4_SERVER tcp dport $PORTS_LDAPS_FORWARD ct original proto-dst $PORTS_LDAPS accept comment "allow ipv4 ldaps connection: SERVER > INFRA"
         ip6 saddr $NET6_SERVER tcp dport $PORTS_LDAPS_FORWARD ct original proto-dst $PORTS_LDAPS accept comment "allow ipv6 ldaps connection: SERVER > INFRA"
         ip saddr $NET4_SERVER tcp dport $PORTS_PROMETHEUS accept comment "allow ipv4 prometheus connection: SERVER > INFRA"
         ip6 saddr $NET6_SERVER tcp dport $PORTS_PROMETHEUS accept comment "allow ipv6 prometheus connection: SERVER > INFRA"
         ip saddr $NET4_SERVER tcp dport $PORTS_LOKI accept comment "allow ipv4 loki connection: SERVER > INFRA"
         ip6 saddr $NET6_SERVER tcp dport $PORTS_LOKI accept comment "allow ipv6 loki connection: SERVER > INFRA"
+        ip saddr $NET4_SERVER tcp dport $PORTS_SMTP_FORWARD ct original proto-dst $PORTS_SMTP accept comment "allow ipv4 smtp connection: SERVER > INFRA"
+        ip6 saddr $NET6_SERVER tcp dport $PORTS_SMTP_FORWARD ct original proto-dst $PORTS_SMTP accept comment "allow ipv6 smtp connection: SERVER > INFRA"
+        ip saddr $NET4_SERVER tcp dport $PORTS_IMAPS_FORWARD ct original proto-dst $PORTS_IMAPS accept comment "allow ipv4 imaps connection: SERVER > INFRA"
+        ip6 saddr $NET6_SERVER tcp dport $PORTS_IMAPS_FORWARD ct original proto-dst $PORTS_IMAPS accept comment "allow ipv6 imaps connection: SERVER > INFRA"
     }
     chain forward {
         type filter hook forward priority 0; policy drop;