chore(script): archive a extract_secret.sh script

archived stack: extract_secret.sh

.gitignore

@@ -1,6 +1,9 @@
+.ansible
 data/bin/*
 data/volumes/*
 data/images/*
+!data/images/containers
+data/images/containers/*
 docs/archives/textfiles/
 docs/notes/*
 *.sql

README.md

@@ -1,8 +1,17 @@
 # ilnmors homelab README
 
-This homelab project implements single-node On-premise IaaS system. The homelab contains virtual machines which are divided by their roles, such as private firewall, DNS, PKI, LDAP and database, SSO\(OIDC\). The standard domain is used to implement this system without specific vendors. All components are defined as code and initiated by IaC \(Ansible\) except hypervisor initial configuration.
+This homelab project implements single-node On-premise IaaS system. The homelab contains virtual machines which are divided by their roles, such as private firewall, DNS, PKI, LDAP and database, SSO(OIDC). The standard domain is used to implement this system without specific vendors. All components are defined as code and initiated by IaC (Ansible) except hypervisor initial configuration.
 
-## RTO times
+## RTO and RPO
+
+### RPO
+- Each backup guarantees 24 hours RPO
+    - DB dumps are backed up at 12:00 AM
+    - Stateful data in app vm is backed up at 03:00 AM
+- The maximum inconsistency window between DB dumps and stateful data can be 27 hours.
+    - The different backup time.
+
+### RTO
 - Feb/25/2026 - Reprovisioning Hypervisor and vms
     - RTO: 1 hour 30 min - verified
         - Manual install and set vmm: 20 min
@@ -15,12 +24,12 @@ This homelab project implements single-node On-premise IaaS system. The homelab
 - Mar/5/2026 - Reprovisioning Hardware and Hypervisor and vms
     - RTO: 2 hour 20 min
         - console: 15min - verified
-        - certificate: 0 min \(When it needs to be created, RTO will be 20 min) - not verified
-        - wireguard: 0 min \(When it needs to be created, RTO will be 1 min) - not verified
-        - hypervisor\(+fw\): 45 min - verified
+        - certificate: 0 min (When it needs to be created, RTO will be 20 min) - not verified
+        - wireguard: 0 min (When it needs to be created, RTO will be 1 min) - not verified
+        - hypervisor(+fw): 45 min - verified
         - switch: 1 min - verified
         - dsm: 30 min - verified
-        - kopia: 0 min \(When it needs to be created, RTO will be 10 min) - verified
+        - kopia: 0 min (When it needs to be created, RTO will be 10 min) - verified
         - Extra vms: 30 min - verified
         - Etc: 30 min
 

ansible/convention.yaml

@@ -33,7 +33,6 @@
   tags:
     - "always"
     - "init"
-    - "upgrade"
-    - "update"
+    - "[service_name]"
 
 # when: "'tags' is not in ansible_run_tags"

ansible/inventory/group_vars/all.yaml

@@ -1,74 +1,189 @@
 ---
 # Global vars
 ansible_ssh_private_key_file: "/etc/secrets/{{ hostvars['console']['node']['uid'] }}/id_console"
+timezone: "Asia/Seoul"
 
-# URL infromation, you can use {{ infra_uri['services'] | split(':') | first|last }} to seperate domain and ports
-infra_uri:
+# CA
+root_cert_filename: "ilnmors_root_ca.crt"
+intermediate_cert_filename: "ilnmors_intermediate_ca.crt"
+intermediate_key_filename: "ilnmors_intermediate_ca.key"
+
+
+# local SAN and SSH SAN should be updated manually on host_vars
+domain:
+  public: "ilnmors.com"
+  internal: "ilnmors.internal"
+  dc: "dc=ilnmors,dc=internal"
+  org: "ilnmors"
+
+# DNS configuration including bind and blocky should be set manually.
+# named.conf.j2 is also set manually.
+# Check the hosts.j2 when cname records are fixed
+
+services:
   crowdsec:
-    domain: "crowdsec.ilnmors.internal"
+    domain: "crowdsec"
     ports:
       https: "8080"
   bind:
-    domain: "bind.ilnmors.internal"
+    domain: "bind"
     ports:
       dns: "53"
   blocky:
-    domain: "blocky.ilnmors.internal"
+    domain: "blocky"
     ports:
       https: "443"
       dns: "53"
   postgresql:
-    domain: "postgresql.ilnmors.internal"
+    domain: "postgresql"
     ports:
       tcp: "5432" # postgresql db connection port
+    subuid: "100998"
   ldap:
-    domain: "ldap.ilnmors.internal"
+    domain: "ldap"
     ports:
       http: "17170"
-      ldaps: "636"
+      ldaps: "6360"
+    subuid: "100999"
   ca:
-    domain: "ca.ilnmors.internal"
+    domain: "ca"
     ports:
       https: "9000"
+    subuid: "100999"
+  x509-exporter:
+    ports:
+      http: "9793"
+    subuid: "165533"
   prometheus:
-    domain: "prometheus.ilnmors.internal"
+    domain: "prometheus"
     ports:
       https: "9090"
+    subuid: "165533"
   loki:
-    domain: "loki.ilnmors.internal"
+    domain: "loki"
     ports:
       https: "3100"
+    subuid: "110000"
+  grafana:
+    domain: "grafana"
+    ports:
+      http: "3000" # Infra server: Internal ports
+    subuid: "100471"
+  caddy:
+    ports:
+      http: "2080"
+      https: "2443"
   nas:
-    domain: "nas.ilnmors.internal"
+    domain: "nas"
     ports:
       https: "5001"
   kopia:
-    domain: "nas.ilnmors.internal"
+    domain: "nas"
     ports:
       https: "51515"
+  authelia:
+    domain: "authelia"
+    ports:
+      http: "9091"
+  redis:
+    subuid: "100998"
+  vaultwarden:
+    domain:
+      public: "vault"
+      internal: "vault.app"
+    ports:
+      http: "8000"
+  gitea:
+    domain:
+      public: "gitea"
+      internal: "gitea.app"
+    ports:
+      http: "3000" # App server: Public ports
+    subuid: "100999"
+  immich:
+    domain:
+      public: "immich"
+      internal: "immich.app"
+    ports:
+      http: "2283"
+      redis: "6379"
+  immich-ml:
+    ports:
+      http: "3003"
+  paperless:
+    domain:
+      public: "paperless"
+      internal: "paperless.app"
+    ports:
+      http: "8001"
+      redis: "6380"
+    subuid: "100999"
+  manticore:
+    subuid: "100998"
+  affine:
+    domain:
+      public: "affine"
+      internal: "affine.app"
+    ports:
+      http: "3010"
+      redis: "6381"
+      manticore: "9308"
+  nextcloud:
+    domain:
+      public: "nextcloud"
+      internal: "nextcloud.app"
+    ports:
+      http: "8002"
+      redis: "6382"
+    subuid: "100032"
+  collabora:
+    domain:
+      public: "collabora"
+      internal: "collabora.app"
+    ports:
+      http: "9980"
+    subuid: "101000"
+  sure:
+    domain:
+      public: "sure"
+      internal: "sure.app"
+    ports:
+      http: "3001"
+      redis: "6383"
+    subuid: "100999"
 
 version:
   packages:
     sops: "3.12.1"
-    step: "0.29.0"
+    step: "0.30.2"
     kopia: "0.22.3"
-    blocky: "0.28.2"
-    alloy: "1.13.0"
-    # telegraf: "1.37.1"
+    blocky: "0.29.0"
+    alloy: "1.16.1"
   containers:
     # common
-    caddy: "2.10.2"
+    caddy: "2.11.2"
     # infra
-    step: "0.29.0"
-    ldap: "v0.6.2"
-    x509-exporter: "3.19.1"
-    prometheus: "v3.9.1"
-    loki: "3.6.5"
-    grafana: "12.3.3"
+    step: "0.30.2"
+    ldap: "v0.6.3"
+    x509-exporter: "4.1.0"
+    prometheus: "v3.11.3"
+    loki: "3.7.1"
+    grafana: "13.0.1"
     ## Postgresql
-    postgresql: "18.2"
+    postgresql: "18.3"
     # For immich - https://github.com/immich-app/base-images/blob/main/postgres/versions.yaml
 #    pgvector: "v0.8.1"
-    vectorchord: "0.5.3"
+    vectorchord: "1.1.1"
     # Auth
-    authelia: "4.39.15"
+    authelia: "4.39.19"
+    # App
+    vaultwarden: "1.36.0"
+    gitea: "1.26.1"
+    redis: "8.6.3"
+    immich: "v2.7.5"
+    paperless: "2.20.15"
+    manticore: "25.0.0"
+    affine: "0.26.3"
+    nextcloud: "33.0.3"
+    collabora: "25.04.9.4.1"
+    sure: "0.7.0-hotfix.2"

ansible/inventory/host_vars/console.yaml

@@ -21,5 +21,6 @@ node:
   config_path: "{{ node.homelab_path }}/config"
   ssh_san: "console,console.ilnmors.internal"
   ssh_users: "vmm,fw,infra,auth,app"
-  local_san: "localhost console.ilnmors.internal"
+  # add the hostname of wsl, it is needed to improve the sudo problem
+  local_san: "localhost console.ilnmors.internal surface"
 # ansible_python_interpreter: "{{ ansible_playbook_python }}"

ansible/playbooks/app/site.yaml

@@ -23,9 +23,9 @@
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -142,8 +142,8 @@
         name: "common"
         tasks_from: "services/set_alloy"
         apply:
-          tags: ["init", "update", "alloy"]
-      tags: ["init", "update", "alloy"]
+          tags: ["init", "alloy"]
+      tags: ["init", "alloy"]
 
     - name: Set kopia
       ansible.builtin.include_role:
@@ -161,6 +161,70 @@
           tags: ["site", "caddy"]
       tags: ["site", "caddy"]
 
+    - name: Set vaultwarden
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_vaultwarden"
+        apply:
+          tags: ["site", "vaultwarden"]
+      tags: ["site", "vaultwarden"]
+
+    - name: Set gitea
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_gitea"
+        apply:
+          tags: ["site", "gitea"]
+      tags: ["site", "gitea"]
+
+    - name: Set immich
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_immich"
+        apply:
+          tags: ["site", "immich"]
+      tags: ["site", "immich"]
+
+    - name: Set paperless
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_paperless"
+        apply:
+          tags: ["site", "paperless"]
+      tags: ["site", "paperless"]
+
+    - name: Set affine
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_affine"
+        apply:
+          tags: ["site", "affine"]
+      tags: ["site", "affine"]
+
+    - name: Set nextcloud
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_nextcloud"
+        apply:
+          tags: ["site", "nextcloud"]
+      tags: ["site", "nextcloud"]
+
+    - name: Set collabora
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_collabora"
+        apply:
+          tags: ["site", "collabora"]
+      tags: ["site", "collabora"]
+
+    - name: Set sure
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_sure"
+        apply:
+          tags: ["site", "sure"]
+      tags: ["site", "sure"]
+
     - name: Flush handlers right now
       ansible.builtin.meta: "flush_handlers"
 

ansible/playbooks/auth/site.yaml

@@ -23,9 +23,9 @@
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 

ansible/playbooks/console/site.yaml

@@ -24,9 +24,9 @@
 
   tasks:
     # init
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -115,7 +115,7 @@
       become: true
       tags: ["init", "site", "install-packages"]
 
-    - name: Install CLI tools
+    - name: Set CLI tools
       ansible.builtin.include_role:
         name: "console"
         tasks_from: "services/set_cli_tools"
@@ -123,10 +123,10 @@
           tags: ["init", "site", "tools"]
       tags: ["init", "site", "tools"]
 
-    - name: Install chromium with font
+    - name: Set kopia
       ansible.builtin.include_role:
-        name: "console"
-        tasks_from: "services/set_chromium"
+        name: "common"
+        tasks_from: "services/set_kopia"
         apply:
-          tags: ["init", "site", "chromium"]
-      tags: ["init", "site", "chromium"]
+          tags: ["init", "site", "kopia"]
+      tags: ["init", "site", "kopia"]

ansible/playbooks/fw/site.yaml

@@ -23,9 +23,9 @@
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 
@@ -162,8 +162,8 @@
         name: "fw"
         tasks_from: "services/set_bind"
         apply:
-          tags: ["init", "update", "bind"]
-      tags: ["init", "update", "bind"]
+          tags: ["init", "bind"]
+      tags: ["init", "bind"]
 
     - name: Set blocky
       ansible.builtin.include_role:

ansible/playbooks/infra/site.yaml

@@ -23,9 +23,9 @@
       tags: ["always"]
 
   tasks:
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 

ansible/playbooks/vmm/site.yaml

@@ -30,9 +30,9 @@
       tags: ["always"]
   tasks:
     # init
-    - name: Set timezone to Asia/Seoul
+    - name: Set timezone
       community.general.timezone:
-        name: Asia/Seoul
+        name: "{{ timezone }}"
       become: true
       tags: ["init", "timezone"]
 

ansible/roles/app/handlers/main.yaml

@@ -0,0 +1,104 @@
+---
+- name: Restart vaultwarden
+  ansible.builtin.systemd:
+    name: "vaultwarden.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_vaultwarden"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart gitea
+  ansible.builtin.systemd:
+    name: "gitea.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_gitea"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart immich
+  ansible.builtin.systemd:
+    name: "immich.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_immich"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart immich-ml
+  ansible.builtin.systemd:
+    name: "immich-ml.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_immich-ml"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart paperless
+  ansible.builtin.systemd:
+    name: "paperless.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_paperless"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart affine
+  ansible.builtin.systemd:
+    name: "affine.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_affine_init.stat.exists
+  changed_when: false
+  listen: "notification_restart_affine"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart nextcloud
+  ansible.builtin.systemd:
+    name: "nextcloud.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_nextcloud_init.stat.exists
+  changed_when: false
+  listen: "notification_restart_nextcloud"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart collabora
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_collabora"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart sure
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  loop:
+    - "sure-web.service"
+    - "sure-worker.service"
+  changed_when: false
+  listen: "notification_restart_sure"
+  ignore_errors: true # noqa: ignore-errors

ansible/roles/app/tasks/node/set_raid.yaml

@@ -68,3 +68,23 @@
     group: "svadmins"
     mode: "0770"
   become: true
+
+- name: Deploy btrfs scrub service and timer
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/app/btrfs/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  loop:
+    - "btrfs-scrub.service"
+    - "btrfs-scrub.timer"
+  become: true
+
+- name: Enable auto btrfs scrub
+  ansible.builtin.systemd:
+    name: "btrfs-scrub.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+  become: true

ansible/roles/app/tasks/services/set_affine.yaml

@@ -0,0 +1,198 @@
+---
+- name: Set manticore service name
+  ansible.builtin.set_fact:
+    manticore_service: "affine"
+
+- name: Create manticore directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['manticore']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/manticore"
+    - "data/containers/manticore/{{ manticore_service }}"
+  become: true
+
+- name: Deploy manticore.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/manticore/manticore.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/manticore_{{ manticore_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_manticore_containerfile"
+
+- name: Enable (Restart) manticore.service
+  ansible.builtin.systemd:
+    name: "manticore_{{ manticore_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_manticore_containerfile.changed # noqa: no-handler
+
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "affine"
+
+- name: Create redis_affine directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create affine directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/affine"
+    - "containers/affine"
+    - "containers/affine/ssl"
+    - "containers/affine/config"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/affine/ssl/{{ root_cert_filename }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0440"
+  notify: "notification_restart_affine"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "AFFINE_PRIVATE_KEY"
+      value: "{{ hostvars['console']['affine']['secret_key'] }}"
+    - name: "AFFINE_DATABASE_URL"
+      value: "postgresql://affine:{{ hostvars['console']['postgresql']['password']['affine'] | urlencode | replace('/', '%2F') }}\
+        @{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}/affine_db?sslmode=verify-full&\
+        sslrootcert=/etc/ssl/affine/{{ root_cert_filename }}"
+  notify: "notification_restart_affine"
+  no_log: true
+
+- name: Check data directory empty
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/affine/.init"
+  register: "is_affine_init"
+
+- name: Initialize affine
+  when: not is_affine_init.stat.exists
+  block:
+    - name: Execute init command (Including pulling image)
+      containers.podman.podman_container:
+        name: "affine_init"
+        image: "ghcr.io/toeverything/affine:{{ version['containers']['affine'] }}"
+        command: ['sh', '-c', 'node ./scripts/self-host-predeploy.js']
+        state: "started"
+        rm: true
+        detach: false
+        secrets:
+          - "AFFINE_DATABASE_URL,type=env,target=DATABASE_URL"
+      no_log: true
+
+    - name: Create .init file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/containers/affine/.init"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
+
+- name: Deploy affine.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/affine/affine.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/affine.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_affine"
+
+- name: Enable affine.service
+  ansible.builtin.systemd:
+    name: "affine.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - image: "docker.io/manticoresearch/manticore:{{ version['containers']['manticore'] }}"
+      file: "docker.io_manticoresearch_manticore_{{ version['containers']['manticore'] }}"
+    - image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - image: "ghcr.io/toeverything/affine:{{ version['containers']['affine'] }}"
+      file: "ghcr.io_toeverything_affine_{{ version['containers']['affine'] }}"
+  loop_control:
+    label: "{{ item.file }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_collabora.yaml

@@ -0,0 +1,37 @@
+---
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/collabora/collabora.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/collabora.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_collabora"
+
+- name: Enable collabora.service
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_collabora_code_{{ version['containers']['collabora'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/collabora/code:{{ version['containers']['collabora'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_collabora_code_{{ version['containers']['collabora'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_collabora_code_{{ version['containers']['collabora'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_gitea.yaml

@@ -0,0 +1,71 @@
+---
+- name: Create gitea directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['gitea']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/gitea"
+    - "containers/gitea"
+    - "containers/gitea/ssl"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/gitea/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['gitea']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_gitea"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "GITEA__database__PASSWD"
+    data: "{{ hostvars['console']['postgresql']['password']['gitea'] }}"
+    state: "present"
+    force: true
+  notify: "notification_restart_gitea"
+  no_log: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/gitea/gitea.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/gitea.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_gitea"
+
+- name: Enable gitea.service
+  ansible.builtin.systemd:
+    name: "gitea.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_gitea_gitea_{{ version['containers']['gitea'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/gitea/gitea:{{ version['containers']['gitea'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_gitea_gitea_{{ version['containers']['gitea'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_gitea_gitea_{{ version['containers']['gitea'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/app/tasks/services/set_immich.yaml

@@ -0,0 +1,155 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "immich"
+
+- name: Create redis_immich directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Add user in video, render group
+  ansible.builtin.user:
+    name: "{{ ansible_user }}"
+    state: "present"
+    groups: "video, render"
+    append: true
+  become: true
+
+- name: Create immich directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/immich"
+    - "containers/immich"
+    - "containers/immich/ssl"
+    - "containers/immich/ml"
+    - "containers/immich/ml/cache"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/immich/ssl/{{ root_cert_filename }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0440"
+  notify: "notification_restart_immich"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "IMMICH_DB_PASSWORD"
+    data: "{{ hostvars['console']['postgresql']['password']['immich'] }}"
+    state: "present"
+    force: true
+  notify: "notification_restart_immich"
+  no_log: true
+
+- name: Deploy immich.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/immich/immich.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/immich.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_immich"
+
+- name: Deploy immich-ml.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/immich/immich-ml.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/immich-ml.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_immich-ml"
+
+- name: Enable immich.service
+  ansible.builtin.systemd:
+    name: "immich.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Enable immich-ml.service
+  ansible.builtin.systemd:
+    name: "immich-ml.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - image: "ghcr.io/immich-app/immich-machine-learning:{{ version['containers']['immich'] }}-openvino"
+      file: "ghcr.io_immich-app_immich-machine-learning_{{ version['containers']['immich'] }}-openvino"
+    - image: "ghcr.io/immich-app/immich-server:{{ version['containers']['immich'] }}"
+      file: "ghcr.io_immich-app_immich-server_{{ version['containers']['immich'] }}"
+  loop_control:
+    label: "{{ item.file }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_nextcloud.yaml

@@ -0,0 +1,209 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "nextcloud"
+
+- name: Create redis_nextcloud directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create nextcloud directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['nextcloud']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/nextcloud"
+    - "data/containers/nextcloud/html"
+    - "containers/nextcloud"
+    - "containers/nextcloud/ssl"
+    - "containers/nextcloud/ini"
+  become: true
+
+- name: Check data directory empty
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/data/containers/nextcloud/.init"
+  register: "is_nextcloud_init"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/nextcloud/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['nextcloud']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_nextcloud"
+  no_log: true
+
+- name: Initialize nextcloud
+  when: not is_nextcloud_init.stat.exists
+  block:
+    - name: Execute init command (Including pulling image)
+      containers.podman.podman_container:
+        name: "nextcloud_init"
+        image: "docker.io/library/nextcloud:{{ version['containers']['nextcloud'] }}"
+        command: "/bin/true"
+        state: "started"
+        rm: true
+        detach: false
+        env:
+          NEXTCLOUD_UPDATE: "1"
+          NEXTCLOUD_ADMIN_USER: "admin-local"
+          NEXTCLOUD_ADMIN_PASSWORD: "{{ hostvars['console']['nextcloud']['admin-local']['password'] }}"
+          POSTGRES_HOST: "{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}"
+          POSTGRES_DB: "nextcloud_db"
+          POSTGRES_USER: "nextcloud"
+          POSTGRES_PASSWORD: "{{ hostvars['console']['postgresql']['password']['nextcloud'] }}"
+          PGSSLMODE: "verify-full"
+          PGSSLROOTCERT: "/etc/ssl/nextcloud/{{ root_cert_filename }}"
+          PGSSLCERTMODE: "disable"
+          REDIS_HOST: "host.containers.internal"
+          REDIS_HOST_PORT: "{{ services['nextcloud']['ports']['redis'] }}"
+        volume:
+          - "{{ node['home_path'] }}/containers/nextcloud/ssl:/etc/ssl/nextcloud:ro"
+          - "{{ node['home_path'] }}/data/containers/nextcloud/html:/var/www/html:rw"
+      no_log: true
+
+    - name: Create .init file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/data/containers/nextcloud/.init"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
+
+- name: Deploy config files
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/config/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/data/containers/nextcloud/html/config/{{ item }}"
+    owner: "{{ services['nextcloud']['subuid'] }}"
+    group: "svadmins"
+    mode: "0640"
+  loop:
+    - "background.config.php"
+    - "cache.config.php"
+    - "domain.config.php"
+    - "local_remote.config.php"
+    - "user_oidc.config.php"
+  become: true
+  notify: "notification_restart_nextcloud"
+
+- name: Deploy opcache.ini file
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/ini/{{ item }}"
+    dest: "{{ node['home_path'] }}/containers/nextcloud/ini/{{ item }}"
+    group: "svadmins"
+    mode: "0644"
+  loop:
+    - "opcache.ini"
+    - "upload.ini"
+  notify: "notification_restart_nextcloud"
+
+- name: Deploy nextcloud.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/nextcloud.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/nextcloud.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_nextcloud"
+
+- name: Deploy nextcloud-cron service
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/nextcloud/systemd/{{ item }}"
+    dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  loop:
+    - "nextcloud-cron.service"
+    - "nextcloud-cron.timer"
+
+- name: Enable nextcloud.service
+  ansible.builtin.systemd:
+    name: "nextcloud.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Enable nextcloud-cron.timer
+  ansible.builtin.systemd:
+    name: "nextcloud-cron.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - image: "docker.io/library/nextcloud:{{ version['containers']['nextcloud'] }}"
+      file: "docker.io_library_nextcloud_{{ version['containers']['nextcloud'] }}"
+  loop_control:
+    label: "{{ item.file }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_paperless.yaml

@@ -0,0 +1,177 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "paperless"
+
+- name: Create redis_paperless directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create paperless directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['paperless']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/paperless"
+    - "data/containers/paperless/data"
+    - "data/containers/paperless/media"
+    - "data/containers/paperless/consume"
+    - "containers/paperless"
+    - "containers/paperless/ssl"
+    - "containers/paperless/build"
+  become: true
+
+- name: Deploy containerfile for build
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/paperless/build/paperless.containerfile.j2"
+    dest: "{{ node['home_path'] }}/containers/paperless/build/Containerfile"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0640"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/paperless/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['paperless']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_paperless"
+  no_log: true
+
+- name: Build paperless container image
+  containers.podman.podman_image:
+    name: "{{ domain['internal'] }}/{{ node['name'] }}/paperless-ngx"
+    # check tags from container file
+    tag: "{{ version['containers']['paperless'] }}"
+    state: "build"
+    path: "{{ node['home_path'] }}/containers/paperless/build"
+
+- name: Prune paperless dangling images
+  containers.podman.podman_prune:
+    image: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "PAPERLESS_SECRET_KEY"
+      value: "{{ hostvars['console']['paperless']['session_secret'] }}"
+    - name: "PAPERLESS_DBPASS"
+      value: "{{ hostvars['console']['postgresql']['password']['paperless'] }}"
+    - name: "PAPERLESS_SOCIALACCOUNT_PROVIDERS"
+      value: |-
+        {
+          "openid_connect": {
+            "SCOPE": ["openid", "profile", "email"],
+            "OAUTH_PKCE_ENABLED": true,
+            "APPS": [
+              {
+                "provider_id": "authelia",
+                "name": "Authelia",
+                "client_id": "paperless",
+                "secret": "{{ hostvars['console']['paperless']['oidc']['secret'] }}",
+                "settings": {
+                  "server_url": "https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}/.well-known/openid-configuration",
+                  "token_auth_method": "client_secret_post"
+                }
+              }
+            ]
+          }
+        }
+  notify: "notification_restart_paperless"
+  no_log: true
+
+- name: Deploy paperless.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/paperless/paperless.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/paperless.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_paperless"
+
+- name: Enable paperless.service
+  ansible.builtin.systemd:
+    name: "paperless.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - image: "{{ domain['internal'] }}/{{ node['name'] }}/paperless-ngx:{{ version['containers']['paperless'] }}"
+      file: "{{ domain['internal'] }}_{{ node['name'] }}_paperless-ngx_{{ version['containers']['paperless'] }}"
+  loop_control:
+    label: "{{ item.file }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_sure.yaml

@@ -0,0 +1,143 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "sure"
+
+- name: Create redis_sure directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create sure directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['sure']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/sure"
+    - "data/containers/sure/storage"
+    - "containers/sure"
+    - "containers/sure/ssl"
+  become: true
+
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/sure/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['sure']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_sure"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "SURE_SECRET_KEY_BASE"
+      value: "{{ hostvars['console']['sure']['session_secret'] }}"
+    - name: "SURE_POSTGRES_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['sure'] }}"
+    - name: "SURE_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['sure']['oidc']['secret'] }}"
+  notify: "notification_restart_sure"
+  no_log: true
+
+- name: Deploy sure.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/sure/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  loop:
+    - "sure-web.container"
+    - "sure-worker.container"
+  notify: "notification_restart_sure"
+
+- name: Enable sure.service
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  loop:
+    - "sure-web.service"
+    - "sure-worker.service"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ item.file }}.tar"
+  loop:
+    - image: "docker.io/library/redis:{{ version['containers']['redis'] }}"
+      file: "docker.io_library_redis_{{ version['containers']['redis'] }}"
+    - image: "ghcr.io/we-promise/sure:{{ version['containers']['sure'] }}"
+      file: "ghcr.io_we-promise_sure_{{ version['containers']['sure'] }}"
+  loop_control:
+    label: "{{ item.file }}"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ item.item.image }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    format: "oci-archive"
+    force: false
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"
+  when: not item.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ item.item.file }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true
+  loop: "{{ container_archive_images.results }}"
+  loop_control:
+    label: "{{ item.item.file }}"

ansible/roles/app/tasks/services/set_vaultwarden.yaml

@@ -0,0 +1,77 @@
+---
+- name: Create vaultwarden directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/vaultwarden"
+    - "containers/vaultwarden"
+    - "containers/vaultwarden/ssl"
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/vaultwarden/ssl/{{ root_cert_filename }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_vaultwarden"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "VW_ADMIN_TOKEN"
+      value: "{{ hostvars['console']['vaultwarden']['admin']['hash'] }}"
+    - name: "VW_DATABASE_URL"
+      value: "postgresql://vaultwarden:{{ hostvars['console']['postgresql']['password']['vaultwarden'] | urlencode | replace('/', '%2F') }}\
+        @{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}/vaultwarden_db?sslmode=verify-full&\
+        sslrootcert=/etc/ssl/vaultwarden/{{ root_cert_filename }}"
+  notify: "notification_restart_vaultwarden"
+  no_log: true
+
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/vaultwarden/vaultwarden.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/vaultwarden.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_vaultwarden"
+
+- name: Enable vaultwarden.service
+  ansible.builtin.systemd:
+    name: "vaultwarden.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_vaultwarden_server_{{ version['containers']['vaultwarden'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/vaultwarden/server:{{ version['containers']['vaultwarden'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_vaultwarden_server_{{ version['containers']['vaultwarden'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_vaultwarden_server_{{ version['containers']['vaultwarden'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/auth/tasks/services/set_authelia.yaml

@@ -27,7 +27,7 @@
   ansible.builtin.copy:
     content: |
       {{ hostvars['console']['ca']['root']['crt'] }}
-    dest: "{{ node['home_path'] }}/containers/authelia/certs/ilnmors_root_ca.crt"
+    dest: "{{ node['home_path'] }}/containers/authelia/certs/{{ root_cert_filename }}"
     owner: "{{ ansible_user }}"
     group: "svadmins"
     mode: "0440"
@@ -76,3 +76,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_authelia_authelia_{{ version['containers']['authelia'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/authelia/authelia:{{ version['containers']['authelia'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_authelia_authelia_{{ version['containers']['authelia'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_authelia_authelia_{{ version['containers']['authelia'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/common/handlers/main.yaml

@@ -27,14 +27,14 @@
   listen: "notification_reload_networkctl"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Reload systemd-resolved.service
+- name: Restart systemd-resolved.service
   ansible.builtin.systemd:
     name: "systemd-resolved.service"
-    state: "reloaded"
+    state: "restarted"
     enabled: true
   become: true
   changed_when: false
-  listen: "notification_reload_resolved"
+  listen: "notification_restart_resolved"
   ignore_errors: true # noqa: ignore-errors
 
 - name: Restart systemd-timesyncd

ansible/roles/common/tasks/node/deploy_root_ca.yaml

@@ -2,7 +2,7 @@
 - name: Deploy root_ca.crt
   ansible.builtin.copy:
     content: "{{ hostvars['console']['ca']['root']['crt'] }}"
-    dest: "/usr/local/share/ca-certificates/ilnmors_root_ca.crt"
+    dest: "/usr/local/share/ca-certificates/{{ root_cert_filename }}"
     owner: "root"
     group: "root"
     mode: "0644"

ansible/roles/common/tasks/node/set_resolved.yaml

@@ -28,7 +28,7 @@
     group: "systemd-resolve"
     mode: "0640"
   become: true
-  notify: "notification_reload_resolved"
+  notify: "notification_restart_resolved"
 
 - name: Restart systemd-resolved.service when it is initiated
   ansible.builtin.systemd:

ansible/roles/common/tasks/services/set_alloy.yaml

@@ -5,9 +5,10 @@
       - hardware
   become: true
 
-- name: Deploy alloy deb file (x86_64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-amd64.deb"
+- name: Download alloy deb file (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.amd64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -15,9 +16,10 @@
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
-- name: Deploy alloy deb file (aarch64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-arm64.deb"
+- name: Download alloy deb file (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.arm64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -30,6 +32,7 @@
     deb: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     state: "present"
   become: true
+  notify: "notification_restart_alloy"
 
 - name: Deploy alloy config
   ansible.builtin.template:
@@ -71,3 +74,10 @@
     enabled: true
     daemon_reload: true
   become: true
+
+- name: Fetch deb bin file
+  ansible.builtin.fetch:
+    src: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/bin/"
+    flat: true
+  become: true

ansible/roles/common/tasks/services/set_caddy.yaml

@@ -54,7 +54,7 @@
 - name: Deploy root crt for build
   ansible.builtin.copy:
     content: "{{ hostvars['console']['ca']['root']['crt'] }}"
-    dest: "{{ node['home_path'] }}/containers/caddy/build/ilnmors_root_ca.crt"
+    dest: "{{ node['home_path'] }}/containers/caddy/build/{{ root_cert_filename }}"
     owner: "{{ ansible_user }}"
     group: "svadmins"
     mode: "0640"
@@ -62,7 +62,7 @@
 
 - name: Build caddy container image
   containers.podman.podman_image:
-    name: "ilnmors.internal/{{ node['name'] }}/caddy"
+    name: "{{ domain['internal'] }}/{{ node['name'] }}/caddy"
     # check tags from container file
     tag: "{{ version['containers']['caddy'] }}"
     state: "build"
@@ -97,3 +97,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/{{ domain['internal'] }}_{{ node['name'] }}_caddy_{{ version['containers']['caddy'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ domain['internal'] }}/{{ node['name'] }}/caddy:{{ version['containers']['caddy'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/{{ domain['internal'] }}_{{ node['name'] }}_caddy_{{ version['containers']['caddy'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/{{ domain['internal'] }}_{{ node['name'] }}_caddy_{{ version['containers']['caddy'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/common/tasks/services/set_crowdsec.yaml

@@ -36,10 +36,15 @@
   ansible.builtin.set_fact:
     acquisd_list:
       fw:
-        collection: "crowdsecurity/suricata"
+        collection:
+          - "crowdsecurity/suricata"
+        parser: []
         config: "suricata.yaml"
       auth:
-        collection: "crowdsecurity/caddy"
+        collection:
+          - "crowdsecurity/caddy"
+        parser:
+          - "crowdsecurity/nextcloud-whitelist"
         config: "caddy.yaml"
 
 - name: Deploy crowdsec-update service files
@@ -181,7 +186,8 @@
   block:
     - name: Install crowdsec collection
       ansible.builtin.command:
-        cmd: "cscli collections install {{ acquisd_list[node['name']]['collection'] }}"
+        cmd: "cscli collections install {{ item }}"
+      loop: "{{ acquisd_list[node['name']]['collection'] }}"
       become: true
       changed_when: "'overwrite' not in is_collection_installed.stderr"
       failed_when:
@@ -189,6 +195,17 @@
         - "'already installed' not in is_collection_installed.stderr"
       register: "is_collection_installed"
 
+    - name: Install crowdsec parser
+      ansible.builtin.command:
+        cmd: "cscli parsers install {{ item }}"
+      loop: "{{ acquisd_list[node['name']]['parser'] }}"
+      become: true
+      changed_when: "'overwrite' not in is_parser_installed.stderr"
+      failed_when:
+        - is_parser_installed.rc != 0
+        - "'already installed' not in is_parser_installed.stderr"
+      register: "is_parser_installed"
+
     - name: Create crowdsec acquis.d directory
       ansible.builtin.file:
         path: "/etc/crowdsec/acquis.d"

ansible/roles/common/tasks/services/set_kopia.yaml

@@ -5,41 +5,43 @@
       - hardware
   become: true
 
-- name: Check kopia installation
-  ansible.builtin.shell: |
-    command -v kopia
-  changed_when: false
-  failed_when: false
-  register: "is_kopia_installed"
-  ignore_errors: true
-
 - name: Set console kopia
   when: node['name'] == 'console'
   block:
-    - name: Apply cli tools (x86_64)
+    - name: Download kopia
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
+        dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
+        mode: "0600"
+      loop:
+        - "amd64"
+        - "arm64"
+
+    - name: Install kopia (x86_64)
       ansible.builtin.apt:
         deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
         state: "present"
       become: true
-      when:
-        - ansible_facts['architecture'] == "x86_64"
-        - is_kopia_installed.rc != 0
-    - name: Apply cli tools (aarch64)
+      when: ansible_facts['architecture'] == "x86_64"
+
+    - name: Install kopia (aarch64)
       ansible.builtin.apt:
         deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
         state: "present"
       become: true
-      when:
-        - ansible_facts['architecture'] == "aarch64"
-        - is_kopia_installed.rc != 0
-    - name: Connect kopia server
+      when: ansible_facts['architecture'] == "aarch64"
+
+    - name: Connect console kopia server
       environment:
         KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['user']['console'] }}"
       ansible.builtin.shell: |
         /usr/bin/kopia repository connect server \
-        --url=https://{{ infra_uri['kopia']['domain'] }}:{{ infra_uri['kopia']['ports']['https'] }} \
+        --url=https://{{ services['kopia']['domain'] }}.{{ domain['internal'] }}:{{ services['kopia']['ports']['https'] }} \
         --override-username=console \
-        --override-hostname=console.ilnmors.internal
+        --override-hostname=console.{{ domain['internal'] }}
       changed_when: false
       failed_when: is_kopia_connected.rc != 0
       register: "is_kopia_connected"
@@ -51,30 +53,36 @@
     - name: Set kopia uid
       ansible.builtin.set_fact:
         kopia_uid: 951
-    - name: Deploy kopia deb file (x86_64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
+
+    - name: Download kopia deb file (x86_64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_amd64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "x86_64"
-    - name: Deploy kopia deb file (aarch64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
+
+    - name: Download kopia deb file (aarch64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_arm64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "aarch64"
+
     - name: Create kopia group
       ansible.builtin.group:
         name: "kopia"
         gid: "{{ kopia_uid }}"
         state: "present"
       become: true
+
     - name: Create kopia user
       ansible.builtin.user:
         name: "kopia"
@@ -85,6 +93,7 @@
         comment: "Kopia backup User"
         state: "present"
       become: true
+
     - name: Create kopia directory
       ansible.builtin.file:
         path: "{{ item.name }}"
@@ -101,12 +110,13 @@
           mode: "0700"
       become: true
       no_log: true
+
     - name: Install kopia
       ansible.builtin.apt:
         deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         state: "present"
       become: true
-      when: is_kopia_installed.rc != 0
+
     - name: Deploy kopia env
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2"
@@ -116,6 +126,7 @@
         mode: "0400"
       become: true
       no_log: true
+
     - name: Deploy kopia service files
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/{{ item }}.j2"
@@ -128,6 +139,7 @@
         - "kopia-backup.service"
         - "kopia-backup.timer"
       become: true
+
     - name: Enable auto kopia rules update
       ansible.builtin.systemd:
         name: "kopia-backup.timer"

ansible/roles/common/tasks/services/set_podman.yaml

@@ -15,7 +15,7 @@
     state: "directory"
     mode: "0700"
 
-- name: Create contaienr data directory for app
+- name: Create container data directory for app
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/data/containers"
     owner: "{{ ansible_user }}"
@@ -24,6 +24,17 @@
     mode: "0770"
   when: node['name'] == "app"
 
+- name: Create container image archive directory
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    state: "directory"
+    mode: "0700"
+  loop:
+    - "{{ node['home_path'] }}/archives"
+    - "{{ node['home_path'] }}/archives/containers"
+
 - name: Install podman and reset ssh connection for initiating
   when: is_podman_installed.rc != 0
   become: true

ansible/roles/console/handlers/main.yaml

@@ -1,8 +1 @@
 ---
-- name: Register font
-  ansible.builtin.shell: |
-    fc-cache -f -v
-  become: true
-  changed_when: false
-  listen: "notification_update_font"
-  ignore_errors: true # noqa: ignore-errors

ansible/roles/console/tasks/node/set_ssh_client.yaml

@@ -23,7 +23,7 @@
   become: true
   ansible.builtin.copy:
     content: |
-      @cert-authority *.ilnmors.internal {{ hostvars['console']['ssh']['ca']['pub'] }}
+      @cert-authority *.{{ domain['internal'] }} {{ hostvars['console']['ssh']['ca']['pub'] }}
     dest: "/etc/ssh/ssh_known_hosts"
     owner: "root"
     group: "root"

ansible/roles/console/tasks/services/set_cli_tools.yaml

@@ -49,42 +49,6 @@
     - "amd64"
     - "arm64"
 
-- name: Download kopia
-  ansible.builtin.get_url:
-    url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
-      kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
-- name: Download blocky
-  ansible.builtin.get_url:
-    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
-      blocky_v{{ version['packages']['blocky'] }}_Linux_{{ item }}.tar.gz"
-    dest: "{{ node['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-{{ item }}.tar.gz"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600" # noqa: line-length
-  loop:
-    - "x86_64"
-    - "arm64"
-
-- name: Download alloy
-  ansible.builtin.get_url:
-    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
-      alloy-{{ version['packages']['alloy'] }}-1.{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
 - name: Apply cli tools (x86_64)
   ansible.builtin.apt:
     deb: "{{ node['data_path'] }}/bin/{{ item }}"
@@ -92,7 +56,6 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-amd64.deb"
     - "step-{{ version['packages']['step'] }}-amd64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-amd64.deb"
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
@@ -103,6 +66,5 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-arm64.deb"
     - "step-{{ version['packages']['step'] }}-arm64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-arm64.deb"
   become: true
   when: ansible_facts['architecture'] == "aarch64"

ansible/roles/fw/tasks/services/set_blocky.yaml

@@ -23,7 +23,7 @@
     state: "present"
   become: true
 
-- name: Create blocky etc directory
+- name: Create blocky directory
   ansible.builtin.file:
     path: "{{ item }}"
     owner: "blocky"
@@ -31,28 +31,38 @@
     mode: "0750"
     state: "directory"
   loop:
+    - "/home/blocky"
+    - "/home/blocky/bin"
     - "/etc/blocky"
     - "/etc/blocky/ssl"
   become: true
 
-- name: Deploy blocky binary file (x86_64)
-  ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
-    dest: "/usr/local/bin/"
-    owner: "root"
-    group: "root"
-    mode: "0755"
-    extra_opts:
-      - "--strip-components=0"
-      - "--wildcards"
-      - "blocky"
+- name: Download blocky (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_x86_64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
   become: true
   when: ansible_facts['architecture'] == "x86_64"
-  notify: "notification_restart_blocky"
 
-- name: Deploy blocky binary file (aarch64)
+- name: Download blocky (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_arm64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "aarch64"
+
+- name: Deploy blocky binary file
   ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    remote_src: true
     dest: "/usr/local/bin/"
     owner: "root"
     group: "root"
@@ -62,7 +72,6 @@
       - "--wildcards"
       - "blocky"
   become: true
-  when: ansible_facts['architecture'] == "aarch64"
   notify: "notification_restart_blocky"
 
 - name: Deploy blocky config
@@ -115,3 +124,10 @@
     enabled: true
     daemon_reload: true
   become: true
+
+- name: Fetch deb bin file
+  ansible.builtin.fetch:
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}.tar.gz"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/bin/"
+    flat: true
+  become: true

ansible/roles/fw/tasks/services/set_ddns.yaml

@@ -21,8 +21,8 @@
   become: true
 
 - name: Deploy ddns service files
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/ddns/{{ item }}"
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/fw/ddns/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/.config/systemd/user/{{ item }}"
     owner: "{{ ansible_user }}"
     group: "svadmins"

ansible/roles/infra/handlers/main.yaml

@@ -12,7 +12,7 @@
 - name: Reload postgresql
   ansible.builtin.command:
     /usr/bin/podman exec -u postgres postgresql sh -c "pg_ctl reload"
-  when: not (is_postgresql_init_run | default(false))
+  when: is_postgresql_init.stat.exists
   changed_when: false
   listen: "notification_reload_postgresql"
   ignore_errors: true # noqa: ignore-errors
@@ -24,7 +24,7 @@
     enabled: true
     daemon_reload: true
     scope: "user"
-  when: not (is_postgresql_init_run | default(false))
+  when: is_postgresql_init.stat.exists
   changed_when: false
   listen: "notification_restart_postgresql"
   ignore_errors: true # noqa: ignore-errors
@@ -73,10 +73,10 @@
   listen: "notification_restart_grafana"
   ignore_errors: true # noqa: ignore-errors
 
-- name: Enable x509-exporter.service
+- name: Restart x509-exporter.service
   ansible.builtin.systemd:
     name: "x509-exporter.service"
-    state: "started"
+    state: "restarted"
     enabled: true
     daemon_reload: true
     scope: "user"

ansible/roles/infra/tasks/services/set_ca_server.yaml

@@ -1,12 +1,8 @@
 ---
-- name: Set ca container subuid
-  ansible.builtin.set_fact:
-    ca_subuid: "100999"
-
 - name: Create ca directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
-    owner: "{{ ca_subuid }}"
+    owner: "{{ services['ca']['subuid'] }}"
     group: "svadmins"
     state: "directory"
     mode: "0770"
@@ -32,7 +28,7 @@
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/ca/config/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/ca/config/{{ item }}"
-    owner: "{{ ca_subuid }}"
+    owner: "{{ services['ca']['subuid'] }}"
     group: "svadmins"
     mode: "0400"
   loop:
@@ -46,19 +42,19 @@
     content: |
       {{ item.value }}
     dest: "{{ item.path }}/{{ item.name }}"
-    owner: "{{ ca_subuid }}"
+    owner: "{{ services['ca']['subuid'] }}"
     group: "svadmins"
     mode: "{{ item.mode }}"
   loop:
-    - name: "ilnmors_root_ca.crt"
+    - name: "{{ root_cert_filename }}"
       value: "{{ hostvars['console']['ca']['root']['crt'] }}"
       path: "{{ node['home_path'] }}/containers/ca/certs"
       mode: "0440"
-    - name: "ilnmors_intermediate_ca.crt"
+    - name: "{{ intermediate_cert_filename }}"
       value: "{{ hostvars['console']['ca']['intermediate']['crt'] }}"
       path: "{{ node['home_path'] }}/containers/ca/certs"
       mode: "0440"
-    - name: "ilnmors_intermediate_ca.key"
+    - name: "{{ intermediate_key_filename }}"
       value: "{{ hostvars['console']['ca']['intermediate']['key'] }}"
       path: "{{ node['home_path'] }}/containers/ca/secrets"
       mode: "0400"
@@ -82,3 +78,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_smallstep_step-ca_{{ version['containers']['step'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/smallstep/step-ca:{{ version['containers']['step'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_smallstep_step-ca_{{ version['containers']['step'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_smallstep_step-ca_{{ version['containers']['step'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_grafana.yaml

@@ -1,12 +1,8 @@
 ---
-- name: Set grafana container subuid
-  ansible.builtin.set_fact:
-    grafana_subuid: "100471"
-
 - name: Create grafana directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
-    owner: "{{ grafana_subuid }}"
+    owner: "{{ services['grafana']['subuid'] }}"
     group: "svadmins"
     state: "directory"
     mode: "0770"
@@ -23,8 +19,8 @@
   ansible.builtin.copy:
     content: |
       {{ hostvars['console']['ca']['root']['crt'] }}
-    dest: "{{ node['home_path'] }}/containers/grafana/ssl/ilnmors_root_ca.crt"
-    owner: "{{ grafana_subuid }}"
+    dest: "{{ node['home_path'] }}/containers/grafana/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['grafana']['subuid'] }}"
     group: "svadmins"
     mode: "0400"
   become: true
@@ -51,7 +47,7 @@
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/grafana/etc/{{ item }}"
-    owner: "{{ grafana_subuid }}"
+    owner: "{{ services['grafana']['subuid'] }}"
     group: "svadmins"
     mode: "0400"
   loop:
@@ -61,11 +57,11 @@
   notify: "notification_restart_grafana"
   no_log: true
 
-- name: Deploy provisioing and dashboard files
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/provisioning/"
-    dest: "{{ node['home_path'] }}/containers/grafana/etc/provisioning/"
-    owner: "{{ grafana_subuid }}"
+- name: Deploy provisioing file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/grafana/etc/provisioning/datasources/datasources.yaml.j2"
+    dest: "{{ node['home_path'] }}/containers/grafana/etc/provisioning/datasources/datasources.yaml"
+    owner: "{{ services['grafana']['subuid'] }}"
     group: "svadmins"
     mode: "0400"
   become: true
@@ -87,3 +83,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_grafana_{{ version['containers']['grafana'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/grafana/grafana:{{ version['containers']['grafana'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_grafana_{{ version['containers']['grafana'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_grafana_{{ version['containers']['grafana'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_ldap.yaml

@@ -1,12 +1,8 @@
 ---
-- name: Set ldap container subuid
-  ansible.builtin.set_fact:
-    ldap_subuid: "100999"
-
 - name: Create ldap directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
-    owner: "{{ ldap_subuid }}"
+    owner: "{{ services['ldap']['subuid'] }}"
     group: "svadmins"
     state: "directory"
     mode: "0770"
@@ -21,11 +17,11 @@
     content: |
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/ldap/ssl/{{ item.name }}"
-    owner: "{{ ldap_subuid }}"
+    owner: "{{ services['ldap']['subuid'] }}"
     group: "svadmins"
     mode: "{{ item.mode }}"
   loop:
-    - name: "ilnmors_root_ca.crt"
+    - name: "{{ root_cert_filename }}"
       value: "{{ hostvars['console']['ca']['root']['crt'] }}"
       mode: "0440"
     - name: "ldap.crt"
@@ -50,7 +46,7 @@
     # urlencode doesn't fix `/` as `%2F`. It needs replace
     - name: "LLDAP_DATABASE_URL"
       value: "postgres://ldap:{{ hostvars['console']['postgresql']['password']['ldap'] | urlencode | replace('/', '%2F') }}\
-        @{{ infra_uri['postgresql']['domain'] }}/ldap_db?sslmode=verify-full&sslrootcert=/etc/ssl/ldap/ilnmors_root_ca.crt"
+        @{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}/ldap_db?sslmode=verify-full&sslrootcert=/etc/ssl/ldap/{{ root_cert_filename }}"
     - name: "LLDAP_KEY_SEED"
       value: "{{ hostvars['console']['ldap']['seed_key'] }}"
     - name: "LLDAP_JWT_SECRET"
@@ -59,6 +55,8 @@
   no_log: true
 
 - name: Initiate ldap (When = false, If DB data does not exist in postgresql, activate this block)
+# The reason why this task doesn't use the way to check ".init" file is this tasks can override original database.
+# Absent of ".init" file cannot guarantee DB is empty.
   when: false
   become: true
   block:
@@ -77,8 +75,8 @@
         rm: true
         detach: false
         env:
-          TZ: "Asia/Seoul"
-          LLDAP_LDAP_BASE_DN: "dc=ilnmors,dc=internal"
+          TZ: "{{ timezone }}"
+          LLDAP_LDAP_BASE_DN: "{{ domain['dc'] }}"
         secrets:
           - "LLDAP_DATABASE_URL,type=env"
           - "LLDAP_KEY_SEED,type=env"
@@ -110,3 +108,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_lldap_lldap_{{ version['containers']['ldap'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/lldap/lldap:{{ version['containers']['ldap'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_lldap_lldap_{{ version['containers']['ldap'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_lldap_lldap_{{ version['containers']['ldap'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_loki.yaml

@@ -1,13 +1,9 @@
 ---
-- name: Set loki container subuid
-  ansible.builtin.set_fact:
-    loki_subuid: "110000" # 10001
-
 - name: Create loki directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
-    owner: "{{ loki_subuid }}"
+    owner: "{{ services['loki']['subuid'] }}"
     group: "svadmins"
     mode: "0770"
   loop:
@@ -18,10 +14,10 @@
   become: true
 
 - name: Deploy loki configuration file
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/loki/etc/loki.yaml"
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/loki/etc/loki.yaml.j2"
     dest: "{{ node['home_path'] }}/containers/loki/etc/loki.yaml"
-    owner: "{{ loki_subuid }}"
+    owner: "{{ services['loki']['subuid'] }}"
     group: "svadmins"
     mode: "0600"
   become: true
@@ -33,11 +29,11 @@
     content: |
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/loki/ssl/{{ item.name }}"
-    owner: "{{ loki_subuid }}"
+    owner: "{{ services['loki']['subuid'] }}"
     group: "svadmins"
     mode: "{{ item.mode }}"
   loop:
-    - name: "ilnmors_root_ca.crt"
+    - name: "{{ root_cert_filename }}"
       value: "{{ hostvars['console']['ca']['root']['crt'] }}"
       mode: "0440"
     - name: "loki.crt"
@@ -68,3 +64,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_loki_{{ version['containers']['loki'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/grafana/loki:{{ version['containers']['loki'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_loki_{{ version['containers']['loki'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_grafana_loki_{{ version['containers']['loki'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_postgresql.yaml

@@ -1,21 +1,23 @@
 ---
-- name: Set postgresql container subuid
-  ansible.builtin.set_fact:
-    postgresql_subuid: "100998"
-
 - name: Set connected services list
   ansible.builtin.set_fact:
-    # telegraf has no database
     connected_services:
       - "ldap"
       - "authelia"
       - "grafana"
+      - "vaultwarden"
+      - "gitea"
+      - "immich"
+      - "paperless"
+      - "affine"
+      - "nextcloud"
+      - "sure"
 
 - name: Create postgresql directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
-    owner: "{{ postgresql_subuid }}"
+    owner: "{{ services['postgresql']['subuid'] }}"
     group: "svadmins"
     mode: "0770"
   loop:
@@ -38,7 +40,7 @@
 
 - name: Build postgresql container image
   containers.podman.podman_image:
-    name: "ilnmors.internal/{{ node['name'] }}/postgres"
+    name: "{{ domain['internal'] }}/{{ node['name'] }}/postgres"
     # check tags from container file
     tag: "pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}"
     state: "build"
@@ -52,7 +54,7 @@
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/config/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/postgresql/config/{{ item }}"
-    owner: "{{ postgresql_subuid }}"
+    owner: "{{ services['postgresql']['subuid'] }}"
     group: "svadmins"
     mode: "0600"
   loop:
@@ -67,11 +69,11 @@
     content: |
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/postgresql/ssl/{{ item.name }}"
-    owner: "{{ postgresql_subuid }}"
+    owner: "{{ services['postgresql']['subuid'] }}"
     group: "svadmins"
     mode: "{{ item.mode }}"
   loop:
-    - name: "ilnmors_root_ca.crt"
+    - name: "{{ root_cert_filename }}"
       value: "{{ hostvars['console']['ca']['root']['crt'] }}"
       mode: "0440"
     - name: "postgresql.crt"
@@ -87,15 +89,13 @@
   no_log: true
 
 - name: Check data directory empty
-  ansible.builtin.find:
-    paths: "{{ node['home_path'] }}/containers/postgresql/data/"
-    hidden: true
-    file_type: "any"
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/containers/postgresql/data/.init"
   become: true
-  register: "is_data_dir_empty"
+  register: "is_postgresql_init"
 
 - name: Prepare initiating DB
-  when: is_data_dir_empty.matched == 0
+  when: not is_postgresql_init.stat.exists
   become: true
   block:
     # `init/pg_cluster.sql` should be fetched from postgresql's backup directory before running initiating
@@ -103,23 +103,28 @@
       ansible.builtin.copy:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/init/pg_cluster.sql"
         dest: "{{ node['home_path'] }}/containers/postgresql/init/0_pg_cluster.sql"
-        owner: "{{ postgresql_subuid }}"
+        owner: "{{ services['postgresql']['subuid'] }}"
         group: "svadmins"
         mode: "0600"
 
-    - name: Deploy resoring data sql files
+    - name: Deploy restoring data sql files
       ansible.builtin.copy:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/init/pg_{{ item }}.sql"
         dest: "{{ node['home_path'] }}/containers/postgresql/init/{{ index_num + 1 }}_pg_{{ item }}.sql"
-        owner: "{{ postgresql_subuid }}"
+        owner: "{{ services['postgresql']['subuid'] }}"
         group: "svadmins"
         mode: "0600"
       loop: "{{ connected_services }}"
       loop_control:
         index_var: index_num
-    - name: Set is_postgresql_init_run
-      ansible.builtin.set_fact:
-        is_postgresql_init_run: true
+
+    - name: Create .init file
+      ansible.builtin.file:
+        path: "{{ node['home_path'] }}/containers/postgresql/data/.init"
+        state: "touch"
+        mode: "0644"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
 
 - name: Deploy container file
   ansible.builtin.template:
@@ -167,3 +172,29 @@
     daemon_reload: true
     scope: "user"
   loop: "{{ connected_services }}"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/\
+      {{ domain['internal'] }}_{{ node['name'] }}_postgres_\
+      pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "{{ domain['internal'] }}/{{ node['name'] }}/postgres:pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/\
+      {{ domain['internal'] }}_{{ node['name'] }}_postgres_\
+      pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/\
+      {{ domain['internal'] }}_{{ node['name'] }}_postgres_\
+      pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_prometheus.yaml

@@ -1,13 +1,9 @@
 ---
-- name: Set prometheus container subuid
-  ansible.builtin.set_fact:
-    prometheus_subuid: "165533" # nobody - 65534
-
 - name: Create prometheus directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
-    owner: "{{ prometheus_subuid }}"
+    owner: "{{ services['prometheus']['subuid'] }}"
     group: "svadmins"
     mode: "0770"
   loop:
@@ -21,7 +17,7 @@
   ansible.builtin.template:
     src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/prometheus/etc/{{ item }}.j2"
     dest: "{{ node['home_path'] }}/containers/prometheus/etc/{{ item }}"
-    owner: "{{ prometheus_subuid }}"
+    owner: "{{ services['prometheus']['subuid'] }}"
     group: "svadmins"
     mode: "0600"
   loop:
@@ -37,11 +33,11 @@
     content: |
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/prometheus/ssl/{{ item.name }}"
-    owner: "{{ prometheus_subuid }}"
+    owner: "{{ services['prometheus']['subuid'] }}"
     group: "svadmins"
     mode: "{{ item.mode }}"
   loop:
-    - name: "ilnmors_root_ca.crt"
+    - name: "{{ root_cert_filename }}"
       value: "{{ hostvars['console']['ca']['root']['crt'] }}"
       mode: "0440"
     - name: "prometheus.crt"
@@ -72,3 +68,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_prom_prometheus_{{ version['containers']['prometheus'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/prom/prometheus:{{ version['containers']['prometheus'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_prom_prometheus_{{ version['containers']['prometheus'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_prom_prometheus_{{ version['containers']['prometheus'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

ansible/roles/infra/tasks/services/set_x509-exporter.yaml

@@ -1,26 +1,33 @@
 ---
-- name: Set x509-exporter container subuid
-  ansible.builtin.set_fact:
-    x509_exporter_subuid: "165533" # nobody - 65534
-
 - name: Create x509-exporter directory
   ansible.builtin.file:
     path: "{{ node['home_path'] }}/containers/{{ item }}"
     state: "directory"
-    owner: "{{ x509_exporter_subuid }}"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
     group: "svadmins"
     mode: "0770"
   loop:
     - "x509-exporter"
+    - "x509-exporter/config"
     - "x509-exporter/certs"
   become: true
 
+- name: Deploy config.yaml
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/x509-exporter/config/config.yaml"
+    dest: "{{ node['home_path'] }}/containers/x509-exporter/config/config.yaml"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_x509-exporter"
+
 - name: Deploy certificates
   ansible.builtin.copy:
     content: |
       {{ item.value }}
     dest: "{{ node['home_path'] }}/containers/x509-exporter/certs/{{ item.name }}"
-    owner: "{{ x509_exporter_subuid }}"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
     group: "svadmins"
     mode: "0440"
   loop:
@@ -61,3 +68,23 @@
     enabled: true
     daemon_reload: true
     scope: "user"
+
+- name: Check container archive images
+  ansible.builtin.stat:
+    path: "{{ node['home_path'] }}/archives/containers/docker.io_enix_x509-certificate-exporter_{{ version['containers']['x509-exporter'] }}.tar"
+  register: container_archive_images
+
+- name: Save container archive images
+  containers.podman.podman_save:
+    image:
+      - "docker.io/enix/x509-certificate-exporter:{{ version['containers']['x509-exporter'] }}"
+    dest: "{{ node['home_path'] }}/archives/containers/docker.io_enix_x509-certificate-exporter_{{ version['containers']['x509-exporter'] }}.tar"
+    format: "oci-archive"
+    force: false
+  when: not container_archive_images.stat.exists
+
+- name: Fetch container archive images
+  ansible.builtin.fetch:
+    src: "{{ node['home_path'] }}/archives/containers/docker.io_enix_x509-certificate-exporter_{{ version['containers']['x509-exporter'] }}.tar"
+    dest: "{{ hostvars['console']['node']['data_path'] }}/images/containers/"
+    flat: true

config/node/common/hosts.j2

@@ -3,32 +3,32 @@
 ::1 {{ node['local_san'] }}
 {% if node['name'] == 'console' %}
 # Hosts IPv4
-{{ hostvars['fw']['network4']['firewall']['server'] }} fw.ilnmors.internal
-{{ hostvars['fw']['network4']['vmm']['client'] }} init.vmm.ilnmors.internal
-{{ hostvars['fw']['network4']['vmm']['server'] }} vmm.ilnmors.internal
-{{ hostvars['fw']['network4']['infra']['server'] }} infra.ilnmors.internal
-{{ hostvars['fw']['network4']['auth']['server'] }} auth.ilnmors.internal
-{{ hostvars['fw']['network4']['app']['server'] }} app.ilnmors.internal
+{{ hostvars['fw']['network4']['firewall']['server'] }} fw.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['vmm']['client'] }} init.vmm.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['vmm']['server'] }} vmm.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['infra']['server'] }} infra.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['auth']['server'] }} auth.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['app']['server'] }} app.{{ domain['internal'] }}
 # Hosts IPv6
-{{ hostvars['fw']['network6']['firewall']['server'] }} fw.ilnmors.internal
-{{ hostvars['fw']['network6']['vmm']['client'] }} init.vmm.ilnmors.internal
-{{ hostvars['fw']['network6']['vmm']['server'] }} vmm.ilnmors.internal
-{{ hostvars['fw']['network6']['infra']['server'] }} infra.ilnmors.internal
-{{ hostvars['fw']['network6']['auth']['server'] }} auth.ilnmors.internal
-{{ hostvars['fw']['network6']['app']['server'] }} app.ilnmors.internal
+{{ hostvars['fw']['network6']['firewall']['server'] }} fw.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['vmm']['client'] }} init.vmm.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['vmm']['server'] }} vmm.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['infra']['server'] }} infra.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['auth']['server'] }} auth.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['app']['server'] }} app.{{ domain['internal'] }}
 {% else %}
 # IPv4
 # Crowdsec, blocky, bind(fw)
-{{ hostvars['fw']['network4']['firewall']['server'] }} ntp.ilnmors.internal crowdsec.ilnmors.internal
-{{ hostvars['fw']['network4']['blocky']['server'] }} blocky.ilnmors.internal
-{{ hostvars['fw']['network4']['bind']['server'] }} bind.ilnmors.internal
+{{ hostvars['fw']['network4']['firewall']['server'] }} ntp.{{ domain['internal'] }} crowdsec.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['blocky']['server'] }} blocky.{{ domain['internal'] }}
+{{ hostvars['fw']['network4']['bind']['server'] }} bind.{{ domain['internal'] }}
 # DB, LDAP, CA, Prometheus, Loki, mail (infra)
-{{ hostvars['fw']['network4']['infra']['server'] }} postgresql.ilnmors.internal ldap.ilnmors.internal prometheus.ilnmors.internal loki.ilnmors.internal mail.ilnmors.internal ca.ilnmors.internal
+{{ hostvars['fw']['network4']['infra']['server'] }} postgresql.{{ domain['internal'] }} ldap.{{ domain['internal'] }} prometheus.{{ domain['internal'] }} loki.{{ domain['internal'] }} mail.{{ domain['internal'] }} ca.{{ domain['internal'] }}
 # IPv6
 # Crowdsec, blocky, bind(fw)
-{{ hostvars['fw']['network6']['firewall']['server'] }} ntp.ilnmors.internal crowdsec.ilnmors.internal
-{{ hostvars['fw']['network6']['blocky']['server'] }} blocky.ilnmors.internal
-{{ hostvars['fw']['network6']['bind']['server'] }} bind.ilnmors.internal
+{{ hostvars['fw']['network6']['firewall']['server'] }} ntp.{{ domain['internal'] }} crowdsec.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['blocky']['server'] }} blocky.{{ domain['internal'] }}
+{{ hostvars['fw']['network6']['bind']['server'] }} bind.{{ domain['internal'] }}
 # DB, LDAP, CA, Prometheus, Loki, mail (infra)
-{{ hostvars['fw']['network6']['infra']['server'] }} postgresql.ilnmors.internal ldap.ilnmors.internal prometheus.ilnmors.internal loki.ilnmors.internal mail.ilnmors.internal ca.ilnmors.internal
+{{ hostvars['fw']['network6']['infra']['server'] }} postgresql.{{ domain['internal'] }} ldap.{{ domain['internal'] }} prometheus.{{ domain['internal'] }} loki.{{ domain['internal'] }} mail.{{ domain['internal'] }} ca.{{ domain['internal'] }}
 {% endif %}

config/node/common/resolved/global.conf.j2

@@ -3,4 +3,4 @@
 DNS=1.1.1.2 1.0.0.2
 DNS=2606:4700:4700::1112 2606:4700:4700::1002
 {% endif %}
-cache=false
+cache=false

config/node/common/timesyncd/local-ntp.conf

@@ -1,3 +1,3 @@
 [Time]
-NTP=ntp.ilnmors.internal
+NTP=ntp.{{ domain['internal'] }}
 FallbackNTP=0.debian.pool.ntp.org 1.debian.pool.ntp.org 2.debian.pool.ntp.org 3.debian.pool.ntp.org

config/node/fw/nftables.conf.j2

@@ -30,6 +30,7 @@ define HOSTS4_INFRA = {{ hostvars['fw']['network4']['infra']['server'] }}
 define HOSTS4_AUTH = {{ hostvars['fw']['network4']['auth']['server'] }}
 define HOSTS4_APP = {{ hostvars['fw']['network4']['app']['server'] }}
 define HOSTS4_NAS = {{ hostvars['fw']['network4']['nas']['client'] }}
+define HOSTS4_PRINTER = {{ hostvars['fw']['network4']['printer']['client'] }}
 
 define HOSTS6_FW = { {{ hostvars['fw']['network6']['firewall'].values() | join(', ') }} }
 define HOSTS6_BLOCKY = {{ hostvars['fw']['network6']['blocky']['server'] }}
@@ -81,6 +82,8 @@ table inet filter {
         chain global {
                 # invalid packets
                 ct state invalid drop comment "deny invalid connection"
+                # VPN connection exception handling
+                udp dport $PORTS_VPN return comment "return vpn connection to input and forward chain"
                 # crowdsec
                 ip saddr @crowdsec-blacklists counter drop comment "deny all crowdsec blacklist"
                 ip6 saddr @crowdsec6-blacklists counter drop comment "deny all ipv6 crowdsec blacklist"
@@ -146,6 +149,8 @@ table inet filter {
                 # Kopia/NAS Console > NAS
                 oifname $IF_CLIENT ip saddr $HOSTS4_CONSOLE ip daddr $HOSTS4_NAS tcp dport { $PORTS_NAS, $PORTS_KOPIA } accept comment "allow ipv4 web connection (DSM, KOPIA): CONSOLE > FW > CLIENT NAS"
                 oifname $IF_CLIENT ip6 saddr $HOSTS6_CONSOLE ip6 daddr $HOSTS6_NAS tcp dport { $PORTS_NAS, $PORTS_KOPIA } accept comment "allow ipv6 web connection (DSM, KOPIA): CONSOLE > FW > CLIENT NAS"
+                # Printer
+                oifname $IF_CLIENT ip saddr $HOSTS4_CONSOLE ip daddr $HOSTS4_PRINTER accept comment "allow ipv4 printer connection: CONSOLE > FW > PRINTER"
 
                 iifname $IF_WAN jump wan comment "set WAN interface rules"
                 iifname $IF_CLIENT jump client comment "set CLIENT interface rules"

config/secrets/secrets.yaml

@@ -113,6 +113,13 @@ postgresql:
         ldap: ENC[AES256_GCM,data:mJrxIhXynHxJhncw3upHpOkXIw+Ka9bmDBJwkDjYl+D9Pg4RDvL6WzBjthw=,iv:y8MUYo6VhgTzbWh/+n7/hf1Jw+L2KcdxKvulPJ67xn8=,tag:4ZFpj1UdOwXmaZjYvC/s3A==,type:str]
         grafana: ENC[AES256_GCM,data:P9okJ7bcsqmeGstkSwbDq/RgnG+lFrgAOvcj8A5lOTpmHaSlXGiKG+ybXa0=,iv:Di1ghnxIbAb/u7uo/mJCC3QYVjdweTHaQDZmXTx8OG4=,tag:DT3a1zgU9sTr0BXpyoZ/SQ==,type:str]
         authelia: ENC[AES256_GCM,data:OqyloAADO6KKEaBjGLsJc9GTe77wn6IvA1VCD2dfCWxx+zgzUYh87fK1XX8=,iv:QIOHNTdNnzcY/f3Co8dPdNHykhBnYRhm43nt35hbALM=,tag:DLQq58GrZd+Ul7MSn6s9uQ==,type:str]
+        vaultwarden: ENC[AES256_GCM,data:BPj5eFo54DTZ82n3yTIqEbm7kb/jWT0n2kZY//oV5q48eRch3C2RBuxn/Ko=,iv:DGC4ipHMyVs25gc4sNMt8LN1RsHjiR/b303vgiFoxMY=,tag:k1eb4DoRPLKvvMstSI1faQ==,type:str]
+        gitea: ENC[AES256_GCM,data:l+pBCzyQa3000SE9z1R4htD0V0ONsBtKy92dfgsVYsZ3XlEyVJDIBOsugwM=,iv:5t/oHW1vFAmV/s2Ze/cV9Vuqo96Qu6QvZeRbio7VX2s=,tag:4zeQaXiXIzBpy+tXsxmN7Q==,type:str]
+        immich: ENC[AES256_GCM,data:11jvxTKA/RL0DGL6y2/X092hnDohj6yTrYGK4IVojqBd1gCOBnDvUjgmx14=,iv:oBfHxsx9nxhyKY/WOuWfybxEX2bf+lHEtsaifFRS9lg=,tag:tAfkBdgQ8ZEkLIFcDICKDw==,type:str]
+        paperless: ENC[AES256_GCM,data:6VBrBbjVoam7SkZCSvoBTdrfkUoDghdGTiBmFLul04X/okXOHeC5zusJffY=,iv:iZumcJ3TWwZD77FzYx8THwCqC+EbnXUBrEKuPh3zgV8=,tag:u2m8SppAdxZ/duNdpuS3oQ==,type:str]
+        affine: ENC[AES256_GCM,data:XPXrcszsV06YqCJZ7CDqc4rCwqqNlbtLCFYfLAQ8jamLtft8L2UVrMA4WZo=,iv:vrWdBeckxB9tmEE628j4jhU+hSpE6TXYMGt0hh1Cg84=,tag:hlWwWUGht8NqWTZREMsa1Q==,type:str]
+        nextcloud: ENC[AES256_GCM,data:ROsximNuWYMTZktmLJPx7W1Qol/uT+APgwoCtFO/6ZYYc3KxKvlk344eqEc=,iv:4d+MrfIHjJKAcwhvZ3g4go66uZcieuL7lngKErJd+fg=,tag:QbWOtxeCbiu62GyrE2atXg==,type:str]
+        sure: ENC[AES256_GCM,data:FULJ2gjJ2gZC3s324itW+CjGRBHIP9RnOqw5TT1UaiUhb7UHAPm1na+LsZk=,iv:c0GnVZkxprJUzPPq3TCQaZvAes9QQuvDXqgVLLaiQIg=,tag:uDxy/Lkd2hNK4AWwMNMslw==,type:str]
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
@@ -131,7 +138,7 @@ ldap:
         authelia: ENC[AES256_GCM,data:G8ZGsLKqEmMzQ5NMAgirF5BQraHNqixtI6dyyaeNhTdXebjJZML52xL36p4=,iv:ZtHAsFYmrQxr+qoQLPW/eme0+nsT148KRsXmW/LNLlU=,tag:Pvjs/eylkgxJpmGBsRmjcw==,type:str]
         grafana: ENC[AES256_GCM,data:vWmU3ZKcolETWAY74C3OMD8gMXDeYk+DqssACL0xefIPi5IkbrhYWmnWAnA=,iv:wcRms3Zp8kPM4USRPVa0UHpCTK36SWhK9C8yHSWu2Cs=,tag:gU5S/6fdMZVd/ih3Yd5uJA==,type:str]
         il: ENC[AES256_GCM,data:/CyMeo1+rIUAYiB25nI0,iv:jsyiiRN5z9GqcUnTZ0CZo4s+umTc2zeY2FPp+tVOC9o=,tag:cwOHcqMysCxX57w3a+Pzpg==,type:str]
-        morsalin: null
+        morsalin: ENC[AES256_GCM,data:YryNch8hF6rx,iv:bNIBur3Jcib8BvKjJ0MejpemsurYTP8rCxo6b2R5yEo=,tag:9dIIgqEPtbeixtgJ1OtMnQ==,type:str]
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
@@ -170,6 +177,80 @@ authelia:
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
+#ENC[AES256_GCM,data:MiobmVqj3dPargzws5q7Cpggnvw=,iv:VzBh1ZWqyByyTDn90pl0//TsjAYO5QLpfbrxnMF5OU4=,tag:7Y16L9i/homdtEpWynzsdg==,type:comment]
+vaultwarden:
+    admin:
+        password: ENC[AES256_GCM,data:FzD80H5lPNyjTRGd/IcIGir35KLF8gN3qmTzEQrlfutHh08mk0Vh9X8irHA=,iv:cfn5432g2MvkuBJLB2zDJmU0sLgMefVvpy0bP/4/oPY=,tag:7MHEv8XswdQZNTARvxs55g==,type:str]
+        hash: ENC[AES256_GCM,data:Ae7bhnL5pxfNV3+r/PCDIpxx0tHcNiB/s4sm7OoxVpp+tcEpXZ9vsv38sZjx+Z8t2vlmiYHc72PEdapcAihleuLMBPM1dAVn8LqzzWtVhfmCOoUZI8Z5L3VL8eyyz6npiw==,iv:c2MEIbrnVsR+bUx3zLLTnKSElFTH7JRl23HmmTlWEBg=,tag:mnMrER9pQEKAdloYMeRa3g==,type:str]
+    il:
+        password: ENC[AES256_GCM,data:mhLb55ENatpE59Rbzk2Uq6iBKc0Jj/9x4fOANJOHR0WvSxTQRKJ94cCq9Ykp/chWbdgYYPrwU5oD0Yo17zqb,iv:6bzKq5WaKhuOsQ8zSSH9ZrQYbDPB1nv/bFoQou1ycL4=,tag:bSmuUodl+/9nzxZ7YBoezA==,type:str]
+    morsalin:
+        password: ENC[AES256_GCM,data:ibg5/MfLH7pSY2pEmjM=,iv:+aV5muP/9BYoKwTGQxKEL+IGY9P+O3GVKGgSuTzT+U0=,tag:rgqcmJvd1RtvWJ91PCxYIg==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
+#ENC[AES256_GCM,data:HAN9BBEl1CW11LAf3x8=,iv:hwqCErtmPqGJ+y86D9MoxJwixvbcJONlyT900Y5DOug=,tag:kCxsG353ltbOCKLe85adCg==,type:comment]
+gitea:
+    il:
+        password: ENC[AES256_GCM,data:Bs5/t5mNbSv5+ek9gNHZp5qqxitM4Kq0Xgh2JV6LiFw9lZJSOVT4JPLRP5M=,iv:z50+naWOTVL8lEgBgm51j6hLjS8ve2UcRRKukvtykM0=,tag:TF9rPxjoLe/vAyvl23PiCg==,type:str]
+        token: ENC[AES256_GCM,data:CiYEXEKLLRDp++iga5YmhyPB2bSvqhhDgSOhmiulp9n9SsBVQ3s5MQ==,iv:+oetgEH/IORz74Xoz5zgDjD3BLyledZTqlYlCWaDrRk=,tag:S9O09YEMuSexuNW3ojEw5Q==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:qjQosi83oaK47wX4VtDktDFlIU3FQgmDpwyiSgJNPDq2Dtc/QEKzdv5/ZpQ=,iv:aUJx5a6Wj7lhaD19aPiZWIE/MWKUgH0muxtTSkwfg8M=,tag:DttjXee7ntxRAbqbFq6qeQ==,type:str]
+        hash: ENC[AES256_GCM,data:iVls8UzdP6JTfRosET6nsk3RcEtFQ8ak6GtPuBjcqAv3lyA9oSLwvvC8MvGaT1aHCB+5y/lspuloHEaAcVGNuzGCIVLg9Adut4a7LYDTHNOZnJIKLj8ldL1ytD5XVdbeBhZGCqoTs088oO0HVCQiDzpJL5NdmM3Ru6egyYBCLLUOlPA=,iv:er63GEC+kxNDscvspvvLiq17VSg0GeZ2w3jmGojF9PM=,tag:F1pDpaQrVdomIlZ8psVpAw==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
+#ENC[AES256_GCM,data:HvMeGuC8JwK50pO1E/nm,iv:5NFTyjesMX0ZnBpH+hEv8jQ0G2NvrDtT23CUyLbQcUo=,tag:qWmPvQpADTeD+W8nWXRQvA==,type:comment]
+immich:
+    il:
+        password: ENC[AES256_GCM,data:PbDFc4m7rNPPN1mCjcvhbKwf/EbiJxdvO/iMspf9jMuCqQyGv7h6VrZqk98=,iv:hlMAp5wXXkFO9+ekq3A2/ioF/EX8Uau0puhb4TAHkRQ=,tag:pfS0W2JNaFNMpBlKgZ3Pjw==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:9ENcp2Ns21OLXDY05zRoSdP+93EiwSH8MGzZZpxK7sToe4QLUXWt9w6xQIE=,iv:Q/VcnArZHs/J2YLRVFXt3Mp+LYfuq4PD/trqO8Simig=,tag:BY/mOyZpYmoAc7NrASlmSA==,type:str]
+        hash: ENC[AES256_GCM,data:mrML2CWFFtGjq8wfWipVpv+pjJRSHe74VGC7Eoa6588R5C/sCnC3W5aI+dsRCZN3LRCjHAkOJJgjeYrwcXYdKRauXsAYR51dNSsHqqSN3WebLxapRDwcYu5e4j5RN1aPHsysr7GaQ4hhe5rKW4ORCGC3Cp3Ob+LChPy4bdCAZG3bN9k=,iv:Q4hmqhq+dvIr7DxCpcqP4E0NKyFZkOeTnDpGctmCxXM=,tag:/gji6AFkHnYwkQf3FSQUxA==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
+#ENC[AES256_GCM,data:McPUAbIUvtC1gdPaxTgAxAMCMWcLfg==,iv:Tp6idRf7he3sYzo8LW596C905JAaoTIhIoDUzSyRT0k=,tag:4mZQ0Swu1X9uuwjsRNhr2A==,type:comment]
+paperless:
+    session_secret: ENC[AES256_GCM,data:siwCs2noeVpg9DCEZybnmo/oz11BdrHSTnHciMOu/6g=,iv:XVjhu10TIujIdUopN9+TVVqRade9EvItDWxym6YXnZs=,tag:TxLYm+4Bo7IMaTQBtMg9pQ==,type:str]
+    il:
+        password: ENC[AES256_GCM,data:9bJHf+chTg1rppgNVafNgEuvwQ69Gx+w5d65hu68q9XeeaVb2pO9HE4BOgg=,iv:1kaXBg/iOoIZxDjEVEdaMJLDtp6zQjep3vxLmIgQN5o=,tag:+MgX8Oa3tmhjx6u9aHkDfQ==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:wjRDVCJsINM4z5946a6uZD+6bhN5BChLMdRzgMEJFGRGFNcXd7A1p2Iqn4I=,iv:Y4QDA09L8ULKr4hhvoiduzCD8Hifo1gAnpzjCr8e520=,tag:R0RvGxYnXo3zwykXJykRug==,type:str]
+        hash: ENC[AES256_GCM,data:pali6WwPNhJA+6QL4O+tKv42PnpGqmojb8JQUZLqxGizv1bJSCgdUN8upCy5Ke0DYZs5P+JY5vh23xfMZFnHduGxGwOuPX6J5lYgvJRV58LqS3/+yIBBprTJyro3MwsurTTEWesgKMr8/2H9lirhaLjWUOSPxAmQ6e4wPNpHycDVyj4=,iv:cg2trI7t1MfIcMo1/M+IY6JEl2msDoKRGgAx/Y5nyGk=,tag:gnOq7sBq9z5zrRY0yhIabg==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
+#ENC[AES256_GCM,data:k55osvepVeB1RC5hZ4IF,iv:AlhfmWwn/DiSESWc+ULJSOLUhnrKAIfWr7MeiwV8qc8=,tag:hOgptwUcY6nVxPIhu+DYgw==,type:comment]
+affine:
+    secret_key: ENC[AES256_GCM,data:LLX78DpYnha1JWhgw0sHLzIVq/oIzvT+nB7zgli4mroGbnt7WZaXCx34zKkYRwYj/+0L4IFFVdkzKtK5DO84SgFkS2Bk2iNdCMqIx80CpyiD8IWAcyRu5d6hh82PlgyxU80T/4nbLbIn0GLubPTTeUX8GC3VxRU=,iv:DnmvbhlygSHes0jAkIm4+WXMUQLzr4R4dNa33rO67v8=,tag:+2wlh+/ekiTyShWM4XBbUw==,type:str]
+    il:
+        password: ENC[AES256_GCM,data:4zxiQAzXTR+fraRjYT657BIwSqrih3lMPFFSibQdardRMjskAbuRYIQA6mo=,iv:ub3giRG9vCFSuwRXDazYTqWbjENzQUWR36290Kruj1o=,tag:C2Ixd2eTEgzBvUNCNBtJuA==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:eRDBrqLZR7MFLlsUwk7Wg7FzxDov7vJLIWQRuKq7vrXbPSJkMcy9jfG2rL4=,iv:UaSoi7gODXgjzihJIDVIdDHJcSAZNV8UKfGeM6YzxqI=,tag:cOUDblcMStP8E4fp+s1WRQ==,type:str]
+        hash: ENC[AES256_GCM,data:jE1CvFo+mjb/Xc3Ft5ky7on03vcnv79cw/5g/xaldXsv94VRrIjmfGMgHAj07r8j5mDpP34A5bYO1PSe9DYrwRcsXa9OUQuzm/8avFy9wVZDhBUUAGR+jiW1BP9hc6nmSpPVPtle+3sbqOB0ZMjXWwlcAcuknOtuhH1mzwmaDP9yf+M=,iv:CSSaXY/6MpHBMhPLUWPkabIeJ9zpZkcVjiEhxVF0zJM=,tag:f72ekkjJs7Qmh1K9wC8L9w==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
+#ENC[AES256_GCM,data:PZS7EbvMHqHGorNUGAWj4dk1,iv:vOE+djRAvBTMM51kHi6kG5Arw3uPXlJt1d/BpcEaD0c=,tag:AuoCHLQz42CYvVVdKFWu1Q==,type:comment]
+nextcloud:
+    admin-local:
+        password: ENC[AES256_GCM,data:mIwF5A09oqYbdK3bOKid9A896Q5J5Q6Ax+vDNqEJFGNdzd/mJ4oQS6rva+s=,iv:QroUMST2wnEJzk6DySe9tPZaWuqdxzJZ0+oi6mW6x00=,tag:3UTzjupK7+omrI3Hvyr8bA==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:Sr4KkKkYdkU0UWdpfUF7PyiGoerjBiw+sOFcENyLxw0FRXGG0Y8gv5uGb4Q=,iv:LbGsNM3+iY7bWFQe88TepVKUdiRQWZ+K7Ubn6ze6lV4=,tag:SbcfIAMW9ZprgahOFU4IQQ==,type:str]
+        hash: ENC[AES256_GCM,data:CkstbIYQmi72QhsbJZN0lQedgCn7TmGpYcYj0n+NvJIoTlol8G9N/88cwGbVoGK9nEISv54FL94cEJFppnMIuj0BHrhasrZsyI2/Lj52YLWdwNJWNQ+iYt+Ifp/1kI0zqmdoajzZ5DS2w/1evCBC1+JdfTRlpVXmSsHUIPIHelBRj90=,iv:vwvT5TTkF4woxXOvrRRqmrdLXf19s47NIDtdT+zLp0U=,tag:KC0MS0DTH6j3zIHOjCFOSA==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
+#ENC[AES256_GCM,data:Fsqc2JDp9dvfgiCjdQ==,iv:3DALKKEXaP8hzXRvxD4CgfFpOiPPsOa16OB94n8WKp8=,tag:K+FF3zGrc0YLXWK/R2L3Ow==,type:comment]
+sure:
+    session_secret: ENC[AES256_GCM,data:InHsz/jld8E9TwI8MWpxk9x2I7dxlIsY9R6jtDK2pBA=,iv:HY5yXEC2Dce26e9/vXTIWELvVd9ZjhcCwFD0jhz5pPw=,tag:LLSJovZ0RH3CUK+se7R4Ag==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:9BSvpcU9BJctSN9bkPIAsRxg8JNHTWvOKdpJFhm//CUDm/Xc7oC/ANHf5no=,iv:JVQLl/rp65kZSK/4SpVXxtiac3Z35XNkxWm2+lEdq/c=,tag:WgfaORiNlrO+wHSdnl4CWQ==,type:str]
+        hash: ENC[AES256_GCM,data:EjJ+1fP7/9wG2jG0Jv2hxMLtErqxjHBstRjru79dd5ZXhqwT7S+jpLfl9WpZU9qi20ps9YP4qe7G08p6NJNXjYhQj852GQxEORRh/9StAZsPt3p8w+ePZSVbivPQH+FpPKWYxoH0VR7y3TnL66R0tKRLh1fNTc5jRy5rU5r1bfs1jZ0=,iv:0y9FxW4QdD7qHz3bPRWlwHFpvOsvlYhVrOItB6BzaE8=,tag:Wc7MZhP3QRYmvZcjpoEWtQ==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
 #ENC[AES256_GCM,data:T4Wtn49AAxPd2QUFTR+q,iv:bH5goGWBDqumAat9dUv2OwfCUJUpuVqncTMqMBZUXhI=,tag:G+W6hHA+yftQ+4RJpXrxHg==,type:comment]
 switch:
     password: ENC[AES256_GCM,data:qu0f9L7A0eFq/UCpaRs=,iv:W8LLOp3MSfd/+EfNEZNf91K8GgI5eUfVPoWTRES2C0Y=,tag:Q5FlAOfwqwJwPvd7k6i+0g==,type:str]
@@ -199,7 +280,7 @@ sops:
             UmliaFNxVTBqRkI1QWJpWGpTRWxETW8KEY/8AfU73UOzCGhny1cNnd5dCNv7bHXt
             k+uyWPPi+enFkVaceSwMFrA66uaWWrwAj11sXEB7yzvGFPrnAGezjQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-03-14T19:40:47Z"
-    mac: ENC[AES256_GCM,data:EUVSxs6FPhKMSSmHe8P/d0IyBZsNb3q7AYj06j98bklAMYYVOludVePdh45MSvn92lDn712Muy6pqcJzDpsPWyxgXngywTu2SGV1yRCyA7U7RloRxlNROuDiugMkJWOtHcKArytVChUHT2PnzagAJR2kBSApbjUsC/xUTMBpsNM=,iv:SsJW2fMNEJHT2M+gjW5TKu6AYoxsf9jKf5T9KgJoF40=,tag:ItVweaSxts2Cm1VKkLp0/w==,type:str]
+    lastmodified: "2026-05-09T14:26:51Z"
+    mac: ENC[AES256_GCM,data:TYs08ZSS2kcO5lYuhQ/IySUSQ3DpL+ba3/uNLyszht4OttR110/W/WQLiRuu/Ql6FwtDtjq6I3iNpOhmCHSv1kMCam1l99GEIYCaPUIY+TY3Zw0j7518dFXe8p/DrKRwIVXfK5lIKLIEd+eizD50HzwXXJFmU+7YDkQ1Dx+55kw=,iv:arJKJ4wO4sdQlu3GZbtultsfM6s8vbhG93tnf2EjJDc=,tag:m95gUqvn4w85XI8qVvCZpQ==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1

config/services/containers/app/affine/affine.container.j2

@@ -0,0 +1,49 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=AFFiNE
+
+After=redis_affine.service manticore_affine.service
+Wants=redis_affine.service manticore_affine.service
+
+[Container]
+Image=ghcr.io/toeverything/affine:{{ version['containers']['affine'] }}
+ContainerName=affine
+HostName=affine
+
+PublishPort={{ services['affine']['ports']['http'] }}:3010
+
+Volume=%h/data/containers/affine:/root/.affine/storage:rw
+Volume=%h/containers/affine/config:/root/.affine/config
+Volume=%h/containers/affine/ssl:/etc/ssl/affine:ro
+
+# General
+Environment="TZ={{ timezone }}"
+## OIDC callback URIs
+Environment="AFFINE_SERVER_HOST={{ services['affine']['domain']['public'] }}.{{ domain['public'] }}"
+Environment="AFFINE_SERVER_EXTERNAL_URL=https://{{ services['affine']['domain']['public'] }}.{{ domain['public'] }}"
+Environment="AFFINE_SERVER_HTTPS=true"
+
+Secret=AFFINE_PRIVATE_KEY,type=env
+
+# Database
+Secret=AFFINE_DATABASE_URL,type=env,target=DATABASE_URL
+## Enable AI function: this needs pgvector
+
+# Redis
+Environment="REDIS_SERVER_HOST=host.containers.internal"
+Environment="REDIS_SERVER_PORT={{ services['affine']['ports']['redis'] }}"
+
+# Indexer
+Environment="AFFINE_INDEXER_ENABLED=true"
+Environment="AFFINE_INDEXER_SEARCH_ENDPOINT=http://host.containers.internal:{{ services['affine']['ports']['manticore'] }}"
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/collabora/collabora.container.j2

@@ -0,0 +1,25 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Collabora Online
+
+[Container]
+Image=docker.io/collabora/code:{{ version['containers']['collabora'] }}
+ContainerName=collabora
+HostName=collabora
+
+PublishPort={{ services['collabora']['ports']['http'] }}:9980/tcp
+
+Environment="TZ={{ timezone }}"
+Environment="aliasgroup1=https://{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}"
+# Environment="aliasgroup2=other_server_FQDN"
+Environment="extra_params=--o:ssl.enable=false --o:ssl.termination=true --o:server_name={{ services['collabora']['domain']['public'] }}.{{ domain['public'] }} --o:admin_console.enable=false"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/gitea/gitea.container.j2

@@ -0,0 +1,51 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Gitea
+
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/gitea/gitea:{{ version['containers']['gitea'] }}
+
+ContainerName=gitea
+HostName=gitea
+
+PublishPort={{ services['gitea']['ports']['http'] }}:3000/tcp
+
+Volume=%h/data/containers/gitea:/data:rw
+Volume=%h/containers/gitea/ssl:/etc/ssl/gitea:ro
+
+# General
+Environment="TZ={{ timezone }}"
+Environment="GITEA__server__DISABLE_SSH=true"
+# Database
+Environment="GITEA__database__DB_TYPE=postgres"
+Environment="GITEA__database__HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}"
+Environment="GITEA__database__NAME=gitea_db"
+Environment="GITEA__database__USER=gitea"
+Secret=GITEA__database__PASSWD,type=env
+Environment="GITEA__database__SSL_MODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/gitea/{{ root_cert_filename }}"
+# OAuth2 client
+Environment="GITEA__oauth2_client__ACCOUNT_LINKING=auto"
+# OIDC configuration
+Environment="GITEA__openid__ENABLE_OPENID_SIGNIN=false"
+Environment="GITEA__openid__ENABLE_OPENID_SIGNUP=true"
+Environment="GITEA__openid__WHITELISTED_URIS={{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+# automatic create user via authelia
+Environment="GITEA__service__DISABLE_REGISTRATION=false"
+Environment="GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION=true"
+Environment="GITEA__service__SHOW_REGISTRATION_BUTTON=false"
+
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/immich/immich-ml.container.j2

@@ -0,0 +1,32 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Immich Machine Learning
+
+After=immich.service
+Wants=immich.service
+
+[Container]
+Image=ghcr.io/immich-app/immich-machine-learning:{{ version['containers']['immich'] }}-openvino
+
+ContainerName=immich-ml
+HostName=immich-ml
+
+PublishPort={{ services['immich-ml']['ports']['http'] }}:3003
+
+# iGPU access for OpenVINO
+AddDevice=/dev/dri:/dev/dri
+PodmanArgs=--group-add keep-groups
+
+Volume=%h/containers/immich/ml/cache:/cache:rw
+
+Environment="TZ={{ timezone }}"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/immich/immich.container.j2

@@ -0,0 +1,53 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Immich
+
+After=redis_immich.service
+Wants=redis_immich.service
+
+[Container]
+Image=ghcr.io/immich-app/immich-server:{{ version['containers']['immich'] }}
+
+ContainerName=immich
+HostName=immich
+
+PublishPort={{ services['immich']['ports']['http'] }}:2283
+
+# iGPU access
+AddDevice=/dev/dri:/dev/dri
+PodmanArgs=--group-add keep-groups
+
+# Volumes
+Volume=%h/data/containers/immich:/data:rw
+Volume=%h/containers/immich/ssl:/etc/ssl/immich:ro
+
+# Environment
+Environment="TZ={{ timezone }}"
+# The new environment from version 2.7.0 to enable CSP
+Environment="IMMICH_HELMET_FILE=true"
+
+# Redis
+Environment="REDIS_HOSTNAME=host.containers.internal"
+Environment="REDIS_PORT={{ services['immich']['ports']['redis'] }}"
+Environment="REDIS_DBINDEX=0"
+
+# Database
+Environment="DB_HOSTNAME={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="DB_PORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="DB_USERNAME=immich"
+Environment="DB_DATABASE_NAME=immich_db"
+Environment="DB_PASSWORD_FILE=/run/secrets/DB_PASSWORD"
+Environment="DB_SSL_MODE=verify-full"
+Environment="NODE_EXTRA_CA_CERTS=/etc/ssl/immich/{{ root_cert_filename }}"
+Secret=IMMICH_DB_PASSWORD,target=/run/secrets/DB_PASSWORD
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/manticore/manticore.container.j2

@@ -0,0 +1,25 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Manticore - {{ manticore_service }}
+
+[Container]
+Image=docker.io/manticoresearch/manticore:{{ version['containers']['manticore'] }}
+ContainerName=manticore_{{ manticore_service }}
+HostName=manticore_{{ manticore_service }}
+
+PublishPort={{ services[manticore_service]['ports']['manticore'] }}:9308
+
+Volume=%h/data/containers/manticore/{{ manticore_service }}:/var/lib/manticore:rw
+
+# General
+Environment="TZ={{ timezone }}"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/nextcloud/config/background.config.php.j2

@@ -0,0 +1,5 @@
+<?php
+$CONFIG = [
+    // Background jobs mode is auto-detected as 'cron' when nextcloud-cron.timer runs cron.php via CLI. No explicit config required.
+    'maintenance_window_start' => 18,
+];

config/services/containers/app/nextcloud/config/cache.config.php.j2

@@ -0,0 +1,12 @@
+<?php
+$CONFIG = [
+    'memcache.local' => '\OC\Memcache\APCu',
+    'memcache.distributed' => '\OC\Memcache\Redis',
+    'memcache.locking' => '\OC\Memcache\Redis',
+    'redis' => [
+        'host' => 'host.containers.internal',
+        'port' => {{ services['nextcloud']['ports']['redis'] }},
+        'timeout' => 1.5,
+        'dbindex' => 0,
+    ],
+];

config/services/containers/app/nextcloud/config/domain.config.php.j2

@@ -0,0 +1,9 @@
+<?php
+$CONFIG = [
+  'trusted_domains' => [
+    '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}',
+  ],
+  'overwritehost' => '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}',
+  'overwriteprotocol' => 'https',
+  'overwrite.cli.url' => 'https://{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}',
+];

config/services/containers/app/nextcloud/config/local_remote.config.php.j2

@@ -0,0 +1,4 @@
+<?php
+$CONFIG = [
+  'allow_local_remote_servers' => true,
+];

config/services/containers/app/nextcloud/config/user_oidc.config.php.j2

@@ -0,0 +1,9 @@
+<?php
+$CONFIG = [
+  'user_oidc' => [
+    'default_token_endpoint_auth_method' => 'client_secret_post',
+    'auto_provision' => true,
+    'soft_auto_provision' => true,
+    'disable_account_creation' => false,
+  ],
+];

config/services/containers/app/nextcloud/ini/opcache.ini

@@ -0,0 +1,14 @@
+; /usr/local/etc/php/conf.d/opcache-recommended.ini
+; OPcache tuning
+opcache.enable=1
+opcache.enable_cli=1
+opcache.memory_consumption=512
+opcache.interned_strings_buffer=32
+opcache.max_accelerated_files=20000
+opcache.validate_timestamps=0
+opcache.save_comments=1
+opcache.revalidate_freq=60
+opcache.fast_shutdown=1
+
+; APCu CLI activate
+apc.enable_cli=1

config/services/containers/app/nextcloud/ini/upload.ini

@@ -0,0 +1,6 @@
+; /usr/local/etc/php/conf.d/nextcloud-upload.ini
+upload_max_filesize=16G
+post_max_size=16G
+memory_limit=1024M
+max_execution_time=3600
+max_input_time=3600

config/services/containers/app/nextcloud/nextcloud.container.j2

@@ -0,0 +1,36 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Nextcloud
+
+[Container]
+Image=docker.io/library/nextcloud:{{ version['containers']['nextcloud'] }}
+ContainerName=nextcloud
+HostName=nextcloud
+
+PublishPort={{ services['nextcloud']['ports']['http'] }}:80
+
+Volume=%h/containers/nextcloud/ssl:/etc/ssl/nextcloud:ro
+Volume=%h/containers/nextcloud/ini/opcache.ini:/usr/local/etc/php/conf.d/opcache-recommended.ini:ro
+Volume=%h/containers/nextcloud/ini/upload.ini:/usr/local/etc/php/conf.d/upload.ini:ro
+Volume=%h/data/containers/nextcloud/html:/var/www/html:rw
+
+# General
+Environment="TZ={{ timezone }}"
+# PostgreSQL
+Environment="PGSSLMODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/nextcloud/{{ root_cert_filename }}"
+## libpq in Nextcloud automatically tries to use a client certificate for mTLS. Therefore, when only TLS is required, then disable the option explicitly.
+Environment="PGSSLCERTMODE=disable"
+# Redis
+Environment="REDIS_HOST=host.containers.internal"
+Environment="REDIS_HOST_PORT={{ services['nextcloud']['ports']['redis'] }}"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/nextcloud/systemd/nextcloud-cron.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Nextcloud cron.php
+Requires=nextcloud.service
+After=nextcloud.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/podman exec -u www-data nextcloud php -f /var/www/html/cron.php

config/services/containers/app/nextcloud/systemd/nextcloud-cron.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run Nextcloud cron every 5 minutes
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=5min
+Unit=nextcloud-cron.service
+
+[Install]
+WantedBy=timers.target

config/services/containers/app/paperless/build/paperless.containerfile.j2

@@ -0,0 +1,13 @@
+FROM ghcr.io/paperless-ngx/paperless-ngx:{{ version['containers']['paperless'] }}
+
+USER root
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends curl ca-certificates \
+  && curl -fsSL https://raw.githubusercontent.com/tesseract-ocr/tessdata_best/main/kor.traineddata \
+    -o /usr/share/tesseract-ocr/5/tessdata/kor.traineddata \
+  && curl -fsSL https://raw.githubusercontent.com/tesseract-ocr/tessdata_best/main/eng.traineddata \
+    -o /usr/share/tesseract-ocr/5/tessdata/eng.traineddata \
+  && rm -rf /var/lib/apt/lists/*
+
+USER paperless

config/services/containers/app/paperless/paperless.container.j2

@@ -0,0 +1,59 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Paperless
+
+After=redis_paperless.service
+Wants=redis_paperless.service
+
+[Container]
+Image={{ domain['internal'] }}/{{ node['name'] }}/paperless-ngx:{{ version['containers']['paperless'] }}
+ContainerName=paperless
+HostName=paperless
+PublishPort={{ services['paperless']['ports']['http'] }}:8000/tcp
+
+# Volumes
+Volume=%h/data/containers/paperless/data:/usr/src/paperless/data:rw
+Volume=%h/data/containers/paperless/media:/usr/src/paperless/media:rw
+Volume=%h/data/containers/paperless/consume:/usr/src/paperless/consume:rw
+Volume=%h/containers/paperless/ssl:/etc/ssl/paperless:ro
+
+# General
+Environment="TZ={{ timezone }}"
+Environment="PAPERLESS_TIME_ZONE={{ timezone }}"
+Environment="PAPERLESS_URL=https://{{ services['paperless']['domain']['public'] }}.{{ domain['public'] }}"
+Environment="PAPERLESS_OCR_LANGUAGE=kor+eng"
+Environment="PAPERLESS_OCR_LANGUAGES=kor"
+# Environment="PAPERLESS_OCR_MODE=force"
+# Environment="PAPERLESS_TASK_WORKERS=1"
+# Environment="PAPERLESS_THREADS_PER_WORKER=1"
+Environment="PAPERLESS_WORKER_TIMEOUT=7200"
+Secret=PAPERLESS_SECRET_KEY,type=env
+
+# Redis
+Environment="PAPERLESS_REDIS=redis://host.containers.internal:{{ services['paperless']['ports']['redis'] }}"
+
+# Database
+Environment="PAPERLESS_DBHOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="PAPERLESS_DBPORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="PAPERLESS_DBNAME=paperless_db"
+Environment="PAPERLESS_DBUSER=paperless"
+Environment="PAPERLESS_DBSSLMODE=verify-full"
+Environment="PAPERLESS_DBSSLROOTCERT=/etc/ssl/paperless/{{ root_cert_filename }}"
+Secret=PAPERLESS_DBPASS,type=env
+
+# OIDC
+Environment="PAPERLESS_APPS=allauth.socialaccount.providers.openid_connect"
+Environment="PAPERLESS_SOCIAL_AUTO_SIGNUP=true"
+Environment="PAPERLESS_SOCIALACCOUNT_ALLOW_SIGNUPS=true"
+Secret=PAPERLESS_SOCIALACCOUNT_PROVIDERS,type=env
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/redis/redis.conf.j2

@@ -0,0 +1,4 @@
+databases 16
+bind 0.0.0.0
+port 6379
+protected-mode no

config/services/containers/app/redis/redis.container.j2

@@ -0,0 +1,31 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Redis - {{ redis_service }}
+
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/library/redis:{{ version['containers']['redis'] }}
+
+ContainerName=redis_{{ redis_service }}
+HostName=redis_{{ redis_service }}
+
+PublishPort={{ services[redis_service]['ports']['redis'] }}:6379
+
+Volume=%h/containers/redis/{{ redis_service }}/data:/data:rw
+Volume=%h/containers/redis/{{ redis_service }}/redis.conf:/usr/local/etc/redis/redis.conf:ro
+
+Exec=redis-server /usr/local/etc/redis/redis.conf
+
+Environment="TZ={{ timezone }}"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/sure/sure-web.container.j2

@@ -0,0 +1,67 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Sure Web
+
+After=network-online.target redis_sure.service
+Wants=network-online.target redis_sure.service
+
+[Container]
+Image=ghcr.io/we-promise/sure:{{ version['containers']['sure'] }}
+ContainerName=sure-web
+HostName=sure-web
+
+PublishPort={{ services['sure']['ports']['http'] }}:3000/tcp
+
+Volume=%h/data/containers/sure/storage:/rails/storage:rw
+Volume=%h/containers/sure/ssl:/etc/ssl/sure:ro
+
+# General
+Environment="TZ={{ timezone }}"
+Environment="SELF_HOSTED=true"
+Environment="ONBOARDING_STATE=closed"
+Environment="RAILS_FORCE_SSL=false"
+Environment="RAILS_ASSUME_SSL=true"
+Environment="APP_DOMAIN={{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+Secret=SURE_SECRET_KEY_BASE,type=env,target=SECRET_KEY_BASE
+
+# PostgreSQL
+Environment="POSTGRES_USER=sure"
+Environment="POSTGRES_DB=sure_db"
+Environment="DB_HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="DB_PORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="PGSSLMODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/sure/{{ root_cert_filename }}"
+Secret=SURE_POSTGRES_PASSWORD,type=env,target=POSTGRES_PASSWORD
+
+# Redis
+Environment="REDIS_URL=redis://host.containers.internal:{{ services['sure']['ports']['redis'] }}/1"
+
+# OIDC - Authelia
+Environment="OIDC_CLIENT_ID=sure"
+Environment="OIDC_ISSUER=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+Environment="OIDC_REDIRECT_URI=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}/auth/openid_connect/callback"
+Secret=SURE_OIDC_CLIENT_SECRET,type=env,target=OIDC_CLIENT_SECRET
+Environment="OIDC_BUTTON_LABEL=Sign in with Authelia"
+Environment="AUTH_JIT_MODE=create_and_link"
+# email's domain, e.g. ilnmors.internal then only user@ilnmors.internal is allowed to sign-up
+Environment="ALLOWED_OIDC_DOMAINS="
+
+# WebAuthn / Passkey
+Environment="WEBAUTHN_RP_ID={{ domain['public'] }}"
+Environment="WEBAUTHN_ALLOWED_ORIGINS=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+
+# Provider
+## Currency
+Environment="EXCHANGE_RATE_PROVIDER=yahoo_finance"
+Environment="SECURITIES_PROVIDER=yahoo_finance"
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/sure/sure-worker.container.j2

@@ -0,0 +1,67 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Sure Worker
+
+After=network-online.target redis_sure.service
+Wants=network-online.target redis_sure.service
+
+[Container]
+Image=ghcr.io/we-promise/sure:{{ version['containers']['sure'] }}
+ContainerName=sure-worker
+HostName=sure-worker
+
+Volume=%h/data/containers/sure/storage:/rails/storage:rw
+Volume=%h/containers/sure/ssl:/etc/ssl/sure:ro
+
+Exec=bundle exec sidekiq
+
+# General
+Environment="TZ={{ timezone }}"
+Environment="SELF_HOSTED=true"
+Environment="ONBOARDING_STATE=closed"
+Environment="RAILS_FORCE_SSL=false"
+Environment="RAILS_ASSUME_SSL=true"
+Environment="APP_DOMAIN={{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+Secret=SURE_SECRET_KEY_BASE,type=env,target=SECRET_KEY_BASE
+
+# PostgreSQL
+Environment="POSTGRES_USER=sure"
+Environment="POSTGRES_DB=sure_db"
+Environment="DB_HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="DB_PORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="PGSSLMODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/sure/{{ root_cert_filename }}"
+Secret=SURE_POSTGRES_PASSWORD,type=env,target=POSTGRES_PASSWORD
+
+# Redis
+Environment="REDIS_URL=redis://host.containers.internal:{{ services['sure']['ports']['redis'] }}/1"
+
+# OIDC - Authelia
+Environment="OIDC_CLIENT_ID=sure"
+Environment="OIDC_ISSUER=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+Environment="OIDC_REDIRECT_URI=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}/auth/openid_connect/callback"
+Secret=SURE_OIDC_CLIENT_SECRET,type=env,target=OIDC_CLIENT_SECRET
+Environment="OIDC_BUTTON_LABEL=Sign in with Authelia"
+Environment="AUTH_JIT_MODE=create_and_link"
+# email's domain, e.g. ilnmors.internal then only user@ilnmors.internal is allowed to sign-up
+Environment="ALLOWED_OIDC_DOMAINS="
+
+# WebAuthn / Passkey
+Environment="WEBAUTHN_RP_ID={{ domain['public'] }}"
+Environment="WEBAUTHN_ALLOWED_ORIGINS=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+
+# Provider
+## Currency
+Environment="EXCHANGE_RATE_PROVIDER=yahoo_finance"
+Environment="SECURITIES_PROVIDER=yahoo_finance"
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/vaultwarden/vaultwarden.container.j2

@@ -0,0 +1,34 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Vaultwarden
+
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/vaultwarden/server:{{ version['containers']['vaultwarden'] }}
+
+ContainerName=vaultwarden
+HostName=vaultwarden
+
+PublishPort={{ services['vaultwarden']['ports']['http'] }}:80/tcp
+
+Volume=%h/data/containers/vaultwarden:/data:rw
+Volume=%h/containers/vaultwarden/ssl:/etc/ssl/vaultwarden:ro
+
+Environment="TZ={{ timezone }}"
+Environment="DOMAIN=https://{{ services['vaultwarden']['domain']['public'] }}.{{ domain['public'] }}"
+Environment="SIGNUPS_ALLOWED=false"
+Secret=VW_ADMIN_TOKEN,type=env,target=ADMIN_TOKEN
+Secret=VW_DATABASE_URL,type=env,target=DATABASE_URL
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/auth/authelia/authelia.container.j2

@@ -15,14 +15,14 @@ ContainerName=authelia
 HostName=authelia
 
 # Web UI
-PublishPort=9091:9091/tcp
+PublishPort={{ services['authelia']['ports']['http'] }}:9091/tcp
 
 
 Volume=%h/containers/authelia/config:/config:rw
 Volume=%h/containers/authelia/certs:/etc/ssl/authelia:ro
 
 # Default
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 # Enable Go template engine
 # !CAUTION!
 {% raw %}# If this environment were enabled, you would have to use {{/* ... /*}} for {{ go_filter }} options. Go engine always processes its own grammar first.
@@ -56,8 +56,9 @@ Exec=--config /config/authelia.yaml
 # Wait for dependency
 # They run as rootless podman container, so their port is not opened until they are normaly running
 # Check their ports with nc command 
-ExecStartPre=/usr/bin/nc -zv {{ infra_uri['postgresql']['domain'] }} {{ infra_uri['postgresql']['ports']['tcp'] }}
-ExecStartPre=/usr/bin/nc -zv {{ infra_uri['ldap']['domain'] }} {{ infra_uri['ldap']['ports']['ldaps'] }}
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+# services['ldap']['ports']['ldaps'] is 6360, but nftables works on 636 the original port
+ExecStartPre=/usr/bin/nc -zv {{ services['ldap']['domain'] }}.{{ domain['internal'] }} 636
 ExecStartPre=sleep 5
 Restart=always
 RestartSec=10s

config/services/containers/auth/authelia/config/authelia.yaml.j2

@@ -10,7 +10,7 @@ theme: 'auto'
 # Server configuration
 server:
   # TLS will be applied on caddy
-  address: 'tcp://:9091/'
+  address: 'tcp://:{{ services['authelia']['ports']['http'] }}/'
 
 # Log configuration
 log:
@@ -20,7 +20,7 @@ log:
 # TOTP configuration
 totp:
   # issure option is for 2FA app. It works as identifier. "My homelab' or 'ilnmors.internal', 'Authelia - ilnmors'
-  issuer: 'ilnmors.internal'
+  issuer: '{{ domain['internal'] }}'
 
 # Identity validation confituration
 identity_validation:
@@ -31,21 +31,21 @@ identity_validation:
 authentication_backend:
   ldap:
     # ldaps uses 636 -> NAT automatically change port 636 in output packet -> 2636 which lldap server uses.
-    address: 'ldaps://ldap.ilnmors.internal'
+    address: 'ldaps://{{ services['ldap']['domain'] }}.{{ domain['internal'] }}'
     implementation: 'lldap'
-    # tls configruation, it uses certificates_directory's /etc/ssl/authelia/ilnmors_root_ca.crt
+    # tls configruation, it uses certificates_directory's /etc/ssl/authelia/{{ root_cert_filename }}
     tls:
-      server_name: 'ldap.ilnmors.internal'
+      server_name: '{{ services['ldap']['domain'] }}.{{ domain['internal'] }}'
       skip_verify: false
     # LLDAP base DN
-    base_dn: 'dc=ilnmors,dc=internal'
+    base_dn: '{{ domain['dc'] }}'
     additional_users_dn: 'ou=people'
     additional_groups_dn: 'ou=groups'
     # LLDAP filters
     users_filter: '(&(|({username_attribute}={input})({mail_attribute}={input}))(objectClass=person))'
     groups_filter: '(&(member={dn})(objectClass=groupOfNames))'
     # LLDAP bind account configuration
-    user: 'uid=authelia,ou=people,dc=ilnmors,dc=internal'
+    user: 'uid=authelia,ou=people,{{ domain['dc'] }}'
     password: '' # $AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD_FILE option is designated in container file
 
 # Access control configuration
@@ -53,14 +53,12 @@ access_control:
   default_policy: 'deny'
   rules:
     # authelia portal
-    - domain: 'authelia.ilnmors.internal'
+    - domain: '{{ services['authelia']['domain'] }}.{{ domain['public'] }}'
       policy: 'bypass'
-    - domain: 'authelia.ilnmors.com'
-      policy: 'bypass'
-    - domain: 'test.ilnmors.com'
-      policy: 'one_factor'
-      subject:
-        - 'group:admins'
+#    - domain: 'test.ilnmors.com'
+#      policy: 'one_factor'
+#      subject:
+#        - 'group:admins'
 # Session provider configuration
 session:
   secret: '' # $AUTHELIA_SESSION_SECRET_FILE  is designated in container file
@@ -68,8 +66,8 @@ session:
   inactivity: '24 hours' # Session maintains for 24 hours without actions
   cookies:
     - name: 'authelia_public_session'
-      domain: 'ilnmors.com'
-      authelia_url: 'https://authelia.ilnmors.com'
+      domain: '{{ domain['public'] }}'
+      authelia_url: 'https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}'
       same_site: 'lax'
 
 # This authelia doesn't use Redis.
@@ -78,12 +76,12 @@ session:
 storage:
   encryption_key: '' # $AUTHELIA_STORAGE_ENCRYPTION_KEY_FILE is designated in container file
   postgres:
-    address: 'tcp://{{ infra_uri['postgresql']['domain'] }}:{{ infra_uri['postgresql']['ports']['tcp'] }}'
+    address: 'tcp://{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}'
     database: 'authelia_db'
     username: 'authelia'
     password: '' # $AUTHELIA_STORAGE_POSTGRES_PASSWORD_FILE is designated in container file
     tls:
-      server_name: '{{ infra_uri['postgresql']['domain'] }}'
+      server_name: '{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}'
       skip_verify: false
 
 # Notification provider
@@ -101,7 +99,7 @@ identity_providers:
         key: {{ secret "/run/secrets/AUTHELIA_JWKS_RS256" | mindent 10 "|" | msquote }}
       - algorithm: 'ES256'
         use: 'sig'
-        key: {{ secret "/run/secrets/AUTHELIA_JWKS_ES256" | mindent 10 "|" | msquote }}{% endraw %}   
+        key: {{ secret "/run/secrets/AUTHELIA_JWKS_ES256" | mindent 10 "|" | msquote }}{% endraw %} 
     clients:
       # https://www.authelia.com/integration/openid-connect/clients/synology-dsm/
       - client_id: 'dsm'
@@ -117,7 +115,7 @@ identity_providers:
         require_pkce: false
         pkce_challenge_method: ''
         redirect_uris:
-          - 'https://{{ infra_uri['nas']['domain'] }}:{{ infra_uri['nas']['ports']['https'] }}'
+          - 'https://{{ services['nas']['domain'] }}.{{ domain['internal'] }}:{{ services['nas']['ports']['https'] }}'
         scopes:
           - 'openid'
           - 'profile'
@@ -131,3 +129,134 @@ identity_providers:
         userinfo_signed_response_alg: 'none'
         # [ client_secret_post | client_secret_basic ]
         token_endpoint_auth_method: 'client_secret_post'
+      # https://www.authelia.com/integration/openid-connect/clients/gitea/
+      - client_id: 'gitea'
+        client_name: 'gitea'
+        client_secret: '{{ hostvars['console']['gitea']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          - 'https://{{ services['gitea']['domain']['public'] }}.{{ domain['public'] }}/user/oauth2/authelia/callback'
+        scopes:
+          - 'openid'
+          - 'email'
+          - 'profile'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
+      # https://www.authelia.com/integration/openid-connect/clients/immich/
+      - client_id: 'immich'
+        client_name: 'immich'
+        client_secret: '{{ hostvars['console']['immich']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          - 'https://{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}/auth/login'
+          - 'https://{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}/user-settings'
+          - 'app.immich:///oauth-callback'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_post'
+      # https://www.authelia.com/integration/openid-connect/clients/paperless/
+      - client_id: 'paperless'
+        client_name: 'Paperless'
+        client_secret: '{{ hostvars['console']['paperless']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://{{ services['paperless']['domain']['public'] }}.{{ domain['public'] }}/accounts/oidc/authelia/login/callback/'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+          - 'groups'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_post'
+      # https://docs.affine.pro/self-host-affine/administer/oauth-2-0
+      - client_id: 'affine'
+        client_name: 'Affine'
+        client_secret: '{{ hostvars['console']['affine']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          - 'https://{{ services['affine']['domain']['public'] }}.{{ domain['public'] }}/oauth/callback'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_post'
+      # https://www.authelia.com/integration/openid-connect/clients/nextcloud/#openid-connect-user-backend-app
+      - client_id: 'nextcloud'
+        client_name: 'Nextcloud'
+        client_secret: '{{ hostvars['console']['nextcloud']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}/apps/user_oidc/code'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+          - 'groups'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_post'
+      # https://www.authelia.com/integration/openid-connect/clients/sure/
+      - client_id: 'sure'
+        client_name: 'Sure'
+        client_secret: '{{ hostvars['console']['sure']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}/auth/openid_connect/callback'
+        scopes:
+          - 'openid'
+          - 'email'
+          - 'profile'
+          - 'groups'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'

config/services/containers/common/caddy/build/caddy.containerfile.j2

@@ -12,6 +12,6 @@ RUN xcaddy build \
 FROM docker.io/library/caddy:{{ version['containers']['caddy'] }}
 
 COPY --from=builder /usr/bin/caddy /usr/bin/caddy
-COPY ./ilnmors_root_ca.crt /usr/local/share/ca-certificates/ilnmors_root_ca.crt
+COPY ./{{ root_cert_filename }} /usr/local/share/ca-certificates/{{ root_cert_filename }}
 
 RUN update-ca-certificates

config/services/containers/common/caddy/caddy.container.j2

@@ -14,18 +14,18 @@ Wants=network-online.target
 
 
 [Container]
-Image=ilnmors.internal/{{ node['name'] }}/caddy:{{ version['containers']['caddy'] }}
+Image={{ domain['internal'] }}/{{ node['name'] }}/caddy:{{ version['containers']['caddy'] }}
 
 ContainerName=caddy_{{ node['name'] }}
 HostName=caddy_{{ node['name'] }}
 {% if node['name'] == 'infra' %}
-AddHost={{ infra_uri['ca']['domain'] }}:host-gateway
-AddHost={{ infra_uri['prometheus']['domain'] }}:host-gateway
-AddHost={{ infra_uri['loki']['domain'] }}:host-gateway
+AddHost={{ services['ca']['domain'] }}.{{ domain['internal'] }}:host-gateway
+AddHost={{ services['prometheus']['domain'] }}.{{ domain['internal'] }}:host-gateway
+AddHost={{ services['loki']['domain'] }}.{{ domain['internal'] }}:host-gateway
 {% endif %}
 
-PublishPort=2080:80/tcp
-PublishPort=2443:443/tcp
+PublishPort={{ services['caddy']['ports']['http'] }}:80/tcp
+PublishPort={{ services['caddy']['ports']['https'] }}:443/tcp
 
 Volume=%h/containers/caddy/etc:/etc/caddy:ro
 Volume=%h/containers/caddy/data:/data:rw
@@ -33,7 +33,7 @@ Volume=%h/containers/caddy/data:/data:rw
 Volume=/var/log/caddy:/log:rw
 {% endif %}
 
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 
 Secret=CADDY_ACME_KEY,target=/run/secrets/CADDY_ACME_KEY
 {% if node['name'] == 'auth' %}

config/services/containers/common/caddy/etc/app/Caddyfile.j2

@@ -8,23 +8,72 @@
 (private_tls) {
         tls {
                 issuer acme {
-                        dir https://{{ infra_uri['ca']['domain'] }}:{{ infra_uri['ca']['ports']['https'] }}/acme/acme@ilnmors.internal/directory
+                        dir https://{{ services['ca']['domain'] }}.{{ domain['internal'] }}:{{ services['ca']['ports']['https'] }}/acme/acme@{{ domain['internal'] }}/directory
                         dns rfc2136 {
-                                server {{ infra_uri['bind']['domain'] }}:{{ infra_uri['bind']['ports']['dns'] }}
+                                server {{ services['bind']['domain'] }}.{{ domain['internal'] }}:{{ services['bind']['ports']['dns'] }}
                                 key_name acme-key
                                 key_alg hmac-sha256
                                 key "{file./run/secrets/CADDY_ACME_KEY}"
                         }
+                        resolvers {{ services['bind']['domain'] }}.{{ domain['internal'] }}
                 }
         }
 }
 
-app.ilnmors.internal {
+{{ node['name'] }}.{{ domain['internal'] }} {
         import private_tls
         metrics
 }
-test.app.ilnmors.internal {
-         import private_tls
-         root * /usr/share/caddy
-         file_server
+# test.app.ilnmors.internal {
+#         import private_tls
+#         root * /usr/share/caddy
+#         file_server
+# }
+{{ services['vaultwarden']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['vaultwarden']['ports']['http'] }} {
+               header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['gitea']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['gitea']['ports']['http'] }} {
+               header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['immich']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['immich']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['paperless']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['paperless']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['affine']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['affine']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['nextcloud']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['nextcloud']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['collabora']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['collabora']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['sure']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['sure']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
 }

config/services/containers/common/caddy/etc/auth/Caddyfile.j2

@@ -1,7 +1,7 @@
 {
         # CrowdSec LAPI connection
         crowdsec {
-                api_url https://{{ infra_uri['crowdsec']['domain'] }}:{{ infra_uri['crowdsec']['ports']['https'] }} 
+                api_url https://{{ services['crowdsec']['domain'] }}.{{ domain['internal'] }}:{{ services['crowdsec']['ports']['https'] }} 
                 api_key "{file./run/secrets/CADDY_CROWDSEC_KEY}"
         }
 }
@@ -15,57 +15,130 @@
                         roll_size 100MiB
                         roll_keep 1
                 }
-        format json
+                format json
         }
 }
 # Private TLS ACME with DNS-01-challenge
 (private_tls) {
         tls {
                 issuer acme {
-                        dir https://{{ infra_uri['ca']['domain'] }}:{{ infra_uri['ca']['ports']['https'] }}/acme/acme@ilnmors.internal/directory
+                        dir https://{{ services['ca']['domain'] }}.{{ domain['internal'] }}:{{ services['ca']['ports']['https'] }}/acme/acme@{{ domain['internal'] }}/directory
                         dns rfc2136 {
-                                server {{ infra_uri['bind']['domain'] }}:{{ infra_uri['bind']['ports']['dns'] }}
+                                server {{ services['bind']['domain'] }}.{{ domain['internal'] }}:{{ services['bind']['ports']['dns'] }}
                                 key_name acme-key
                                 key_alg hmac-sha256
                                 key "{file./run/secrets/CADDY_ACME_KEY}"
                         }
+                        resolvers {{ services['bind']['domain'] }}.{{ domain['internal'] }}
                 }
         }
 }
 
 # Public domain
-authelia.ilnmors.com {
+{{ services['authelia']['domain'] }}.{{ domain['public'] }} {
         import crowdsec_log
         route {
                 crowdsec
-                reverse_proxy host.containers.internal:9091
+                reverse_proxy host.containers.internal:{{ services['authelia']['ports']['http'] }}
         }
 }
-test.ilnmors.com {
+# test.ilnmors.com {
+#         import crowdsec_log
+#         route {
+#                 crowdsec
+#                 forward_auth host.containers.internal:9091 {
+#                         # Authelia Forward Auth endpoint URI
+#                         uri /api/authz/forward-auth
+#                         copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
+#                 }
+#                 root * /usr/share/caddy
+#                 file_server
+#         }
+# }
+# test.app.ilnmors.com {
+#         import crowdsec_log
+#         route {
+#                 crowdsec
+#                 reverse_proxy https://test.app.ilnmors.internal {
+#                         header_up Host {http.reverse_proxy.upstream.host}
+#                 }
+#         }
+# }
+{{ services['vaultwarden']['domain']['public'] }}.{{ domain['public'] }} {
         import crowdsec_log
         route {
                 crowdsec
-                forward_auth host.containers.internal:9091 {
-                        # Authelia Forward Auth endpoint URI
-                        uri /api/authz/forward-auth
-                        copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
+                reverse_proxy https://{{ services['vaultwarden']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
                 }
-                root * /usr/share/caddy
-                file_server
         }
 }
-test.app.ilnmors.com {
+{{ services['gitea']['domain']['public'] }}.{{ domain['public'] }} {
         import crowdsec_log
         route {
                 crowdsec
-                reverse_proxy https://test.app.ilnmors.internal {
+                reverse_proxy https://{{ services['gitea']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
+{{ services['immich']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{ services['immich']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
+{{ services['paperless']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{ services['paperless']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }  
+}
+{{ services['affine']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{ services['affine']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
+{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{services['nextcloud']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
+{{ services['collabora']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{services['collabora']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
+{{ services['sure']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{services['sure']['domain']['internal'] }}.{{ domain['internal'] }} {
                         header_up Host {http.reverse_proxy.upstream.host}
                 }
         }
 }
 
 # Internal domain
-auth.ilnmors.internal {
+{{ node['name'] }}.{{ domain['internal'] }} {
         import private_tls
         metrics
 }

config/services/containers/common/caddy/etc/infra/Caddyfile.j2

@@ -2,39 +2,40 @@
 (private_tls) {
         tls {
                 issuer acme {
-                        dir https://{{ infra_uri['ca']['domain'] }}:{{ infra_uri['ca']['ports']['https'] }}/acme/acme@ilnmors.internal/directory
+                        dir https://{{ services['ca']['domain'] }}.{{ domain['internal'] }}:{{ services['ca']['ports']['https'] }}/acme/acme@{{ domain['internal'] }}/directory
                         dns rfc2136 {
-                                server {{ infra_uri['bind']['domain'] }}:{{ infra_uri['bind']['ports']['dns'] }}
+                                server {{ services['bind']['domain'] }}.{{ domain['internal'] }}:{{ services['bind']['ports']['dns'] }}
                                 key_name acme-key
                                 key_alg hmac-sha256
                                 key "{file./run/secrets/CADDY_ACME_KEY}"
                         }
+                        resolvers {{ services['bind']['domain'] }}.{{ domain['internal'] }}
                 }
         }
 }
 
-infra.ilnmors.internal {
+{{ node['name'] }}.{{ domain['internal'] }} {
         import private_tls
         metrics
 }
 
-{{ infra_uri['ldap']['domain'] }} {
+{{ services['ldap']['domain'] }}.{{ domain['internal'] }} {
         import private_tls
         route {
-            reverse_proxy host.containers.internal:{{ infra_uri['ldap']['ports']['http'] }}
+            reverse_proxy host.containers.internal:{{ services['ldap']['ports']['http'] }}
         }
 }
 
-{{ infra_uri['prometheus']['domain'] }} {
+{{ services['prometheus']['domain'] }}.{{ domain['internal'] }} {
         import private_tls
         route {
-                reverse_proxy https://{{ infra_uri['prometheus']['domain'] }}:{{ infra_uri['prometheus']['ports']['https'] }}
+                reverse_proxy https://{{ services['prometheus']['domain'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}
         }
 }
 
-grafana.ilnmors.internal {
+{{ services['grafana']['domain'] }}.{{ domain['internal'] }} {
         import private_tls
         route {
-                reverse_proxy host.containers.internal:3000
+                reverse_proxy host.containers.internal:{{ services['grafana']['ports']['http'] }}
         }
 }

config/services/containers/infra/ca/ca.container.j2

@@ -13,7 +13,7 @@ Image=docker.io/smallstep/step-ca:{{ version['containers']['step'] }}
 ContainerName=ca
 HostName=ca
 
-PublishPort=9000:9000/tcp
+PublishPort={{ services['ca']['ports']['https'] }}:9000/tcp
 
 Volume=%h/containers/ca/certs:/home/step/certs:ro
 Volume=%h/containers/ca/secrets:/home/step/secrets:ro
@@ -21,15 +21,18 @@ Volume=%h/containers/ca/config:/home/step/config:rw
 Volume=%h/containers/ca/db:/home/step/db:rw
 Volume=%h/containers/ca/templates:/home/step/templates:rw
 
-Environment="TZ=Asia/Seoul"
-Environment="PWDPATH=/run/secrets/STEP_CA_PASSWORD"
+Environment="TZ={{ timezone }}"
+# Since 0.30.0, Docker CMD no longer expands PWDPATH.
+#Environment="PWDPATH=/run/secrets/STEP_CA_PASSWORD"
 
 Secret=STEP_CA_PASSWORD,target=/run/secrets/STEP_CA_PASSWORD
 
+Exec=/usr/local/bin/step-ca --password-file /run/secrets/STEP_CA_PASSWORD /home/step/config/ca.json
+
 [Service]
 Restart=always
 RestartSec=10s
 TimeoutStopSec=120
 
 [Install]
-WantedBy=default.target
+WantedBy=default.target

config/services/containers/infra/ca/config/ca.json.j2

@@ -1,12 +1,12 @@
 {
-	"root": "/home/step/certs/ilnmors_root_ca.crt",
+	"root": "/home/step/certs/{{ root_cert_filename }}",
 	"federatedRoots": null,
-	"crt": "/home/step/certs/ilnmors_intermediate_ca.crt",
-	"key": "/home/step/secrets/ilnmors_intermediate_ca.key",
+	"crt": "/home/step/certs/{{ intermediate_cert_filename }}",
+	"key": "/home/step/secrets/{{ intermediate_key_filename }}",
 	"address": ":9000",
 	"insecureAddress": "",
 	"dnsNames": [
-		"{{ infra_uri['ca']['domain'] }}"
+		"{{ services['ca']['domain'] }}.{{ domain['internal'] }}"
 	],
 	"logger": {
 		"format": "text"
@@ -21,9 +21,9 @@
 			"x509": {
 				"allow": {
 					"dns": [
-						"ilnmors.internal",
-						"*.ilnmors.internal",
-						"*.app.ilnmors.internal"
+						"{{ domain['internal'] }}",
+						"*.{{ domain['internal'] }}",
+						"*.app.{{ domain['internal'] }}"
 					]
 				},
 				"allowWildcardNames": true
@@ -32,7 +32,7 @@
 		"provisioners": [
 			{
 				"type": "ACME",
-				"name": "acme@ilnmors.internal",
+				"name": "acme@{{ domain['internal'] }}",
 				"claims": {
 					"defaultTLSCertDuration": "2160h0m0s",
 					"enableSSHCA": true,
@@ -58,5 +58,5 @@
 		"maxVersion": 1.3,
 		"renegotiation": false
 	},
-	"commonName": "ilnmors Online CA"
+	"commonName": "{{ domain['internal'] }} Online CA"
 }

config/services/containers/infra/ca/config/defaults.json.j2

@@ -1,6 +1,6 @@
 {
-	"ca-url": "https://{{ infra_uri['ca']['domain'] }}:{{ infra_uri['ca']['ports']['https'] }}",
+	"ca-url": "https://{{ services['ca']['domain'] }}.{{ domain['internal'] }}:{{ services['ca']['ports']['https'] }}",
 	"ca-config": "/home/step/config/ca.json",
 	"fingerprint": "215c851d2d0d2dbf90fc3507425207c29696ffd587c640c94a68dddb1d84d8e8",
-	"root": "/home/step/certs/ilnmors_root_ca.crt"
+	"root": "/home/step/certs/{{ root_cert_filename }}"
 }

config/services/containers/infra/grafana/etc/grafana.ini.j2

@@ -7,19 +7,19 @@ provisioning = /etc/grafana/provisioning
 
 [server]
 protocol = http
-http_port = 3000
-domain = grafana.ilnmors.internal
-root_url = http://grafana.ilnmors.internal/
+http_port = {{ services['grafana']['ports']['http'] }}
+domain = {{ services['grafana']['domain'] }}.{{ domain['internal'] }}
+root_url = http://{{ services['grafana']['domain'] }}.{{ domain['internal'] }}/
 router_logging = false
 
 [database]
 type = postgres
-host = {{ infra_uri['postgresql']['domain'] }}:{{ infra_uri['postgresql']['ports']['tcp'] }}
+host = {{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}
 name = grafana_db
 user = grafana
 password = $__file{/run/secrets/GF_DB_PASSWORD}
 ssl_mode = verify-full
-ca_cert_path = /etc/ssl/grafana/ilnmors_root_ca.crt
+ca_cert_path = /etc/ssl/grafana/{{ root_cert_filename }}
 
 [auth.ldap]
 enabled = true

config/services/containers/infra/grafana/etc/ldap.toml.j2

@@ -1,7 +1,7 @@
 # https://github.com/lldap/lldap/blob/main/example_configs/grafana_ldap_config.toml
 [[servers]]
-host = "{{ infra_uri['ldap']['domain'] }}"
-port = {{ infra_uri['ldap']['ports']['ldaps'] }}
+host = "{{ services['ldap']['domain'] }}.{{ domain['internal'] }}"
+port = {{ services['ldap']['ports']['ldaps'] }}
 # Activate STARTTLS or LDAPS
 use_ssl = true
 # true = STARTTLS, false = LDAPS
@@ -9,16 +9,16 @@ start_tls = false
 tls_ciphers = []
 min_tls_version = ""
 ssl_skip_verify = false
-root_ca_cert = "/etc/ssl/grafana/ilnmors_root_ca.crt"
+root_ca_cert = "/etc/ssl/grafana/{{ root_cert_filename }}"
 # mTLS option, it is not needed
 # client_cert = "/path/to/client.crt"
 # client_key = "/path/to/client.key"
 
-bind_dn = "uid=grafana,ou=people,dc=ilnmors,dc=internal"
+bind_dn = "uid=grafana,ou=people,{{ domain['dc'] }}"
 bind_password = "$__file{/run/secrets/LDAP_BIND_PASSWORD}"
 
 search_filter = "(|(uid=%s)(mail=%s))"
-search_base_dns = ["dc=ilnmors,dc=internal"]
+search_base_dns = ["{{ domain['dc'] }}"]
 
 [servers.attributes]
 member_of = "memberOf"
@@ -28,20 +28,20 @@ surname = "sn"
 username = "uid"
 
 group_search_filter = "(&(objectClass=groupOfUniqueNames)(uniqueMember=%s))"
-group_search_base_dns = ["ou=groups,dc=ilnmors,dc=internal"]
+group_search_base_dns = ["ou=groups,{{ domain['dc'] }}"]
 group_search_filter_user_attribute = "uid"
 
 [[servers.group_mappings]]
-group_dn = "cn=lldap_admin,ou=groups,dc=ilnmors,dc=internal"
+group_dn = "cn=lldap_admin,ou=groups,{{ domain['dc'] }}"
 org_role = "Admin"
 grafana_admin = true
 
 [[servers.group_mappings]]
-group_dn = "cn=admins,ou=groups,dc=ilnmors,dc=internal"
+group_dn = "cn=admins,ou=groups,{{ domain['dc'] }}"
 org_role = "Editor"
 grafana_admin = false
 
 [[servers.group_mappings]]
-group_dn = "cn=users,ou=groups,dc=ilnmors,dc=internal"
+group_dn = "cn=users,ou=groups,{{ domain['dc'] }}"
 org_role = "Viewer"
 grafana_admin = false

config/services/containers/infra/grafana/etc/provisioning/datasources/datasources.yaml.j2

@@ -4,7 +4,7 @@ apiVersion: 1
 datasources:
   - name: Prometheus
     type: prometheus
-    url: https://prometheus.ilnmors.internal:9090
+    url: https://{{ services['prometheus']['domain'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}
     access: proxy
     isDefault: true
     jsonData:
@@ -12,11 +12,11 @@ datasources:
       tlsAuthWithCACert: true
       httpMethod: POST
     secureJsonData:
-      tlsCACert: "$__file{/etc/ssl/grafana/ilnmors_root_ca.crt}"
+      tlsCACert: "$__file{/etc/ssl/grafana/{{ root_cert_filename }}}"
 
   - name: Loki
     type: loki
-    url: https://loki.ilnmors.internal:3100
+    url: https://{{ services['loki']['domain'] }}.{{ domain['internal'] }}:{{ services['loki']['ports']['https'] }}
     access: proxy
     jsonData:
       tlsAuth: false
@@ -25,5 +25,5 @@ datasources:
       httpHeaderName1: "X-Scope-OrgID"
       maxLines: 1000
     secureJsonData:
-      tlsCACert: "$__file{/etc/ssl/grafana/ilnmors_root_ca.crt}"
-      httpHeaderValue1: "ilnmors.internal"
+      tlsCACert: "$__file{/etc/ssl/grafana/{{ root_cert_filename }}}"
+      httpHeaderValue1: "{{ domain['internal'] }} "

config/services/containers/infra/grafana/grafana.container.j2

@@ -13,18 +13,18 @@ Image=docker.io/grafana/grafana:{{ version['containers']['grafana'] }}
 ContainerName=grafana
 HostName=grafana
 
-AddHost={{ infra_uri['postgresql']['domain'] }}:host-gateway
-AddHost={{ infra_uri['ldap']['domain'] }}:host-gateway
-AddHost={{ infra_uri['prometheus']['domain'] }}:host-gateway
-AddHost={{ infra_uri['loki']['domain'] }}:host-gateway
+AddHost={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:host-gateway
+AddHost={{ services['ldap']['domain'] }}.{{ domain['internal'] }}:host-gateway
+AddHost={{ services['prometheus']['domain'] }}.{{ domain['internal'] }}:host-gateway
+AddHost={{ services['loki']['domain'] }}.{{ domain['internal'] }}:host-gateway
 
-PublishPort=3000:3000/tcp
+PublishPort={{ services['grafana']['ports']['http'] }}:3000/tcp
 
 Volume=%h/containers/grafana/data:/var/lib/grafana:rw
 Volume=%h/containers/grafana/etc:/etc/grafana:ro
 Volume=%h/containers/grafana/ssl:/etc/ssl/grafana:ro
 
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 Environment="GF_PATHS_CONFIG=/etc/grafana/grafana.ini"
 # plugin
 # Environment="GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource"

config/services/containers/infra/ldap/ldap.container.j2

@@ -13,21 +13,21 @@ Image=docker.io/lldap/lldap:{{ version['containers']['ldap'] }}
 ContainerName=ldap
 HostName=ldap
 # They are at the same host (for Pasta, it is needed)
-AddHost={{ infra_uri['postgresql']['domain'] }}:host-gateway
+AddHost={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:host-gateway
 # For LDAPS - 636 > 6360 nftables
-PublishPort=6360:6360/tcp
+PublishPort={{ services['ldap']['ports']['ldaps'] }}:6360/tcp
 # Web UI
-PublishPort=17170:17170/tcp
+PublishPort={{ services['ldap']['ports']['http'] }}:17170/tcp
 
 
 Volume=%h/containers/ldap/data:/data:rw
 Volume=%h/containers/ldap/ssl:/etc/ssl/ldap:ro
 
 # Default
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 
 # Domain
-Environment="LLDAP_LDAP_BASE_DN=dc=ilnmors,dc=internal"
+Environment="LLDAP_LDAP_BASE_DN={{ domain['dc'] }}"
 
 # LDAPS
 Environment="LLDAP_LDAPS_OPTIONS__ENABLED=true"

config/services/containers/infra/loki/etc/loki.yaml.j2

@@ -1,7 +1,7 @@
 ---
 server:
   http_listen_address: "::"
-  http_listen_port: 3100
+  http_listen_port: {{ services['loki']['ports']['https'] }}
   http_tls_config:
     cert_file: /etc/ssl/loki/loki.crt
     key_file: /etc/ssl/loki/loki.key

config/services/containers/infra/loki/loki.container.j2

@@ -13,13 +13,13 @@ Image=docker.io/grafana/loki:{{ version['containers']['loki'] }}
 ContainerName=loki
 HostName=loki
 
-PublishPort=3100:3100/tcp
+PublishPort={{ services['loki']['ports']['https'] }}:3100/tcp
 
 Volume=%h/containers/loki/data:/loki:rw
 Volume=%h/containers/loki/etc:/etc/loki:ro
 Volume=%h/containers/loki/ssl:/etc/ssl/loki:ro
 
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 
 Exec=--config.file=/etc/loki/loki.yaml
 

config/services/containers/infra/postgresql/config/postgresql.conf.j2

@@ -8,11 +8,11 @@ listen_addresses = '*'
 # Max connections
 max_connections = 250
 # listen_port
-port = 5432
+port = {{ services['postgresql']['ports']['tcp'] }}
 
 # SSL
 ssl = on
-ssl_ca_file = '/etc/ssl/postgresql/ilnmors_root_ca.crt'
+ssl_ca_file = '/etc/ssl/postgresql/{{ root_cert_filename }}'
 ssl_cert_file = '/etc/ssl/postgresql/postgresql.crt'
 ssl_key_file = '/etc/ssl/postgresql/postgresql.key'
 ssl_ciphers = 'HIGH:!aNULL:!MD5'

config/services/containers/infra/postgresql/postgresql.container.j2

@@ -8,12 +8,12 @@ After=network-online.target
 Wants=network-online.target
 
 [Container]
-Image=ilnmors.internal/{{ node['name'] }}/postgres:pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}
+Image={{ domain['internal'] }}/{{ node['name'] }}/postgres:pg{{ version['containers']['postgresql'] }}-vectorchord{{ version['containers']['vectorchord'] }}
 
 ContainerName=postgresql
 HostName=postgresql
 
-PublishPort=5432:5432/tcp
+PublishPort={{ services['postgresql']['ports']['tcp'] }}:5432/tcp
 
 Volume=%h/containers/postgresql/data:/var/lib/postgresql:rw
 Volume=%h/containers/postgresql/config:/config:ro
@@ -21,7 +21,7 @@ Volume=%h/containers/postgresql/ssl:/etc/ssl/postgresql:ro
 Volume=%h/containers/postgresql/init:/docker-entrypoint-initdb.d/:ro
 Volume=%h/containers/postgresql/backups:/backups:rw
 
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 # This option is only for init process, after init custom config file `pg_hba.conf` will control this option.
 Environment="POSTGRES_HOST_AUTH_METHOD=trust"
 

config/services/containers/infra/prometheus/etc/prometheus.yaml.j2

@@ -23,8 +23,8 @@ scrape_configs:
     # metrics_path defaults to '/metrics'
     scheme: "https"
     tls_config:
-      ca_file: "/etc/ssl/prometheus/ilnmors_root_ca.crt"
-      server_name: "{{ infra_uri['prometheus']['domain'] }}"
+      ca_file: "/etc/ssl/prometheus/{{ root_cert_filename }}"
+      server_name: "{{ services['prometheus']['domain'] }}.{{ domain['internal'] }}"
     static_configs:
       - targets: ["localhost:9090"]
        # The label name is added as a label `label_name=<label_value>` to any timeseries scraped from this config.

config/services/containers/infra/prometheus/prometheus.container.j2

@@ -13,13 +13,13 @@ Image=docker.io/prom/prometheus:{{ version['containers']['prometheus'] }}
 ContainerName=prometheus
 HostName=prometheus
 
-PublishPort=9090:9090/tcp
+PublishPort={{ services['prometheus']['ports']['https'] }}:9090/tcp
 
 Volume=%h/containers/prometheus/data:/prometheus:rw
 Volume=%h/containers/prometheus/etc:/etc/prometheus:ro
 Volume=%h/containers/prometheus/ssl:/etc/ssl/prometheus:ro
 
-Environment="TZ=Asia/Seoul"
+Environment="TZ={{ timezone }}"
 
 Exec=--config.file=/etc/prometheus/prometheus.yaml \
      --web.config.file=/etc/prometheus/web-config.yaml \

config/services/containers/infra/x509-exporter/config/config.yaml

@@ -0,0 +1,11 @@
+server:
+  listen: :9793
+
+sources:
+  - kind: file
+    name: homelab-certs
+    paths:
+      - /certs/*.crt
+      - /certs/*.pem
+      - /certs/*.cer
+    refreshInterval: 1m

config/services/containers/infra/x509-exporter/x509-exporter.container.j2

@@ -11,11 +11,12 @@ Image=docker.io/enix/x509-certificate-exporter:{{ version['containers']['x509-ex
 ContainerName=x509-exporter
 HostName=X509-exporter
 
+Volume=%h/containers/x509-exporter/config/config.yaml:/etc/config.yaml:ro
 Volume=%h/containers/x509-exporter/certs:/certs:ro
 
-PublishPort=9793:9793
+PublishPort={{ services['x509-exporter']['ports']['http'] }}:9793
 
-Exec=--listen-address :9793 --watch-dir=/certs
+Exec=--config /etc/config.yaml
 
 [Service]
 Restart=always

config/services/systemd/app/btrfs/btrfs-scrub.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=BTRFS auto scrub
+ConditionPathIsMountPoint={{ storage['btrfs']['mount_point'] }}
+RequiresMountsFor={{ storage['btrfs']['mount_point'] }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/btrfs scrub start -Bd {{ storage['btrfs']['mount_point'] }}
+Nice=19
+IOSchedulingClass=idle

config/services/systemd/app/btrfs/btrfs-scrub.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Monthly BTRFS auto scrub 
+
+[Timer]
+OnCalendar=*-*-01 04:00:00
+Persistent=true
+RandomizedDelaySec=300
+
+[Install]
+WantedBy=timers.target

config/services/systemd/common/alloy/config.alloy.j2

@@ -6,7 +6,7 @@
 //// Metric ouput
 prometheus.remote_write "prometheus" {
     endpoint {
-        url = "https://{{ infra_uri['prometheus']['domain'] }}:{{ infra_uri['prometheus']['ports']['https'] }}/api/v1/write"
+        url = "https://{{ services['prometheus']['domain'] }}.{{ domain['internal'] }}:{{ services['prometheus']['ports']['https'] }}/api/v1/write"
     }
 }
 
@@ -71,8 +71,8 @@ prometheus.scrape "system" {
 ////// For Crowdsec metrics
 prometheus.scrape "crowdsec" {
     targets = [
-        { "__address__" = "{{ infra_uri['crowdsec']['domain'] }}:6060", "job" = "crowdsec" },
-        { "__address__" = "{{ infra_uri['crowdsec']['domain'] }}:60601", "job" = "crowdsec-bouncer" },
+        { "__address__" = "{{ services['crowdsec']['domain'] }}.{{ domain['internal'] }}:6060", "job" = "crowdsec" },
+        { "__address__" = "{{ services['crowdsec']['domain'] }}.{{ domain['internal'] }}:60601", "job" = "crowdsec-bouncer" },
     ]
     honor_labels = true
     forward_to = [prometheus.relabel.default_label.receiver]
@@ -83,7 +83,7 @@ prometheus.scrape "crowdsec" {
 ////// For postgresql metrics
 prometheus.exporter.postgres "postgresql" {
     data_source_names = [
-        "postgres://alloy@{{ infra_uri['postgresql']['domain'] }}:{{ infra_uri['postgresql']['ports']['tcp'] }}/postgres?sslmode=verify-full",
+        "postgres://alloy@{{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}/postgres?sslmode=verify-full",
     ]
 }
 prometheus.scrape "postgresql" {
@@ -93,7 +93,7 @@ prometheus.scrape "postgresql" {
 ///// For certificates metrics
 prometheus.scrape "x509" {
     targets = [
-        { "__address__" = "{{ node['name'] }}.ilnmors.internal:9793" },
+        { "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:{{ services['x509-exporter']['ports']['http'] }}" },
     ]
     forward_to = [prometheus.relabel.default_label.receiver]
 }
@@ -103,7 +103,7 @@ prometheus.scrape "x509" {
 ////// For Input Caddy metrics
 prometheus.scrape "caddy" {
     targets = [
-        { "__address__" = "{{ node['name'] }}.ilnmors.internal:443" },
+        { "__address__" = "{{ node['name'] }}.{{ domain['internal'] }}:443" },
     ]
     scheme = "https"
     forward_to = [prometheus.relabel.default_label.receiver]
@@ -114,8 +114,8 @@ prometheus.scrape "caddy" {
 //// Logs output
 loki.write "loki" {
     endpoint {
-        url = "https://{{ infra_uri['loki']['domain'] }}:{{ infra_uri['loki']['ports']['https'] }}/loki/api/v1/push"
-        tenant_id = "ilnmors.internal"
+        url = "https://{{ services['loki']['domain'] }}.{{ domain['internal'] }}:{{ services['loki']['ports']['https'] }}/loki/api/v1/push"
+        tenant_id = "{{ domain['internal'] }}"
     }
 }
 //// Logs relabel
@@ -203,12 +203,11 @@ loki.relabel "caddy_relabel" {
 loki.process "journal_parser" {
         forward_to = [loki.write.loki.receiver]
         // Severity parsing
-        // If content of log includes "level" information, change the level
-        stage.logfmt {
-            mapping = {
-                "content_level" = "level",
-            }
+        stage.regex {
+            // Regex to extract the log level from the content.
+            expression = "(?i)(?:level[\"\\s:=]+|\\[|\\s|^)(?P<content_level>info|warn|warning|error|debug|fatal|critical|trace)(?:[\"\\]\\s]|$)"
         }
+        
         stage.labels {
             values = {
                 "level" = "content_level",

config/services/systemd/common/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml.j2

@@ -8,7 +8,7 @@ log_compression: true
 log_max_size: 100
 log_max_backups: 3
 log_max_age: 30
-api_url: "https://{{ infra_uri['crowdsec']['domain'] }}:{{ infra_uri['crowdsec']['ports']['https'] }}"
+api_url: "https://{{ services['crowdsec']['domain'] }}.{{ domain['internal'] }}:{{ services['crowdsec']['ports']['https'] }}"
 api_key: "{{ hostvars['console']['crowdsec']['bouncer']['fw'] }}"
 insecure_skip_verify: false
 disable_ipv6: false

config/services/systemd/common/crowdsec/bouncers/whitelists.yaml.j2

@@ -1,11 +1,21 @@
 name: crowdsecurity/whitelists
-description: "Whitelist console/admin hosts only"
+description: "Local whitelist policy"
 whitelist:
-    reason: "trusted admin hosts"
+    reason: "rules"
     ip:
+        # Console IP lists
         - "127.0.0.1"
         - "::1"
         - "{{ hostvars['fw']['network4']['console']['client'] }}"
         - "{{ hostvars['fw']['network4']['console']['wg'] }}"
         - "{{ hostvars['fw']['network6']['console']['client'] }}"
         - "{{ hostvars['fw']['network6']['console']['wg'] }}"
+{% if node['name'] == 'auth' %}
+    expression:
+        # immich thumbnail request 404 error false positive
+        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
+        # nextcloud thumbnail/preview request error false positive
+        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'"
+        # nextcloud chunks.mjs request false positive
+        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains 'chunk.mjs'"
+{% endif %}

config/services/systemd/common/crowdsec/etc/local_api_credentials.yaml.j2

@@ -1,3 +1,3 @@
-url: https://{{ infra_uri['crowdsec']['domain'] }}:{{ infra_uri['crowdsec']['ports']['https'] }}
+url: https://{{ services['crowdsec']['domain'] }}.{{ domain['internal'] }}:{{ services['crowdsec']['ports']['https'] }}
 login: {{ node['name'] }}
 password: {{ hostvars['console']['crowdsec']['machine'][node['name']] }}