chore(app): archive removed stacks from app

archived stacks: - actual-budget - ezbookkeeping - opencloud - trilium - vikunja - wikijs
feat(trilium): release trilium
2026-05-10 00:07:51 +09:00 · 2026-05-09 22:38:57 +09:00 · 2026-05-09 21:44:11 +09:00 · 2026-05-09 20:54:32 +09:00 · 2026-05-09 17:50:05 +09:00 · 2026-05-09 17:42:35 +09:00
117 changed files with 1858 additions and 627 deletions
@@ -1,6 +1,6 @@
 # ilnmors homelab README

-This homelab project implements single-node On-premise IaaS system. The homelab contains virtual machines which are divided by their roles, such as private firewall, DNS, PKI, LDAP and database, SSO\(OIDC\). The standard domain is used to implement this system without specific vendors. All components are defined as code and initiated by IaC \(Ansible\) except hypervisor initial configuration.
+This homelab project implements single-node On-premise IaaS system. The homelab contains virtual machines which are divided by their roles, such as private firewall, DNS, PKI, LDAP and database, SSO(OIDC). The standard domain is used to implement this system without specific vendors. All components are defined as code and initiated by IaC (Ansible) except hypervisor initial configuration.

 ## RTO times
 - Feb/25/2026 - Reprovisioning Hypervisor and vms
@@ -15,12 +15,12 @@ This homelab project implements single-node On-premise IaaS system. The homelab
 - Mar/5/2026 - Reprovisioning Hardware and Hypervisor and vms
    - RTO: 2 hour 20 min
        - console: 15min - verified
-        - certificate: 0 min \(When it needs to be created, RTO will be 20 min) - not verified
-        - wireguard: 0 min \(When it needs to be created, RTO will be 1 min) - not verified
-        - hypervisor\(+fw\): 45 min - verified
+        - certificate: 0 min (When it needs to be created, RTO will be 20 min) - not verified
+        - wireguard: 0 min (When it needs to be created, RTO will be 1 min) - not verified
+        - hypervisor(+fw): 45 min - verified
        - switch: 1 min - verified
        - dsm: 30 min - verified
-        - kopia: 0 min \(When it needs to be created, RTO will be 10 min) - verified
+        - kopia: 0 min (When it needs to be created, RTO will be 10 min) - verified
        - Extra vms: 30 min - verified
        - Etc: 30 min

@@ -66,7 +66,7 @@ services:
  grafana:
    domain: "grafana"
    ports:
-      http: "3000"
+      http: "3000" # Infra server: Internal ports
    subuid: "100471"
  caddy:
    ports:
@@ -97,7 +97,7 @@ services:
      public: "gitea"
      internal: "gitea.app"
    ports:
-      http: "3000"
+      http: "3000" # App server: Public ports
    subuid: "100999"
  immich:
    domain:
@@ -109,13 +109,6 @@ services:
  immich-ml:
    ports:
      http: "3003"
-  actualbudget:
-    domain:
-      public: "budget"
-      internal: "budget.app"
-    ports:
-      http: "5006"
-    subuid: "101000"
  paperless:
    domain:
      public: "paperless"
@@ -124,20 +117,6 @@ services:
      http: "8001"
      redis: "6380"
    subuid: "100999"
-  vikunja:
-    domain:
-      public: "vikunja"
-      internal: "vikunja.app"
-    ports:
-      http: "3456"
-    subuid: "100999"
-  opencloud:
-    domain:
-      public: "opencloud"
-      internal: "opencloud.app"
-    ports:
-      http: "9200"
-    subuid: "100999"
  manticore:
    subuid: "100998"
  affine:
@@ -156,6 +135,21 @@ services:
      http: "8002"
      redis: "6382"
    subuid: "100032"
+  collabora:
+    domain:
+      public: "collabora"
+      internal: "collabora.app"
+    ports:
+      http: "9980"
+    subuid: "101000"
+  sure:
+    domain:
+      public: "sure"
+      internal: "sure.app"
+    ports:
+      http: "3001"
+      redis: "6383"
+    subuid: "100999"

 version:
  packages:
@@ -163,33 +157,32 @@ version:
    step: "0.30.2"
    kopia: "0.22.3"
    blocky: "0.29.0"
-    alloy: "1.13.0"
+    alloy: "1.16.1"
  containers:
    # common
    caddy: "2.11.2"
    # infra
    step: "0.30.2"
-    ldap: "v0.6.2"
-    x509-exporter: "3.21.0"
-    prometheus: "v3.9.1"
-    loki: "3.6.5"
-    grafana: "12.3.3"
+    ldap: "v0.6.3"
+    x509-exporter: "4.1.0"
+    prometheus: "v3.11.3"
+    loki: "3.7.1"
+    grafana: "13.0.1"
    ## Postgresql
-    postgresql: "18.2"
+    postgresql: "18.3"
    # For immich - https://github.com/immich-app/base-images/blob/main/postgres/versions.yaml
 #    pgvector: "v0.8.1"
-    vectorchord: "0.5.3"
+    vectorchord: "1.1.1"
    # Auth
    authelia: "4.39.19"
    # App
-    vaultwarden: "1.35.8"
+    vaultwarden: "1.36.0"
    gitea: "1.26.1"
-    redis: "8.6.1"
+    redis: "8.6.3"
    immich: "v2.7.5"
-    actualbudget: "26.3.0"
    paperless: "2.20.15"
-    vikunja: "2.2.2"
-    opencloud: "4.0.6"
    manticore: "25.0.0"
    affine: "0.26.3"
    nextcloud: "33.0.3"
+    collabora: "25.04.9.4.1"
+    sure: "0.7.0-hotfix.2"
@@ -185,14 +185,6 @@
          tags: ["site", "immich"]
      tags: ["site", "immich"]

-    - name: Set actual budget
-      ansible.builtin.include_role:
-        name: "app"
-        tasks_from: "services/set_actual-budget"
-        apply:
-          tags: ["site", "actual-budget"]
-      tags: ["site", "actual-budget"]
-
    - name: Set paperless
      ansible.builtin.include_role:
        name: "app"
@@ -201,22 +193,6 @@
          tags: ["site", "paperless"]
      tags: ["site", "paperless"]

-    - name: Set vikunja
-      ansible.builtin.include_role:
-        name: "app"
-        tasks_from: "services/set_vikunja"
-        apply:
-          tags: ["site", "vikunja"]
-      tags: ["site", "vikunja"]
-
-    - name: Set opencloud
-      ansible.builtin.include_role:
-        name: "app"
-        tasks_from: "services/set_opencloud"
-        apply:
-          tags: ["site", "opencloud"]
-      tags: ["site", "opencloud"]
-
    - name: Set affine
      ansible.builtin.include_role:
        name: "app"
@@ -233,6 +209,21 @@
          tags: ["site", "nextcloud"]
      tags: ["site", "nextcloud"]

+    - name: Set collabora
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_collabora"
+        apply:
+          tags: ["site", "collabora"]
+      tags: ["site", "collabora"]
+
+    - name: Set sure
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_sure"
+        apply:
+          tags: ["site", "sure"]
+      tags: ["site", "sure"]

    - name: Flush handlers right now
      ansible.builtin.meta: "flush_handlers"
@@ -122,3 +122,11 @@
        apply:
          tags: ["init", "site", "tools"]
      tags: ["init", "site", "tools"]
+
+    - name: Set kopia
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_kopia"
+        apply:
+          tags: ["init", "site", "kopia"]
+      tags: ["init", "site", "kopia"]
@@ -43,17 +43,6 @@
  listen: "notification_restart_immich-ml"
  ignore_errors: true # noqa: ignore-errors

- name: Restart actual-budget
-  ansible.builtin.systemd:
-    name: "actual-budget.service"
-    state: "restarted"
-    enabled: true
-    scope: "user"
-    daemon_reload: true
-  changed_when: false
-  listen: "notification_restart_actual-budget"
-  ignore_errors: true # noqa: ignore-errors
-
 - name: Restart paperless
  ansible.builtin.systemd:
    name: "paperless.service"
@@ -65,29 +54,6 @@
  listen: "notification_restart_paperless"
  ignore_errors: true # noqa: ignore-errors

- name: Restart vikunja
-  ansible.builtin.systemd:
-    name: "vikunja.service"
-    state: "restarted"
-    enabled: true
-    scope: "user"
-    daemon_reload: true
-  changed_when: false
-  listen: "notification_restart_vikunja"
-  ignore_errors: true # noqa: ignore-errors
-
- name: Restart opencloud
-  ansible.builtin.systemd:
-    name: "opencloud.service"
-    state: "restarted"
-    enabled: true
-    daemon_reload: true
-    scope: "user"
-  when: is_opencloud_init.stat.exists
-  changed_when: false
-  listen: "notification_restart_opencloud"
-  ignore_errors: true # noqa: ignore-errors
-
 - name: Restart affine
  ansible.builtin.systemd:
    name: "affine.service"
@@ -111,3 +77,28 @@
  changed_when: false
  listen: "notification_restart_nextcloud"
  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart collabora
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_collabora"
+  ignore_errors: true # noqa: ignore-errors
+
+- name: Restart sure
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  loop:
+    - "sure-web.service"
+    - "sure-worker.service"
+  changed_when: false
+  listen: "notification_restart_sure"
+  ignore_errors: true # noqa: ignore-errors
@@ -68,3 +68,23 @@
    group: "svadmins"
    mode: "0770"
  become: true
+
+- name: Deploy btrfs scrub service and timer
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/app/btrfs/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  loop:
+    - "btrfs-scrub.service"
+    - "btrfs-scrub.timer"
+  become: true
+
+- name: Enable auto btrfs scrub
+  ansible.builtin.systemd:
+    name: "btrfs-scrub.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+  become: true
@@ -0,0 +1,17 @@
+---
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/collabora/collabora.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/collabora.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_collabora"
+
+- name: Enable collabora.service
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
@@ -0,0 +1,110 @@
+---
+- name: Set redis service name
+  ansible.builtin.set_fact:
+    redis_service: "sure"
+
+- name: Create redis_sure directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['redis']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "containers/redis"
+    - "containers/redis/{{ redis_service }}"
+    - "containers/redis/{{ redis_service }}/data"
+  become: true
+
+- name: Deploy redis config file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.conf.j2"
+    dest: "{{ node['home_path'] }}/containers/redis/{{ redis_service }}/redis.conf"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_conf"
+
+- name: Deploy redis container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/redis/redis.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/redis_{{ redis_service }}.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  register: "is_redis_containerfile"
+
+- name: Enable (Restart) redis service
+  ansible.builtin.systemd:
+    name: "redis_{{ redis_service }}.service"
+    state: "restarted"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  when: is_redis_conf.changed or is_redis_containerfile.changed # noqa: no-handler
+
+- name: Create sure directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['sure']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/sure"
+    - "data/containers/sure/storage"
+    - "containers/sure"
+    - "containers/sure/ssl"
+  become: true
+
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/sure/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['paperless']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_sure"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "SURE_SECRET_KEY_BASE"
+      value: "{{ hostvars['console']['sure']['session_secret'] }}"
+    - name: "SURE_POSTGRES_PASSWORD"
+      value: "{{ hostvars['console']['postgresql']['password']['sure'] }}"
+    - name: "SURE_OIDC_CLIENT_SECRET"
+      value: "{{ hostvars['console']['sure']['oidc']['secret'] }}"
+  notify: "notification_restart_sure"
+  no_log: true
+
+- name: Deploy sure.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/sure/{{ item }}.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/{{ item }}"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  loop:
+    - "sure-web.container"
+    - "sure-worker.container"
+  notify: "notification_restart_sure"
+
+- name: Enable paperless.service
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
+  loop:
+    - "sure-web.service"
+    - "sure-worker.service"
@@ -5,9 +5,10 @@
      - hardware
  become: true

- name: Deploy alloy deb file (x86_64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-amd64.deb"
+- name: Download alloy deb file (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.amd64.deb"
    dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
    owner: "root"
    group: "root"
@@ -15,9 +16,10 @@
  become: true
  when: ansible_facts['architecture'] == "x86_64"

- name: Deploy alloy deb file (aarch64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-arm64.deb"
+- name: Download alloy deb file (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.arm64.deb"
    dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
    owner: "root"
    group: "root"
@@ -30,6 +32,7 @@
    deb: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
    state: "present"
  become: true
+  notify: "notification_restart_alloy"

 - name: Deploy alloy config
  ansible.builtin.template:
@@ -5,34 +5,36 @@
      - hardware
  become: true

- name: Check kopia installation
-  ansible.builtin.shell: |
-    command -v kopia
-  changed_when: false
-  failed_when: false
-  register: "is_kopia_installed"
-  ignore_errors: true
-
 - name: Set console kopia
  when: node['name'] == 'console'
  block:
-    - name: Apply cli tools (x86_64)
+    - name: Download kopia
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
+        dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
+        mode: "0600"
+      loop:
+        - "amd64"
+        - "arm64"
+
+    - name: Install kopia (x86_64)
      ansible.builtin.apt:
        deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
        state: "present"
      become: true
-      when:
-        - ansible_facts['architecture'] == "x86_64"
-        - is_kopia_installed.rc != 0
-    - name: Apply cli tools (aarch64)
+      when: ansible_facts['architecture'] == "x86_64"
+
+    - name: Install kopia (aarch64)
      ansible.builtin.apt:
        deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
        state: "present"
      become: true
-      when:
-        - ansible_facts['architecture'] == "aarch64"
-        - is_kopia_installed.rc != 0
-    - name: Connect kopia server
+      when: ansible_facts['architecture'] == "aarch64"
+
+    - name: Connect console kopia server
      environment:
        KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['user']['console'] }}"
      ansible.builtin.shell: |
@@ -51,30 +53,36 @@
    - name: Set kopia uid
      ansible.builtin.set_fact:
        kopia_uid: 951
-    - name: Deploy kopia deb file (x86_64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
+
+    - name: Download kopia deb file (x86_64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_amd64.deb"
        dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
        owner: "root"
        group: "root"
        mode: "0644"
      become: true
      when: ansible_facts['architecture'] == "x86_64"
-    - name: Deploy kopia deb file (aarch64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
+
+    - name: Download kopia deb file (aarch64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_arm64.deb"
        dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
        owner: "root"
        group: "root"
        mode: "0644"
      become: true
      when: ansible_facts['architecture'] == "aarch64"
+
    - name: Create kopia group
      ansible.builtin.group:
        name: "kopia"
        gid: "{{ kopia_uid }}"
        state: "present"
      become: true
+
    - name: Create kopia user
      ansible.builtin.user:
        name: "kopia"
@@ -85,6 +93,7 @@
        comment: "Kopia backup User"
        state: "present"
      become: true
+
    - name: Create kopia directory
      ansible.builtin.file:
        path: "{{ item.name }}"
@@ -101,12 +110,13 @@
          mode: "0700"
      become: true
      no_log: true
+
    - name: Install kopia
      ansible.builtin.apt:
        deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
        state: "present"
      become: true
-      when: is_kopia_installed.rc != 0
+
    - name: Deploy kopia env
      ansible.builtin.template:
        src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2"
@@ -116,6 +126,7 @@
        mode: "0400"
      become: true
      no_log: true
+
    - name: Deploy kopia service files
      ansible.builtin.template:
        src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/{{ item }}.j2"
@@ -128,6 +139,7 @@
        - "kopia-backup.service"
        - "kopia-backup.timer"
      become: true
+
    - name: Enable auto kopia rules update
      ansible.builtin.systemd:
        name: "kopia-backup.timer"
@@ -49,42 +49,6 @@
    - "amd64"
    - "arm64"

- name: Download kopia
-  ansible.builtin.get_url:
-    url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
-      kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
- name: Download blocky
-  ansible.builtin.get_url:
-    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
-      blocky_v{{ version['packages']['blocky'] }}_Linux_{{ item }}.tar.gz"
-    dest: "{{ node['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-{{ item }}.tar.gz"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "x86_64"
-    - "arm64"
-
- name: Download alloy
-  ansible.builtin.get_url:
-    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
-      alloy-{{ version['packages']['alloy'] }}-1.{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
 - name: Apply cli tools (x86_64)
  ansible.builtin.apt:
    deb: "{{ node['data_path'] }}/bin/{{ item }}"
@@ -92,7 +56,6 @@
  loop:
    - "sops-{{ version['packages']['sops'] }}-amd64.deb"
    - "step-{{ version['packages']['step'] }}-amd64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-amd64.deb"
  become: true
  when: ansible_facts['architecture'] == "x86_64"

@@ -103,6 +66,5 @@
  loop:
    - "sops-{{ version['packages']['sops'] }}-arm64.deb"
    - "step-{{ version['packages']['step'] }}-arm64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-arm64.deb"
  become: true
  when: ansible_facts['architecture'] == "aarch64"
@@ -23,7 +23,7 @@
    state: "present"
  become: true

- name: Create blocky etc directory
+- name: Create blocky directory
  ansible.builtin.file:
    path: "{{ item }}"
    owner: "blocky"
@@ -31,13 +31,38 @@
    mode: "0750"
    state: "directory"
  loop:
+    - "/home/blocky"
+    - "/home/blocky/bin"
    - "/etc/blocky"
    - "/etc/blocky/ssl"
  become: true

+- name: Download blocky (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_x86_64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "x86_64"
+
+- name: Download blocky (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_arm64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "aarch64"
+
 - name: Deploy blocky binary file (x86_64)
  ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    remote_src: true
    dest: "/usr/local/bin/"
    owner: "root"
    group: "root"
@@ -52,7 +77,8 @@

 - name: Deploy blocky binary file (aarch64)
  ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    remote_src: true
    dest: "/usr/local/bin/"
    owner: "root"
    group: "root"
@@ -9,9 +9,9 @@
      - "gitea"
      - "immich"
      - "paperless"
-      - "vikunja"
      - "affine"
      - "nextcloud"
+      - "sure"

 - name: Create postgresql directory
  ansible.builtin.file:
@@ -8,9 +8,20 @@
    mode: "0770"
  loop:
    - "x509-exporter"
+    - "x509-exporter/config"
    - "x509-exporter/certs"
  become: true

+- name: Deploy config.yaml
+  ansible.builtin.copy:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/x509-exporter/config/config.yaml"
+    dest: "{{ node['home_path'] }}/containers/x509-exporter/config/config.yaml"
+    owner: "{{ services['x509-exporter']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_x509-exporter"
+
 - name: Deploy certificates
  ansible.builtin.copy:
    content: |
@@ -82,6 +82,8 @@ table inet filter {
        chain global {
                # invalid packets
                ct state invalid drop comment "deny invalid connection"
+                # VPN connection exception handling
+                udp dport $PORTS_VPN return comment "return vpn connection to input and forward chain"
                # crowdsec
                ip saddr @crowdsec-blacklists counter drop comment "deny all crowdsec blacklist"
                ip6 saddr @crowdsec6-blacklists counter drop comment "deny all ipv6 crowdsec blacklist"
@@ -117,9 +117,9 @@ postgresql:
        gitea: ENC[AES256_GCM,data:l+pBCzyQa3000SE9z1R4htD0V0ONsBtKy92dfgsVYsZ3XlEyVJDIBOsugwM=,iv:5t/oHW1vFAmV/s2Ze/cV9Vuqo96Qu6QvZeRbio7VX2s=,tag:4zeQaXiXIzBpy+tXsxmN7Q==,type:str]
        immich: ENC[AES256_GCM,data:11jvxTKA/RL0DGL6y2/X092hnDohj6yTrYGK4IVojqBd1gCOBnDvUjgmx14=,iv:oBfHxsx9nxhyKY/WOuWfybxEX2bf+lHEtsaifFRS9lg=,tag:tAfkBdgQ8ZEkLIFcDICKDw==,type:str]
        paperless: ENC[AES256_GCM,data:6VBrBbjVoam7SkZCSvoBTdrfkUoDghdGTiBmFLul04X/okXOHeC5zusJffY=,iv:iZumcJ3TWwZD77FzYx8THwCqC+EbnXUBrEKuPh3zgV8=,tag:u2m8SppAdxZ/duNdpuS3oQ==,type:str]
-        vikunja: ENC[AES256_GCM,data:/+wQdoFPTBG2elI9kZbAVWrHZ0DhMaYr4dc+2z9QNdb3TcDS2PEia0JuSAg=,iv:MViZTyUD8YqMmxSTWCQpJ30f/KQdQGOzPlRHHsQ8lAw=,tag:zov3POno139dkMxFDpj2gg==,type:str]
        affine: ENC[AES256_GCM,data:XPXrcszsV06YqCJZ7CDqc4rCwqqNlbtLCFYfLAQ8jamLtft8L2UVrMA4WZo=,iv:vrWdBeckxB9tmEE628j4jhU+hSpE6TXYMGt0hh1Cg84=,tag:hlWwWUGht8NqWTZREMsa1Q==,type:str]
        nextcloud: ENC[AES256_GCM,data:ROsximNuWYMTZktmLJPx7W1Qol/uT+APgwoCtFO/6ZYYc3KxKvlk344eqEc=,iv:4d+MrfIHjJKAcwhvZ3g4go66uZcieuL7lngKErJd+fg=,tag:QbWOtxeCbiu62GyrE2atXg==,type:str]
+        sure: ENC[AES256_GCM,data:FULJ2gjJ2gZC3s324itW+CjGRBHIP9RnOqw5TT1UaiUhb7UHAPm1na+LsZk=,iv:c0GnVZkxprJUzPPq3TCQaZvAes9QQuvDXqgVLLaiQIg=,tag:uDxy/Lkd2hNK4AWwMNMslw==,type:str]
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
@@ -210,14 +210,6 @@ immich:
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
-#ENC[AES256_GCM,data:bzMt0Ox0Za4dOhoo7S6dYCdK32JI9Q==,iv:PRTryIJk0tR545XY0LoHwklvsJp5+A5bEljNmzUvRhY=,tag:EVsjRUGMOadaNbMu0Xr4XA==,type:comment]
-actualbudget:
-    oidc:
-        secret: ENC[AES256_GCM,data:TE2umZ9Vvr7cSfA2+TAfRadIWZN3hyOKQ6U9NqJFm5e9iiw1avI+QlnYcKI=,iv:rUWoclBRqh0tsGnMq29395Fn2NP7AXnSCd0s+S8jQ6I=,tag:qPX/TcdIo6BJeex7wmi02Q==,type:str]
-        hash: ENC[AES256_GCM,data:UjhNkGj+sxbnmPUx1V5kVYwZnzsB0aEvN8YV29lcvMbSnf9xpQWwD5C93Zu8SYrnS/p88qZpGBgAjr9Pcly3y0H1YMRt9zzbHZU3Uo0DPDrSWRQdeB/8LkcM/cwMAs8arS6PO03ECNnN5Z6aTmFdFnLjUkvUuSWMFscItAzMzhWCpeY=,iv:B06LI7Cq3NN8haOLfN3gWIpUFnvdUlq6D2XmARojDpk=,tag:MflE8qcY5j/aAA7xfPCqng==,type:str]
-#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
-#
-#
 #ENC[AES256_GCM,data:McPUAbIUvtC1gdPaxTgAxAMCMWcLfg==,iv:Tp6idRf7he3sYzo8LW596C905JAaoTIhIoDUzSyRT0k=,tag:4mZQ0Swu1X9uuwjsRNhr2A==,type:comment]
 paperless:
    session_secret: ENC[AES256_GCM,data:siwCs2noeVpg9DCEZybnmo/oz11BdrHSTnHciMOu/6g=,iv:XVjhu10TIujIdUopN9+TVVqRade9EvItDWxym6YXnZs=,tag:TxLYm+4Bo7IMaTQBtMg9pQ==,type:str]
@@ -229,22 +221,6 @@ paperless:
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
-#ENC[AES256_GCM,data:V7DJHA2JQirfBsrCGhXrhg==,iv:+jYqX9hGNnuyYj9o9LpCYFVOoD6nSrtc4t40Ag0mMzo=,tag:1wSxKtkJm42reUxdwYDvlg==,type:comment]
-vikunja:
-    session_secret: ENC[AES256_GCM,data:CMyw8JGHyTczGsrOJJwQBKfXMU4Sudvwkur1Lgx4o64=,iv:F2VmpqddiDT4jGaGDKGl6FARsQOt3lLz3X6TjC2MIVU=,tag:UJYyzrl/FX1BNwY4ROFncA==,type:str]
-    oidc:
-        secret: ENC[AES256_GCM,data:QwqndYsfr+fh9OLkHYtLYCa6WUdhnL7A4btz1d1eelTwq3Kps5S6BUN5qZg=,iv:51N8byIAAUh4ky7YBAuEJOBEWu1d9AX5W1m37/cLlCM=,tag:GD7jbxNGd748TCPgqsxyMg==,type:str]
-        hash: ENC[AES256_GCM,data:ORifyT4u1V2CyBCNBgF72wwS2i05mlzA4iIVEa1cH9aaE69PdiQvGGzMHK+tmlfpVaVQEENSt1QDUSSlMyeuZT/3a0JwAvlz+XDbpS7bicL2cB6DCa4JyEd/rbGRXs0/COfxPxXzYv7jq9gd2uSJ+cCGYb/93WuEXSEI6PHi+FF7N94=,iv:FVSGySa4YB2vwenqSagBzxeIexg91ewvcQMix+etmng=,tag:yyQtOgzOZypba+rV3A1K9g==,type:str]
-#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
-#
-#
-#ENC[AES256_GCM,data:EsRGZP7snPchEAMoQN5PoQpiOA==,iv:A/8POGq3pIw7aX5S2vyKtI2vPqH0FT6yZnpe/vVbifw=,tag:BgUYHX2zxIL7yLS0JbI1Yg==,type:comment]
-opencloud:
-    admin:
-        password: ENC[AES256_GCM,data:VKG7sNTTLHCXRGf4SAlR91+hvc7PaNrnpJX/4kItVcT9W1Hdl/yKgHHD7M8=,iv:WwWnx9KuN+i/Ugwv+HY4IGDZrLHk71hsobGFOn9kml0=,tag:SS6ihrtZjLnlAJR59lw+gw==,type:str]
-#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
-#
-#
 #ENC[AES256_GCM,data:k55osvepVeB1RC5hZ4IF,iv:AlhfmWwn/DiSESWc+ULJSOLUhnrKAIfWr7MeiwV8qc8=,tag:hOgptwUcY6nVxPIhu+DYgw==,type:comment]
 affine:
    secret_key: ENC[AES256_GCM,data:LLX78DpYnha1JWhgw0sHLzIVq/oIzvT+nB7zgli4mroGbnt7WZaXCx34zKkYRwYj/+0L4IFFVdkzKtK5DO84SgFkS2Bk2iNdCMqIx80CpyiD8IWAcyRu5d6hh82PlgyxU80T/4nbLbIn0GLubPTTeUX8GC3VxRU=,iv:DnmvbhlygSHes0jAkIm4+WXMUQLzr4R4dNa33rO67v8=,tag:+2wlh+/ekiTyShWM4XBbUw==,type:str]
@@ -266,6 +242,15 @@ nextcloud:
 #ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
 #
 #
+#ENC[AES256_GCM,data:Fsqc2JDp9dvfgiCjdQ==,iv:3DALKKEXaP8hzXRvxD4CgfFpOiPPsOa16OB94n8WKp8=,tag:K+FF3zGrc0YLXWK/R2L3Ow==,type:comment]
+sure:
+    session_secret: ENC[AES256_GCM,data:InHsz/jld8E9TwI8MWpxk9x2I7dxlIsY9R6jtDK2pBA=,iv:HY5yXEC2Dce26e9/vXTIWELvVd9ZjhcCwFD0jhz5pPw=,tag:LLSJovZ0RH3CUK+se7R4Ag==,type:str]
+    oidc:
+        secret: ENC[AES256_GCM,data:9BSvpcU9BJctSN9bkPIAsRxg8JNHTWvOKdpJFhm//CUDm/Xc7oC/ANHf5no=,iv:JVQLl/rp65kZSK/4SpVXxtiac3Z35XNkxWm2+lEdq/c=,tag:WgfaORiNlrO+wHSdnl4CWQ==,type:str]
+        hash: ENC[AES256_GCM,data:EjJ+1fP7/9wG2jG0Jv2hxMLtErqxjHBstRjru79dd5ZXhqwT7S+jpLfl9WpZU9qi20ps9YP4qe7G08p6NJNXjYhQj852GQxEORRh/9StAZsPt3p8w+ePZSVbivPQH+FpPKWYxoH0VR7y3TnL66R0tKRLh1fNTc5jRy5rU5r1bfs1jZ0=,iv:0y9FxW4QdD7qHz3bPRWlwHFpvOsvlYhVrOItB6BzaE8=,tag:Wc7MZhP3QRYmvZcjpoEWtQ==,type:str]
+#ENC[AES256_GCM,data:ODXFUxxxdQ==,iv:s9zJVx6wo6x517tbNvC+FZ0dFzqbjqeLI6rXBq72hQA=,tag:bXoV2I3LbpmQyddJrtS3Qg==,type:comment]
+#
+#
 #ENC[AES256_GCM,data:T4Wtn49AAxPd2QUFTR+q,iv:bH5goGWBDqumAat9dUv2OwfCUJUpuVqncTMqMBZUXhI=,tag:G+W6hHA+yftQ+4RJpXrxHg==,type:comment]
 switch:
    password: ENC[AES256_GCM,data:qu0f9L7A0eFq/UCpaRs=,iv:W8LLOp3MSfd/+EfNEZNf91K8GgI5eUfVPoWTRES2C0Y=,tag:Q5FlAOfwqwJwPvd7k6i+0g==,type:str]
@@ -295,7 +280,7 @@ sops:
            UmliaFNxVTBqRkI1QWJpWGpTRWxETW8KEY/8AfU73UOzCGhny1cNnd5dCNv7bHXt
            k+uyWPPi+enFkVaceSwMFrA66uaWWrwAj11sXEB7yzvGFPrnAGezjQ==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-02T04:55:25Z"
-    mac: ENC[AES256_GCM,data:4U/SGYS9eNRgRvUEvZh9E0JSctkZzSpdoUYEAbnOVyU+5u8NcG9lbMUAB4kFXb9kHVGBUI5wMwnzg102g96q1IYw5m/k4lrpePceGVNAxxKpWTnkLROhJlL3Z/Bylgq2mj7PVDcGCGEB0xPDgN+ffa7ldCxIikYmSKktISguwYU=,iv:zqS9iJ54FIaNQhnfOl4YY9QcaZLbPekTxlY1AEp3m/s=,tag:TckIRAKRVyxf/UD+jejNng==,type:str]
+    lastmodified: "2026-05-09T14:26:51Z"
+    mac: ENC[AES256_GCM,data:TYs08ZSS2kcO5lYuhQ/IySUSQ3DpL+ba3/uNLyszht4OttR110/W/WQLiRuu/Ql6FwtDtjq6I3iNpOhmCHSv1kMCam1l99GEIYCaPUIY+TY3Zw0j7518dFXe8p/DrKRwIVXfK5lIKLIEd+eizD50HzwXXJFmU+7YDkQ1Dx+55kw=,iv:arJKJ4wO4sdQlu3GZbtultsfM6s8vbhG93tnf2EjJDc=,tag:m95gUqvn4w85XI8qVvCZpQ==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.12.1
@@ -0,0 +1,25 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Collabora Online
+
+[Container]
+Image=docker.io/collabora/code:{{ version['containers']['collabora'] }}
+ContainerName=collabora
+HostName=collabora
+
+PublishPort={{ services['collabora']['ports']['http'] }}:9980/tcp
+
+Environment="TZ=Asia/Seoul"
+Environment="aliasgroup1=https://{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}"
+# Environment="aliasgroup2=other_server_FQDN"
+Environment="extra_params=--o:ssl.enable=false --o:ssl.termination=true --o:server_name={{ services['collabora']['domain']['public'] }}.{{ domain['public'] }} --o:admin_console.enable=false"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target
@@ -1,4 +1,5 @@
 <?php
 $CONFIG = [
+    // Background jobs mode is auto-detected as 'cron' when nextcloud-cron.timer runs cron.php via CLI. No explicit config required.
    'maintenance_window_start' => 18,
 ];
@@ -0,0 +1,67 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Sure Web
+
+After=network-online.target redis_sure.service
+Wants=network-online.target redis_sure.service
+
+[Container]
+Image=ghcr.io/we-promise/sure:{{ version['containers']['sure'] }}
+ContainerName=sure-web
+HostName=sure-web
+
+PublishPort={{ services['sure']['ports']['http'] }}:3000/tcp
+
+Volume=%h/data/containers/sure/storage:/rails/storage:rw
+Volume=%h/containers/sure/ssl:/etc/ssl/sure:ro
+
+# General
+Environment="TZ=Asia/Seoul"
+Environment="SELF_HOSTED=true"
+Environment="ONBOARDING_STATE=closed"
+Environment="RAILS_FORCE_SSL=false"
+Environment="RAILS_ASSUME_SSL=true"
+Environment="APP_DOMAIN={{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+Secret=SURE_SECRET_KEY_BASE,type=env,target=SECRET_KEY_BASE
+
+# PostgreSQL
+Environment="POSTGRES_USER=sure"
+Environment="POSTGRES_DB=sure_db"
+Environment="DB_HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="DB_PORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="PGSSLMODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/sure/{{ root_cert_filename }}"
+Secret=SURE_POSTGRES_PASSWORD,type=env,target=POSTGRES_PASSWORD
+
+# Redis
+Environment="REDIS_URL=redis://host.containers.internal:{{ services['sure']['ports']['redis'] }}/1"
+
+# OIDC - Authelia
+Environment="OIDC_CLIENT_ID=sure"
+Environment="OIDC_ISSUER=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+Environment="OIDC_REDIRECT_URI=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}/auth/openid_connect/callback"
+Secret=SURE_OIDC_CLIENT_SECRET,type=env,target=OIDC_CLIENT_SECRET
+Environment="OIDC_BUTTON_LABEL=Sign in with Authelia"
+Environment="AUTH_JIT_MODE=create_and_link"
+# email's domain, e.g. ilnmors.internal then only user@ilnmors.internal is allowed to sign-up
+Environment="ALLOWED_OIDC_DOMAINS="
+
+# WebAuthn / Passkey
+Environment="WEBAUTHN_RP_ID={{ domain['public'] }}"
+Environment="WEBAUTHN_ALLOWED_ORIGINS=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+
+# Provider
+## Currency
+Environment="EXCHANGE_RATE_PROVIDER=yahoo_finance"
+Environment="SECURITIES_PROVIDER=yahoo_finance"
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,67 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Sure Worker
+
+After=network-online.target redis_sure.service
+Wants=network-online.target redis_sure.service
+
+[Container]
+Image=ghcr.io/we-promise/sure:{{ version['containers']['sure'] }}
+ContainerName=sure-worker
+HostName=sure-worker
+
+Volume=%h/data/containers/sure/storage:/rails/storage:rw
+Volume=%h/containers/sure/ssl:/etc/ssl/sure:ro
+
+Exec=bundle exec sidekiq
+
+# General
+Environment="TZ=Asia/Seoul"
+Environment="SELF_HOSTED=true"
+Environment="ONBOARDING_STATE=closed"
+Environment="RAILS_FORCE_SSL=false"
+Environment="RAILS_ASSUME_SSL=true"
+Environment="APP_DOMAIN={{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+Secret=SURE_SECRET_KEY_BASE,type=env,target=SECRET_KEY_BASE
+
+# PostgreSQL
+Environment="POSTGRES_USER=sure"
+Environment="POSTGRES_DB=sure_db"
+Environment="DB_HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="DB_PORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="PGSSLMODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/sure/{{ root_cert_filename }}"
+Secret=SURE_POSTGRES_PASSWORD,type=env,target=POSTGRES_PASSWORD
+
+# Redis
+Environment="REDIS_URL=redis://host.containers.internal:{{ services['sure']['ports']['redis'] }}/1"
+
+# OIDC - Authelia
+Environment="OIDC_CLIENT_ID=sure"
+Environment="OIDC_ISSUER=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+Environment="OIDC_REDIRECT_URI=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}/auth/openid_connect/callback"
+Secret=SURE_OIDC_CLIENT_SECRET,type=env,target=OIDC_CLIENT_SECRET
+Environment="OIDC_BUTTON_LABEL=Sign in with Authelia"
+Environment="AUTH_JIT_MODE=create_and_link"
+# email's domain, e.g. ilnmors.internal then only user@ilnmors.internal is allowed to sign-up
+Environment="ALLOWED_OIDC_DOMAINS="
+
+# WebAuthn / Passkey
+Environment="WEBAUTHN_RP_ID={{ domain['public'] }}"
+Environment="WEBAUTHN_ALLOWED_ORIGINS=https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}"
+
+# Provider
+## Currency
+Environment="EXCHANGE_RATE_PROVIDER=yahoo_finance"
+Environment="SECURITIES_PROVIDER=yahoo_finance"
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target
@@ -93,17 +93,6 @@ notifier:
 identity_providers:
  oidc:
    hmac_secret: '' # $AUTHELIA_IDENTITY_PROVIDERS_OIDC_HMAC_SECRET_FILE
-    # For the app which doesn't use secret.
-    cors:
-      endpoints:
-        - 'authorization'
-        - 'token'
-        - 'revocation'
-        - 'introspection'
-        - 'userinfo'
-      allowed_origins:
-        - 'https://{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}'
-      allowed_origins_from_client_redirect_uris: true
    jwks:{% raw %}
      - algorithm: 'RS256'
        use: 'sig'
@@ -184,28 +173,6 @@ identity_providers:
        access_token_signed_response_alg: 'none'
        userinfo_signed_response_alg: 'none'
        token_endpoint_auth_method: 'client_secret_post'
-      # https://www.authelia.com/integration/openid-connect/clients/actual-budget/
-      - client_id: 'actual-budget'
-        client_name: 'Actual Budget'
-        client_secret: '{{ hostvars['console']['actualbudget']['oidc']['hash'] }}'
-        public: false
-        authorization_policy: 'one_factor'
-        require_pkce: false
-        pkce_challenge_method: ''
-        redirect_uris:
-          - 'https://{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}/openid/callback'
-        scopes:
-          - 'openid'
-          - 'profile'
-          - 'groups'
-          - 'email'
-        response_types:
-          - 'code'
-        grant_types:
-          - 'authorization_code'
-        access_token_signed_response_alg: 'none'
-        userinfo_signed_response_alg: 'none'
-        token_endpoint_auth_method: 'client_secret_basic'
      # https://www.authelia.com/integration/openid-connect/clients/paperless/
      - client_id: 'paperless'
        client_name: 'Paperless'
@@ -228,122 +195,6 @@ identity_providers:
        access_token_signed_response_alg: 'none'
        userinfo_signed_response_alg: 'none'
        token_endpoint_auth_method: 'client_secret_post'
-      # https://www.authelia.com/integration/openid-connect/clients/vikunja/
-      - client_id: 'vikunja'
-        client_name: 'Vikunja'
-        client_secret: '{{ hostvars['console']['vikunja']['oidc']['hash'] }}'
-        public: false
-        authorization_policy: 'one_factor'
-        require_pkce: false
-        pkce_challenge_method: ''
-        redirect_uris:
-          - 'https://{{ services['vikunja']['domain']['public'] }}.{{ domain['public'] }}/auth/openid/authelia'
-        scopes:
-          - 'openid'
-          - 'profile'
-          - 'email'
-        response_types:
-          - 'code'
-        grant_types:
-          - 'authorization_code'
-        access_token_signed_response_alg: 'none'
-        userinfo_signed_response_alg: 'none'
-        token_endpoint_auth_method: 'client_secret_basic'
-      # OpenCloud configuration
-      ## https://docs.opencloud.eu/docs/admin/configuration/authentication-and-user-management/external-idp/
-      ## Web
-      - client_id: 'opencloud'
-        client_name: 'OpenCloud'
-        public: true
-        authorization_policy: 'one_factor'
-        require_pkce: true
-        pkce_challenge_method: 'S256'
-        redirect_uris:
-          - 'https://{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}/'
-          - 'https://{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}/oidc-callback.html'
-          - 'https://{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}/oidc-silent-redirect.html'
-        scopes:
-          - 'openid'
-          - 'profile'
-          - 'email'
-          - 'groups'
-        response_types:
-          - 'code'
-        grant_types:
-          - 'authorization_code'
-        access_token_signed_response_alg: 'RS256'
-        userinfo_signed_response_alg: 'none'
-        token_endpoint_auth_method: 'none'
-      ## desktop
-      - client_id: 'OpenCloudDesktop'
-        client_name: 'OpenCloud'
-        public: true
-        authorization_policy: 'one_factor'
-        require_pkce: true
-        pkce_challenge_method: 'S256'
-        redirect_uris:
-          - 'http://localhost'
-          - 'http://127.0.0.1'
-        scopes:
-          - 'openid'
-          - 'profile'
-          - 'email'
-          - 'groups'
-          - 'offline_access'
-        response_types:
-          - 'code'
-        grant_types:
-          - 'authorization_code'
-          - 'refresh_token'
-        access_token_signed_response_alg: 'RS256'
-        userinfo_signed_response_alg: 'none'
-        token_endpoint_auth_method: 'none'   
-      ## Android
-      - client_id: 'OpenCloudAndroid'
-        client_name: 'OpenCloud'
-        public: true
-        authorization_policy: 'one_factor'
-        require_pkce: true
-        pkce_challenge_method: 'S256'
-        redirect_uris:
-          - 'oc://android.opencloud.eu'
-        scopes:
-          - 'openid'
-          - 'profile'
-          - 'email'
-          - 'groups'
-          - 'offline_access'
-        response_types:
-          - 'code'
-        grant_types:
-          - 'authorization_code'
-          - 'refresh_token'
-        access_token_signed_response_alg: 'RS256'
-        userinfo_signed_response_alg: 'none'
-        token_endpoint_auth_method: 'none'
-      ## IOS
-      - client_id: 'OpenCloudIOS'
-        client_name: 'OpenCloud'
-        public: true
-        authorization_policy: 'one_factor'
-        require_pkce: true
-        pkce_challenge_method: 'S256'
-        redirect_uris:
-          - 'oc://ios.opencloud.eu'
-        scopes:
-          - 'openid'
-          - 'profile'
-          - 'email'
-          - 'groups'
-          - 'offline_access'
-        response_types:
-          - 'code'
-        grant_types:
-          - 'authorization_code'
-          - 'refresh_token'
-        access_token_signed_response_alg: 'RS256'
-        userinfo_signed_response_alg: 'none'
-        token_endpoint_auth_method: 'none'
      # https://docs.affine.pro/self-host-affine/administer/oauth-2-0
      - client_id: 'affine'
        client_name: 'Affine'
@@ -387,3 +238,25 @@ identity_providers:
        access_token_signed_response_alg: 'none'
        userinfo_signed_response_alg: 'none'
        token_endpoint_auth_method: 'client_secret_post'
+      # https://www.authelia.com/integration/openid-connect/clients/sure/
+      - client_id: 'sure'
+        client_name: 'Sure'
+        client_secret: '{{ hostvars['console']['sure']['oidc']['hash'] }}'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://{{ services['sure']['domain']['public'] }}.{{ domain['public'] }}/auth/openid_connect/callback'
+        scopes:
+          - 'openid'
+          - 'email'
+          - 'profile'
+          - 'groups'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
@@ -47,30 +47,12 @@
                header_up Host {http.request.header.X-Forwarded-Host}
        }
 }
-{{ services['actualbudget']['domain']['internal'] }}.{{ domain['internal'] }} {
-        import private_tls
-        reverse_proxy host.containers.internal:{{ services['actualbudget']['ports']['http'] }} {
-                header_up Host {http.request.header.X-Forwarded-Host}
-        }
-}
 {{ services['paperless']['domain']['internal'] }}.{{ domain['internal'] }} {
        import private_tls
        reverse_proxy host.containers.internal:{{ services['paperless']['ports']['http'] }} {
                header_up Host {http.request.header.X-Forwarded-Host}
        }
 }
-{{ services['vikunja']['domain']['internal'] }}.{{ domain['internal'] }} {
-        import private_tls
-        reverse_proxy host.containers.internal:{{ services['vikunja']['ports']['http'] }} {
-                header_up Host {http.request.header.X-Forwarded-Host}
-        }
-}
-{{ services['opencloud']['domain']['internal'] }}.{{ domain['internal'] }} {
-        import private_tls
-        reverse_proxy host.containers.internal:{{ services['opencloud']['ports']['http'] }} {
-                header_up Host {http.request.header.X-Forwarded-Host}
-        }
-}
 {{ services['affine']['domain']['internal'] }}.{{ domain['internal'] }} {
        import private_tls
        reverse_proxy host.containers.internal:{{ services['affine']['ports']['http'] }} {
@@ -83,3 +65,15 @@
                header_up Host {http.request.header.X-Forwarded-Host}
        }
 }
+{{ services['collabora']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['collabora']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
+{{ services['sure']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['sure']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}
@@ -91,15 +91,6 @@
                }
        }
 }
-{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }} {
-        import crowdsec_log
-        route {
-                crowdsec
-                reverse_proxy https://{{ services['actualbudget']['domain']['internal'] }}.{{ domain['internal'] }} {
-                        header_up Host {http.reverse_proxy.upstream.host}
-                }
-        }
-}
 {{ services['paperless']['domain']['public'] }}.{{ domain['public'] }} {
        import crowdsec_log
        route {
@@ -109,24 +100,6 @@
                }
        }  
 }
-{{ services['vikunja']['domain']['public'] }}.{{ domain['public'] }} {
-        import crowdsec_log
-        route {
-                crowdsec
-                reverse_proxy https://{{ services['vikunja']['domain']['internal'] }}.{{ domain['internal'] }} {
-                        header_up Host {http.reverse_proxy.upstream.host}
-                }
-        }
-}
-{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }} {
-        import crowdsec_log
-        route {
-                crowdsec
-                reverse_proxy https://{{ services['opencloud']['domain']['internal'] }}.{{ domain['internal'] }} {
-                        header_up Host {http.reverse_proxy.upstream.host}
-                }
-        }
-}
 {{ services['affine']['domain']['public'] }}.{{ domain['public'] }} {
        import crowdsec_log
        route {
@@ -145,6 +118,24 @@
                }
        }
 }
+{{ services['collabora']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{services['collabora']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
+{{ services['sure']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{services['sure']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}

 # Internal domain
 {{ node['name'] }}.{{ domain['internal'] }} {
@@ -0,0 +1,11 @@
+server:
+  listen: :9793
+
+sources:
+  - kind: file
+    name: homelab-certs
+    paths:
+      - /certs/*.crt
+      - /certs/*.pem
+      - /certs/*.cer
+    refreshInterval: 1m
@@ -11,11 +11,12 @@ Image=docker.io/enix/x509-certificate-exporter:{{ version['containers']['x509-ex
 ContainerName=x509-exporter
 HostName=X509-exporter

+Volume=%h/containers/x509-exporter/config/config.yaml:/etc/config.yaml:ro
 Volume=%h/containers/x509-exporter/certs:/certs:ro

 PublishPort={{ services['x509-exporter']['ports']['http'] }}:9793

-Exec=--listen-address :9793 --watch-dir=/certs
+Exec=--config /etc/config.yaml

 [Service]
 Restart=always
@@ -0,0 +1,10 @@
+[Unit]
+Description=BTRFS auto scrub
+ConditionPathIsMountPoint={{ storage['btrfs']['mount_point'] }}
+RequiresMountsFor={{ storage['btrfs']['mount_point'] }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/btrfs scrub start -Bd {{ storage['btrfs']['mount_point'] }}
+Nice=19
+IOSchedulingClass=idle
@@ -0,0 +1,10 @@
+[Unit]
+Description=Monthly BTRFS auto scrub 
+
+[Timer]
+OnCalendar=*-*-01 04:00:00
+Persistent=true
+RandomizedDelaySec=300
+
+[Install]
+WantedBy=timers.target
@@ -12,10 +12,8 @@ whitelist:
        - "{{ hostvars['fw']['network6']['console']['wg'] }}"
 {% if node['name'] == 'auth' %}
    expression:
-        # budget local-first sql scrap rule
-        - "evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'"
        # immich thumbnail request 404 error false positive
-        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
-        # opencloud chunk request false positive
-        - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
+        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
+        # nextcloud thumbnail/preview request error false positive
+        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'"
 {% endif %}
@@ -21,9 +21,9 @@ ProtectHome=tmpfs
 InaccessiblePaths=/boot /root

 {% if node['name'] == 'infra' %}
-BindReadOnlyPaths=%h/containers/postgresql/backups
+BindReadOnlyPaths={{ node['home_path'] }}/containers/postgresql/backups
 {% elif node['name'] == 'app' %}
-BindReadOnlyPaths=%h/data
+BindReadOnlyPaths={{ node['home_path'] }}/data
 {% endif %}
 # In root namescope, %u always bring 0
 BindPaths=/etc/kopia
@@ -38,10 +38,10 @@ ExecStartPre=/usr/bin/kopia repository connect server \

 {% if node['name'] == 'infra' %}
 ExecStart=/usr/bin/kopia snapshot create \
-    /home/infra/containers/postgresql/backups
+    {{ node['home_path'] }}/containers/postgresql/backups
 {% elif node['name'] == 'app' %}
 ExecStart=/usr/bin/kopia snapshot create \
-    /home/app/data
+    {{ node['home_path'] }}/data
 {% endif %}


@@ -14,22 +14,22 @@
 ## Context

 - Maintaining multi nodes requires a huge amount of resources, including hardware, electricity, even administrative efforts
- All units which responsible for a single role should follow the Principle of Least Privilege \(PoLP\).
+- All units which responsible for a single role should follow the Principle of Least Privilege (PoLP).
 - All units should be interchangeable on standard to avoid vendor lock-in.

 ## Consideration

 ### Hypervisor

- Proxmox Virutal Environment \(PVE\)
+- Proxmox Virutal Environment (PVE)
    - Based on Debian.
    - PVE uses `qm` command which is not a standard to implement the virtual environment.
 - VMware ESXi
-    - Based on UNIX, deveoped by VMware \(Licence is not free\)
+    - Based on UNIX, deveoped by VMware (Licence is not free)
 - Hyper-V
-    - Based on Microsoft Windows \(Licence is not free\)
+    - Based on Microsoft Windows (Licence is not free)
 - Debian Stable
-    - Based on standard linux \(conservative\)
+    - Based on standard linux (conservative)
    - Standard virtualization technology 'Libvirt, QEMU, KVM'

 ### Container
@@ -37,7 +37,7 @@
 - Docker
    - Daemon is used to run containers
    - Root authority required
-    - Socket and network problem is complex \(Docker bridge\)
+    - Socket and network problem is complex (Docker bridge)
    - docker-compose is an orchestration tool
 - Rootless Podman
    - Daemonless design
@@ -58,7 +58,7 @@

 ## Decisions

- Use Libvirt/KVM/QEMU on pure linux \(Debian stable\).
+- Use Libvirt/KVM/QEMU on pure linux (Debian stable).
 - Separate all services by VM, and podman rootless containers without K3S.
    - Orchestration stack is not needed in single node system
    - Services will be defined by Quadelt to integrate into systemd and to manage them declaratively
@@ -23,15 +23,15 @@

 - OPNSense/pfSense
    - vendor lock-in
-    - GUI environment \(WebGUI\) can contain vulnerability
+    - GUI environment (WebGUI) can contain vulnerability
    - It is hard to manage configurations by IaC
 - iptables
    - Previous standard of Linux
-    - IPv4 and IPv6 configuration is separated \(no inet\)
+    - IPv4 and IPv6 configuration is separated (no inet)
 - nftables
    - New standard of Linux
    - English grammar friendly
-    - IPv4 and IPv6 configuration can be set on the same table \(inet\)
+    - IPv4 and IPv6 configuration can be set on the same table (inet)

 ### Flat network structure
 - LAN only
@@ -48,8 +48,8 @@
    - VLAN 20: user (DHCP allocated devices)
    - wg0: VPN connections
 - Manage the rules based on roles fundamentally, furthermore manage them based on ip and ports when it is needed
- All L3 communication which needs to pass gateway should be on control of firewall \(fw\)
- All nodes including firewall uses nftables \(modern standard\) to manage the packets based on zone concept
+- All L3 communication which needs to pass gateway should be on control of firewall (fw)
+- All nodes including firewall uses nftables (modern standard) to manage the packets based on zone concept
 - IPv6 has two track strategy
    - Client and server, wg nodes has static ULA IP, and use NAT66 for permanency
    - User nodes has GUA SLAAC IP from ISP for compatibility
@@ -24,7 +24,7 @@
 ### Automate protocol

 - JWK/JWT provisioner
-    - It is hard to manage pre-shared secret values than ACME \(Especially nsupdate\)
+    - It is hard to manage pre-shared secret values than ACME (Especially nsupdate)
 - authorized_keys
    - When the nodes are increased, it is hard to manage authorized_key.
    - SSH ca.pub allow all the certificates signed by ca key, so it is not needed to manage authroized_keys from each hosts.
@@ -39,19 +39,19 @@
 ## Decisions

 - Operate private CA
-    - Root CA \(Store on coldstorage\) - 10 years
-    - Intermediate CA \(Online server as Step-CA\) - 5 years
+    - Root CA (Store on coldstorage) - 10 years
+    - Intermediate CA (Online server as Step-CA) - 5 years
    - SSH CA - No period
 - Manage certificates with two track
-    - ACME with nsupdate \(using private DNS\) for web services via Caddy - 90 days
+    - ACME with nsupdate (using private DNS) for web services via Caddy - 90 days
    - Manual issuing and managing leaf certificate for infra services for independency - 2.5 years
    - All manual issuing leaf certificate expiry date is observed by x509-exporter on infra vm
 - Manage SSH certificates
-    - *-cert.pub for host \(with -h options\)
-    - *-cert.pub for client \(without -h options\)
+    - *-cert.pub for host (with -h options)
+    - *-cert.pub for client (without -h options)

 ## Consequences

 - Private PKI is operated
 - Private SSH CA is operated
- All external/internal communication is encrypted as TLS re-encryption. \(E2EE\)
+- All external/internal communication is encrypted as TLS re-encryption. (E2EE)
@@ -12,9 +12,9 @@

 ## Context

- Private authoritative DNS is required to use private reserved root domain \(.internal\)
+- Private authoritative DNS is required to use private reserved root domain (.internal)
 - Split horizon DNS needs DNS resolver, because authoritative DNS must not send queries to other DNS.
- Automatical issuing certificates needs private authoritative DNS which supports nsupdate \(RFC 2136\)
+- Automatical issuing certificates needs private authoritative DNS which supports nsupdate (RFC 2136)

 ## Consideration

@@ -22,13 +22,13 @@
 - AdGuard Home
    - More powerful query routing than blocky
    - Web UI dependency
-    - Extra function which is not useful \(DHCP, etc ..\)
+    - Extra function which is not useful (DHCP, etc ..)
 - Unbound DNS
    - Cache and forward zone management is powerful
    - more complex than blocky
    - cache function is not that needed in this environment
        - Internal authoritative DNS only takes charge of internal communication
-        - All security function is delegated to public DNS like cloudflare \(DNSSEC, etc\) 
+        - All security function is delegated to public DNS like cloudflare (DNSSEC, etc) 

 ## Decisions

@@ -29,8 +29,8 @@
    - Regex based log parsing is less structured than CrowdSec's parser/scenario model

 - Crowdsec
-    - Community based rules and sinario \(CAPI\)
-    - Prevention based on local machines and parsers \(LAPI\)
+    - Community based rules and sinario (CAPI)
+    - Prevention based on local machines and parsers (LAPI)
    - Bouncers can use nftables to prevent threats
    - Parser can detect even L7 attack under TLS

@@ -43,7 +43,7 @@
 - Operate Crowdsec as IPS
    - CrowdSec uses two API server, CAPI, LAPI.
        - CAPI updates malicious IPs based on community decisions
-        - LAPI decides malicious attack based on log from its parser and scenario \(Suricata, caddy, etc\)
+        - LAPI decides malicious attack based on log from its parser and scenario (Suricata, caddy, etc)
        - When CAPI, and LAPI decides block some IP based on log parsed by parser and scenarios, bouncer block the malicious accesses.
    - Crowdsec register blacklist on nftables or iptables.

@@ -20,7 +20,7 @@

 - HashiCorp Vault or Infisical
    - Very powerful, but introduces significant compute/memory overhead.
-    - Creates a "Secret Zero" problem for a single-node homelab environment because of dependency \(DB, or etc\).
+    - Creates a "Secret Zero" problem for a single-node homelab environment because of dependency (DB, or etc).
    - It is hard to operate hardware separated key servers.

 ### Systemd-credential
@@ -37,10 +37,10 @@
 ## Decisions

 - All secret data which has yaml format is encrypted by sops with age-key in `secret.yaml`.
- age-key is encrypted by gpg and ansible vault with master key \(including upper, lower case, number, special letters) above 40 characters.
+- age-key is encrypted by gpg and ansible vault with master key (including upper, lower case, number, special letters) above 40 characters.
    - All secret data always decrypt by `edit_secret.sh` script or ansible tasks from secrets.yaml using age-key encrypted by ansible-vault.
    - decrypted secret data is always processed on ramfs, they are never saved on disk.
- Master key is never saved on disk, but only cold storage \(USB, M-DISC, operators' memory\)
+- Master key is never saved on disk, but only cold storage (USB, M-DISC, operators' memory)
 - The secret data will be saved on each servers specific directory or podman secret.
    - OS:
        - path: /etc/secrets
@@ -6,7 +6,10 @@
    - First documentation

 - Feb/27/2026
-    - Status changed from Deffered to Accepted
+    - Status changed from Deferred to Accepted
+
+- May/06/2026
+    - Add backup checking rules
    
 ## Status

@@ -14,7 +17,7 @@

 ## Context

- All configuration file is managed by git \(IaC\)
+- All configuration file is managed by git (IaC)
 - All data file should be backed up by kopia
 - All backup should follow 3-2-1 backup cycle

@@ -30,20 +33,26 @@
    - Backing up the `/var/lib/postgresql` directory directly while the DB is running can lead to severe data corruption and inconsistency.
    - Logical dumps (`pg_dump`) are much safer, database-agnostic, and easier to restore in a homelab environment.

+### Silent failure problem
+
+- May/06/2026, the fact that backups haven't run since commit '9f236b6fa5' because of '%h' in system service unit.
+    - Operator couldn't realize backup doesn't run because the system service was failed silently.
+    - Therefore, set the checking rule.
+

 ## Decisions

 - All configuration files are managed by Git
    - Configuration files are based on text
    - It is necessary to version, history management.
-    - Local git -> private Gitea -> github private project \(mirrored\) 
+    - Local git -> private Gitea -> github private project (mirrored) 
    - This fulfills 3-2-1 backup rules
 - Data files are managed by Kopia and DSM
    - Local storage - kopia -> DSM's Kopia repository server - CloudSync -> Cloud server such as OneDrive or Google Drive
    - This fulfills 3-2-1 backup rules
 - Data files which needs backup
    - DB data files: dump
-        - DB data files are located on infra:/home/infra/containers/postgresql/backups/\{cluster,$service\}/
+        - DB data files are located on infra:/home/infra/containers/postgresql/backups/{cluster,$service}/
    - App data files: Photos, Media, etc ..
        - App data files are located on app:/home/app/data/
    - Backed up files: kopia
@@ -51,11 +60,18 @@
 - Kopia over DSM configuration is managed by runbook with equivalent CLI commands due to vendor limitation
 - Restore will be processed manually
    - DB data files
-        - From kopia server to console:$HOMELAB_PATH/data/volume/infra/postgresql/\{cluster,data\}
+        - From kopia server to console:$HOMELAB_PATH/data/volume/infra/postgresql/{cluster,data}
    - APP data files
        - From kopia server to APP vm after initiating before deploy services
- Automative backup does not guarantee integrity of data system, so before reset the system conduct manual backup after making sure all services are shutdown.
+- Automatic backup does not guarantee integrity of data system, so before reset the system conduct manual backup after making sure all services are shutdown.
+- Check the repository once a week (Every monday)
+    - Check the snapshot in repository with `kopia snapshot list --all`
+    - Mount the snapshot respectively with `kopia mount $SNAPSHOT_ID $DESTINATION`
+    - Copy random file from snapshot and check the values.
+    - If there's some failure, check the backup service and conduct backup immediately.
+    - Repeat the check flow.
+    - When everything is done, umount the kopia mount with `ctrl+c`

 ## Consequences

- All files including configuration and data back ups will fulfill 3-2-1 \(3 Copies, 2 different media, 1 offsite\) back up rules
+- All files including configuration and data back ups will fulfill 3-2-1 (3 Copies, 2 different media, 1 offsite) back up rules
@@ -11,7 +11,7 @@

 ## Context

- App VM needs GPU for heavy workloads like Immich \(hardware transcoding and machine learning\)
+- App VM needs GPU for heavy workloads like Immich (hardware transcoding and machine learning)
 - App VM needs huge data storage for its own services 

 ## Considerations
@@ -18,7 +18,7 @@
 ### Hypervisor

 - As a pure hypervisor, it should only operate virtualization for VM.
- Hypervisor just provides resources and dummy hub \(br\)
+- Hypervisor just provides resources and dummy hub (br)

 ### VM

@@ -30,7 +30,7 @@

 ### Services

- Services should be distinguished based on their needs \(Privilege\)
+- Services should be distinguished based on their needs (Privilege)
    - Network stack, backup stack needs special privilege for low level ACL or networks.
    - application stack doesn't need low level privilege usually

@@ -27,7 +27,7 @@
    - Removing
    - Formatting
    - Destroying
-    - Certificates and CA \([ADR-003](./003-pki.md)\)
+    - Certificates and CA ([ADR-003](./003-pki.md))
    - Etc. what operator decides that is sensitive

 ## Consequences
@@ -19,7 +19,7 @@
 ### Apply mTLS

 - implementing mTLS needs both client certificate and server certificate
- Managing a number of certificates makes a huge operational burden \(expiry date, revocation, etc ..\)
+- Managing a number of certificates makes a huge operational burden (expiry date, revocation, etc ..)

 ## Decisions

@@ -30,4 +30,4 @@

 - The policy is set simple
 - The overhead is increased little
- Exclude the exceptions on operation \(For the administrator\)
+- Exclude the exceptions on operation (For the administrator)
@@ -0,0 +1,26 @@
+---
+identity_providers:
+  oidc:
+    clients:
+      # https://www.authelia.com/integration/openid-connect/clients/actual-budget/
+      - client_id: 'actual-budget'
+        client_name: 'Actual Budget'
+        client_secret: 'secret'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          - 'https://actualbudget.example.com/openid/callback'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'groups'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
@@ -0,0 +1,6 @@
+name: crowdsecurity/whitelists
+description: "Local whitelist policy"
+whitelist:
+    expression:
+        # budget local-first sql scrap rule
+        - "evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/data/migrations/'"
@@ -0,0 +1,13 @@
+---
+services:
+    actualbudget:
+      domain:
+        public: ""
+        internal: ""
+      ports:
+        http: ""
+      subuid: "101000"
+
+version:
+  containers:
+    actualbudget: "26.3.0"
@@ -0,0 +1,5 @@
+---
+actualbudget:
+  oidc:
+    secret: ""
+    hash: ""
@@ -0,0 +1,25 @@
+---
+identity_providers:
+  oidc:
+    clients:
+      # https://www.authelia.com/integration/openid-connect/clients/ezbookkeeping/
+      - client_id: 'ezbookkeeping'
+        client_name: 'ezBookkeeping'
+        client_secret: 'hash'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://ezbookkeeping.example.com/oauth2/callback'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
@@ -0,0 +1,61 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=ezBookkeeping
+
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/mayswind/ezbookkeeping:{{ version['containers']['ezbookkeeping'] }}
+ContainerName=ezbookkeeping
+HostName=ezbookkeeping
+
+PublishPort={{ services['ezbookkeeping']['ports']['http'] }}:8080/tcp
+
+Volume=%h/data/containers/ezbookkeeping/data:/data:rw
+Volume=%h/containers/ezbookkeeping/ssl:/etc/ssl/ezbookkeeping:ro
+
+# General
+Environment="TZ=Asia/Seoul"
+Environment="EBK_SERVER_DOMAIN={{ services['ezbookkeeping']['domain']['public'] }}.{{ domain['public'] }}"
+Environment="EBK_SERVER_ROOT_URL=https://{{ services['ezbookkeeping']['domain']['public'] }}.{{ domain['public'] }}/"
+Environment="EBK_LOG_MODE=console"
+
+# Database
+Environment="EBK_DATABASE_TYPE=postgres"
+Environment="EBK_DATABASE_HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}:{{ services['postgresql']['ports']['tcp'] }}"
+Environment="EBK_DATABASE_NAME=ezbookkeeping_db"
+Environment="EBK_DATABASE_USER=ezbookkeeping"
+Secret=EBK_DATABASE_PASSWD,type=env
+Environment="EBK_DATABASE_SSL_MODE=verify-full"
+Environment="PGSSLROOTCERT=/etc/ssl/ezbookkeeping/{{ root_cert_filename }}"
+
+# OIDC
+Environment="EBK_AUTH_ENABLE_OAUTH2_AUTH=true"
+Environment="EBK_AUTH_OAUTH2_PROVIDER=oidc"
+Environment="EBK_AUTH_OAUTH2_CLIENT_ID=ezbookkeeping"
+Secret=EBK_AUTH_OAUTH2_CLIENT_SECRET,type=env
+Environment="EBK_AUTH_OAUTH2_USE_PKCE=true"
+Environment="EBK_AUTH_OIDC_PROVIDER_BASE_URL=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+Environment="EBK_AUTH_ENABLE_OIDC_DISPLAY_NAME=true"
+Environment="EBK_AUTH_OIDC_CUSTOM_DISPLAY_NAME=Authelia"
+
+# Registration / auth policy
+Environment="EBK_AUTH_ENABLE_INTERNAL_AUTH=false"
+Environment="EBK_USER_ENABLE_REGISTER=true"
+Environment="EBK_AUTH_OAUTH2_AUTO_REGISTER=true"
+
+# AI / MCP disabled by default
+Environment="EBK_MCP_ENABLE_MCP=false"
+Environment="EBK_LLM_TRANSACTION_FROM_AI_IMAGE_RECOGNITION=false"
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,35 @@
+# ezBookkeeping
+
+## Prerequisite
+
+### Create database
+
+- Create the password with `openssl rand -base64 32`
+  - Save this value in secrets.yaml in `postgresql.password.ezbookkeeping`
+  - Access infra server to create paperless_db with `podman exec -it postgresql psql -U postgres`
+
+```SQL
+CREATE USER ezbookkeeping WITH PASSWORD 'postgresql.password.ezbookkeeping';
+CREATE DATABASE ezbookkeeping_db;
+ALTER DATABASE ezbookkeeping_db OWNER TO ezbookkeeping;
+```
+
+### Create oidc secret and hash
+
+- Create the secret with `openssl rand -base64 32`
+- access to auth vm
+  - `podman exec -it authelia sh`
+  - `authelia crypto hash generate pbkdf2 --password 'ezbookkeeping.oidc.secret'`
+- Save this value in secrets.yaml in `ezbookkeeping.oidc.secret` and `ezbookkeeping.oidc.hash`
+
+### Add postgresql dump backup list
+
+- [set_postgresql.yaml](../../../ansible/roles/infra/tasks/services/set_postgresql.yaml)
+
+```yaml
+- name: Set connected services list
+  ansible.builtin.set_fact:
+    connected_services:
+      - ...
+      - "ezbookkeeping"
+```
@@ -0,0 +1,13 @@
+---
+services:
+  ezbookkeeping:
+    domain:
+      public: ""
+      internal: ""
+    ports:
+      http: ""
+    subuid: "100999"
+
+version:
+  containers:
+    ezbookkeeping: "1.4.0"
@@ -0,0 +1,8 @@
+---
+postgresql:
+  password:
+    ezbookkeeping: ""
+ezbookkeeping:
+  oidc:
+    secret: ""
+    hash: ""
@@ -0,0 +1,58 @@
+---
+- name: Create ezbookkeeping directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['ezbookkeeping']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/ezbookkeeping"
+    - "data/containers/ezbookkeeping/data"
+    - "containers/ezbookkeeping"
+    - "containers/ezbookkeeping/ssl"
+  become: true
+
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/ezbookkeeping/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['ezbookkeeping']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_ezbookkeeping"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "{{ item.name }}"
+    data: "{{ item.value }}"
+    state: "present"
+    force: true
+  loop:
+    - name: "EBK_AUTH_OAUTH2_CLIENT_SECRET"
+      value: "{{ hostvars['console']['ezbookkeeping']['oidc']['secret'] }}"
+    - name: "EBK_DATABASE_PASSWD"
+      value: "{{ hostvars['console']['postgresql']['password']['ezbookkeeping'] }}"
+  notify: "notification_restart_ezbookkeeping"
+  no_log: true
+
+- name: Deploy ezbookkeeping.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/ezbookkeeping/ezbookkeeping.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/ezbookkeeping.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_ezbookkeeping"
+
+- name: Enable ezbookkeeping.service
+  ansible.builtin.systemd:
+    name: "ezbookkeeping.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
@@ -0,0 +1,110 @@
+---
+identity_providers:
+  oidc:
+    # For the app which doesn't use secret.
+    cors:
+      endpoints:
+        - 'authorization'
+        - 'token'
+        - 'revocation'
+        - 'introspection'
+        - 'userinfo'
+      allowed_origins:
+        - 'https://opencloud.example.com'
+      allowed_origins_from_client_redirect_uris: true
+    clients:
+# OpenCloud configuration
+      ## https://docs.opencloud.eu/docs/admin/configuration/authentication-and-user-management/external-idp/
+      ## Web
+      - client_id: 'opencloud'
+        client_name: 'OpenCloud'
+        public: true
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://opencloud.example.com/'
+          - 'https://opencloud.example.com/oidc-callback.html'
+          - 'https://opencloud.example.com/oidc-silent-redirect.html'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+          - 'groups'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'RS256'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'none'
+      ## desktop
+      - client_id: 'OpenCloudDesktop'
+        client_name: 'OpenCloud'
+        public: true
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'http://localhost'
+          - 'http://127.0.0.1'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+          - 'groups'
+          - 'offline_access'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+          - 'refresh_token'
+        access_token_signed_response_alg: 'RS256'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'none'   
+      ## Android
+      - client_id: 'OpenCloudAndroid'
+        client_name: 'OpenCloud'
+        public: true
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'oc://android.opencloud.eu'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+          - 'groups'
+          - 'offline_access'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+          - 'refresh_token'
+        access_token_signed_response_alg: 'RS256'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'none'
+      ## IOS
+      - client_id: 'OpenCloudIOS'
+        client_name: 'OpenCloud'
+        public: true
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'oc://ios.opencloud.eu'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+          - 'groups'
+          - 'offline_access'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+          - 'refresh_token'
+        access_token_signed_response_alg: 'RS256'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'none'
@@ -0,0 +1,6 @@
+name: crowdsecurity/whitelists
+description: "Local whitelist policy"
+whitelist:
+    expression:
+        # opencloud chunk request false positive
+        - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'"
@@ -0,0 +1,13 @@
+---
+services:
+  opencloud:
+    domain:
+      public: ""
+      internal: ""
+    ports:
+      http: ""
+    subuid: "100999"
+
+version:
+  containers:
+    opencloud: "4.0.6"
@@ -13,8 +13,8 @@

 ## Configuration

- **!CAUTION!** OpenCloud application \(Android, IOS, Desktop\) doesn't support standard OIDC. Every scopes and client id is hardcoded.
-  - WEBFINGER_\[DESKTOP|ANDROID|IOS\]_OIDC_CLIENT_ID, WEBFINGER_\[DESKTOP|ANDROID|IOS\]_OIDC_CLIENT_SCOPES don't work on official app.
+- **!CAUTION!** OpenCloud application (Android, IOS, Desktop) doesn't support standard OIDC. Every scopes and client id is hardcoded.
+  - `WEBFINGER_[DESKTOP|ANDROID|IOS]_OIDC_CLIENT_ID`, `WEBFINGER_[DESKTOP|ANDROID|IOS]_OIDC_CLIENT_SCOPES` don't work on official app.
  - It is impossible to set group claim in scopes. Therefore, it is hard to control roles with token including group claim.
 - When authelia doesn't work, annotate `OC_EXCLUDE_RUN_SERVICES=idp` and restart to container to use local admin.
 - This app doesn't support regex on role_assignment mapping.
@@ -0,0 +1,3 @@
+---
+opencloud:
+  admin: ""
@@ -0,0 +1,36 @@
+---
+identity_providers:
+  oidc:
+    claims_policies:
+      # trilium expects name/email value in id token, but authelia doesn't send it basically
+      trilium:
+        id_token:
+          - email
+          - email_verified
+          - preferred_username
+          - name
+    clients:
+      # https://www.authelia.com/integration/openid-connect/clients/trillium/
+      # The name is trilium, not trillium
+      - client_id: 'trilium'
+        client_name: 'Trilium Notes'
+        client_secret: 'hash'
+        public: false
+        authorization_policy: 'one_factor'
+        # claims policy above
+        claims_policy: 'trilium'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          - 'https://trilium.example.com/callback'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
@@ -0,0 +1,13 @@
+---
+services:
+  trilium:
+    domain:
+      public: ""
+      internal: ""
+    ports:
+      http: ""
+    subuid: "100999"
+
+version:
+  containers:
+    trilium: "v0.102.2"
@@ -0,0 +1,6 @@
+---
+trilium:
+  admin: ""
+  oidc:
+    secret: ""
+    hash: ""
@@ -0,0 +1,38 @@
+---
+- name: Create trilium directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['trilium']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/trilium"
+    - "data/containers/trilium/data"
+  become: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "TRILIUM_OAUTH_CLIENT_SECRET"
+    data: "{{ hostvars['console']['trilium']['oidc']['secret'] }}"
+    state: "present"
+    force: true
+  notify: "notification_restart_trilium"
+  no_log: true
+
+- name: Deploy trilium.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/trilium/trilium.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/trilium.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_trilium"
+
+- name: Enable trilium.service
+  ansible.builtin.systemd:
+    name: "trilium.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
@@ -0,0 +1,46 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=trilium
+
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/triliumnext/trilium:{{ version['containers']['trilium'] }}
+
+ContainerName=trilium
+HostName=trilium
+
+PublishPort={{ services['trilium']['ports']['http'] }}:8080/tcp
+
+Volume=%h/data/containers/trilium/data:/home/node/trilium-data:rw
+
+# General
+Environment="TZ=Asia/Seoul"
+Environment="TRILIUM_DATA_DIR=/home/node/trilium-data"
+Environment="TRILIUM_NO_UPLOAD_LIMIT=true"
+
+# OIDC
+## Short Alias doesn't work now.
+#Environment="TRILIUM_OAUTH_BASE_URL=https://{{ services['trilium']['domain']['public'] }}.{{ domain['public'] }}"
+#Environment="TRILIUM_OAUTH_CLIENT_ID=trilium"
+#Environment="TRILIUM_OAUTH_ISSUER_BASE_URL=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+#Environment="TRILIUM_OAUTH_ISSUER_NAME=Authelia"
+#Environment="TRILIUM_OAUTH_ISSUER_ICON=https://www.authelia.com/images/branding/logo-cropped.png"
+#Secret="TRILIUM_OAUTH_CLIENT_SECRET",type=env
+Environment="TRILIUM_MULTIFACTORAUTHENTICATION_OAUTHBASEURL=https://{{ services['trilium']['domain']['public'] }}.{{ domain['public'] }}"
+Environment="TRILIUM_MULTIFACTORAUTHENTICATION_OAUTHCLIENTID=trilium"
+Environment="TRILIUM_MULTIFACTORAUTHENTICATION_OAUTHISSUERBASEURL=https://{{ services['authelia']['domain'] }}.{{ domain['public'] }}"
+Environment="TRILIUM_MULTIFACTORAUTHENTICATION_OAUTHISSUERNAME=Authelia"
+Environment="TRILIUM_MULTIFACTORAUTHENTICATION_OAUTHISSUERICON=https://www.authelia.com/images/branding/logo-cropped.png"
+Secret="TRILIUM_OAUTH_CLIENT_SECRET",type=env,target=TRILIUM_MULTIFACTORAUTHENTICATION_OAUTHCLIENTSECRET
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,33 @@
+# trilium
+
+## Prerequisite
+
+### Create oidc secret and hash
+
+- Create the secret with `openssl rand -base64 32`
+- access to auth vm
+  - `podman exec -it authelia sh`
+  - `authelia crypto hash generate pbkdf2 --password 'trilium.oidc.secret'`
+- Save this value in secrets.yaml in `trilium.oidc.secret` and `trilium.oidc.hash`
+
+## Configuration
+
+### Access
+
+- https://notes.ilnmors.com
+    - `[x]` I'm a new user, and I want to create a new Trilium document for my notes
+    - Next
+    - Password configuration
+    - local password login
+
+### OIDC
+
+- Menu: Options: MFA
+    - `[x]` Enable MFA
+    - `[x]` OAuth/OpenID
+    - logout
+- Authelia
+
+### about ERRORS
+
+- This is so unstable to use, especially OIDC is one of terrible experience.
@@ -0,0 +1,25 @@
+---
+identity_providers:
+  oidc:
+    clients:
+      # https://www.authelia.com/integration/openid-connect/clients/vikunja/
+      - client_id: 'vikunja'
+        client_name: 'Vikunja'
+        client_secret: 'hash'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          - 'https://vikunja.example.com/auth/openid/authelia'
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
@@ -0,0 +1,13 @@
+---
+services:
+  vikunja:
+    domain:
+      public: ""
+      internal: ""
+    ports:
+      http: ""
+    subuid: "100999"
+
+version:
+  containers:
+    vikunja: "2.2.2"
@@ -0,0 +1,9 @@
+---
+postgresql:
+  password:
+    vikunja: ""
+vikunja:
+  session_secret: ""
+  oidc:
+    secret: ""
+    hash: ""
@@ -0,0 +1,26 @@
+---
+identity_providers:
+  oidc:
+    clients:
+      # https://www.authelia.com/integration/openid-connect/clients/wikijs/
+      - client_id: 'wikijs'
+        client_name: 'Wiki'
+        client_secret: 'hash'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: false
+        pkce_challenge_method: ''
+        redirect_uris:
+          # add Callback URL / Redirect URI HERE
+          - 'https://wikijs.example.com/login/$UUID/callback'  # Note this must be copied during step 7 of the Application configuration.
+        scopes:
+          - 'openid'
+          - 'profile'
+          - 'email'
+        response_types:
+          - 'code'
+        grant_types:
+          - 'authorization_code'
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_post'
@@ -0,0 +1,13 @@
+---
+services:
+  wikijs:
+    domain:
+      public: ""
+      internal: ""
+    ports:
+      http: ""
+    subuid: "100999"
+
+version:
+  containers:
+    wikijs: "2.5.314"
@@ -0,0 +1,9 @@
+---
+postgresql:
+  password: 
+    wikijs: ""
+wikijs:
+  admin: ""
+  oidc:
+    secret: ""
+    hash: ""
@@ -0,0 +1,53 @@
+---
+- name: Create wiki.js directory
+  ansible.builtin.file:
+    path: "{{ node['home_path'] }}/{{ item }}"
+    state: "directory"
+    owner: "{{ services['wikijs']['subuid'] }}"
+    group: "svadmins"
+    mode: "0770"
+  loop:
+    - "data/containers/wikijs"
+    - "data/containers/wikijs/data"
+    - "data/containers/wikijs/export"
+    - "containers/wikijs"
+    - "containers/wikijs/ssl"
+  become: true
+
+- name: Deploy root certificate
+  ansible.builtin.copy:
+    content: |
+      {{ hostvars['console']['ca']['root']['crt'] }}
+    dest: "{{ node['home_path'] }}/containers/wikijs/ssl/{{ root_cert_filename }}"
+    owner: "{{ services['wikijs']['subuid'] }}"
+    group: "svadmins"
+    mode: "0440"
+  become: true
+  notify: "notification_restart_wikijs"
+  no_log: true
+
+- name: Register secret value to podman secret
+  containers.podman.podman_secret:
+    name: "WIKIJS_DB_PASS"
+    data: "{{ hostvars['console']['postgresql']['password']['wikijs'] }}"
+    state: "present"
+    force: true
+  notify: "notification_restart_wikijs"
+  no_log: true
+
+- name: Deploy wikijs.container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/wikijs/wikijs.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/wikijs.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_wikijs"
+
+- name: Enable wikijs.service
+  ansible.builtin.systemd:
+    name: "wikijs.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"
@@ -0,0 +1,41 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Wiki.js
+
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=ghcr.io/requarks/wiki:{{ version['containers']['wikijs'] }}
+ContainerName=wikijs
+HostName=wikijs
+PublishPort={{ services['wikijs']['ports']['http'] }}:3000/tcp
+
+# Volumes
+Volume=%h/data/containers/wikijs/data:/wiki/data:rw
+Volume=%h/data/containers/wikijs/export:/wiki/export:rw
+Volume=%h/containers/wikijs/ssl:/etc/ssl/wiki:ro
+
+# General
+Environment="TZ=Asia/Seoul"
+
+# Database
+Environment="DB_TYPE=postgres"
+Environment="DB_HOST={{ services['postgresql']['domain'] }}.{{ domain['internal'] }}"
+Environment="DB_PORT={{ services['postgresql']['ports']['tcp'] }}"
+Environment="DB_USER=wikijs"
+Environment="DB_NAME=wikijs_db"
+Environment="DB_SSL=true"
+Environment="NODE_EXTRA_CA_CERTS=/etc/ssl/wiki/{{ root_cert_filename }}"
+Secret=WIKIJS_DB_PASS,type=env,target=DB_PASS
+
+[Service]
+ExecStartPre=/usr/bin/nc -zv {{ services['postgresql']['domain'] }}.{{ domain['internal'] }} {{ services['postgresql']['ports']['tcp'] }}
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,106 @@
+# wiki.js
+
+## Prerequisite
+
+### Create database
+
+- Create the password with `openssl rand -base64 32`
+  - Save this value in secrets.yaml in `postgresql.password.wikijs`
+  - Access infra server to create wikijs_db with `podman exec -it postgresql psql -U postgres`
+
+```SQL
+CREATE USER wikijs WITH PASSWORD 'postgresql.password.wikijs';
+CREATE DATABASE wikijs_db;
+ALTER DATABASE wikijs_db OWNER TO wikijs;
+```
+
+### Create oidc secret and hash
+
+- Create the secret with `openssl rand -base64 32`
+- access to auth vm
+  - `podman exec -it authelia sh`
+  - `authelia crypto hash generate pbkdf2 --password 'wikijs.oidc.secret'`
+- Save this value in secrets.yaml in `wikijs.oidc.secret` and `wikijs.oidc.hash`
+- !CAUTION! Don't update authelia with ansible-playbook before configuration
+
+### Add postgresql dump backup list
+
+- [set_postgresql.yaml](../../../ansible/roles/infra/tasks/services/set_postgresql.yaml)
+
+```yaml
+- name: Set connected services list
+  ansible.builtin.set_fact:
+    connected_services:
+      - ...
+      - "wikijs"
+```
+
+## Configuration
+
+### Access
+
+- https://wiki.ilnmors.com
+    - Administrator Email: admin@wiki.ilnmors.internal
+    - Password: wikijs.il.password
+    - Site URL: https://wiki.ilnmors.com
+    - INSTALL
+
+### Group configuration
+
+- Administration: Groups: Guests: PERMISSIONS
+    - Remove all permissions
+- Administration: Groups: NEW GROUP
+    - Users
+- Administration: Groups: Users: PERMISSIONS
+    - Grant all permission in CONTENT
+
+- Administration: Groups: Users: PAGE RULES
+    - Allow / Deny: Allow
+    - Match: Path starts with
+    - Path: empty value
+    - Locale: Any / All
+    - Permissions:
+        - Grant all permission
+    - Update Group
+
+### OIDC configuration
+
+- Administration: Modules: Authentication
+    - Add Strategy: Generic OpenID Connect / OAuth2
+        - Display Name: Authelia
+        - client id: wikijs
+        - client secret: wikijs.oidc.secret
+        - Authorization Endpoint URL: https://authelia.ilnmors.com/api/oidc/authorization
+        - Token Endpoint URL: https://authelia.ilnmors.com/api/oidc/token
+        - User info Endpoint URL: https://authelia.ilnmors.com/api/oidc/userinfo
+        - Skip User Profile: untoggled
+        - Issure: https://authelia.ilnmors.com
+        - Email Claim: email
+        - Display Name Claim: displayName
+        - Picture Claim: picture
+        - Map Groups: untoggled
+        - Groups Claim: groups
+        - Registration: Allow self-registration: toggled
+        - Assign to group: Users
+    - Check: Callback URL / Redirect URI
+    - Apply
+
+- add Callback URL / Redirect URI to [authelia config](../../../config/services/containers/auth/authelia/config/authelia.yaml.j2)
+    - update authelia
+
+- logout from administrator
+
+- login: Select Authentication Provider: Authelia
+
+### Storage
+
+- Administration: Modules: Stroage
+    - Local File System
+    - Path: /wiki/export
+    - Apply
+
+### Locale
+
+- Administration: Site: Locale
+    - Download what you needs.
+    - Korean, Arabic, French ...
@@ -21,13 +21,14 @@
 ## Timeline
 - 2026-03-21: Release actual budget
 - 2026-03-21: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression

 ## Solution
 - Access to fw
  - Check the ban list with `sudo cscli alerts list`
  - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'
+  - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/data/migrations/'
 - Delete false positive decision
  - Check false positive decision with `sudo cscli decision list`
  - Delete false positive decision with `sudo cscli decision delete --id $ID`
@@ -20,13 +20,14 @@
 ## Timeline
 - 2026-03-21: Release Immich
 - 2026-03-21: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression

 ## Solution
 - Access to fw
  - Check the ban list with `sudo cscli alerts list`
  - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
 - Delete false positive decision
  - Check false positive decision with `sudo cscli decision list`
  - Delete false positive decision with `sudo cscli decision delete --id $ID`
@@ -20,13 +20,14 @@
 ## Timeline
 - 2026-04-04: Release OpenCloud
 - 2026-04-04: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression

 ## Solution
 - Access to fw
  - Check the ban list with `sudo cscli alerts list`
  - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'
+  - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'
 - Delete false positive decision
  - Check false positive decision with `sudo cscli decision list`
  - Delete false positive decision with `sudo cscli decision delete --id $ID`
@@ -21,9 +21,13 @@
 - 2026-05-02: Find the false positive case, and add whitelist
 - 2026-05-03: Install crowdsecurity/nextcloud-whitelist parser
 - 2026-05-03: Make previous expressions annotation
+- 2026-05-07: Find the false positive case, which is not on `crowdsecurity/nextcloud-whitelist`
+- 2026-05-07: Set whitelist expression

 ## Solution
 - Install crowdsecurity/nextcloud-whitelist on auth node
+- Add expression on whitelist
+  - evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'

 ### Deprecated solution
 - Access to fw
@@ -22,7 +22,11 @@ When the migration is decided, the manual backup after shutting all services dow
 Only when kopia repository exists.

 ```bash
-kopia repository connect --override-username="console" --override-hostname="console.ilnmors.internal"
+kopia repository connect server \
+--url=https://nas.ilnmors.internal:51515 \
+--override-username=console \
+--override-hostname=console.ilnmors.internal
+# Enter the password

 kopia snapshot list --all

@@ -38,7 +42,7 @@ cp ~/workspace/homelab/volumes/infra/cluster/cluster.sql ~/workspace/homelab/con

 ### Provisioning

-Ansible playbooks should be declarative. This won't contain complex branch logics \(Declarative over imperative\). Playbooks describes what should be there, not how to. The basic rule is manual destroy and auto reprovisioning.
+Ansible playbooks should be declarative. This won't contain complex branch logics (Declarative over imperative). Playbooks describes what should be there, not how to. The basic rule is manual destroy and auto reprovisioning.

 #### vmm and fw

@@ -63,6 +63,10 @@ Set-Service DiagTrack -StartupType Disable
 Stop-Service dmwappushservice
 Set-Service dmwappushservice -StartupType Disable

+## Disable - WorkloadsSessionHost which is the service for AI function
+Stop-Service -Name "WSAIFabricSvc" -Force -ErrorAction SilentlyContinue; Set-Service -Name "WSAIFabricSvc" -StartupType Disabled
+Stop-Process -Name "WorkloadsSessionHost" -Force -ErrorAction SilentlyContinue
+
 ## Compact OS configuration
 compact /compactos:always
 ```
@@ -102,10 +106,10 @@ sign in on app only

 - WindowsDefender Firewall:Inbound Rules:
    - File and Printer Sharing (Echo Request - ICMPv4-In) - Profile: Private, Public
-        - General: \[x\] Enable
+        - General: `[x]` Enable
        - Scope: 192.168.1.0/24, 192.168.10.0/24, 192.168.99.0/24
    - File and Printer Sharing (Echo Request - ICMPv6-In) - Profile: Private, Public
-        - General: \[x\] Enable
+        - General: `[x]` Enable
        - Scope: fd00::/8
    - Apply

@@ -119,7 +123,8 @@ sign in on app only

 ### Create wsl config

- C:\Users\$USERNAME\.wslconfig
+- `C:\Users\$USERNAME\.wslconfig`
+
 ```ini
 [wsl2]
 processors=4
@@ -201,13 +206,13 @@ mkdir ~/workspace

 #### VS Code configuration

- WSL extension\(`Ctrl + shift + x`\)
+- WSL extension(`Ctrl + shift + x`)
    - Install `WSL` by Microsoft
        - Remote Explorer:Debian:Connect in Current Windows
 - `Ctrl + k` and `Ctrl + o`
    - Open folder: `/home/console/workspace`
 - `` Ctrl + shift + ` `` for Terminal
- Extensions\(`Ctrl + shift + x`\)
+- Extensions(`Ctrl + shift + x`)
    - Install `Ansible` by RedHat

 ### Playbooks
@@ -244,9 +249,9 @@ ansible-playbook playbooks/console/site.yaml --tags "init"
    - encrypted by gpg and ansible vault with master key
 - Master key
    - The key which has above 40 characters containing upper and lower letters, numbers, and special letters
-    - managed by physical media \(Mind, MDisc, paper\) as file, string, and QR
+    - managed by physical media (Mind, MDisc, paper) as file, string, and QR
    - This value is never saved in server or console.
- Root CA \(including ssh CA\) must not be deployed.
+- Root CA (including ssh CA) must not be deployed.
     - The tasks with root CA must be performed manually. The source of Trust is the most important in security.
 - Intermediate CA can be deployed.
    - Intermediate CA is operated as a live server. 
@@ -12,7 +12,7 @@ openssl rand -base64 32 > /run/user/$UID/root_ca_password
 openssl rand -base64 32 > /run/user/$UID/intermediate_ca_password
 # Save the values in `secrets.yaml`

-# Create CAs \(Key and cert)
+# Create CAs (Key and cert)
 # Root CA
 step certificate create \
 "ilnmors.internal Root CA" /run/user/$UID/root_ca.crt /run/user/$UID/root_ca.key \
@@ -1,4 +1,4 @@
-# Hypervisor \(vmm\)
+# Hypervisor (vmm)

 Initiating hypervisor doesn't use ansible. Hypervisor is working on hardware itself, so there is a lot of possible variables like IOMMU id, MAC addresses, etc.

@@ -19,29 +19,29 @@ Hypervisor is initiated manually with the configuration files which are stored i
    - Hostname: vmm
    - Domain: ilnmors.internal
 - User:
-    - Root Password: \[blank\]
+    - Root Password: `[blank]`
    - Full name for the new user: vmm
    - User Name: bootstrap
    - User Password: debian
 - Partition setting: manual
-    - 512MiB - EFI system partition \(Booting flag: on\) 
-    - 1GiB - Ext4 Journaling \(Mount: /boot)
+    - 512MiB - EFI system partition (Booting flag: on) 
+    - 1GiB - Ext4 Journaling (Mount: /boot)
    - 800 GiB -LVM
-        - 64GiB: vmm-root - Ext4 Journaling \(Mount: /\)
-        - 700GiB: vmm-libvirt - Ext4 \(Mount: /var/lib/libvirt\)
+        - 64GiB: vmm-root - Ext4 Journaling (Mount: /)
+        - 700GiB: vmm-libvirt - Ext4 (Mount: /var/lib/libvirt)
 - Debian package manager setting
    - Scan extra installation media: no
    - Mirror country: South Korea
    - Archive mirror: deb.debian.org
-    - Proxy: \[blank\]
+    - Proxy: `[blank]`
    - Popularity-contest: no
 - Installing packages setting
-    - \[\*\] SSH server
-    - \[\*\] Standard system utilities
+    - `[*]` SSH server
+    - `[*]` Standard system utilities

 ### Initial configuration

-Hypervisor operates pure L2 switch for fw and it never can access WAN without fw after initial configuration. This means, there is an air-gap which means hypervisor cannot access to WAN for a while \(from end of initial setting to the beginning of fw setting\).
+Hypervisor operates pure L2 switch for fw and it never can access WAN without fw after initial configuration. This means, there is an air-gap which means hypervisor cannot access to WAN for a while (from end of initial setting to the beginning of fw setting).

 Hypervisor operates on hardware. Hardware information is always uncertain, and it is set only once. Managing this process as IaC is over engineering.

@@ -6,19 +6,19 @@ All hardware configuration is set after fw vm. The MAC address of hardware is re

 ### Access VLAN switch

- http://switch.ilnmors.internal \(192.168.1.2, KEA-DHCP, Only IPv4 support\)
+- http://switch.ilnmors.internal (192.168.1.2, KEA-DHCP, Only IPv4 support)
    - before set ipv6, use ip4 address instead of FQDN
    - id: admin, password: admin
    - new password: switch.password

 ### Set VLAN
 - VLAN:802.1Q VLAN
-    - \[x\] Enable - Apply
+    - `[x]` Enable - Apply
    - VLAN client
        - id 1
        - name default > client
-        - member \(Untagged\)
-            - Port 1 \(Trunk, untagged\): Linux bridge is already process untagged packet as id 1
+        - member (Untagged)
+            - Port 1 (Trunk, untagged): Linux bridge is already process untagged packet as id 1
            - Port 3
            - Port 4
            - Port 5
@@ -29,13 +29,13 @@ All hardware configuration is set after fw vm. The MAC address of hardware is re
        - id 10
        - name server
        - member
-            - Port 1 \(Trunk, tagged\)
+            - Port 1 (Trunk, tagged)
    - VLAN user
        - id 20
        - name user
        - member
-            - Port 1 \(Trunk, tagged\)
-            - Port 2 \(Not a member of client vlan, untagged\)
+            - Port 1 (Trunk, tagged)
+            - Port 2 (Not a member of client vlan, untagged)

 - VLAN:802.1Q VLAN PVID setting
    - Port 2
@@ -48,9 +48,9 @@ All hardware configuration is set after fw vm. The MAC address of hardware is re
 - Check internet connection


-## DSM \(DS124\)
+## DSM (DS124)

- https://finds.synology.com/# \(192.168.1.11, KEA-DHCP\)
+- https://finds.synology.com/# (192.168.1.11, KEA-DHCP)
    - Install DSM

 ### Initial configuration
@@ -83,7 +83,7 @@ Kea in fw already reserved DSM's IP. However it is necessary to set IP address s
        - Certificate
        - Intermediate certificate
    - Edit: For: Set as default certificate
-    - Setting \(!CAUTION!\)
+    - Setting (!CAUTION!)
        - Even though you set the certificate as default, you have to set certificate for each services.
        - configure: service: certificate: nas.ilnmors.internal

@@ -92,20 +92,20 @@ Kea in fw already reserved DSM's IP. However it is necessary to set IP address s
 - **!CAUTION!** It can be set after authelia is implemented
 - Following [here](../../config/services/containers/auth/authelia/config/authelia.yaml.j2) for Authelia configuration
 - Control Panel:Domain/LDAP:SSO Client
-    - Login Settings: \[x\] Select SSO by default on the login page
+    - Login Settings: `[x]` Select SSO by default on the login page
    - Services
-        - \[x\] Enable OpenID Connect SSO service
+        - `[x]` Enable OpenID Connect SSO service
        - OpenID Connect SSO Settings
            - Profile: OIDC
            - Account type: Domain/LDAP/local
            - Name: Authelia
            - Well-Known URL: https://authelia.ilnmors.com/.well-known/openid-configuration
-            - Application ID: dsm \(what you designated\)
+            - Application ID: dsm (what you designated)
            - Application Secret: secret value
            - Redirect URI: https://nas.ilnmors.internal:5001
            - Authorization scope: openid profile groups email
            - Username claim: preferred_username
- Match the user name \(ID\) in DSM and lldap id.
+- Match the user name (ID) in DSM and lldap id.

 ### Kopia in DSM

@@ -123,15 +123,15 @@ Kea in fw already reserved DSM's IP. However it is necessary to set IP address s

 - Add certificate - DSM reverse proxy cannot deal with gRPC
    - /docker/kopia/config/ssl/nas.key
-    - /docker/kopia/config/ssl/nas.crt \(including intermediate crt\)
+    - /docker/kopia/config/ssl/nas.crt (including intermediate crt)

 - container manager:images:import
    - kopia/kopia
-    - tags: \{\{ version['packages']['kopia'] \}\}
+    - tags: {{ version['packages']['kopia'] }}
 - run
    - image: kopia/kopia
    - containername: kopia-server
-    - \[x\] Enable auto restart
+    - `[x]` Enable auto restart
    - port: 51515:51515
    - volume: /docker/kopia/config:/app/config:rw
    - volume: /docker/kopia/cache:/app/cache:rw
@@ -159,7 +159,7 @@ Repository directory - encrypted by server KOPIA_PASSWORD as master key of repos

 Server manage ACL with user password, user's KOPIA_PASSWORD. When server verify user with their password, server works with its repository password.

-Repository - \(Repository key; master key\) -  Server - \(User key; access key\) - Client
+Repository - (Repository key; master key) -  Server - (User key; access key) - Client

 - Client knows its access password as KOPIA_PASSWORD to access server. It doesn't know master key, server's KOPIA_PASSWORD. server will control repository by its KOPIA_PASSWORD. their name is the same but it is different.

@@ -8,11 +8,11 @@

 # *! CAUTION !*
 # THIS PROCESS CONTAINING SECRET VALUES.
-# WHEN YOU TYPE THE COMMAND ON SHELL, YOU MUST USE [BLANK] BEFORE COMMAND
+# WHEN YOU TYPE THE COMMAND ON SHELL, YOU MUST USE [blank] BEFORE COMMAND
 # e.g.
 # shell@shell$ command (X)
-# shell@shell$ [BLANK]command (O)
-# BLANK prevent the command to save on .bash_history
+# shell@shell$ [blank]command (O)
+# [blank] prevent the command to save on .bash_history
 # After finish this process, use `history -c` and `clear` for just in case.


@@ -70,6 +70,13 @@ git stash pop # get temporary save
 # After git switch
 git switch service
 git rebase --ignore-date main # set date as current time on main branch
+
+# Example
+
+git show HEAD
+git show HEAD~$NUM
+git diff HEAD~$NUM HEAD
+git diff $PREVIOUS_HASH $CURRENT_HASH
 ```

 ## Add Service with git
@@ -90,6 +97,10 @@ git merge caddy-app
 - Set this after gitea is implemented

 ```bash
+# Copy git from remote
+git clone --mirror https://gitea.ilnmors.com/il/ilnmors-homelab.git
+# If the tag doesn't come then
+git fetch --tags
 # Add git remote repository
 git config --global credential.helper store
 git remote add origin https://gitea.ilnmors.com/il/ilnmors-homelab.git
@@ -54,8 +54,8 @@ CREATE EXTENSION IF NOT EXISTS vector;
 ### About community edition limitation

 - Workspace seats
-  - The number of members itself \(account\) are unlimited.
-  - However the number of members who work on the same workspace simultaneously \(seats\) are designated as 10 members.
+  - The number of members itself (account) are unlimited.
+  - However the number of members who work on the same workspace simultaneously (seats) are designated as 10 members.
 - Workspace storage quota
  - Originally, self-hosted version has no limitation in storage quota and uploading file size.
  - Now, there is some limitation even in the self-hosted version.
@@ -85,8 +85,8 @@ CREATE EXTENSION IF NOT EXISTS vector;

 #### Auth

- [ ] Whether allow new registrations
- [x] Whether allow new registration via configured oauth
+- `[ ]` Whether allow new registrations
+- `[x]` Whether allow new registration via configured oauth
 - Minimum length requirement of password: 8
 - Maximum length requirement of password: 50
 - save
@@ -117,5 +117,5 @@ Environment="AFFINE_SERVER_HTTPS=true"

 #### Flags

- [x] Whether allow guest users to create demo workspaces
+- `[x]` Whether allow guest users to create demo workspaces
 - save
@@ -0,0 +1,28 @@
+# Collabora office
+
+## Prerequisite
+
+- Nothing
+
+## Configuration
+
+- Admin page is disabled by Environment options
+    - `admin_console.enable=false`
+
+### Link to nextcloud
+
+- https://nextcloud.ilnmors.com
+    - login with admin account
+
+- Profile: Apps: Nextcloud Office 
+    - Check installation and enable
+
+- Profile: Administration Settings: Nextcloud Office: Your own server
+    - http://host.containers.internal:9980 (collabora container port)
+    - Public FQDN is set automatically
+    - save
+
+- Files
+    - Verify document opening (verified)
+    - The basic font `Noto Sans KR` exists
+        - Korean is presented very well
@@ -61,7 +61,7 @@ CREATE EXTENSION IF NOT EXISTS earthdistance CASCADE;
  - map
  - version check
 - User privacy
-  - google cast \(disable\)
+  - google cast (disable)
 - Storage template
  - `{{y}}/{{MM}}/{{y}}{{MM}}{{dd}}_{{hh}}{{mm}}{{ss}}`
 - Backups
@@ -61,7 +61,7 @@ ALTER DATABASE nextcloud_db OWNER TO nextcloud;
  - Mail
  - Nextcloud Office

-### Configuration
+### OIDC and DB Configuration

 ```bash
 podman exec -u www-data nextcloud php occ user_oidc:provider Authelia \
@@ -86,3 +86,21 @@ podman exec -u www-data nextcloud php occ db:add-missing-primary-keys

 - Profile: Accounts:
  - allocate admin group for admin users
+
+#### Disable System addressbook expose
+
+- Profile: Administration Settings: Groupware: System Address Book
+  - Disable `Enable system address book` option
+
+## Security warning in Nextcloud (ignored)
+
+### trusted_proxies option
+- Nextcloud wants admin to set `trusted_proxies` via forwarded ip header.
+  - In current system, app vm explicitly prevents access the nextcloud container outside of vm.
+  - trusted_proxy ip address will be definitely 169.254.1.2 (caddy's APIPA address which is used in PASTA network), so it is not distinguished from other containers.
+  - Therefore, it doesn't need to be set.
+
+### HSTS option
+- This system is already main - sidecar reverse proxy system, and main proxy automatically changes http requests to https request (Caddyfile listens https).
+  - main - sidecar communication is also on https via internal certificate.
+  - Therefore, it doesn't need to be set.
@@ -67,8 +67,8 @@ ALTER DATABASE paperless_db OWNER TO paperless;
  - Mode: skip
    - When the archive file has broken ocr text, then conduct replcae command manually
  - Skip archive File: never
-  - Deskew: disable \(toggle to enable and once more to active disable option\)
-  - rotate: disable \(toggle to enable and once more to active disable option\)
+  - Deskew: disable (toggle to enable and once more to active disable option)
+  - rotate: disable (toggle to enable and once more to active disable option)

 ## The non-standard pdf file

@@ -0,0 +1,67 @@
+# sure
+
+## Prerequisite
+
+### Create database
+
+- Create the password with `openssl rand -base64 32`
+  - Save this value in secrets.yaml in `postgresql.password.sure`
+  - Access infra server to create sure_db with `podman exec -it postgresql psql -U postgres`
+
+```SQL
+CREATE USER sure WITH PASSWORD 'postgresql.password.sure';
+CREATE DATABASE sure_db;
+ALTER DATABASE sure_db OWNER TO sure;
+```
+
+### Create oidc secret and hash
+
+- Create the secret with `openssl rand -base64 32`
+- access to auth vm
+  - `podman exec -it authelia sh`
+  - `authelia crypto hash generate pbkdf2 --password 'sure.oidc.secret'`
+- Save this value in secrets.yaml in `sure.oidc.secret` and `sure.oidc.hash`
+
+### Create session secret value
+
+- Create the secret with `LC_ALL=C tr -dc 'A-Za-z0-9!#%&()*+,-./:;<=>?@[\]^_{|}~' </dev/urandom | head -c 32`
+  - Save this value in secrets.yaml in `sure.session_secret`
+
+### Add postgresql dump backup list
+
+- [set_postgresql.yaml](../../../ansible/roles/infra/tasks/services/set_postgresql.yaml)
+
+```yaml
+- name: Set connected services list
+  ansible.builtin.set_fact:
+    connected_services:
+      - ...
+      - "sure"
+```
+
+## Configuration
+
+### Access to sure
+
+- https://sure.ilnmors.com
+  - Sign in with Authelia
+  - Create account
+
+### Account configuration
+
+- Setup: 
+  - First name and last name
+  - Will be using sure with
+    - `[x]` Family members
+  - Country: South Korea
+- Preference: 
+  - South Korean Won (KRW)
+  - date: YYYY-MM-DD
+- Goals:
+  - Next
+
+### Group and user configuration
+
+- Profile: Settings: Profile info:
+  - Household: Set name
+  - Invite member: input email of member
@@ -2,7 +2,7 @@

 ## Communication

-Alloy runs on systemd \(host\), and postgresql runs as container \(rootless podman\). When host system and container communicate, container recognizes host system as host-gateway \(Link local address\).
+Alloy runs on systemd (host), and postgresql runs as container (rootless podman). When host system and container communicate, container recognizes host system as host-gateway (Link local address).

 ## postgresql monitor
 
@@ -6,10 +6,10 @@ This is not a perfect E2EE communication theorogically, however technically it i

 ### .com public domain

-WAN - \(Let's Encrypt certificate\) -> Caddy \(auth\) - \(ilnmors internal certificate\) -> Caddy \(app\) or https services - http -> app's local service
+WAN - (Let's Encrypt certificate) -> Caddy (auth) - (ilnmors internal certificate) -> Caddy (app) or https services - http -> app's local service

 ### .internal private domain
-client - \(ilnmors internal certificate\) -> Caddy \(Infra\) - http -> local services
+client - (ilnmors internal certificate) -> Caddy (Infra) - http -> local services

 ### DNS record

@@ -3,16 +3,16 @@
 ## LAPI

 ### Detecting
-Host logs \> CrowdSec Agent\(parser\) > CrowdSec LAPI
+Host logs > CrowdSec Agent(parser) > CrowdSec LAPI

 ### Decision
-CrowdSec LAPI \(Decision + Register\)
+CrowdSec LAPI (Decision + Register)

 ### Block
-CrowdSec LAPI \> CrowdSec Bouncer \(Block\)
+CrowdSec LAPI > CrowdSec Bouncer (Block)

 ## CAPI
-CrowdSec CAPI \> crowdsec LAPI \(local\) \> CrowdSec Bouncer \(Block\)
+CrowdSec CAPI > crowdsec LAPI (local) > CrowdSec Bouncer (Block)

 ## Ansible Deployment

@@ -20,34 +20,34 @@ CrowdSec CAPI \> crowdsec LAPI \(local\) \> CrowdSec Bouncer \(Block\)

 - Deploy fw's config.yaml
 - Deploy crowdsec certificates
- Register machines \(Agents\)
- Register bouncers \(Bouncers\)
+- Register machines (Agents)
+- Register bouncers (Bouncers)

 ### Set Bouncer (fw/roles/tasks/set_crowdsec_bouncer.yaml)

 - Deploy crowdsec-firewall-bouncer.yaml
- Install suricata collection \(parser\) with cscli
+- Install suricata collection (parser) with cscli
 - Set acquis.d for suricata
 - set-only: bouncer can't get metrics from the chain and rules count result which it doesn't make. - It means, it is impossible to use prometheus metric with set-only true option.
 - chain or rules matched count reasults are able to check on nftables. 
-  - use sudo nft list chain inet filter global to check packet blocked. \(counter command is required\)
+  - use sudo nft list chain inet filter global to check packet blocked. (counter command is required)

 ### Set Machines; agents (common/tasks/set_crowdsec_agent.yaml)

- Deploy config.yaml except fw \(disable LAPI, online_api_credentials\)
+- Deploy config.yaml except fw (disable LAPI, online_api_credentials)
 - Deploy local_api_credentials.yaml

 ### Set caddy host (auth/tasks/set_caddy.yaml)

 - Set caddy CrowdSec module
 - Set caddy log directory
- Install caddy collection \(parser\) with cscli
+- Install caddy collection (parser) with cscli
 - Set acquis.d for caddy

 ### Set whitelist (/etc/crowdsec/parser/s02-enrich/whitelists.yaml)

 - Set only local console IP address
- This can block local VM to the other subnet, but the communication between vms is possible because they are in the same subnet\(L2\) - packets don't pass the fw.
+- This can block local VM to the other subnet, but the communication between vms is possible because they are in the same subnet(L2) - packets don't pass the fw.
 - Crowdsec bouncer only conducts blocks forward chain which pass Firewall, it is blocked by crowdsec bouncer based on lapi

 ## Test
@@ -234,9 +234,9 @@ fw@fw:~$ sudo cscli alerts inspect 230 -d

 - check the log and analyze and make expression
  - e.g. immich
-    - evt.Meta.target_fqdn == 'immich.ilnmors.com' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+    - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
  - e.g. opencloud
-    - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
+    - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'"
 - free false positive decision

 fw@fw:~$ sudo cscli decision list
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
il	4527e39d0f	chore(app): archive removed stacks from app archived stacks: - actual-budget - ezbookkeeping - opencloud - trilium - vikunja - wikijs	2026-05-10 00:07:51 +09:00
il	02fa912cb1	feat(trilium): release trilium deployment notes: - oidc error (users cannot access at once, it needs login twice when using oidc	2026-05-09 22:38:57 +09:00
il	aceef4bdaa	refactor(authelia): update authelia.yaml.j2 to fix redirect_uris from hardcoded uris to ansible variables	2026-05-09 21:44:11 +09:00
il	64aad4fcf0	docs(all): fix markdown syntax and snippets	2026-05-09 20:54:32 +09:00
il	81244d55a7	feat(wiki.js): release wiki.js deployment notes: - use this as personal/family wiki system - compare to affine / memos and triliumNext	2026-05-09 17:50:05 +09:00
il	1cfd024285	refactor(x509-exporter): update notification to restart x509-exporter when its config.yaml is changed	2026-05-09 17:42:35 +09:00
il	26115c5660	feat(redis): update redis from 8.6.1 to 8.6.3 update notes: - run 'ansible-playbook playbooks/app/site.yaml --tags "site"' so that update all redis at onece	2026-05-09 13:55:28 +09:00
il	acef35ca8b	feat(postgresql): update postgresql and vectorchord extension update notes: - update postgresql version from 18.2 to 18.3 - update vectorchord version from 0.5.3 to 1.1.1 - add update flow and notice to postgresql.md	2026-05-09 13:54:10 +09:00
il	b531170bd7	feat(vaultwardne): update vaultwarden from 1.35.8 to 1.36.0	2026-05-09 12:56:22 +09:00
il	ad586c3cd3	feat(grafana): update grafana from 12.3.3 to 13.0.1	2026-05-09 12:50:36 +09:00
il	6dfef08f7b	feat(prometheus): update prometheus from v3.9.1 to v3.11.3	2026-05-09 12:44:44 +09:00
il	934dd314a8	feat(x509-exporter): update x509-exporter from 3.21.0 to 4.1.0 update notes: - '--listen-address' and '--watch-dir' cli flags are deprecated - add '--config' cli flag and config.yaml	2026-05-09 12:44:05 +09:00
il	2529a918df	feat(loki): update loki version from 3.6.5 to 3.7.1	2026-05-09 12:17:16 +09:00
il	7dfa20d3dd	feat(ldap): update lldap version from 0.6.2 to 0.6.3	2026-05-09 11:56:19 +09:00
il	329620c7d7	feat(alloy): update alloy version from 1.13.0 to 1.16.1	2026-05-09 11:52:25 +09:00
il	f820e89cf6	refactor(roles): update binary application installation flow update notes: - keep set_cli_tools responsible only for console CLI tools - download and install kopia from the kopia role - download and install blocky from the blocky role - download and install alloy from the alloy role - reduce console artifact staging for service binaries	2026-05-09 10:46:29 +09:00
il	a05951f883	fix(crowdsec): optimize whitelist expressions update notes: - add http_status and http_verb for each expressions (actual budget, immich, opencloud) - fix crowdsec and issues documents	2026-05-07 10:32:11 +09:00
il	b404a9e459	fix(crowdsec): update whitelist.yaml to prevent false positive false positive: - nextcloud thumbnail/preview 404 problem (crowdsecurity/http-probing)	2026-05-07 10:27:34 +09:00
il	3b4b56f53f	fix(nftables): update fw nftables to allow vpn connection regardless of crowdsec ban	2026-05-07 09:22:49 +09:00
il	f697715065	feat(sure): release sure (we-promise/sure) deployment notes: - let's try three of budget apps, actual budget, ezbookkeeping, and sure	2026-05-06 18:52:31 +09:00
il	be7f215380	feat(ezbookkeeping): release ezbookkeeping deployment notes: - use ezbookkeeping for budget - compare to actual budget - it has no RBAC and sharing budget, try to sure (we-promise/sure)	2026-05-06 15:56:19 +09:00
il	26e0fe4f8b	docs(ADR): update ADR 007 - backup to add checking rule and flows	2026-05-06 14:30:25 +09:00
il	2bb1f015e0	fix(kopia): update the bound home path from %h to ansible variable update note: - hotfix - backups haven't run since commit '9f236b6fa5' - the root service unit's %h always indicates root's home path - backup service is verified	2026-05-06 14:06:22 +09:00
il	0f546e13b3	fix(btrfs): update btrfs scrub path update notes: - from '{{ node['home_path'] }}/data' to '{{ storage['btrfs']['mount_point'] }}'	2026-05-06 10:33:57 +09:00
il	ba8b312bf2	feat(btrfs): update btrfs scrub service and timer on app vm	2026-05-06 08:15:53 +09:00
il	6fcedd9162	feat(collabora): release collabora deployment note: - link to nextcloud - document opening is verified (including korean fonts)	2026-05-05 21:20:31 +09:00
il	6ca4f61d50	docs(nextcloud): update security warning decisions and background job annotation update notes: - trusted_proxies warning - HSTS option warning - background job mode annotation	2026-05-05 20:09:00 +09:00
il	15c09cb899	docs(nextcloud): update how to disable auto generated contacts from nextcloud account	2026-05-03 12:05:11 +09:00