docs(ADR): update ADR 007 - backup to add checking rule and flows

update note:
- hotfix
- backups haven't run since commit '9f236b6fa5'
- the root service unit's %h always indicates root's home path
- backup service is verified

ansible/inventory/group_vars/all.yaml

@@ -156,6 +156,13 @@ services:
       http: "8002"
       redis: "6382"
     subuid: "100032"
+  collabora:
+    domain:
+      public: "collabora"
+      internal: "collabora.app"
+    ports:
+      http: "9980"
+    subuid: "101000"
 
 version:
   packages:
@@ -193,3 +200,4 @@ version:
     manticore: "25.0.0"
     affine: "0.26.3"
     nextcloud: "33.0.3"
+    collabora: "25.04.9.4.1"

ansible/playbooks/app/site.yaml

@@ -233,6 +233,13 @@
           tags: ["site", "nextcloud"]
       tags: ["site", "nextcloud"]
 
+    - name: Set collabora
+      ansible.builtin.include_role:
+        name: "app"
+        tasks_from: "services/set_collabora"
+        apply:
+          tags: ["site", "collabora"]
+      tags: ["site", "collabora"]
 
     - name: Flush handlers right now
       ansible.builtin.meta: "flush_handlers"

ansible/roles/app/handlers/main.yaml

@@ -111,3 +111,14 @@
   changed_when: false
   listen: "notification_restart_nextcloud"
   ignore_errors: true # noqa: ignore-errors
+
+- name: Restart collabora
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "restarted"
+    enabled: true
+    scope: "user"
+    daemon_reload: true
+  changed_when: false
+  listen: "notification_restart_collabora"
+  ignore_errors: true # noqa: ignore-errors

ansible/roles/app/tasks/node/set_raid.yaml

@@ -68,3 +68,23 @@
     group: "svadmins"
     mode: "0770"
   become: true
+
+- name: Deploy btrfs scrub service and timer
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/app/btrfs/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  loop:
+    - "btrfs-scrub.service"
+    - "btrfs-scrub.timer"
+  become: true
+
+- name: Enable auto btrfs scrub
+  ansible.builtin.systemd:
+    name: "btrfs-scrub.timer"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+  become: true

ansible/roles/app/tasks/services/set_collabora.yaml

@@ -0,0 +1,17 @@
+---
+- name: Deploy container file
+  ansible.builtin.template:
+    src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/app/collabora/collabora.container.j2"
+    dest: "{{ node['home_path'] }}/.config/containers/systemd/collabora.container"
+    owner: "{{ ansible_user }}"
+    group: "svadmins"
+    mode: "0644"
+  notify: "notification_restart_collabora"
+
+- name: Enable collabora.service
+  ansible.builtin.systemd:
+    name: "collabora.service"
+    state: "started"
+    enabled: true
+    daemon_reload: true
+    scope: "user"

ansible/roles/common/tasks/services/set_crowdsec.yaml

@@ -36,10 +36,15 @@
   ansible.builtin.set_fact:
     acquisd_list:
       fw:
-        collection: "crowdsecurity/suricata"
+        collection:
+          - "crowdsecurity/suricata"
+        parser: []
         config: "suricata.yaml"
       auth:
-        collection: "crowdsecurity/caddy"
+        collection:
+          - "crowdsecurity/caddy"
+        parser:
+          - "crowdsecurity/nextcloud-whitelist"
         config: "caddy.yaml"
 
 - name: Deploy crowdsec-update service files
@@ -181,7 +186,8 @@
   block:
     - name: Install crowdsec collection
       ansible.builtin.command:
-        cmd: "cscli collections install {{ acquisd_list[node['name']]['collection'] }}"
+        cmd: "cscli collections install {{ item }}"
+      loop: "{{ acquisd_list[node['name']]['collection'] }}"
       become: true
       changed_when: "'overwrite' not in is_collection_installed.stderr"
       failed_when:
@@ -189,6 +195,17 @@
         - "'already installed' not in is_collection_installed.stderr"
       register: "is_collection_installed"
 
+    - name: Install crowdsec parser
+      ansible.builtin.command:
+        cmd: "cscli parsers install {{ item }}"
+      loop: "{{ acquisd_list[node['name']]['parser'] }}"
+      become: true
+      changed_when: "'overwrite' not in is_parser_installed.stderr"
+      failed_when:
+        - is_parser_installed.rc != 0
+        - "'already installed' not in is_parser_installed.stderr"
+      register: "is_parser_installed"
+
     - name: Create crowdsec acquis.d directory
       ansible.builtin.file:
         path: "/etc/crowdsec/acquis.d"

config/services/containers/app/collabora/collabora.container.j2

@@ -0,0 +1,25 @@
+[Quadlet]
+DefaultDependencies=false
+
+[Unit]
+Description=Collabora Online
+
+[Container]
+Image=docker.io/collabora/code:{{ version['containers']['collabora'] }}
+ContainerName=collabora
+HostName=collabora
+
+PublishPort={{ services['collabora']['ports']['http'] }}:9980/tcp
+
+Environment="TZ=Asia/Seoul"
+Environment="aliasgroup1=https://{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}"
+# Environment="aliasgroup2=other_server_FQDN"
+Environment="extra_params=--o:ssl.enable=false --o:ssl.termination=true --o:server_name={{ services['collabora']['domain']['public'] }}.{{ domain['public'] }} --o:admin_console.enable=false"
+
+[Service]
+Restart=always
+RestartSec=10s
+TimeoutStopSec=120
+
+[Install]
+WantedBy=default.target

config/services/containers/app/nextcloud/config/background.config.php.j2

@@ -1,4 +1,5 @@
 <?php
 $CONFIG = [
+    // Background jobs mode is auto-detected as 'cron' when nextcloud-cron.timer runs cron.php via CLI. No explicit config required.
     'maintenance_window_start' => 18,
 ];

config/services/containers/common/caddy/etc/app/Caddyfile.j2

@@ -83,3 +83,9 @@
                 header_up Host {http.request.header.X-Forwarded-Host}
         }
 }
+{{ services['collabora']['domain']['internal'] }}.{{ domain['internal'] }} {
+        import private_tls
+        reverse_proxy host.containers.internal:{{ services['collabora']['ports']['http'] }} {
+                header_up Host {http.request.header.X-Forwarded-Host}
+        }
+}

config/services/containers/common/caddy/etc/auth/Caddyfile.j2

@@ -145,6 +145,15 @@
                 }
         }
 }
+{{ services['collabora']['domain']['public'] }}.{{ domain['public'] }} {
+        import crowdsec_log
+        route {
+                crowdsec
+                reverse_proxy https://{{services['collabora']['domain']['internal'] }}.{{ domain['internal'] }} {
+                        header_up Host {http.reverse_proxy.upstream.host}
+                }
+        }
+}
 
 # Internal domain
 {{ node['name'] }}.{{ domain['internal'] }} {

config/services/systemd/app/btrfs/btrfs-scrub.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=BTRFS auto scrub
+ConditionPathIsMountPoint={{ storage['btrfs']['mount_point'] }}
+RequiresMountsFor={{ storage['btrfs']['mount_point'] }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/btrfs scrub start -Bd {{ storage['btrfs']['mount_point'] }}
+Nice=19
+IOSchedulingClass=idle

config/services/systemd/app/btrfs/btrfs-scrub.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Monthly BTRFS auto scrub 
+
+[Timer]
+OnCalendar=*-*-01 04:00:00
+Persistent=true
+RandomizedDelaySec=300
+
+[Install]
+WantedBy=timers.target

config/services/systemd/common/crowdsec/bouncers/whitelists.yaml.j2

@@ -18,9 +18,4 @@ whitelist:
         - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
         # opencloud chunk request false positive
         - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
-        # nextcloud chunk request false positive (crowdsecurity/http-crawl-non_statics)
-        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/apps/viewer/js/'"
-        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/dist/'"
-        # nextcloud upload directory request 404 error false positive (crowdsecurity/http-probing)
-        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/remote.php/dav/files/'"
 {% endif %}

config/services/systemd/common/kopia/kopia-backup.service.j2

@@ -21,9 +21,9 @@ ProtectHome=tmpfs
 InaccessiblePaths=/boot /root
 
 {% if node['name'] == 'infra' %}
-BindReadOnlyPaths=%h/containers/postgresql/backups
+BindReadOnlyPaths={{ node['home_path'] }}/containers/postgresql/backups
 {% elif node['name'] == 'app' %}
-BindReadOnlyPaths=%h/data
+BindReadOnlyPaths={{ node['home_path'] }}/data
 {% endif %}
 # In root namescope, %u always bring 0
 BindPaths=/etc/kopia
@@ -38,10 +38,10 @@ ExecStartPre=/usr/bin/kopia repository connect server \
 
 {% if node['name'] == 'infra' %}
 ExecStart=/usr/bin/kopia snapshot create \
-    /home/infra/containers/postgresql/backups
+    {{ node['home_path'] }}/containers/postgresql/backups
 {% elif node['name'] == 'app' %}
 ExecStart=/usr/bin/kopia snapshot create \
-    /home/app/data
+    {{ node['home_path'] }}/data
 {% endif %}
 
 

docs/adr/007-backup.md

@@ -6,7 +6,10 @@
     - First documentation
 
 - Feb/27/2026
-    - Status changed from Deffered to Accepted
+    - Status changed from Deferred to Accepted
+
+- May/06/2026
+    - Add backup checking rules
     
 ## Status
 
@@ -30,6 +33,12 @@
     - Backing up the `/var/lib/postgresql` directory directly while the DB is running can lead to severe data corruption and inconsistency.
     - Logical dumps (`pg_dump`) are much safer, database-agnostic, and easier to restore in a homelab environment.
 
+### Silent failure problem
+
+- May/06/2026, the fact that backups haven't run since commit '9f236b6fa5' because of '%h' in system service unit.
+    - Operator couldn't realize backup doesn't run because the system service was failed silently.
+    - Therefore, set the checking rule.
+
 
 ## Decisions
 
@@ -54,7 +63,14 @@
         - From kopia server to console:$HOMELAB_PATH/data/volume/infra/postgresql/\{cluster,data\}
     - APP data files
         - From kopia server to APP vm after initiating before deploy services
-- Automative backup does not guarantee integrity of data system, so before reset the system conduct manual backup after making sure all services are shutdown.
+- Automatic backup does not guarantee integrity of data system, so before reset the system conduct manual backup after making sure all services are shutdown.
+- Check the repository once a week (Every monday)
+    - Check the snapshot in repository with `kopia snapshot list --all`
+    - Mount the snapshot respectively with `kopia mount $SNAPSHOT_ID $DESTINATION`
+    - Copy random file from snapshot and check the values.
+    - If there's some failure, check the backup service and conduct backup immediately.
+    - Repeat the check flow.
+    - When everything is done, umount the kopia mount with `ctrl+c`
 
 ## Consequences
 

docs/issues/crowdsec/260321_actual_budget.md

@@ -26,7 +26,7 @@
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
-- Add regex on whitelist
+- Add expressions on whitelist
   - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`

docs/issues/crowdsec/260321_immich.md

@@ -25,8 +25,8 @@
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
-- Add regex on whitelist
+- Add expressions on whitelist
-  - evt.Meta.target_fqdn == 'Immich.ilnmors.com' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260404_opencloud.md

@@ -25,7 +25,7 @@
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
-- Add regex on whitelist
+- Add expressions on whitelist
   - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`

docs/issues/crowdsec/260502_nextcloud.md

@@ -14,18 +14,18 @@
   - fw ban users' IP address.
 
 ## Reason
-- Nextcloud uses chunks for actions, and uploading and downloading
+- Nextcloud has a lot of workflows which can be caught from crowdsec
-    - chunks on '/apps/viewer/js', '/dist/'
-    - `crowdsecurity/http-crawl-non_statics`
-- Nextcloud keeps checking directory which is uploading
-    - upload directory '/remote.php/dav/files/'
-    - `crowdsecurity/http-probing`
 
 ## Timeline
 - 2026-05-02: Release nextcloud
 - 2026-05-02: Find the false positive case, and add whitelist
+- 2026-05-03: Install crowdsecurity/nextcloud-whitelist parser
+- 2026-05-03: Make previous expressions annotation
 
 ## Solution
+- Install crowdsecurity/nextcloud-whitelist on auth node
+
+### Deprecated solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`

docs/services/app/collabora.md

@@ -0,0 +1,28 @@
+# Collabora office
+
+## Prerequisite
+
+- Nothing
+
+## Configuration
+
+- Admin page is disabled by Environment options
+    - `admin_console.enable=false`
+
+### Link to nextcloud
+
+- https://nextcloud.ilnmors.com
+    - login with admin account
+
+- Profile: Apps: Nextcloud Office 
+    - Check installation and enable
+
+- Profile: Administration Settings: Nextcloud Office: Your own server
+    - http://host.containers.internal:9980 (collabora container port)
+    - Public FQDN is set automatically
+    - save
+
+- Files
+    - Verify document opening (verified)
+    - The basic font `Noto Sans KR` exists
+        - Korean is presented very well

docs/services/app/nextcloud.md

@@ -61,7 +61,7 @@ ALTER DATABASE nextcloud_db OWNER TO nextcloud;
   - Mail
   - Nextcloud Office
 
-### Configuration
+### OIDC and DB Configuration
 
 ```bash
 podman exec -u www-data nextcloud php occ user_oidc:provider Authelia \
@@ -86,3 +86,21 @@ podman exec -u www-data nextcloud php occ db:add-missing-primary-keys
 
 - Profile: Accounts:
   - allocate admin group for admin users
+
+#### Disable System addressbook expose
+
+- Profile: Administration Settings: Groupware: System Address Book
+  - Disable `Enable system address book` option
+
+## Security warning in Nextcloud (ignored)
+
+### trusted_proxies option
+- Nextcloud wants admin to set `trusted_proxies` via forwarded ip header.
+  - In current system, app vm explicitly prevents access the nextcloud container outside of vm.
+  - trusted_proxy ip address will be definitely 169.254.1.2 (caddy's APIPA address which is used in PASTA network), so it is not distinguished from other containers.
+  - Therefore, it doesn't need to be set.
+
+### HSTS option
+- This system is already main - sidecar reverse proxy system, and main proxy automatically changes http requests to https request (Caddyfile listens https).
+  - main - sidecar communication is also on https via internal certificate.
+  - Therefore, it doesn't need to be set.

docs/specifications/environments.md

@@ -123,7 +123,7 @@
         - [x] OpenCloud
         - [x] affine \(Notion substitution\)
         - [x] Nextcloud \(Use nextcloud as CalDAV and CardDav, kanban and todo\)
-        - [ ] Collabora office
+        - [x] Collabora office \(Link to Nextcloud, it works well\)
         - WriteFreely
         - MediaCMS
         - Funkwhale