feat(loki): update loki version from 3.6.5 to 3.7.1

update notes:
- keep set_cli_tools responsible only for console CLI tools
- download and install kopia from the kopia role
- download and install blocky from the blocky role
- download and install alloy from the alloy role
- reduce console artifact staging for service binaries

ansible/inventory/group_vars/all.yaml

@@ -185,16 +185,16 @@ version:
     step: "0.30.2"
     kopia: "0.22.3"
     blocky: "0.29.0"
-    alloy: "1.13.0"
+    alloy: "1.16.1"
   containers:
     # common
     caddy: "2.11.2"
     # infra
     step: "0.30.2"
-    ldap: "v0.6.2"
+    ldap: "v0.6.3"
     x509-exporter: "3.21.0"
     prometheus: "v3.9.1"
-    loki: "3.6.5"
+    loki: "3.7.1"
     grafana: "12.3.3"
     ## Postgresql
     postgresql: "18.2"

ansible/playbooks/console/site.yaml

@@ -122,3 +122,11 @@
         apply:
           tags: ["init", "site", "tools"]
       tags: ["init", "site", "tools"]
+
+    - name: Set kopia
+      ansible.builtin.include_role:
+        name: "common"
+        tasks_from: "services/set_kopia"
+        apply:
+          tags: ["init", "site", "kopia"]
+      tags: ["init", "site", "kopia"]

ansible/roles/common/tasks/services/set_alloy.yaml

@@ -5,9 +5,10 @@
       - hardware
   become: true
 
-- name: Deploy alloy deb file (x86_64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-amd64.deb"
+- name: Download alloy deb file (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.amd64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -15,9 +16,10 @@
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
-- name: Deploy alloy deb file (aarch64)
-  ansible.builtin.copy:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-arm64.deb"
+- name: Download alloy deb file (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
+      alloy-{{ version['packages']['alloy'] }}-1.arm64.deb"
     dest: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     owner: "root"
     group: "root"
@@ -30,6 +32,7 @@
     deb: "/var/cache/apt/archives/alloy-{{ version['packages']['alloy'] }}.deb"
     state: "present"
   become: true
+  notify: "notification_restart_alloy"
 
 - name: Deploy alloy config
   ansible.builtin.template:

ansible/roles/common/tasks/services/set_kopia.yaml

@@ -5,34 +5,36 @@
       - hardware
   become: true
 
-- name: Check kopia installation
-  ansible.builtin.shell: |
-    command -v kopia
-  changed_when: false
-  failed_when: false
-  register: "is_kopia_installed"
-  ignore_errors: true
-
 - name: Set console kopia
   when: node['name'] == 'console'
   block:
-    - name: Apply cli tools (x86_64)
+    - name: Download kopia
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
+        dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
+        owner: "{{ ansible_user }}"
+        group: "svadmins"
+        mode: "0600"
+      loop:
+        - "amd64"
+        - "arm64"
+
+    - name: Install kopia (x86_64)
       ansible.builtin.apt:
         deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
         state: "present"
       become: true
-      when:
-        - ansible_facts['architecture'] == "x86_64"
-        - is_kopia_installed.rc != 0
-    - name: Apply cli tools (aarch64)
+      when: ansible_facts['architecture'] == "x86_64"
+
+    - name: Install kopia (aarch64)
       ansible.builtin.apt:
         deb: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
         state: "present"
       become: true
-      when:
-        - ansible_facts['architecture'] == "aarch64"
-        - is_kopia_installed.rc != 0
-    - name: Connect kopia server
+      when: ansible_facts['architecture'] == "aarch64"
+
+    - name: Connect console kopia server
       environment:
         KOPIA_PASSWORD: "{{ hostvars['console']['kopia']['user']['console'] }}"
       ansible.builtin.shell: |
@@ -51,30 +53,36 @@
     - name: Set kopia uid
       ansible.builtin.set_fact:
         kopia_uid: 951
-    - name: Deploy kopia deb file (x86_64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-amd64.deb"
+
+    - name: Download kopia deb file (x86_64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_amd64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "x86_64"
-    - name: Deploy kopia deb file (aarch64)
-      ansible.builtin.copy:
-        src: "{{ hostvars['console']['node']['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-arm64.deb"
+
+    - name: Download kopia deb file (aarch64)
+      ansible.builtin.get_url:
+        url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
+          kopia_{{ version['packages']['kopia'] }}_linux_arm64.deb"
         dest: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         owner: "root"
         group: "root"
         mode: "0644"
       become: true
       when: ansible_facts['architecture'] == "aarch64"
+
     - name: Create kopia group
       ansible.builtin.group:
         name: "kopia"
         gid: "{{ kopia_uid }}"
         state: "present"
       become: true
+
     - name: Create kopia user
       ansible.builtin.user:
         name: "kopia"
@@ -85,6 +93,7 @@
         comment: "Kopia backup User"
         state: "present"
       become: true
+
     - name: Create kopia directory
       ansible.builtin.file:
         path: "{{ item.name }}"
@@ -101,12 +110,13 @@
           mode: "0700"
       become: true
       no_log: true
+
     - name: Install kopia
       ansible.builtin.apt:
         deb: "/var/cache/apt/archives/kopia-{{ version['packages']['kopia'] }}.deb"
         state: "present"
       become: true
-      when: is_kopia_installed.rc != 0
+
     - name: Deploy kopia env
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2"
@@ -116,6 +126,7 @@
         mode: "0400"
       become: true
       no_log: true
+
     - name: Deploy kopia service files
       ansible.builtin.template:
         src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/{{ item }}.j2"
@@ -128,6 +139,7 @@
         - "kopia-backup.service"
         - "kopia-backup.timer"
       become: true
+
     - name: Enable auto kopia rules update
       ansible.builtin.systemd:
         name: "kopia-backup.timer"

ansible/roles/console/tasks/services/set_cli_tools.yaml

@@ -49,42 +49,6 @@
     - "amd64"
     - "arm64"
 
-- name: Download kopia
-  ansible.builtin.get_url:
-    url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\
-      kopia_{{ version['packages']['kopia'] }}_linux_{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/kopia-{{ version['packages']['kopia'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
-- name: Download blocky
-  ansible.builtin.get_url:
-    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
-      blocky_v{{ version['packages']['blocky'] }}_Linux_{{ item }}.tar.gz"
-    dest: "{{ node['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-{{ item }}.tar.gz"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "x86_64"
-    - "arm64"
-
-- name: Download alloy
-  ansible.builtin.get_url:
-    url: "https://github.com/grafana/alloy/releases/download/v{{ version['packages']['alloy'] }}/\
-      alloy-{{ version['packages']['alloy'] }}-1.{{ item }}.deb"
-    dest: "{{ node['data_path'] }}/bin/alloy-{{ version['packages']['alloy'] }}-{{ item }}.deb"
-    owner: "{{ ansible_user }}"
-    group: "svadmins"
-    mode: "0600"
-  loop:
-    - "amd64"
-    - "arm64"
-
 - name: Apply cli tools (x86_64)
   ansible.builtin.apt:
     deb: "{{ node['data_path'] }}/bin/{{ item }}"
@@ -92,7 +56,6 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-amd64.deb"
     - "step-{{ version['packages']['step'] }}-amd64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-amd64.deb"
   become: true
   when: ansible_facts['architecture'] == "x86_64"
 
@@ -103,6 +66,5 @@
   loop:
     - "sops-{{ version['packages']['sops'] }}-arm64.deb"
     - "step-{{ version['packages']['step'] }}-arm64.deb"
-    - "kopia-{{ version['packages']['kopia'] }}-arm64.deb"
   become: true
   when: ansible_facts['architecture'] == "aarch64"

ansible/roles/fw/tasks/services/set_blocky.yaml

@@ -23,7 +23,7 @@
     state: "present"
   become: true
 
-- name: Create blocky etc directory
+- name: Create blocky directory
   ansible.builtin.file:
     path: "{{ item }}"
     owner: "blocky"
@@ -31,13 +31,38 @@
     mode: "0750"
     state: "directory"
   loop:
+    - "/home/blocky"
+    - "/home/blocky/bin"
     - "/etc/blocky"
     - "/etc/blocky/ssl"
   become: true
 
+- name: Download blocky (x86_64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_x86_64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "x86_64"
+
+- name: Download blocky (aarch64)
+  ansible.builtin.get_url:
+    url: "https://github.com/0xERR0R/blocky/releases/download/v{{ version['packages']['blocky'] }}/\
+      blocky_v{{ version['packages']['blocky'] }}_Linux_arm64.tar.gz"
+    dest: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    owner: "blocky"
+    group: "blocky"
+    mode: "0600"
+  become: true
+  when: ansible_facts['architecture'] == "aarch64"
+
 - name: Deploy blocky binary file (x86_64)
   ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-x86_64.tar.gz"
+    remote_src: true
     dest: "/usr/local/bin/"
     owner: "root"
     group: "root"
@@ -52,7 +77,8 @@
 
 - name: Deploy blocky binary file (aarch64)
   ansible.builtin.unarchive:
-    src: "{{ hostvars['console']['node']['data_path'] }}/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    src: "/home/blocky/bin/blocky-{{ version['packages']['blocky'] }}-arm64.tar.gz"
+    remote_src: true
     dest: "/usr/local/bin/"
     owner: "root"
     group: "root"

config/node/fw/nftables.conf.j2

@@ -82,6 +82,8 @@ table inet filter {
         chain global {
                 # invalid packets
                 ct state invalid drop comment "deny invalid connection"
+                # VPN connection exception handling
+                udp dport $PORTS_VPN return comment "return vpn connection to input and forward chain"
                 # crowdsec
                 ip saddr @crowdsec-blacklists counter drop comment "deny all crowdsec blacklist"
                 ip6 saddr @crowdsec6-blacklists counter drop comment "deny all ipv6 crowdsec blacklist"

config/services/systemd/common/crowdsec/bouncers/whitelists.yaml.j2

@@ -13,9 +13,11 @@ whitelist:
 {% if node['name'] == 'auth' %}
     expression:
         # budget local-first sql scrap rule
-        - "evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'"
+        - "evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/data/migrations/'"
         # immich thumbnail request 404 error false positive
-        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
+        - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
         # opencloud chunk request false positive
-        - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
+        - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'"
+        # nextcloud thumbnail/preview request error false positive
+        - "evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'"
 {% endif %}

docs/issues/crowdsec/260321_actual_budget.md

@@ -21,13 +21,14 @@
 ## Timeline
 - 2026-03-21: Release actual budget
 - 2026-03-21: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression
 
 ## Solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/data/migrations/'
+  - evt.Meta.target_fqdn == '{{ services['actualbudget']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/data/migrations/'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260321_immich.md

@@ -20,13 +20,14 @@
 ## Timeline
 - 2026-03-21: Release Immich
 - 2026-03-21: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression
 
 ## Solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+  - evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260404_opencloud.md

@@ -20,13 +20,14 @@
 ## Timeline
 - 2026-04-04: Release OpenCloud
 - 2026-04-04: Find the false positive case, and add whitelist
+- 2026-05-07: Optimize whitelist expression
 
 ## Solution
 - Access to fw
   - Check the ban list with `sudo cscli alerts list`
   - Read the ban case with `sudo cscli alerts inspect $NUMBER`
 - Add expressions on whitelist
-  - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'
+  - evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'
 - Delete false positive decision
   - Check false positive decision with `sudo cscli decision list`
   - Delete false positive decision with `sudo cscli decision delete --id $ID`

docs/issues/crowdsec/260502_nextcloud.md

@@ -21,9 +21,13 @@
 - 2026-05-02: Find the false positive case, and add whitelist
 - 2026-05-03: Install crowdsecurity/nextcloud-whitelist parser
 - 2026-05-03: Make previous expressions annotation
+- 2026-05-07: Find the false positive case, which is not on `crowdsecurity/nextcloud-whitelist`
+- 2026-05-07: Set whitelist expression
 
 ## Solution
 - Install crowdsecurity/nextcloud-whitelist on auth node
+- Add expression on whitelist
+  - evt.Meta.target_fqdn == '{{ services['nextcloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' &&  evt.Meta.http_path startsWith '/index.php/core/preview?'
 
 ### Deprecated solution
 - Access to fw

docs/services/common/crowdsec.md

@@ -234,9 +234,9 @@ fw@fw:~$ sudo cscli alerts inspect 230 -d
 
 - check the log and analyze and make expression
   - e.g. immich
-    - evt.Meta.target_fqdn == 'immich.ilnmors.com' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'
+    - "evt.Meta.target_fqdn == '{{ services['immich']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status == '404' && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/api/assets/' && evt.Meta.http_path contains '/thumbnail'"
   - e.g. opencloud
-    - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_path contains '/js/chunks/'"
+    - "evt.Meta.target_fqdn == '{{ services['opencloud']['domain']['public'] }}.{{ domain['public'] }}' && evt.Meta.http_status in ['200', '304'] && evt.Meta.http_verb == 'GET' && evt.Meta.http_path contains '/js/chunks/'"
 - free false positive decision
 
 fw@fw:~$ sudo cscli decision list