diff --git a/ansible/inventory/group_vars/all.yaml b/ansible/inventory/group_vars/all.yaml index 1debfe5..bef81ac 100644 --- a/ansible/inventory/group_vars/all.yaml +++ b/ansible/inventory/group_vars/all.yaml @@ -45,6 +45,7 @@ services: internal: "kopia" ports: https: "51515" + uid: "951" dependency: cert: true caddy: @@ -333,26 +334,26 @@ version: ## maddy maddy: "0.9.5" x509-exporter: "4.1.0" - prometheus: "v3.13.0" - loki: "3.7.3" - grafana: "13.1.0" + prometheus: "v3.13.2" + loki: "3.7.4" + grafana: "13.1.1" # Auth authelia: "4.39.20" # App - vaultwarden: "1.36.0" - gitea: "1.26.4" - redis: "8.8.0" - immich: "v3.0.1" - nextcloud: "34.0.1" - collabora: "26.04.2.1.1" + vaultwarden: "1.37.1" + gitea: "1.27.1" + redis: "8.10.0" + immich: "v3.1.0" + nextcloud: "34.0.2" + collabora: "26.04.2.4.1" sure: "0.7.2" - outline: "1.8.1" - memos: "0.29.1" - mealie: "v3.20.1" + outline: "1.9.2" + memos: "0.30.0" + mealie: "v3.22.0" homebox: "0.26.2" homepage: "v1.13.2" snappymail: "v2.38.2" - mastodon: "v4.6.3" + mastodon: "v4.6.4" booklogr: "v1.11.1" papra: "26.6.1-rootless" - romm: "5.0.0-beta.2" + romm: "5.1.0" diff --git a/ansible/roles/common/tasks/services/set_kopia.yaml b/ansible/roles/common/tasks/services/set_kopia.yaml index 5d9389c..538dabb 100644 --- a/ansible/roles/common/tasks/services/set_kopia.yaml +++ b/ansible/roles/common/tasks/services/set_kopia.yaml @@ -85,11 +85,6 @@ notify: "notification_restart_kopia_server" no_log: true - - name: Check repository existance - ansible.builtin.stat: - path: "{{ node['home_path'] }}/data/kopia/repository" - register: "is_kopia_repository" - - name: Create repository directory ansible.builtin.file: path: "{{ node['home_path'] }}/data/kopia/repository" @@ -97,7 +92,14 @@ owner: "{{ ansible_user }}" group: "{{ node['group'] }}" mode: "0770" - when: not is_kopia_repository.stat.exists + + - name: Check repository directory empty + ansible.builtin.find: + path: "{{ node['home_path'] }}/data/kopia/repository/" + file_type: "any" + hidden: true + recurse: false + register: "is_kopia_repository_empty" - name: Create kopia repository ansible.builtin.shell: | @@ -106,11 +108,9 @@ set +a /usr/bin/kopia repository create filesystem \ --path="{{ node['home_path'] }}/data/kopia/repository" \ - --description="{{ domain['internal'] }} kopia repository" \ - --create-only - when: not is_kopia_repository.stat.exists + --description="{{ domain['internal'] }} kopia repository" + when: is_kopia_repository_empty.matched == 0 changed_when: true - register: "is_repository_created" no_log: true - name: Connect to kopia repository @@ -122,10 +122,11 @@ --path="{{ node['home_path'] }}/data/kopia/repository" \ --override-username="il" \ --override-hostname="{{ node['name'] }}.{{ domain['internal'] }}" - when: - - not is_kopia_repository.stat.exists - - is_repository_created.rc == 0 - changed_when: true + when: is_kopia_repository_empty.matched > 0 + changed_when: false + failed_when: + - is_kopia_repository_empty.matched > 0 + - is_repository_connected.rc != 0 register: "is_repository_connected" no_log: true @@ -144,10 +145,27 @@ password: "{{ hostvars['console']['kopia']['users']['infra'] }}" - name: "app@app.{{ domain['internal'] }}" password: "{{ hostvars['console']['kopia']['users']['app'] }}" - when: - - not is_kopia_repository.stat.exists - - is_repository_connected.rc == 0 - changed_when: true + when: is_kopia_repository_empty.matched == 0 + changed_when: false + no_log: true + + - name: Update kopia user + ansible.builtin.shell: | + set -a + . /etc/secrets/{{ node['uid'] }}/kopia.env + set +a + /usr/bin/kopia server users set \ + --user-password {{ item.password | quote }} \ + {{ item.name }} + loop: + - name: "console@console.{{ domain['internal'] }}" + password: "{{ hostvars['console']['kopia']['users']['console'] }}" + - name: "infra@infra.{{ domain['internal'] }}" + password: "{{ hostvars['console']['kopia']['users']['infra'] }}" + - name: "app@app.{{ domain['internal'] }}" + password: "{{ hostvars['console']['kopia']['users']['app'] }}" + when: is_kopia_repository_empty.matched > 0 + changed_when: false no_log: true - name: Set kopia console ACL @@ -158,17 +176,15 @@ /usr/bin/kopia server acl add \ --user console@console.{{ domain['internal'] }} \ --target {{ item }} \ - --access FULL + --access FULL \ + --overwrite loop: - "type=snapshot" - "type=policy" - "type=user" - "type=acl" - "type=content" - when: - - not is_kopia_repository.stat.exists - - is_repository_connected.rc == 0 - changed_when: true + changed_when: false no_log: true - name: Set kopia infra ACL @@ -179,7 +195,8 @@ /usr/bin/kopia server acl add \ --user infra@infra.{{ domain['internal'] }} \ --target {{ item.target }} \ - --access {{ item.access }} + --access {{ item.access }} \ + --overwrite loop: - target: "type=snapshot" access: "FULL" @@ -189,10 +206,7 @@ access: "FULL" - target: "type=content" access: "APPEND" - when: - - not is_kopia_repository.stat.exists - - is_repository_connected.rc == 0 - changed_when: true + changed_when: false no_log: true - name: Set kopia app ACL @@ -203,7 +217,8 @@ /usr/bin/kopia server acl add \ --user app@app.{{ domain['internal'] }} \ --target {{ item.target }} \ - --access {{ item.access }} + --access {{ item.access }} \ + --overwrite loop: - target: "type=snapshot" access: "FULL" @@ -213,10 +228,7 @@ access: "FULL" - target: "type=content" access: "APPEND" - when: - - not is_kopia_repository.stat.exists - - is_repository_connected.rc == 0 - changed_when: true + changed_when: false no_log: true - name: Set kopia policy @@ -248,10 +260,7 @@ --keep-weekly=4 \ --keep-monthly=6 \ --keep-annual=1 - when: - - not is_kopia_repository.stat.exists - - is_repository_connected.rc == 0 - changed_when: true + changed_when: false no_log: true - name: Set kopia parameters @@ -261,10 +270,7 @@ set +a /usr/bin/kopia repository set-parameters \ --max-pack-size-mb=100 - when: - - not is_kopia_repository.stat.exists - - is_repository_connected.rc == 0 - changed_when: true + changed_when: false no_log: true - name: Deploy kopia server service file @@ -289,10 +295,6 @@ - name: Set infra/app backup kopia when: node['name'] in ["infra", "app"] block: - - name: Set kopia uid - ansible.builtin.set_fact: - kopia_uid: 951 - - name: Download kopia deb file (x86_64) ansible.builtin.get_url: url: "https://github.com/kopia/kopia/releases/download/v{{ version['packages']['kopia'] }}/\ @@ -318,14 +320,14 @@ - name: Create kopia group ansible.builtin.group: name: "kopia" - gid: "{{ kopia_uid }}" + gid: "{{ services['kopia']['uid'] }}" state: "present" become: true - name: Create kopia user ansible.builtin.user: name: "kopia" - uid: "{{ kopia_uid }}" + uid: "{{ services['kopia']['uid'] }}" group: "kopia" shell: "/usr/sbin/nologin" password_lock: true @@ -359,8 +361,8 @@ - name: Deploy kopia env ansible.builtin.template: src: "{{ hostvars['console']['node']['config_path'] }}/services/systemd/common/kopia/kopia.env.j2" - dest: "/etc/secrets/{{ kopia_uid }}/kopia.env" - owner: "{{ kopia_uid }}" + dest: "/etc/secrets/{{ services['kopia']['uid'] }}/kopia.env" + owner: "{{ services['kopia']['uid'] }}" group: "root" mode: "0400" become: true @@ -413,7 +415,7 @@ become: true when: (is_kopia_connected.rc | default(1)) == 0 changed_when: false - failed_when: false + failed_when: is_snapshot_exist.rc != 0 register: "is_snapshot_exist" no_log: true diff --git a/ansible/roles/infra/tasks/services/set_postgresql.yaml b/ansible/roles/infra/tasks/services/set_postgresql.yaml index 6aaf8fe..7c0d669 100644 --- a/ansible/roles/infra/tasks/services/set_postgresql.yaml +++ b/ansible/roles/infra/tasks/services/set_postgresql.yaml @@ -34,6 +34,11 @@ - "containers/postgresql/build" become: true +- name: Check postgresql .init marker + ansible.builtin.stat: + path: "{{ node['home_path'] }}/containers/postgresql/.init" + register: "is_postgresql_init_marker" + - name: Deploy containerfile for build ansible.builtin.template: src: "{{ hostvars['console']['node']['config_path'] }}/services/containers/infra/postgresql/build/postgresql.containerfile.j2" @@ -95,38 +100,56 @@ notify: "notification_reload_postgresql" no_log: true -- name: Check postgresql .init marker - ansible.builtin.stat: - path: "{{ node['home_path'] }}/containers/postgresql/.init" - register: "is_postgresql_init_marker" +- name: Initiating Postgresql + when: not is_postgresql_init_marker.stat.exists + block: + - name: Check initiating SQL files + ansible.builtin.find: + path: "{{ node['home_path'] }}/data/containers/postgresql/backups/sql/" + file_type: "file" + patterns: + - "pg_*.sql" + hidden: true + recurse: false + become: true + register: "postgresql_init_sql_files" -- name: Check initiating SQL file - ansible.builtin.find: - path: "{{ node['home_path'] }}/data/containers/postgresql/backups/sql/" - file_type: "file" - patterns: - - "pg_*.sql" - hidden: true - recurse: false - become: true - register: "postgresql_init_sql_files" + - name: Set initiating SQL file lists + ansible.builtin.set_fact: + postgresql_init_sql_file_list: >- + {{ + postgresql_init_sql_files.files + | map(attribute='path') + | map('basename') + | map('regex_replace', '^pg_', '') + | map('regex_replace', '\\.sql$', '') + | list + }} -- name: Remote copy the initiating SQL - ansible.builtin.copy: - src: "{{ node['home_path'] }}/data/containers/postgresql/backups/sql/pg_{{ item }}.sql" - remote_src: true - dest: "{{ node['home_path'] }}/containers/postgresql/init/{{ '%02d' | format(index_num) }}_pg_{{ item }}.sql" - owner: "{{ services['postgresql']['subuid'] }}" - group: "{{ node['group'] }}" - mode: "0600" - loop: "{{ ['cluster'] + connected_services }}" - loop_control: - index_var: index_num - become: true - when: - - (connected_services | length) > 0 - - not is_postgresql_init_marker.stat.exists - - postgresql_init_sql_files.matched == (connected_services | length) + 1 + - name: Fail on invalid initiating SQL file set + ansible.builtin.fail: + when: + - postgresql_init_sql_files.matched > 0 + - ((['cluster'] + connected_services) + | symmetric_difference(postgresql_init_sql_file_list) + | length) > 0 + + - name: Remote copy the initiating SQL + ansible.builtin.copy: + src: "{{ node['home_path'] }}/data/containers/postgresql/backups/sql/pg_{{ item }}.sql" + remote_src: true + dest: "{{ node['home_path'] }}/containers/postgresql/init/{{ '%02d' | format(index_num) }}_pg_{{ item }}.sql" + owner: "{{ services['postgresql']['subuid'] }}" + group: "{{ node['group'] }}" + mode: "0600" + loop: "{{ ['cluster'] + connected_services }}" + loop_control: + index_var: index_num + become: true + when: + - ((['cluster'] + connected_services) + | symmetric_difference(postgresql_init_sql_file_list) + | length) == 0 - name: Deploy container file ansible.builtin.template: diff --git a/config/node/fw/nftables.conf.j2 b/config/node/fw/nftables.conf.j2 index 5daff0e..e0f01bd 100644 --- a/config/node/fw/nftables.conf.j2 +++ b/config/node/fw/nftables.conf.j2 @@ -112,6 +112,7 @@ table inet filter { jump global comment "set global condition" iifname "lo" accept comment "allow local connection: FW > FW" udp dport $PORTS_VPN accept comment "allow vpn connection: > FW" + ip6 saddr $NET6_LLA meta l4proto icmpv6 accept comment "allow icmpv6 connection: ipv6 LLA > FW" iifname { $IF_CLIENT, $IF_SERVER, $IF_USER, $IF_WG } meta l4proto { icmp, icmpv6 } accept comment "allow icmp, icmpv6 connection: CLIENT/SERVER/USER/WG > FW" iifname { $IF_CLIENT, $IF_SERVER, $IF_WG } udp dport $PORTS_NTP accept comment "allow ntp connection: CLIENT/SERVER/WG > FW" # Global chain contains "WAN !> :SSH_PORT" diff --git a/config/services/containers/app/homepage/images/.gitkeep b/config/services/containers/app/homepage/images/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/services/containers/infra/grafana/etc/dashboards/.gitkeep b/config/services/containers/infra/grafana/etc/dashboards/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/services/containers/infra/postgresql/services/postgresql-cluster-backup.service b/config/services/containers/infra/postgresql/services/postgresql-cluster-backup.service index b6e8a15..99f7d54 100644 --- a/config/services/containers/infra/postgresql/services/postgresql-cluster-backup.service +++ b/config/services/containers/infra/postgresql/services/postgresql-cluster-backup.service @@ -15,7 +15,8 @@ ExecStartPre=/usr/bin/podman exec postgresql sh -c "mkdir -p /backups/sql && cho # Run the script -ExecStart=/usr/bin/podman exec -u postgres postgresql sh -c 'pg_dumpall -U postgres --schema-only | grep -v -E "CREATE ROLE postgres" > /backups/cluster/pg_cluster_$(date "+%%Y-%%m-%%d").sql' +ExecStart=/usr/bin/podman exec -u postgres postgresql sh -c 'pg_dumpall -U postgres --schema-only > /backups/cluster/pg_cluster_$(date "+%%Y-%%m-%%d").sql' +ExecStart=/usr/bin/podman exec -u postgres postgresql sh -c 'sed -i \'/CREATE ROLE postgres/d\' /backups/cluster/pg_cluster_$(date "+%%Y-%%m-%%d").sql' ExecStart=/usr/bin/podman exec -u postgres postgresql sh -c 'cp /backups/cluster/pg_cluster_$(date "+%%Y-%%m-%%d").sql /backups/sql/pg_cluster.sql' ExecStart=/usr/bin/podman exec -u postgres postgresql sh -c "find /backups/cluster -maxdepth 1 -type f -name 'pg_cluster_*.sql' -mtime +7 -delete" ExecStart=/usr/bin/podman exec postgresql sh -c "chown -R postgres:root /backups/cluster && chmod 660 /backups/cluster/*" diff --git a/config/services/systemd/common/alloy/config.alloy.j2 b/config/services/systemd/common/alloy/config.alloy.j2 index c9ab450..91e7c2f 100644 --- a/config/services/systemd/common/alloy/config.alloy.j2 +++ b/config/services/systemd/common/alloy/config.alloy.j2 @@ -346,7 +346,7 @@ loki.process "journal_parser" { } } stage.match { - selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\berror\\b|\\berr\\b|\\bfailed\\b|\\bfailure\\b|\\bexception\\b|traceback|exit-code|container died|permission denied|access denied|connection refused|connection reset|timed out|timeout exceeded|no such file|authentication failed|sqlstate|duplicate key|relation .* does not exist|column .* does not exist|database .* does not exist|relation .* already exists|column .* already exists|x509|unknown authority|certificate .*expired|tls handshake|bad gateway|upstream .*failed|upstream .*error)\"" + selector = "{level=~\"info|debug\", job!~\"loki|prometheus|grafana\"} |~ \"(?i)(\\berror\\b|\\berr\\b|\\bfailed\\b|\\bfailure\\b|\\bexception\\b|traceback|exit-code|container died|permission denied|access denied|connection refused|connection reset|timed out|timeout exceeded|no such file|authentication failed|sqlstate|duplicate key|relation .* does not exist|column .* does not exist|database .* does not exist|relation .* already exists|column .* already exists|\bx509:|unknown authority|certificate .*expired|tls handshake|bad gateway|upstream .*failed|upstream .*error)\"" stage.static_labels { values = { level = "error", diff --git a/config/services/systemd/common/kopia/kopia-backup.service.j2 b/config/services/systemd/common/kopia/kopia-backup.service.j2 index 7cf0606..b3cfb93 100644 --- a/config/services/systemd/common/kopia/kopia-backup.service.j2 +++ b/config/services/systemd/common/kopia/kopia-backup.service.j2 @@ -24,9 +24,9 @@ BindReadOnlyPaths={{ node['home_path'] }}/data # In root namescope, %u always bring 0 BindPaths=/etc/kopia -BindPaths=/etc/secrets/{{ kopia_uid }} +BindPaths=/etc/secrets/{{ services['kopia']['uid'] }} BindPaths=/var/cache/kopia -EnvironmentFile=/etc/secrets/{{ kopia_uid }}/kopia.env +EnvironmentFile=/etc/secrets/{{ services['kopia']['uid'] }}/kopia.env ExecStartPre=/usr/bin/nc -zv {{ services['kopia']['domain']['internal'] }}.{{ domain['internal'] }} 443