diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8065b8c4..7329a60e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -19,3 +19,6 @@ jobs: - name: Run statix run: nix run nixpkgs#statix -- check . + + - name: Check restic prune singleton + run: scripts/check-restic-prune-singleton.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 700ab742..0653ec63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,3 +24,10 @@ repos: pass_filenames: false args: ["check", "."] + - id: check-restic-prune-singleton + name: check-restic-prune-singleton + entry: scripts/check-restic-prune-singleton.sh + language: system + files: ^modules/hosts/.*\.nix$ + pass_filenames: false + diff --git a/modules/genebean/home/default.nix b/modules/genebean/home/default.nix index adea2ba3..761b9f79 100644 --- a/modules/genebean/home/default.nix +++ b/modules/genebean/home/default.nix @@ -58,6 +58,7 @@ ./services/chromium-kiosk.nix ./services/flatpak.nix ./services/kiosk-backups.nix + ./services/restic.nix ./services/tailscale.nix ]; } diff --git a/modules/genebean/home/services/restic.nix b/modules/genebean/home/services/restic.nix new file mode 100644 index 00000000..c899a19e --- /dev/null +++ b/modules/genebean/home/services/restic.nix @@ -0,0 +1,12 @@ +{ lib, ... }: +{ + options.genebean.services.restic = { + enable = lib.mkEnableOption "shared restic backup infrastructure (backup + cheap per-host forget)"; + + enablePruneJob = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Run the expensive, fleet-wide restic prune (data reclaim) job on this host. Must be true on EXACTLY ONE host across the whole fleet - prune is a whole-repository operation (it has to know what data every host's remaining snapshots still reference), so running it on more than one host wastes redundant work and running it on zero means space is never reclaimed. Enforced by a pre-commit check (scripts/check-restic-prune-singleton.sh)."; + }; + }; +} diff --git a/modules/genebean/nixos/default.nix b/modules/genebean/nixos/default.nix index b6d01c63..0fce95a6 100644 --- a/modules/genebean/nixos/default.nix +++ b/modules/genebean/nixos/default.nix @@ -13,6 +13,7 @@ ./services/chromium-kiosk.nix ./services/flatpak.nix ./services/kiosk-backups.nix + ./services/restic.nix ./services/tailscale.nix ]; } diff --git a/modules/genebean/nixos/services/kiosk-backups.nix b/modules/genebean/nixos/services/kiosk-backups.nix index 205e3716..4fd905ce 100644 --- a/modules/genebean/nixos/services/kiosk-backups.nix +++ b/modules/genebean/nixos/services/kiosk-backups.nix @@ -1,11 +1,13 @@ { config, lib, + pkgs, username, ... }: let cfg = config.home-manager.users.${username}.genebean.services.kiosk-backups; + resticCfg = config.home-manager.users.${username}.genebean.services.restic; usesImpermanence = config.environment.persistence."/persist".enable or false; prefixedPaths = map (path: if usesImpermanence then "/persist${path}" else path) cfg.paths; in @@ -16,34 +18,43 @@ in assertion = !(lib.any (lib.hasPrefix "/persist") cfg.paths); message = "genebean.services.kiosk-backups.paths must not include a /persist prefix - it's added automatically when the host uses impermanence."; } + { + assertion = resticCfg.enable; + message = "genebean.services.kiosk-backups requires genebean.services.restic.enable = true - it depends on that module's restic_env/restic_repo/restic_password secrets and base daily backup job."; + } ]; - services.restic.backups = { - daily = { - paths = prefixedPaths; - # Avoid a boot-triggered catch-up backup capturing the wrong state - # as "latest" before a chance to restore after a reinstall - - # confirmed on kiosk-gene-desk hardware, see its original comment. - timerConfig = { - OnCalendar = "daily"; - Persistent = false; - }; + services.restic.backups.daily = { + paths = prefixedPaths; + # Avoid a boot-triggered catch-up backup capturing the wrong state + # as "latest" before a chance to restore after a reinstall - + # confirmed on kiosk-gene-desk hardware, see its original comment. + timerConfig = { + OnCalendar = "daily"; + Persistent = false; }; + }; - pre-reinstall-cleanup = { - environmentFile = config.sops.secrets.restic_env.path; - passwordFile = config.sops.secrets.restic_password.path; - repositoryFile = config.sops.secrets.restic_repo.path; - pruneOpts = [ - "--tag pre-reinstall" - "--host ${config.networking.hostName}" - "--keep-within 45d" - ]; - timerConfig = { - OnCalendar = "daily"; - Persistent = false; - }; + systemd.services.restic-forget-kiosk-pre-reinstall = { + description = "Expire this kiosk's own pre-reinstall-tagged snapshots older than 45d (cheap - no prune)"; + serviceConfig = { + Type = "oneshot"; + EnvironmentFile = config.sops.secrets.restic_env.path; + # Reuses genebean.services.restic's own cache dir (same repo) - + # see that module's resticEnv comment for why this matters. + CacheDirectory = "restic-backups-daily"; + CacheDirectoryMode = "0700"; + }; + environment = { + RESTIC_REPOSITORY_FILE = config.sops.secrets.restic_repo.path; + RESTIC_PASSWORD_FILE = config.sops.secrets.restic_password.path; + RESTIC_CACHE_DIR = "/var/cache/restic-backups-daily"; }; + script = '' + ${lib.getExe pkgs.restic} forget \ + --tag pre-reinstall --host ${config.networking.hostName} --keep-within 45d + ''; + startAt = "daily"; }; genebean.programs.kiosk-restic-full-restore = { diff --git a/modules/genebean/nixos/services/restic.nix b/modules/genebean/nixos/services/restic.nix new file mode 100644 index 00000000..4c07f3f3 --- /dev/null +++ b/modules/genebean/nixos/services/restic.nix @@ -0,0 +1,103 @@ +{ + config, + lib, + pkgs, + username, + ... +}: +let + cfg = config.home-manager.users.${username}.genebean.services.restic; + # Reuses the same cache the module-generated restic-backups-daily.service + # already maintains at /var/cache/restic-backups-daily (its own + # CacheDirectory), rather than each raw unit below getting no cache at + # all - confirmed on nixnuc: without RESTIC_CACHE_DIR, restic falls back + # to "unable to locate cache directory: neither $XDG_CACHE_HOME nor + # $HOME are defined" and warns "running prune without a cache, this may + # be very slow!" - restic-fleet-prune was still stuck on the read-only + # scan phase 5+ hours in when this was caught. Each unit below also + # declares its own CacheDirectory (same name) rather than relying on + # restic-backups-daily.service having already run first to create it - + # CacheDirectory is idempotent, so multiple units declaring the same one + # just share it safely. + resticEnv = { + RESTIC_REPOSITORY_FILE = config.sops.secrets.restic_repo.path; + RESTIC_PASSWORD_FILE = config.sops.secrets.restic_password.path; + RESTIC_CACHE_DIR = "/var/cache/restic-backups-daily"; + }; +in +{ + config = lib.mkIf cfg.enable { + environment.systemPackages = [ pkgs.restic ]; + + sops.secrets = { + restic_env.sopsFile = ../../../shared/secrets.yaml; + restic_repo.sopsFile = ../../../shared/secrets.yaml; + restic_password.sopsFile = ../../../shared/secrets.yaml; + }; + + services.restic.backups.daily = { + initialize = true; + environmentFile = config.sops.secrets.restic_env.path; + repositoryFile = config.sops.secrets.restic_repo.path; + passwordFile = config.sops.secrets.restic_password.path; + extraBackupArgs = [ "--retry-lock 2h" ]; + # Deliberately no pruneOpts - the module always bundles that into + # `forget --prune`, and prune is a whole-repo operation. See + # restic-forget-daily below for the cheap, host-scoped replacement. + }; + + systemd.services = { + restic-forget-daily = { + description = "Forget old restic snapshots for this host only (cheap - no prune)"; + after = [ "restic-backups-daily.service" ]; + wants = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; + EnvironmentFile = config.sops.secrets.restic_env.path; + CacheDirectory = "restic-backups-daily"; + CacheDirectoryMode = "0700"; + }; + environment = resticEnv; + script = '' + ${lib.getExe pkgs.restic} forget \ + --host ${config.networking.hostName} \ + --keep-daily 7 --keep-weekly 5 --keep-monthly 6 --keep-tag pre-reinstall + ''; + startAt = "daily"; + }; + + # Deliberately host-UNscoped: this is the fleet-wide safety net + # for pre-reinstall tags, including from a host that's since been + # fully decommissioned and can no longer run its own forget. + restic-forget-pre-reinstall-backstop = { + description = "Fleet-wide backstop: expire any pre-reinstall-tagged snapshot older than 1y, any host (cheap - no prune)"; + serviceConfig = { + Type = "oneshot"; + EnvironmentFile = config.sops.secrets.restic_env.path; + CacheDirectory = "restic-backups-daily"; + CacheDirectoryMode = "0700"; + }; + environment = resticEnv; + script = '' + ${lib.getExe pkgs.restic} forget --tag pre-reinstall --keep-within 1y + ''; + startAt = "daily"; + }; + } + // lib.optionalAttrs cfg.enablePruneJob { + restic-fleet-prune = { + description = "Reclaim space for the whole shared restic repo (all hosts' data) - the one place this runs fleet-wide"; + after = [ "restic-forget-daily.service" ]; # best-effort ordering only - prune is always safe regardless of timing, this just makes it more effective + serviceConfig = { + Type = "oneshot"; + EnvironmentFile = config.sops.secrets.restic_env.path; + CacheDirectory = "restic-backups-daily"; + CacheDirectoryMode = "0700"; + }; + environment = resticEnv; + script = "${lib.getExe pkgs.restic} prune"; + startAt = "04:00"; # later than every host's own daily/forget schedule + }; + }; + }; +} diff --git a/modules/hosts/nixos/default.nix b/modules/hosts/nixos/default.nix index 5fa05152..e4ab7bca 100644 --- a/modules/hosts/nixos/default.nix +++ b/modules/hosts/nixos/default.nix @@ -113,6 +113,18 @@ restartIfChanged = lib.mkForce false; }; + # Same bug, same fix, but for the per-user session bus - confirmed + # separately on nixnuc: its user-session dbus-broker (distinct systemd + # instance from the system one above) hit the identical reload hang + # during a deploy, which cascaded into the whole activation failing and + # even the automatic rollback tripping over the same hang on its way + # back to the previous generation. NixOS's systemd.user.services.* mirrors + # systemd.services.* for the user instance, so the same override applies. + systemd.user.services.dbus-broker = { + reloadIfChanged = lib.mkForce false; + restartIfChanged = lib.mkForce false; + }; + time.timeZone = "America/New_York"; users.defaultUserShell = pkgs.zsh; diff --git a/modules/hosts/nixos/hetznix01/home-gene.nix b/modules/hosts/nixos/hetznix01/home-gene.nix index 8a7ca7bb..4c5d3f56 100644 --- a/modules/hosts/nixos/hetznix01/home-gene.nix +++ b/modules/hosts/nixos/hetznix01/home-gene.nix @@ -3,6 +3,8 @@ genebean = { services = { + restic.enable = true; + tailscale = { advertiseExitNode = true; useRoutingFeatures = "both"; diff --git a/modules/hosts/nixos/hetznix01/post-install/default.nix b/modules/hosts/nixos/hetznix01/post-install/default.nix index 0f760e6d..f509661a 100644 --- a/modules/hosts/nixos/hetznix01/post-install/default.nix +++ b/modules/hosts/nixos/hetznix01/post-install/default.nix @@ -12,7 +12,6 @@ in { imports = [ ../../../../shared/nixos/lets-encrypt.nix - ../../../../shared/nixos/restic.nix ./containers/emqx.nix ./matrix-synapse.nix ./monitoring.nix diff --git a/modules/hosts/nixos/kiosk-entryway/default.nix b/modules/hosts/nixos/kiosk-entryway/default.nix index 8e92e55a..db08ad09 100644 --- a/modules/hosts/nixos/kiosk-entryway/default.nix +++ b/modules/hosts/nixos/kiosk-entryway/default.nix @@ -8,7 +8,6 @@ ./disk-config.nix ./hardware-configuration.nix ./monitoring.nix - ../../../shared/nixos/restic.nix ]; system.stateVersion = "24.11"; diff --git a/modules/hosts/nixos/kiosk-entryway/home-gene.nix b/modules/hosts/nixos/kiosk-entryway/home-gene.nix index 15edfc69..b1ba7306 100644 --- a/modules/hosts/nixos/kiosk-entryway/home-gene.nix +++ b/modules/hosts/nixos/kiosk-entryway/home-gene.nix @@ -19,6 +19,7 @@ wirelessInterface = config.genebean.kiosk-hardware.wirelessInterface; }; kiosk-backups.enable = true; + restic.enable = true; }; }; } diff --git a/modules/hosts/nixos/kiosk-gene-desk/default.nix b/modules/hosts/nixos/kiosk-gene-desk/default.nix index 3711c744..fdd81cfe 100644 --- a/modules/hosts/nixos/kiosk-gene-desk/default.nix +++ b/modules/hosts/nixos/kiosk-gene-desk/default.nix @@ -8,7 +8,6 @@ imports = [ ./disko.nix ./persistence.nix - ../../../shared/nixos/restic.nix ]; system.stateVersion = "24.11"; diff --git a/modules/hosts/nixos/kiosk-gene-desk/home-gene.nix b/modules/hosts/nixos/kiosk-gene-desk/home-gene.nix index 812b098b..a14e1592 100644 --- a/modules/hosts/nixos/kiosk-gene-desk/home-gene.nix +++ b/modules/hosts/nixos/kiosk-gene-desk/home-gene.nix @@ -19,6 +19,7 @@ wirelessInterface = config.genebean.kiosk-hardware.wirelessInterface; }; kiosk-backups.enable = true; + restic.enable = true; }; }; diff --git a/modules/hosts/nixos/nixnuc/default.nix b/modules/hosts/nixos/nixnuc/default.nix index 5c06fdcb..e9d94f30 100644 --- a/modules/hosts/nixos/nixnuc/default.nix +++ b/modules/hosts/nixos/nixnuc/default.nix @@ -24,7 +24,6 @@ in ./social-reader-mcp.nix ./zfs-datasets.nix ../../../shared/nixos/lets-encrypt.nix - ../../../shared/nixos/restic.nix ]; system.stateVersion = "23.11"; diff --git a/modules/hosts/nixos/nixnuc/home-gene.nix b/modules/hosts/nixos/nixnuc/home-gene.nix index fa67450a..1bc06427 100644 --- a/modules/hosts/nixos/nixnuc/home-gene.nix +++ b/modules/hosts/nixos/nixnuc/home-gene.nix @@ -3,6 +3,11 @@ genebean = { services = { + restic = { + enable = true; + enablePruneJob = true; + }; + tailscale = { advertiseExitNode = true; advertiseRoutes = [ "192.168.20.0/22" ]; diff --git a/modules/shared/nixos/restic.nix b/modules/shared/nixos/restic.nix deleted file mode 100644 index 19f12ca6..00000000 --- a/modules/shared/nixos/restic.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ config, pkgs, ... }: -{ - environment.systemPackages = with pkgs; [ - restic - ]; - - sops.secrets = { - restic_env.sopsFile = ../secrets.yaml; - restic_repo.sopsFile = ../secrets.yaml; - restic_password.sopsFile = ../secrets.yaml; - }; - - services.restic.backups = { - daily = { - initialize = true; - - environmentFile = config.sops.secrets.restic_env.path; - repositoryFile = config.sops.secrets.restic_repo.path; - passwordFile = config.sops.secrets.restic_password.path; - - extraBackupArgs = [ - "--retry-lock 2h" - ]; - - pruneOpts = [ - "--keep-daily 7" - "--keep-weekly 5" - "--keep-monthly 6" - # Every host shares this one repository and none of these forget - # invocations are scoped by --host, so any host's prune run - # processes retention for every host's snapshots, not just its - # own. This needs to live here (not on whichever host actually - # uses the tag) so it's honored no matter which host's prune job - # happens to run first - see kiosk-gene-desk/persistence.nix for - # where pre-reinstall gets applied. - "--keep-tag pre-reinstall" - ]; - }; - - # Fleet-wide backstop, deliberately not scoped by --host: whatever - # per-host cleanup job manages a normal expiry for its own - # pre-reinstall-tagged snapshots (see kiosk-gene-desk/persistence.nix) - # is the primary mechanism, but this catches anything that slips - # through - a host decommissioned before its own cleanup job ran, one - # disabled/misconfigured, etc. - so pre-reinstall tags can never pin a - # snapshot forever fleet-wide, only for up to a year. - pre-reinstall-backstop = { - environmentFile = config.sops.secrets.restic_env.path; - repositoryFile = config.sops.secrets.restic_repo.path; - passwordFile = config.sops.secrets.restic_password.path; - pruneOpts = [ - "--tag pre-reinstall" - "--keep-within 1y" - ]; - # Confirmed on hardware: the module default (Persistent = true) fired - # this immediately on the same reboot it was first created on, - # pegging CPU on a resource-constrained Pi actively running its - # kiosk display, right alongside two other newly-created jobs doing - # the same thing. A missed day here is a non-issue - see - # kiosk-gene-desk/persistence.nix's daily/pre-reinstall-cleanup - # timerConfig overrides for the same reasoning. - timerConfig = { - OnCalendar = "daily"; - Persistent = false; - }; - }; - }; -} diff --git a/scripts/check-restic-prune-singleton.sh b/scripts/check-restic-prune-singleton.sh new file mode 100755 index 00000000..7a1f7de6 --- /dev/null +++ b/scripts/check-restic-prune-singleton.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# restic prune is a whole-repository operation (see +# modules/genebean/nixos/services/restic.nix) - it must run on exactly one +# host across the fleet, or space is either never reclaimed (zero hosts) or +# reclaimed redundantly (more than one). This is a fast text check, not a +# full `nix eval` of every host's resolved config - it can be fooled by a +# stray comment or duplicate line, but catches the common mistake cheaply +# on every commit. +set -euo pipefail + +matches=$(grep -rl 'enablePruneJob = true' modules/hosts/ || true) +count=$(printf '%s' "$matches" | grep -c . || true) + +if [ "$count" -ne 1 ]; then + echo "error: expected exactly one host with genebean.services.restic.enablePruneJob = true, found $count" >&2 + if [ -n "$matches" ]; then + printf '%s\n' "$matches" >&2 + fi + exit 1 +fi