Fold restic backups into genebean.services.restic, fix fleet-wide prune performance

.github/workflows/validate.yml

@@ -19,3 +19,6 @@ jobs:
 
       - name: Run statix
         run: nix run nixpkgs#statix -- check .
+
+      - name: Check restic prune singleton
+        run: scripts/check-restic-prune-singleton.sh

.pre-commit-config.yaml

@@ -24,3 +24,10 @@ repos:
         pass_filenames: false
         args: ["check", "."]
 
+      - id: check-restic-prune-singleton
+        name: check-restic-prune-singleton
+        entry: scripts/check-restic-prune-singleton.sh
+        language: system
+        files: ^modules/hosts/.*\.nix$
+        pass_filenames: false
+

modules/genebean/home/default.nix

@@ -58,6 +58,7 @@
     ./services/chromium-kiosk.nix
     ./services/flatpak.nix
     ./services/kiosk-backups.nix
+    ./services/restic.nix
     ./services/tailscale.nix
   ];
 }

modules/genebean/home/services/restic.nix

@@ -0,0 +1,12 @@
+{ lib, ... }:
+{
+  options.genebean.services.restic = {
+    enable = lib.mkEnableOption "shared restic backup infrastructure (backup + cheap per-host forget)";
+
+    enablePruneJob = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Run the expensive, fleet-wide restic prune (data reclaim) job on this host. Must be true on EXACTLY ONE host across the whole fleet - prune is a whole-repository operation (it has to know what data every host's remaining snapshots still reference), so running it on more than one host wastes redundant work and running it on zero means space is never reclaimed. Enforced by a pre-commit check (scripts/check-restic-prune-singleton.sh).";
+    };
+  };
+}

modules/genebean/nixos/default.nix

@@ -13,6 +13,7 @@
     ./services/chromium-kiosk.nix
     ./services/flatpak.nix
     ./services/kiosk-backups.nix
+    ./services/restic.nix
     ./services/tailscale.nix
   ];
 }

modules/genebean/nixos/services/kiosk-backups.nix

@@ -1,11 +1,13 @@
 {
   config,
   lib,
+  pkgs,
   username,
   ...
 }:
 let
   cfg = config.home-manager.users.${username}.genebean.services.kiosk-backups;
+  resticCfg = config.home-manager.users.${username}.genebean.services.restic;
   usesImpermanence = config.environment.persistence."/persist".enable or false;
   prefixedPaths = map (path: if usesImpermanence then "/persist${path}" else path) cfg.paths;
 in
@@ -16,34 +18,43 @@ in
         assertion = !(lib.any (lib.hasPrefix "/persist") cfg.paths);
         message = "genebean.services.kiosk-backups.paths must not include a /persist prefix - it's added automatically when the host uses impermanence.";
       }
+      {
+        assertion = resticCfg.enable;
+        message = "genebean.services.kiosk-backups requires genebean.services.restic.enable = true - it depends on that module's restic_env/restic_repo/restic_password secrets and base daily backup job.";
+      }
     ];
 
-    services.restic.backups = {
-      daily = {
-        paths = prefixedPaths;
-        # Avoid a boot-triggered catch-up backup capturing the wrong state
-        # as "latest" before a chance to restore after a reinstall -
-        # confirmed on kiosk-gene-desk hardware, see its original comment.
-        timerConfig = {
-          OnCalendar = "daily";
-          Persistent = false;
-        };
+    services.restic.backups.daily = {
+      paths = prefixedPaths;
+      # Avoid a boot-triggered catch-up backup capturing the wrong state
+      # as "latest" before a chance to restore after a reinstall -
+      # confirmed on kiosk-gene-desk hardware, see its original comment.
+      timerConfig = {
+        OnCalendar = "daily";
+        Persistent = false;
       };
+    };
 
-      pre-reinstall-cleanup = {
-        environmentFile = config.sops.secrets.restic_env.path;
-        passwordFile = config.sops.secrets.restic_password.path;
-        repositoryFile = config.sops.secrets.restic_repo.path;
-        pruneOpts = [
-          "--tag pre-reinstall"
-          "--host ${config.networking.hostName}"
-          "--keep-within 45d"
-        ];
-        timerConfig = {
-          OnCalendar = "daily";
-          Persistent = false;
-        };
+    systemd.services.restic-forget-kiosk-pre-reinstall = {
+      description = "Expire this kiosk's own pre-reinstall-tagged snapshots older than 45d (cheap - no prune)";
+      serviceConfig = {
+        Type = "oneshot";
+        EnvironmentFile = config.sops.secrets.restic_env.path;
+        # Reuses genebean.services.restic's own cache dir (same repo) -
+        # see that module's resticEnv comment for why this matters.
+        CacheDirectory = "restic-backups-daily";
+        CacheDirectoryMode = "0700";
+      };
+      environment = {
+        RESTIC_REPOSITORY_FILE = config.sops.secrets.restic_repo.path;
+        RESTIC_PASSWORD_FILE = config.sops.secrets.restic_password.path;
+        RESTIC_CACHE_DIR = "/var/cache/restic-backups-daily";
       };
+      script = ''
+        ${lib.getExe pkgs.restic} forget \
+          --tag pre-reinstall --host ${config.networking.hostName} --keep-within 45d
+      '';
+      startAt = "daily";
     };
 
     genebean.programs.kiosk-restic-full-restore = {

modules/genebean/nixos/services/restic.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  username,
+  ...
+}:
+let
+  cfg = config.home-manager.users.${username}.genebean.services.restic;
+  # Reuses the same cache the module-generated restic-backups-daily.service
+  # already maintains at /var/cache/restic-backups-daily (its own
+  # CacheDirectory), rather than each raw unit below getting no cache at
+  # all - confirmed on nixnuc: without RESTIC_CACHE_DIR, restic falls back
+  # to "unable to locate cache directory: neither $XDG_CACHE_HOME nor
+  # $HOME are defined" and warns "running prune without a cache, this may
+  # be very slow!" - restic-fleet-prune was still stuck on the read-only
+  # scan phase 5+ hours in when this was caught. Each unit below also
+  # declares its own CacheDirectory (same name) rather than relying on
+  # restic-backups-daily.service having already run first to create it -
+  # CacheDirectory is idempotent, so multiple units declaring the same one
+  # just share it safely.
+  resticEnv = {
+    RESTIC_REPOSITORY_FILE = config.sops.secrets.restic_repo.path;
+    RESTIC_PASSWORD_FILE = config.sops.secrets.restic_password.path;
+    RESTIC_CACHE_DIR = "/var/cache/restic-backups-daily";
+  };
+in
+{
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.restic ];
+
+    sops.secrets = {
+      restic_env.sopsFile = ../../../shared/secrets.yaml;
+      restic_repo.sopsFile = ../../../shared/secrets.yaml;
+      restic_password.sopsFile = ../../../shared/secrets.yaml;
+    };
+
+    services.restic.backups.daily = {
+      initialize = true;
+      environmentFile = config.sops.secrets.restic_env.path;
+      repositoryFile = config.sops.secrets.restic_repo.path;
+      passwordFile = config.sops.secrets.restic_password.path;
+      extraBackupArgs = [ "--retry-lock 2h" ];
+      # Deliberately no pruneOpts - the module always bundles that into
+      # `forget --prune`, and prune is a whole-repo operation. See
+      # restic-forget-daily below for the cheap, host-scoped replacement.
+    };
+
+    systemd.services = {
+      restic-forget-daily = {
+        description = "Forget old restic snapshots for this host only (cheap - no prune)";
+        after = [ "restic-backups-daily.service" ];
+        wants = [ "network-online.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          EnvironmentFile = config.sops.secrets.restic_env.path;
+          CacheDirectory = "restic-backups-daily";
+          CacheDirectoryMode = "0700";
+        };
+        environment = resticEnv;
+        script = ''
+          ${lib.getExe pkgs.restic} forget \
+            --host ${config.networking.hostName} \
+            --keep-daily 7 --keep-weekly 5 --keep-monthly 6 --keep-tag pre-reinstall
+        '';
+        startAt = "daily";
+      };
+
+      # Deliberately host-UNscoped: this is the fleet-wide safety net
+      # for pre-reinstall tags, including from a host that's since been
+      # fully decommissioned and can no longer run its own forget.
+      restic-forget-pre-reinstall-backstop = {
+        description = "Fleet-wide backstop: expire any pre-reinstall-tagged snapshot older than 1y, any host (cheap - no prune)";
+        serviceConfig = {
+          Type = "oneshot";
+          EnvironmentFile = config.sops.secrets.restic_env.path;
+          CacheDirectory = "restic-backups-daily";
+          CacheDirectoryMode = "0700";
+        };
+        environment = resticEnv;
+        script = ''
+          ${lib.getExe pkgs.restic} forget --tag pre-reinstall --keep-within 1y
+        '';
+        startAt = "daily";
+      };
+    }
+    // lib.optionalAttrs cfg.enablePruneJob {
+      restic-fleet-prune = {
+        description = "Reclaim space for the whole shared restic repo (all hosts' data) - the one place this runs fleet-wide";
+        after = [ "restic-forget-daily.service" ]; # best-effort ordering only - prune is always safe regardless of timing, this just makes it more effective
+        serviceConfig = {
+          Type = "oneshot";
+          EnvironmentFile = config.sops.secrets.restic_env.path;
+          CacheDirectory = "restic-backups-daily";
+          CacheDirectoryMode = "0700";
+        };
+        environment = resticEnv;
+        script = "${lib.getExe pkgs.restic} prune";
+        startAt = "04:00"; # later than every host's own daily/forget schedule
+      };
+    };
+  };
+}

modules/hosts/nixos/default.nix

@@ -113,6 +113,18 @@
     restartIfChanged = lib.mkForce false;
   };
 
+  # Same bug, same fix, but for the per-user session bus - confirmed
+  # separately on nixnuc: its user-session dbus-broker (distinct systemd
+  # instance from the system one above) hit the identical reload hang
+  # during a deploy, which cascaded into the whole activation failing and
+  # even the automatic rollback tripping over the same hang on its way
+  # back to the previous generation. NixOS's systemd.user.services.* mirrors
+  # systemd.services.* for the user instance, so the same override applies.
+  systemd.user.services.dbus-broker = {
+    reloadIfChanged = lib.mkForce false;
+    restartIfChanged = lib.mkForce false;
+  };
+
   time.timeZone = "America/New_York";
 
   users.defaultUserShell = pkgs.zsh;

modules/hosts/nixos/hetznix01/home-gene.nix

@@ -3,6 +3,8 @@
 
   genebean = {
     services = {
+      restic.enable = true;
+
       tailscale = {
         advertiseExitNode = true;
         useRoutingFeatures = "both";

modules/hosts/nixos/hetznix01/post-install/default.nix

@@ -12,7 +12,6 @@ in
 {
   imports = [
     ../../../../shared/nixos/lets-encrypt.nix
-    ../../../../shared/nixos/restic.nix
     ./containers/emqx.nix
     ./matrix-synapse.nix
     ./monitoring.nix

modules/hosts/nixos/kiosk-entryway/default.nix

@@ -8,7 +8,6 @@
     ./disk-config.nix
     ./hardware-configuration.nix
     ./monitoring.nix
-    ../../../shared/nixos/restic.nix
   ];
 
   system.stateVersion = "24.11";

modules/hosts/nixos/kiosk-entryway/home-gene.nix

@@ -19,6 +19,7 @@
         wirelessInterface = config.genebean.kiosk-hardware.wirelessInterface;
       };
       kiosk-backups.enable = true;
+      restic.enable = true;
     };
   };
 }

modules/hosts/nixos/kiosk-gene-desk/default.nix

@@ -8,7 +8,6 @@
   imports = [
     ./disko.nix
     ./persistence.nix
-    ../../../shared/nixos/restic.nix
   ];
 
   system.stateVersion = "24.11";

modules/hosts/nixos/kiosk-gene-desk/home-gene.nix

@@ -19,6 +19,7 @@
         wirelessInterface = config.genebean.kiosk-hardware.wirelessInterface;
       };
       kiosk-backups.enable = true;
+      restic.enable = true;
     };
   };
 

modules/hosts/nixos/nixnuc/default.nix

@@ -24,7 +24,6 @@ in
     ./social-reader-mcp.nix
     ./zfs-datasets.nix
     ../../../shared/nixos/lets-encrypt.nix
-    ../../../shared/nixos/restic.nix
   ];
 
   system.stateVersion = "23.11";

modules/hosts/nixos/nixnuc/home-gene.nix

@@ -3,6 +3,11 @@
 
   genebean = {
     services = {
+      restic = {
+        enable = true;
+        enablePruneJob = true;
+      };
+
       tailscale = {
         advertiseExitNode = true;
         advertiseRoutes = [ "192.168.20.0/22" ];