diff --git a/backup.md b/backup.md new file mode 100644 index 00000000..92c10ee2 --- /dev/null +++ b/backup.md @@ -0,0 +1,103 @@ +# Backup + +This backup solution is designed for a SQLite local database and single-instance VaultWarden deployment. It can also be used for attachments, while persistent PostgreSQL/MySQL backups should be handled separately. + +The backup utilizes the [vaultwarden-backup](https://github.com/ttionya/vaultwarden-backup) solution. Each scheduled backup creates an additional sidecar vaultwarden-backup container where the data folder is shared between the VaultWarden container and the backup sidecar container. + +## VaultWarden Settings + +For the backup to access the data, VaultWarden must run with an appropriate security context, matching the backup container. For example: + +```yaml +podSecurityContext: + ## @param runAsGroup group ID for VaultWarden and backup run with + ## Same as default user for vaultwarden-backup + runAsUser: 1100 + runAsGroup: 1100 + fsGroup: 1100 +``` + +## Parameters + + +| Name | Description | Value | +|-------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| +| `backup.enabled` | Enable the backup | true/false | +| `backup.image` | Docker image, see https://hub.docker.com/r/ttionya/vaultwarden-backup | | +| `backup.rcloneConfig` | [rClone config](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#configure-rclone-%EF%B8%8F-must-read-%EF%B8%8F). Recommended to keep it secure, sops, helm secrets | | +| `backup.remoteName` | Backup remote name, see [RCLONE_REMOTE_NAME](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#rclone_remote_name) | | +| `backup.globalFlags` | rClone global flags [RCLONE_GLOBAL_FLAG](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#rclone_global_flag) | | +| `backup.zipPassword` | Password to encrypt backup archive with, see [ZIP_PASSWORD](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#zip_password) | | +| `backup.healthcheckPingKey` | See [Ping, Healthchecks.io](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#ping) | | +| `backup.timezone` | Backup timezone, see [TIMEZONE](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#timezone). | UTC | +| `backup.smtp...` | SMTP parameters, [Mail](https://github.com/ttionya/vaultwarden-backup?tab=readme-ov-file#mail) | | +| `backup.backups` | Array of backups, each of it's own schedule | | +| `backup.backups[].name` | Backup name, container to be named after it | "hourly", "weekly" etc | +| `backup.backups[].schedule` | Cron job syntax schedule | "5 * * * *" | +| `backup.backups[].keepDays` | Backup to be delete after these N days. 0 - keep forever | 7, 0 ... | +| `backup.backups[].fileDateSuffix` | Suffix for the each archive file | "-%H-%M-%S" | +| `backup.backups[].healthCheckPing` | healthchecks.io ping url, see [Ping](). Such as `https://hc-ping.com/{ping_key}/vaultwarden-` Set it on most-frequent backup | https://hc-ping.com/{ping_key}/vaultwarden-main | +| | | | +| | | | + +### Example +```yaml +podSecurityContext: + ## @param runAsGroup group ID for VaultWarden and backup run with + ## Same as default user for vaultwarden-backup + runAsUser: 1100 + runAsGroup: 1100 + fsGroup: 1100 + +backup: + enabled: true + backups: + - name: hourly + remoteDir: "/vaultWarden-main/hourly/" + # Every hour at 5 mins + cron: "5 * * * *" + keepDays: 7 + fileDateSuffix: "-%H-%M-%S" + healthCheckPing: "https://hc-ping.com//vaultwarden-main" + + - name: daily + remoteDir: "/vaultWarden-main/daily/" + # every day at 03:23 + cron: "23 3 * * *" + keepDays: 60 + fileDateSuffix: "-%H-%M-%S" + + - name: monthly + remoteDir: "/vaultWarden-main/monthly/" + # every 20th day at 03:47 + cron: "47 3 20 * *" + # Keep forever + keepDays: 0 + fileDateSuffix: "-%H-%M-%S" +``` + +## Restore + +If the deployment is lost, including PVs, it can be restored from the backup: + +1. Download the backup archive from remote storage. +2. Have the `zipPassword` ready for unzipping the archive. +3. Run the restore script (requires functional kubectl): + +zipFile is a part of Helm chart secret + +`./restore.sh --archive --release --storage-class ` + +Example: + + `./restore.sh --archive /tmp/backup.20221103-19-05-01.zip --release vaultwarden --storage-class "local-path"` + +Kubernetes Namespace and context can be set with kubectl beforehand or passed as arguments. Create one if needed + +The script will create a PV and PVC in the target cluster and namespace. When the VaultWarden helm chart is deployed, it will use this PV and PVC. + +Use kubectl to check if another PVC is created in case of a mismatch. Adjust script parameters accordingly. + + + + diff --git a/charts/vaultwarden/templates/_helpers.tpl b/charts/vaultwarden/templates/_helpers.tpl index cdf5f397..f4313f69 100644 --- a/charts/vaultwarden/templates/_helpers.tpl +++ b/charts/vaultwarden/templates/_helpers.tpl @@ -93,6 +93,13 @@ Determine whether to use deployment or statefulset {{- end }} {{- end }} +{{/* +Do backup? Needs backup enabled and persistence +*/}} +{{- define "vaultwarden.doBackup" -}} + {{- and .Values.backup.enabled (hasKey .Values.storage "data") -}} +{{- end }} + {{/* Return true when the HIBP API key should be sourced from a Kubernetes Secret. */}} @@ -109,4 +116,4 @@ Return the legacy hibpApiKey string value when hibpApiKey is set and hibp is not {{- if and (not (include "vaultwarden.hibpUseSecret" .)) (kindIs "string" .Values.hibpApiKey) .Values.hibpApiKey -}} {{- .Values.hibpApiKey -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/vaultwarden/templates/_podSpec.tpl b/charts/vaultwarden/templates/_podSpec.tpl index c831d7e1..738ce09e 100644 --- a/charts/vaultwarden/templates/_podSpec.tpl +++ b/charts/vaultwarden/templates/_podSpec.tpl @@ -22,13 +22,30 @@ priorityClassName: {{ . | quote }} securityContext: {{- toYaml . | nindent 2 }} {{- end }} -{{- with .Values.initContainers }} +{{- if or .Values.initContainers (eq (include "vaultwarden.doBackup" .) "true") }} initContainers: +{{- with .Values.initContainers }} {{- toYaml . | nindent 2 }} {{- end }} + +{{- if eq (include "vaultwarden.doBackup" .) "true" }} + # Copy rclone config from read-only secret mount to writable share + # https://github.com/rclone/rclone/issues/3655 + - name: copy-config + image: busybox:latest + command: ["sh", "-c", "cp -v /src-config/rclone.conf /config/"] + volumeMounts: + - name: backup-secret-conf + mountPath: "/src-config/" + readOnly: true + - name: config + mountPath: "/config/" +{{- end }} +{{- end }} {{- if not .Values.enableServiceLinks }} enableServiceLinks: false {{- end }} + containers: - image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} @@ -253,6 +270,55 @@ containers: successThreshold: {{ .Values.startupProbe.successThreshold }} failureThreshold: {{ .Values.startupProbe.failureThreshold }} {{- end }} + {{- if eq (include "vaultwarden.doBackup" .) "true" }} + {{- range .Values.backup.backups }} + - image: {{ $.Values.backup.image }} + name: backup-{{ .name }} + securityContext: + allowPrivilegeEscalation: false + env: + - name: DATA_DIR + value: {{ default "/data" .path | quote }} + - name: RCLONE_REMOTE_NAME + value: {{ $.Values.backup.remoteName | quote }} + - name: RCLONE_REMOTE_DIR + value: {{ .remoteDir | quote }} + - name: RCLONE_GLOBAL_FLAG + value: "{{ $.Values.backup.globalFlags }} --config /config/rclone.conf" + - name: CRON + value: {{ .cron | quote }} + - name: ZIP_PASSWORD + value: {{ $.Values.backup.zipPassword | quote }} + - name: BACKUP_KEEP_DAYS + value: {{ .keepDays | quote }} + - name: BACKUP_FILE_DATE_SUFFIX + value: {{ .fileDateSuffix | quote }} + - name: TIMEZONE + value: {{ $.Values.backup.timezone | quote }} + {{- if .healthCheckPing }} + - name: PING_URL + value: {{ (tpl .healthCheckPing $) | quote }} + {{- end }} + {{- if $.Values.backup.smtp.enabled }} + - name: MAIL_SMTP_ENABLE + value: "true" + - name: MAIL_SMTP_VARIABLES + value: {{ $.Values.backup.smtp.smtpVariables | quote }} + - name: MAIL_TO + value: {{ $.Values.backup.smtp.mailTo | quote }} + - name: MAIL_WHEN_SUCCESS + value: {{ $.Values.backup.smtp.mailWhenSuccess | quote }} + - name: MAIL_WHEN_FAILURE + value: {{ $.Values.backup.smtp.mailWhenFailure | quote }} + {{- end }} + # When run as non-root, script cannot create crontabs in home folcer + volumeMounts: + - name: vaultwarden-data + mountPath: {{ default "/data" $.Values.storage.data.path }} + - name: config + mountPath: "/config/" + {{- end }} + {{- end }} {{- with .Values.sidecars }} {{- toYaml . | nindent 2 }} {{- end }} diff --git a/charts/vaultwarden/templates/deployment.yaml b/charts/vaultwarden/templates/deployment.yaml index 91e7419b..8d387e36 100644 --- a/charts/vaultwarden/templates/deployment.yaml +++ b/charts/vaultwarden/templates/deployment.yaml @@ -40,7 +40,7 @@ spec: {{- end }} spec: {{- include "vaultwarden.podSpec" . | nindent 6 }} - {{- if or (.Values.storage.existingVolumeClaim) (.Values.storage.data) (.Values.storage.attachments) (.Values.rocket.tls.secretName) (.Values.extraVolumes) }} + {{- if or (.Values.storage.existingVolumeClaim) (.Values.storage.data) (.Values.storage.attachments) (.Values.rocket.tls.secretName) (.Values.extraVolumes) (eq (include "vaultwarden.doBackup" .) "true") }} volumes: {{- if .Values.storage.existingVolumeClaim }} {{- with .Values.storage.existingVolumeClaim }} @@ -64,5 +64,15 @@ spec: {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if eq (include "vaultwarden.doBackup" .) "true" }} + - name: backup-secret-conf + secret: + secretName: {{ include "vaultwarden.fullname" . }}-rclone + optional: false + # readable by user/owner + defaultMode: 0400 + - name: config + emptyDir: {} + {{- end }} {{- end }} {{- end }} diff --git a/charts/vaultwarden/templates/secret-backup.yaml b/charts/vaultwarden/templates/secret-backup.yaml new file mode 100644 index 00000000..b77d11f6 --- /dev/null +++ b/charts/vaultwarden/templates/secret-backup.yaml @@ -0,0 +1,12 @@ +{{- if eq (include "vaultwarden.doBackup" .) "true" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "vaultwarden.fullname" . }}-rclone + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: vaultwarden +type: Opaque +data: + rclone.conf: {{ .Values.backup.rcloneConfig | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/vaultwarden/templates/statefulset.yaml b/charts/vaultwarden/templates/statefulset.yaml index 4fd74350..e555268e 100644 --- a/charts/vaultwarden/templates/statefulset.yaml +++ b/charts/vaultwarden/templates/statefulset.yaml @@ -41,7 +41,7 @@ spec: {{- end }} spec: {{- include "vaultwarden.podSpec" . | nindent 6 }} - {{- if or (.Values.storage.existingVolumeClaim) (.Values.rocket.tls.secretName) (.Values.extraVolumes) }} + {{- if or (.Values.storage.existingVolumeClaim) (.Values.rocket.tls.secretName) (.Values.extraVolumes) (eq (include "vaultwarden.doBackup" .) "true") }} volumes: {{- with .Values.storage.existingVolumeClaim }} - name: vaultwarden-data @@ -56,6 +56,16 @@ spec: {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if eq (include "vaultwarden.doBackup" .) "true" }} + - name: backup-secret-conf + secret: + secretName: {{ include "vaultwarden.fullname" . }}-rclone + optional: false + # readable by user/owner + defaultMode: 0400 + - name: config + emptyDir: {} + {{- end }} {{- end }} persistentVolumeClaimRetentionPolicy: whenDeleted: Retain diff --git a/charts/vaultwarden/values.yaml b/charts/vaultwarden/values.yaml index 34e752b9..1650d541 100644 --- a/charts/vaultwarden/values.yaml +++ b/charts/vaultwarden/values.yaml @@ -976,3 +976,52 @@ sso: ## @param sso.clientSecret.existingSecretKey When using an existing secret, specify the key which contains the password. ## existingSecretKey: "" + +## @section Backup Configuration +## https://github.com/ttionya/vaultwarden-backup +backup: + ## @param backup.enabled Enable backup + ## + enabled: false + + ## @param backup.image Backup image FQDN + ## + image: "ttionya/vaultwarden-backup:1.22.0" + + ## @param backup.rcloneConfig Raw rClone backup + ## Recommended to provide as an encrypted content + rcloneConfig: "" + + remoteName: "vaultWarden" + + globalFlags: "" + + # cron: "5 * * * *" + + zipPassword: "changeme" + + # https://blog.healthchecks.io/2021/09/new-feature-slug-urls/ + # Obtain from https://healthchecks.io/projects//settings/ + healthcheckPingKey: "get-from-healthchecks.io" + + timezone: "UTC" + + smtp: + enable: false + smtpVariables: "" + mailTo: "" + mailWhenSuccess: false + mailWhenFailure: true + + ## An array for backups + ## Make sure they not happen at the same time! + ## e.g: + ## backups: + ## - name: hourly + ## remoteDir: "/BitwardenBackup/hourly/" + ## cron: "5 * * * *" + ## keepDays: 7 + ## fileDateSuffix: "-%H-%M-%S" + ## healthCheckPing: https://hc-ping.com/{ping_key}/vaultwarden-utility + ## + backups: [] diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..82da1c52 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +restore_path="/bitwarden/restore" +data_path="/bitwarden/data" +pod_name="restore" +image="ttionya/vaultwarden-backup:1.22.0" + + +usage () +{ + echo "Usage: ${0##*/} --archive some-backup.zip --release vaultwarden-helm-release --storage-class storage-class [--chart vaultvarden-chart] [--capacity pv-size] [--namespace kubernetes-namespace] [--context kubernetes-context]" + echo "Example: ${0##*/} --archive \"/tmp/20221011-19-05-01.zip\" --release main --capacity 10Gi --storage-class \"local-path\"" + echo "Capacity defaults to \"5Gi\"" + exit 1 +} + + +if ! OPTS=$(getopt --options "h" --longoptions archive:,release:,capacity:,chart:,storage-class:,namespace:,context:,help --name 'parse-options' -- "$@"); then + echo "Failed parsing options." >&2 + exit 1 +fi + + +eval set -- "${OPTS}" + +while true; do + case "$1" in + --archive) + archive="$2" + shift 2 + ;; + --release) + release="$2" + shift 2 + ;; + --chart) + chart="$2" + shift 2 + ;; + --capacity) + capacity="$2" + shift 2 + ;; + --storage-class) + storage_class="$2" + shift 2 + ;; + --namespace) + ns="$2" + shift 2 + ;; + --context) + context="$2" + shift 2 + ;; + -h | --help ) usage; ;; + --) + shift + break + ;; + *) break ;; + esac +done + +# Let's presume kubectl is installed +command -v jq >/dev/null 2>&1 || { echo >&2 "jq is required. Aborting."; exit 1; } + +if [ -z "${archive+set}" ]; then + echo "Archive file is not set" + usage +fi + +if [ ! -f "${archive}" ]; then + echo "Archive file does not exist" + usage +fi + +if [ -z "${release+set}" ]; then + echo "Vaultwarden helm release name is not provided" + usage +fi + +if [ -z "${storage_class+set}" ]; then + echo "Storage class for PVC is not provided" + usage +fi + +if [ -z "${chart+set}" ]; then + chart="vaultwarden" +fi + +if [ -z "${capacity+set}" ]; then + capacity="5Gi" +fi + +kubeconf=$(kubectl config view -o json) + +if [ -z "${context+set}" ]; then + context=$(echo "${kubeconf}" | jq -r '.["current-context"]') + if [ -z "${context}" ]; then + echo "Cannot get current context" + exit 1 + fi +fi + +if [ -z "${ns+set}" ]; then + ns=$(echo "${kubeconf}" | jq -r --arg ctx "${context}" '.contexts[] | select(.name==$ctx) | .context.namespace') + if [ -z "${ns}" ]; then + echo "Cannot get current namespace" + exit 1 + fi +fi + + +echo "Kubernetes context: [${context}], namespace: [${ns}]" + +if ! kubectl get namespace "${ns}" > /dev/null; then + echo "Namespace \"${ns}\" does not exist, creating it" + kubectl create namespace "${ns}" +fi + +#{{- if contains $name .Release.Name -}} +#{{- .Release.Name | trunc 20 | trimSuffix "-" -}} +#{{- else -}} +#{{- printf "%s-%s" .Release.Name $name | trunc 20 | trimSuffix "-" -}} +#{{- end -}} + +# bit simplified condition +if [[ "${release}" =~ .*"${chart}".* ]]; then + # release name contained in chart name + pvc_name="vaultwarden-data-${chart}-0" +else + # release name not contained in chart name + pvc_name="vaultwarden-data-${release}-${chart}-0" +fi + + +echo "Creating PVC: \"${pvc_name}\" ..." + +pvc=$(cat <<"EOF" +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": $claim + }, + "spec": { + "storageClassName": $class, + "accessModes": ["ReadWriteOnce"], + "resources": { + "requests": { + "storage": $capacity + } + } + } +} +EOF +) + +jq -n --arg claim "${pvc_name}" --arg capacity "${capacity}" --arg class "${storage_class}" "${pvc}" | kubectl apply --context "${context}" --namespace "${ns}" -f - + +echo "Creating restore pod ..." + +pod_template=$(cat <<"EOF" +{ + "apiVersion": "v1", + "spec": { + "containers": [{ + "name": $name, + "image": $image, + "command": ["sh", "-c", "mkdir "+$restore_path+" && sleep infinite"], + "volumeMounts" : [{ + "name": "data", + "mountPath": $data_path + }] + }], + "volumes": [{ + "name": "data", + "persistentVolumeClaim": { + "claimName": $claim + } + }] + } +} +EOF +) + +overrides=$(jq -n --arg name "${pod_name}" --arg image "${image}" --arg restore_path "${restore_path}" --arg claim "${pvc_name}" --arg data_path "${data_path}" "${pod_template}") +kubectl run "${pod_name}" --context "${context}" --namespace "${ns}" --image="${image}" --restart=Never --overrides="${overrides}" +kubectl wait --context "${context}" --namespace "${ns}" --for=condition=Ready pod/"${pod_name}" + +echo "Copying archive to restore pod ..." +kubectl cp --context "${context}" --namespace "${ns}" "${archive}" "${pod_name}:${restore_path}" + +echo "Performing the restore, if you have a zip password on archive, it will be prompted during the restore ..." + +archive_name=$(basename "${archive}") + +echo "Sleeping 5 seconds to make sure pod is ready" + +sleep 5 + +kubectl exec --context "${context}" --namespace "${ns}" -ti "${pod_name}" -- sh -c "/app/entrypoint.sh restore --zip-file "${restore_path}/${archive_name}" --force-restore && chown -R backuptool:backuptool \"${data_path}\"" + +echo "Data is restored, deleting the pod. It can take up to 30 seconds ..." +kubectl delete pod --context "${context}" --namespace "${ns}" "${pod_name}" + +echo "Done !"