feat(hcpctl): add velero backup queries to snapshot gatherer

tooling/hcpctl/pkg/snapshot/gatherer.go

@@ -276,6 +276,11 @@ func (g *Gatherer) Gather(ctx context.Context, input GatherInput, outputDir stri
 		PhaseStartTime: input.TimeWindow.Start,
 		PhaseEndTime:   input.TimeWindow.End,
 	}
+	// Seed the management-cluster list from the PR-job hint. When absent, the
+	// velero/mgmtCluster discovery query fills it from container logs.
+	if seedData.ManagementClusterName != "" {
+		seedData.ManagementClusterNames = []string{seedData.ManagementClusterName}
+	}
 
 	pool := &queryPool{gatherer: g, input: input}
 

tooling/hcpctl/pkg/snapshot/queries.go

@@ -124,6 +124,13 @@ type queryData struct {
 	ServiceClusterName    string
 	ManagementClusterName string
 
+	// ManagementClusterNames lists the management (AKS) cluster names that host
+	// the target HCP's control plane. It is seeded from ManagementClusterName
+	// (PR jobs) and otherwise discovered by the velero/mgmtCluster query. Velero
+	// runs per management cluster, so cluster-scoped velero queries (serverLogs)
+	// filter "| where cluster in (...)" on this list.
+	ManagementClusterNames []string
+
 	// FullStartTime and FullEndTime define the entire snapshot window. Use
 	// these for broad timestamp pre-filters (Kusto partition pruning) and
 	// for discovery queries that must see the complete time range.
@@ -416,6 +423,31 @@ var allQueries = []querySpec{
 			return nil
 		},
 	},
+	{
+		// Discovers which management cluster(s) host this HCP's velero backups, so the
+		// cross-HCP velero serverLogs query can scope to them. Reads the cluster from the
+		// HCP's Velero Backup CRs (matched by the ARM resource-id annotation) — that is
+		// exactly the management cluster where velero runs for this HCP.
+		component:    "velero",
+		queryName:    "mgmtCluster",
+		templatePath: "queries/velero/mgmtCluster/query.kql",
+		database:     "service",
+		category:     categoryResourceDiscovery,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && d.ClusterResourceID != "" && len(d.ManagementClusterNames) == 0
+		},
+		prerequisites: "ClusterResourceID, ResourceType is cluster, ManagementClusterNames not already seeded",
+		storeResult: func(d *queryData, rows []resultRow) error {
+			names := make([]string, 0, len(rows))
+			for _, r := range rows {
+				if len(r.values) > 0 && r.values[0] != "" {
+					names = append(names, r.values[0])
+				}
+			}
+			d.ManagementClusterNames = names
+			return nil
+		},
+	},
 
 	// --- State: time-windowed, resource-scoped ---
 	{
@@ -802,6 +834,74 @@ var allQueries = []querySpec{
 		},
 		prerequisites: "HostedControlPlaneNamespace, ResourceType is cluster",
 	},
+	// --- Velero: backup CRs and component logs (ARO-28989) ---
+	{
+		component:    "velero",
+		queryName:    "backups",
+		templatePath: "queries/velero/backups/query.kql",
+		database:     "service",
+		category:     categoryState,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && d.ClusterResourceID != ""
+		},
+		prerequisites: "ClusterResourceID, ResourceType is cluster",
+	},
+	{
+		component:    "velero",
+		queryName:    "schedules",
+		templatePath: "queries/velero/schedules/query.kql",
+		database:     "service",
+		category:     categoryState,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && d.ClusterResourceID != ""
+		},
+		prerequisites: "ClusterResourceID, ResourceType is cluster",
+	},
+	{
+		component:    "velero",
+		queryName:    "dataUploads",
+		templatePath: "queries/velero/dataUploads/query.kql",
+		database:     "service",
+		category:     categoryState,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && d.HostedControlPlaneNamespace != ""
+		},
+		prerequisites: "HostedClusterNamespace, HostedControlPlaneNamespace, ResourceType is cluster",
+	},
+	{
+		component:    "velero",
+		queryName:    "deleteBackupRequests",
+		templatePath: "queries/velero/deleteBackupRequests/query.kql",
+		database:     "service",
+		category:     categoryState,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && d.HostedClusterNamespace != ""
+		},
+		prerequisites: "HostedClusterNamespace, ResourceType is cluster",
+	},
+	{
+		component:    "velero",
+		queryName:    "logs",
+		templatePath: "queries/velero/logs/query.kql",
+		database:     "service",
+		category:     categoryLogs,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && d.HostedControlPlaneNamespace != ""
+		},
+		prerequisites: "HostedClusterNamespace, HostedControlPlaneNamespace, ResourceType is cluster",
+	},
+	{
+		component:    "velero",
+		queryName:    "serverLogs",
+		templatePath: "queries/velero/serverLogs/query.kql",
+		database:     "service",
+		category:     categoryLogs,
+		ready: func(d queryData) bool {
+			return isClusterType(d) && len(d.ManagementClusterNames) > 0
+		},
+		prerequisites: "ManagementClusterNames (seeded or discovered), ResourceType is cluster",
+	},
+
 	{
 		component:    "alerts",
 		queryName:    "cluster",

tooling/hcpctl/pkg/snapshot/queries/velero/backups/README.md

@@ -0,0 +1,22 @@
+# velero / backups
+
+## Summary
+
+Full snapshot history of this HCP's Velero Backup CRs, captured on the management cluster,
+one row per captured snapshot event so phase transitions are visible over time. Pinned to
+this HCP via the `azure.microsoft.com/hcp-cluster-azure-resource-id` annotation (a
+management cluster hosts many HCPs). `isFailed` flags failed / partially-failed phases.
+
+## What to Look For
+
+- `phase` progression `New -> InProgress -> Completed`. Terminal `Failed`,
+  `PartiallyFailed`, or `FailedValidation` (see `isFailed`) means the backup did not
+  fully succeed.
+- `errors` / `warnings` counts and `failureReason` / `validationErrors` for the cause.
+- Gaps in `backup` rows vs the schedule cadence — backups that never started.
+
+## Where to Go Next
+
+- `state/velero/schedules.md` — is a schedule producing these backups at all?
+- `state/velero/dataUploads.md` — per-volume upload failures behind a `PartiallyFailed`.
+- `logs/velero/logs.md` — velero component logs for this HCP around the failure time.

tooling/hcpctl/pkg/snapshot/queries/velero/backups/query.kql

@@ -0,0 +1,22 @@
+cluster('{{ .ClusterURI }}').database('{{ .ServiceDatabase }}').table('kubernetesResourceSnapshots')
+| where timestamp between ({{ kqlDatetime .PhaseStartTime }} .. {{ kqlDatetime .PhaseEndTime }})
+{{- if and .ServiceClusterName .ManagementClusterName }}
+| where cluster in ('{{ .ServiceClusterName }}', '{{ .ManagementClusterName }}')
+{{- end }}
+| where objectKind == 'Backup'
+| where namespace == 'velero'
+| where tostring(object.metadata.annotations['azure.microsoft.com/hcp-cluster-azure-resource-id']) =~ '{{ .ClusterResourceID }}'
+| extend
+    backup = name,
+    schedule = tostring(object.metadata.labels['velero.io/schedule-name']),
+    phase = tostring(object.status.phase),
+    startTimestamp = todatetime(object.status.startTimestamp),
+    completionTimestamp = todatetime(object.status.completionTimestamp),
+    expiration = todatetime(object.status.expiration),
+    errors = toint(object.status.errors),
+    warnings = toint(object.status.warnings),
+    failureReason = tostring(object.status.failureReason),
+    validationErrors = object.status.validationErrors
+| extend isFailed = phase in ('Failed', 'PartiallyFailed', 'FailedValidation', 'WaitingForPluginOperationsPartiallyFailed', 'FinalizingPartiallyFailed')
+| project timestamp, event, cluster, backup, schedule, phase, isFailed, startTimestamp, completionTimestamp, expiration, errors, warnings, failureReason, validationErrors
+| order by timestamp asc

tooling/hcpctl/pkg/snapshot/queries/velero/dataUploads/README.md

@@ -0,0 +1,21 @@
+# velero / dataUploads
+
+## Summary
+
+Per-volume data-mover upload operations (CSI snapshot -> object store) for this HCP, where
+PersistentVolume backup failures surface (e.g. etcd data volumes). DataUpload CRs carry no
+ARM annotation, so they are matched by `spec.sourceNamespace` against this HCP's namespaces
+(`ocm-<env>-<id>` and `-<name>`). `isFailed` flags Failed / Canceled uploads.
+
+## What to Look For
+
+- `phase == Failed` / `Canceled` (`isFailed`) — the volume upload did not complete; this is
+  the usual cause behind a `PartiallyFailed` backup.
+- `bytesDone` stalled well below `totalBytes` — an upload that hung.
+- `sourcePVC` / `sourceNamespace` to identify which volume failed; `node` for where the
+  data-mover pod ran (correlate with node problems).
+
+## Where to Go Next
+
+- `state/velero/backups.md` — the parent Backup (`backup` column) this upload belongs to.
+- `logs/velero/logs.md` — data-mover pod logs (the per-backup pod names embed the backup name).

tooling/hcpctl/pkg/snapshot/queries/velero/dataUploads/query.kql

@@ -0,0 +1,22 @@
+cluster('{{ .ClusterURI }}').database('{{ .ServiceDatabase }}').table('kubernetesResourceSnapshots')
+| where timestamp between ({{ kqlDatetime .PhaseStartTime }} .. {{ kqlDatetime .PhaseEndTime }})
+{{- if and .ServiceClusterName .ManagementClusterName }}
+| where cluster in ('{{ .ServiceClusterName }}', '{{ .ManagementClusterName }}')
+{{- end }}
+| where objectKind == 'DataUpload'
+| where namespace == 'velero'
+| where tostring(object.spec.sourceNamespace) in ('{{ .HostedClusterNamespace }}', '{{ .HostedControlPlaneNamespace }}')
+| extend
+    dataUpload = name,
+    backup = tostring(object.metadata.labels['velero.io/backup-name']),
+    sourceNamespace = tostring(object.spec.sourceNamespace),
+    sourcePVC = tostring(object.spec.sourcePVC),
+    phase = tostring(object.status.phase),
+    node = tostring(object.status.node),
+    startTimestamp = todatetime(object.status.startTimestamp),
+    completionTimestamp = todatetime(object.status.completionTimestamp),
+    bytesDone = tolong(object.status.progress.bytesDone),
+    totalBytes = tolong(object.status.progress.totalBytes)
+| extend isFailed = phase in ('Failed', 'Canceled')
+| project timestamp, event, cluster, dataUpload, backup, sourceNamespace, sourcePVC, phase, isFailed, node, startTimestamp, completionTimestamp, bytesDone, totalBytes
+| order by timestamp asc

tooling/hcpctl/pkg/snapshot/queries/velero/deleteBackupRequests/README.md

@@ -0,0 +1,20 @@
+# velero / deleteBackupRequests
+
+## Summary
+
+Backup-deletion requests for this HCP — deletion / garbage-collection intent and outcome.
+A durable record complementing the backup snapshot stream: a DeleteBackupRequest captures a
+deletion that a Backup `event: "Delete"` row may have missed. Matched to this HCP via the
+`velero.io/backup-name` label, whose value embeds the HCP namespace token.
+
+## What to Look For
+
+- `phase == Processed` with non-empty `errors` — the deletion failed; the backup (and its
+  object-store data) may be orphaned.
+- Unexpected deletions of backups you still expect to exist.
+- `backup` names correlate each request to a row in `state/velero/backups.md`.
+
+## Where to Go Next
+
+- `state/velero/backups.md` — whether the referenced backup still appears afterwards.
+- `logs/velero/serverLogs.md` — velero server errors around the deletion.

tooling/hcpctl/pkg/snapshot/queries/velero/deleteBackupRequests/query.kql

@@ -0,0 +1,15 @@
+cluster('{{ .ClusterURI }}').database('{{ .ServiceDatabase }}').table('kubernetesResourceSnapshots')
+| where timestamp between ({{ kqlDatetime .PhaseStartTime }} .. {{ kqlDatetime .PhaseEndTime }})
+{{- if and .ServiceClusterName .ManagementClusterName }}
+| where cluster in ('{{ .ServiceClusterName }}', '{{ .ManagementClusterName }}')
+{{- end }}
+| where objectKind == 'DeleteBackupRequest'
+| where namespace == 'velero'
+| where tostring(object.metadata.labels['velero.io/backup-name']) has '{{ .HostedClusterNamespace }}'
+| extend
+    request = name,
+    backup = tostring(object.spec.backupName),
+    phase = tostring(object.status.phase),
+    errors = object.status.errors
+| project timestamp, event, cluster, request, backup, phase, errors
+| order by timestamp asc

tooling/hcpctl/pkg/snapshot/queries/velero/logs/README.md

@@ -0,0 +1,23 @@
+# velero / logs
+
+## Summary
+
+Velero component logs (server, node-agent, and per-backup data-mover pods, all in the
+`velero` namespace on the management cluster) attributed to this HCP by matching its
+namespace tokens in the log body. Aggregated by `container_name`, `level`, and `msg` with
+first/last occurrence and count — raw per-line output would be far too large for a snapshot.
+
+## What to Look For
+
+- `level == error` / `warning` rows, highest `occurrences` first, around a failing backup's
+  time window (from `state/velero/backups.md`).
+- `container_name` distinguishes the emitting component: `velero` (server),
+  `node-agent`, or a per-backup data-mover pod.
+- A repeating `msg` spanning `first_occurrence`..`last_occurrence` — a persistent failure vs
+  a one-off.
+
+## Where to Go Next
+
+- `state/velero/backups.md` / `state/velero/dataUploads.md` — correlate error times to a
+  specific backup or upload.
+- `logs/velero/serverLogs.md` — shared velero-server errors not attributable to one HCP.

tooling/hcpctl/pkg/snapshot/queries/velero/logs/query.kql

@@ -0,0 +1,16 @@
+cluster('{{ .ClusterURI }}').database('{{ .ServiceDatabase }}').table('containerLogs')
+| where timestamp between ({{ kqlDatetime .PhaseStartTime }} .. {{ kqlDatetime .PhaseEndTime }})
+{{- if and .ServiceClusterName .ManagementClusterName }}
+| where cluster in ('{{ .ServiceClusterName }}', '{{ .ManagementClusterName }}')
+{{- end }}
+| where namespace_name == 'velero'
+| where log has_any ('{{ .HostedClusterNamespace }}', '{{ .HostedControlPlaneNamespace }}')
+| extend
+    level = tostring(extract('level=([a-zA-Z]+)', 1, tostring(log))),
+    msg = tostring(extract('msg="([^"]*)"', 1, tostring(log)))
+| summarize
+    first_occurrence = min(timestamp),
+    last_occurrence = max(timestamp),
+    occurrences = count()
+  by container_name, level, msg
+| order by first_occurrence asc

tooling/hcpctl/pkg/snapshot/queries/velero/mgmtCluster/README.md

@@ -0,0 +1,19 @@
+# velero / mgmtCluster
+
+## Summary
+
+Discovers which management (AKS) cluster(s) host this HCP's velero backups, by reading the
+`cluster` from the HCP's Velero Backup CRs (matched by the ARM resource-id annotation) —
+exactly the management cluster where velero runs for this HCP. Velero runs per management
+cluster, so this list scopes the cross-HCP `logs/velero/serverLogs.md` query. Discovery-only
+— its result feeds other queries and is not primary triage output.
+
+## What to Look For
+
+Normally a single management cluster. More than one means the control plane moved
+between management clusters during the window (a migration or failover), which is worth
+noting when reading velero results.
+
+## Where to Go Next
+
+`logs/velero/serverLogs.md` — shared velero server errors/warnings on these clusters.

tooling/hcpctl/pkg/snapshot/queries/velero/mgmtCluster/query.kql

@@ -0,0 +1,6 @@
+cluster('{{ .ClusterURI }}').database('{{ .ServiceDatabase }}').table('kubernetesResourceSnapshots')
+| where timestamp between ({{ kqlDatetime .FullStartTime }} .. {{ kqlDatetime .FullEndTime }})
+| where objectKind == 'Backup'
+| where namespace == 'velero'
+| where tostring(object.metadata.annotations['azure.microsoft.com/hcp-cluster-azure-resource-id']) =~ '{{ .ClusterResourceID }}'
+| distinct cluster

tooling/hcpctl/pkg/snapshot/queries/velero/schedules/README.md

@@ -0,0 +1,20 @@
+# velero / schedules
+
+## Summary
+
+Snapshot history of this HCP's Velero Schedule CRs — the cron definitions that drive
+backup creation. Pinned to this HCP via the `azure.microsoft.com/hcp-cluster-azure-resource-id`
+annotation. Answers the first triage question: *is the HCP being backed up at all?*
+
+## What to Look For
+
+- `paused == true` — backups are suspended; no new Backup CRs will be created.
+- `phase != Enabled` (`isHealthy == false`) or non-empty `validationErrors` — the schedule
+  itself is broken.
+- `cron` cadence vs `lastBackup` — a stale `lastBackup` relative to the cron means backups
+  stopped firing.
+- No rows at all — no schedule exists for this HCP.
+
+## Where to Go Next
+
+`state/velero/backups.md` — the Backup CRs this schedule should have produced.

tooling/hcpctl/pkg/snapshot/queries/velero/schedules/query.kql

@@ -0,0 +1,18 @@
+cluster('{{ .ClusterURI }}').database('{{ .ServiceDatabase }}').table('kubernetesResourceSnapshots')
+| where timestamp between ({{ kqlDatetime .PhaseStartTime }} .. {{ kqlDatetime .PhaseEndTime }})
+{{- if and .ServiceClusterName .ManagementClusterName }}
+| where cluster in ('{{ .ServiceClusterName }}', '{{ .ManagementClusterName }}')
+{{- end }}
+| where objectKind == 'Schedule'
+| where namespace == 'velero'
+| where tostring(object.metadata.annotations['azure.microsoft.com/hcp-cluster-azure-resource-id']) =~ '{{ .ClusterResourceID }}'
+| extend
+    schedule = name,
+    cron = tostring(object.spec.schedule),
+    paused = tobool(object.spec.paused),
+    phase = tostring(object.status.phase),
+    lastBackup = todatetime(object.status.lastBackup),
+    validationErrors = object.status.validationErrors
+| extend isHealthy = phase == 'Enabled'
+| project timestamp, event, cluster, schedule, cron, paused, phase, isHealthy, lastBackup, validationErrors
+| order by timestamp asc

tooling/hcpctl/pkg/snapshot/queries/velero/serverLogs/README.md

@@ -0,0 +1,23 @@
+# velero / serverLogs
+
+## Summary
+
+Velero **server** error/warning logs on the management cluster(s) hosting this HCP (from
+`discovery/velero/mgmtCluster.md`, or the PR-job hint). Deliberately **not** filtered to
+this HCP: the velero server is shared per management cluster, so an infrastructure-level
+failure (object-store auth, plugin crash, BackupStorageLocation unavailable) breaks backups
+for every HCP on the cluster and carries no per-HCP token. Aggregated by `cluster`, `level`,
+`msg`, and extracted `err`.
+
+## What to Look For
+
+- Repeated `error`-level `msg` / `err` spanning the failure window — a shared-plane problem
+  affecting all HCPs on that management cluster.
+- BackupStorageLocation / object-store / credential errors — these explain backups that fail
+  before any per-HCP work starts.
+
+## Where to Go Next
+
+- `logs/velero/logs.md` — the same window filtered to this HCP's namespaces (server +
+  node-agent + data-mover), to tell shared failures from HCP-specific ones.
+- `state/velero/backups.md` — whether backup phases correlate with these server errors.