Skip to content

cnv-must-gather: add promtool backfill validation to test-incident CI step #83036

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tiraboschi wants to merge 1 commit into
base: main
Choose a base branch
from
+50 −0
Open

cnv-must-gather: add promtool backfill validation to test-incident CI step #83036

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions ci-operator/config/kubevirt/must-gather/kubevirt-must-gather-main.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,37 @@ tests:
commands: |
INCIDENT_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
oc adm must-gather --image="${KMG_IMAGE}" --dest-dir=${ARTIFACT_DIR}/must-gather-incident-output -- NS=ns001 VM=testvm-ns001-vm001 /usr/bin/gather --vm-incident --incident-time=${INCIDENT_TIME}
METRICS_FILE=$(find ${ARTIFACT_DIR}/must-gather-incident-output -name "incident-metrics.txt" -type f | head -1)
if [[ -f "${METRICS_FILE}" ]]; then
PROM_IMAGE=$(oc get statefulset -n openshift-monitoring prometheus-k8s -o jsonpath='{.spec.template.spec.containers[?(@.name=="prometheus")].image}' 2>/dev/null)
if [[ -z "${PROM_IMAGE}" ]]; then
echo "WARNING: openshift-monitoring/prometheus-k8s not found, skipping backfill validation"
else
echo "Validating OpenMetrics backfill with promtool..."
PROMTOOL_DIR=$(mktemp -d)
BACKFILL_DIR=$(mktemp -d)
PROMTOOL_LOG=$(mktemp)
_promtool_cleanup() { rm -rf "${PROMTOOL_DIR}" "${BACKFILL_DIR}" "${PROMTOOL_LOG}"; }
trap _promtool_cleanup EXIT
oc image extract "${PROM_IMAGE}" --path=/bin/promtool:"${PROMTOOL_DIR}" --confirm
if "${PROMTOOL_DIR}/promtool" tsdb create-blocks-from openmetrics "${METRICS_FILE}" "${BACKFILL_DIR}" > "${PROMTOOL_LOG}" 2>&1; then
echo "promtool backfill validation passed"
else
echo "ERROR: promtool backfill validation failed (exit code $?). Redacted output saved to artifacts."
sed -E 's/="[^"]*"/="<redacted>"/g' "${PROMTOOL_LOG}" > "${ARTIFACT_DIR}/promtool-backfill.log"
exit 1
fi
fi
else
echo "WARNING: incident-metrics.txt not found, skipping backfill validation"
fi
mkdir -p ${ARTIFACT_DIR}/junit
(cd ${ARTIFACT_DIR}; KUBECONFIG=${KUBECONFIG} INCIDENT_NS=ns001 MG_INCIDENT_OUTPUT_DIR=must-gather-incident-output /go/bin/must-gather.test --ginkgo.label-filter=level:incident --ginkgo.junit-report=${ARTIFACT_DIR}/junit/report.xml --ginkgo.v)
dependencies:
- env: KMG_IMAGE
name: kubevirt-must-gather
from: test-bin
grace_period: 1m0s
resources:
requests:
cpu: 100m
Expand Down Expand Up @@ -254,12 +279,37 @@ tests:
commands: |
INCIDENT_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
oc adm must-gather --image="${KMG_IMAGE}" --dest-dir=${ARTIFACT_DIR}/must-gather-incident-output -- NS=ns001 VM=testvm-ns001-vm001 /usr/bin/gather --vm-incident --incident-time=${INCIDENT_TIME}
METRICS_FILE=$(find ${ARTIFACT_DIR}/must-gather-incident-output -name "incident-metrics.txt" -type f | head -1)
if [[ -f "${METRICS_FILE}" ]]; then
PROM_IMAGE=$(oc get statefulset -n openshift-monitoring prometheus-k8s -o jsonpath='{.spec.template.spec.containers[?(@.name=="prometheus")].image}' 2>/dev/null)
if [[ -z "${PROM_IMAGE}" ]]; then
echo "WARNING: openshift-monitoring/prometheus-k8s not found, skipping backfill validation"
else
echo "Validating OpenMetrics backfill with promtool..."
PROMTOOL_DIR=$(mktemp -d)
BACKFILL_DIR=$(mktemp -d)
PROMTOOL_LOG=$(mktemp)
_promtool_cleanup() { rm -rf "${PROMTOOL_DIR}" "${BACKFILL_DIR}" "${PROMTOOL_LOG}"; }
trap _promtool_cleanup EXIT
oc image extract "${PROM_IMAGE}" --path=/bin/promtool:"${PROMTOOL_DIR}" --confirm
if "${PROMTOOL_DIR}/promtool" tsdb create-blocks-from openmetrics "${METRICS_FILE}" "${BACKFILL_DIR}" > "${PROMTOOL_LOG}" 2>&1; then
echo "promtool backfill validation passed"
else
echo "ERROR: promtool backfill validation failed (exit code $?). Redacted output saved to artifacts."
sed -E 's/="[^"]*"/="<redacted>"/g' "${PROMTOOL_LOG}" > "${ARTIFACT_DIR}/promtool-backfill.log"
exit 1
fi
fi
else
echo "WARNING: incident-metrics.txt not found, skipping backfill validation"
fi
mkdir -p ${ARTIFACT_DIR}/junit
(cd ${ARTIFACT_DIR}; KUBECONFIG=${KUBECONFIG} INCIDENT_NS=ns001 MG_INCIDENT_OUTPUT_DIR=must-gather-incident-output /go/bin/must-gather.test --ginkgo.label-filter=level:incident --ginkgo.junit-report=${ARTIFACT_DIR}/junit/report.xml --ginkgo.v)
dependencies:
- env: KMG_IMAGE
name: kubevirt-must-gather
from: test-bin
grace_period: 1m0s
resources:
requests:
cpu: 100m
Expand Down