OPCT-403: rebase release-0.6 from main for v0.6.6 release

.github/workflows/ci.yaml

@@ -38,7 +38,7 @@ jobs:
       - reviewer
     env:
       VERSION: "v0.0.0-devel-pr.${{ github.event.pull_request.number }}"
-      VERSION_TOOLS: v0.5.0
+      VERSION_TOOLS: v0.6.0
       PLATFORMS: linux/amd64
       EXPIRATION: 1d
       PUSH: false

artifacts-collector/Containerfile

@@ -5,10 +5,20 @@
 ARG PLUGIN_REPO=quay.io/opct/plugin-openshift-tests
 FROM ${PLUGIN_REPO}:${BUILD_VERSION} AS plugin
 
-FROM quay.io/opct/tools:v0.5.0 AS tools
+FROM quay.io/opct/tools:v0.6.0 AS tools
+
+FROM quay.io/fedora/fedora-minimal:42 AS mgc
+ARG MGC_VERSION=0.0.5
+ARG TARGETARCH=amd64
+RUN microdnf install -y tar gzip curl && \
+    set -euo pipefail && \
+    curl -sSLf -o /tmp/mgc.tar.gz https://github.com/openshift/must-gather-clean/releases/download/v${MGC_VERSION}/must-gather-clean-linux-${TARGETARCH}.tar.gz && \
+    curl -sSLf -o /tmp/mgc.sha256 https://github.com/openshift/must-gather-clean/releases/download/v${MGC_VERSION}/SHA256_SUM && \
+    echo "$(grep must-gather-clean-linux-${TARGETARCH}.tar.gz /tmp/mgc.sha256 | awk '{print $1}')  /tmp/mgc.tar.gz" | sha256sum --check --strict - && \
+    tar xz -C /usr/local/bin -f /tmp/mgc.tar.gz must-gather-clean
 
 # Main image
-FROM quay.io/fedora/fedora-minimal:41
+FROM quay.io/fedora/fedora-minimal:42
 ARG QUAY_EXPIRATION=never
 LABEL io.k8s.display-name="OPCT Clients" \
         io.k8s.description="OPCT Clients is the base image for most of OPCT plugins." \
@@ -25,13 +35,17 @@
 COPY --from=tools /usr/bin/oc /usr/bin/oc
 COPY --from=tools /usr/bin/jq /usr/bin/jq
 COPY --from=tools /usr/bin/camgi /usr/bin/camgi
+COPY --from=mgc /usr/local/bin/must-gather-clean /usr/bin/must-gather-clean
+COPY --from=tools /usr/bin/mco-sanitize /usr/bin/mco-sanitize
 
 # plugin openshift-tests must be built first (dependency)
 COPY --from=plugin /usr/bin/openshift-tests-plugin /usr/bin/openshift-tests-plugin
 
 COPY ./*.sh ./
 COPY ./collector.sh ./
+COPY ./mgc-config-mustgather.yaml /plugin/
+COPY ./mgc-config-e2e.yaml /plugin/
 
 RUN ln -svf /usr/bin/oc /usr/bin/kubectl
 
 ENTRYPOINT /plugin/entrypoint-collector.sh

artifacts-collector/collector.sh

@@ -33,9 +33,54 @@ send_test_progress() {
 # the steps here must ensure edge scenariois not added
 # in must-gather workflow.
 clean_must_gather() {
-    # clean registry credentials
+    # always clean internalRegistryPullSecret first (safety net before MGC)
     sed -i 's/\(internalRegistryPullSecret:\s*\).*/\1"<sensitive>"/' \
-        ${MUST_GATHER_DIR}/*/cluster-scoped-resources/machineconfiguration.openshift.io/controllerconfigs/machine-config-controller.yaml >/dev/null
+        ${MUST_GATHER_DIR}/*/cluster-scoped-resources/machineconfiguration.openshift.io/controllerconfigs/machine-config-controller.yaml >/dev/null 2>&1
+
+    # sanitize MCO resources (machineconfigs contain pull secrets and JWTs in ignition config)
+    if command -v mco-sanitize &>/dev/null; then
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] Running mco-sanitize on must-gather"
+        if ! mco-sanitize --input="${MUST_GATHER_DIR}"; then
+            os_log_info "[executor][PluginID#${PLUGIN_ID}] mco-sanitize failed, falling back to manual redaction"
+            find "${MUST_GATHER_DIR}" -type f -path '*/cluster-scoped-resources/machineconfiguration.openshift.io/*' ! -name '*.redacted' \
+                -exec sh -c 'echo "REDACTED" > "$1" && mv "$1" "$1.redacted"' _ {} \;
+        fi
+    else
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] mco-sanitize not available, falling back to manual redaction"
+        find "${MUST_GATHER_DIR}" -type f -path '*/cluster-scoped-resources/machineconfiguration.openshift.io/*' ! -name '*.redacted' \
+            -exec sh -c 'echo "REDACTED" > "$1" && mv "$1" "$1.redacted"' _ {} \;
+    fi
+
+    os_log_info "[executor][PluginID#${PLUGIN_ID}] Cleaning must-gather with must-gather-clean"
+    local mg_clean_dir="${MUST_GATHER_DIR}-clean"
+    must-gather-clean -c /plugin/mgc-config-mustgather.yaml \
+        -i "${MUST_GATHER_DIR}" \
+        -o "${mg_clean_dir}" \
+        -v4 >./artifacts_log-mgc-must-gather.log 2>&1 || {
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] must-gather-clean failed, sed already applied"
+        return
+    }
+    mv "${MUST_GATHER_DIR}" "${MUST_GATHER_DIR}-orig" && \
+        mv "${mg_clean_dir}" "${MUST_GATHER_DIR}" && \
+        rm -rf "${MUST_GATHER_DIR}-orig"
+}
+
+clean_e2e_metadata() {
+    os_log_info "[executor][PluginID#${PLUGIN_ID}] Cleaning e2e metadata archives"
+    for archive in "${RESULTS_DIR}"/artifacts_e2e-metadata-*.tar.gz; do
+        [ -f "${archive}" ] || continue
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] Cleaning ${archive##*/}"
+        local tmpdir
+        tmpdir=$(mktemp -d)
+        local cleandir="${tmpdir}-clean"
+        local tmparchive="${archive}.tmp"
+        tar xzf "${archive}" -C "${tmpdir}" || { rm -rf "${tmpdir}"; continue; }
+        must-gather-clean -c /plugin/mgc-config-e2e.yaml \
+            -i "${tmpdir}" -o "${cleandir}" \
+            -v4 >>./artifacts_log-mgc-e2e-metadata.log 2>&1 || { rm -rf "${tmpdir}" "${cleandir}"; continue; }
+        tar czf "${tmparchive}" -C "${cleandir}" . && mv "${tmparchive}" "${archive}" || rm -f "${tmparchive}"
+        rm -rf "${tmpdir}" "${cleandir}"
+    done
 }
 
 # Collect must-gather and pre-process any data* from it, then create a tarball file.
@@ -158,8 +203,30 @@ collect_metrics() {
         return
     }
 
+    os_log_info "${msg_prefix} Cleaning must-gather-metrics with must-gather-clean"
+    local metrics_src_dir
+    metrics_src_dir=$(ls -d must-gather-metrics/*/ | head -1)
+    local metrics_clean_dir="${metrics_src_dir%/}-clean"
+    if must-gather-clean -c /plugin/mgc-config-mustgather.yaml \
+        -i "${metrics_src_dir}" -o "${metrics_clean_dir}" \
+        -v4 >./artifacts_log-mgc-must-gather-metrics.log 2>&1; then
+        cp -v must-gather-metrics/timestamp must-gather-metrics/event-filter.html "${metrics_clean_dir}/monitoring/" || true
+        if mv "${metrics_src_dir}" "${metrics_src_dir%/}-orig" && \
+           mv "${metrics_clean_dir}" "${metrics_src_dir}"; then
+            rm -rf "${metrics_src_dir%/}-orig"
+        else
+            os_log_info "${msg_prefix} WARNING: moving files failed, restoring original"
+            [ -d "${metrics_src_dir%/}-orig" ] && mv "${metrics_src_dir%/}-orig" "${metrics_src_dir}"
+            rm -rf "${metrics_clean_dir}"
+        fi
+    else
+        os_log_info "${msg_prefix} WARNING: must-gather-clean failed for metrics, skipping metrics packing"
+        rm -rf "${metrics_clean_dir}"
+        return
+    fi
+
     os_log_info "${msg_prefix} Packing must-gather-metrics..."
-    cp -v must-gather-metrics/timestamp must-gather-metrics/event-filter.html must-gather-metrics/*/monitoring/
+    cp -v must-gather-metrics/timestamp must-gather-metrics/event-filter.html must-gather-metrics/*/monitoring/ || true
     tar cfJ artifacts_must-gather-metrics.tar.xz -C must-gather-metrics/*/ monitoring/
 
     os_log_info "${msg_prefix} finished!"
@@ -264,6 +331,10 @@ run_plugin_collector() {
         collect_kube_burner || true
     fi
 
+    # Clean sensitive data from e2e metadata archives
+    send_test_progress "status=running=cleaning sensitive data";
+    clean_e2e_metadata || true
+
     # Create result file used to publish to sonobuoy aggregator. (must be the last step)
     send_test_progress "status=running=saving artifacts";
     os_log_info "[executor][PluginID#${PLUGIN_ID}] Packing all results..."

artifacts-collector/mgc-config-e2e.yaml

@@ -0,0 +1,11 @@
+config:
+  obfuscate:
+    - type: Regex
+      target: FileContents
+      regex: "sha256~[A-Za-z0-9_-]+"
+    - type: Regex
+      target: FileContents
+      regex: "eyJ[A-Za-z0-9_-]{10,}\\.eyJ[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]+"
+    - type: Regex
+      target: FileContents
+      regex: "(?i)Authorization:\\s+\\w+\\s+[A-Za-z0-9+/=_-]{20,}"

artifacts-collector/mgc-config-mustgather.yaml

@@ -0,0 +1,14 @@
+config:
+  obfuscate:
+    - type: Regex
+      target: FileContents
+      regex: "sha256~[A-Za-z0-9_-]+"
+    - type: Regex
+      target: FileContents
+      regex: "eyJ[A-Za-z0-9_-]{10,}\\.eyJ[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]+"
+    - type: Regex
+      target: FileContents
+      regex: "(?i)Authorization:\\s+\\w+\\s+[A-Za-z0-9+/=_-]{20,}"
+    - type: Regex
+      target: FileContents
+      regex: "(?i)(secret|httpSecret):\\s+[A-Za-z0-9+/=_-]{20,}"

build.sh

@@ -92,6 +92,9 @@ function build_tools() {
         oc_archive="openshift-client-linux.tar.gz"
         oc_url="https://mirror.openshift.com/pub/openshift-v4/${TARGET_ARCH}/clients/ocp/${ocp_version}/${oc_archive}"
 
+        mco_arch=$([ "${TARGET_ARCH}" = "amd64" ] && echo "x86_64" || echo "${TARGET_ARCH}")
+        mco_sanitize_url="https://openshift-mirror-list.ci-systems.workers.dev/pub/ci/${mco_arch}/mco-sanitize/mco-sanitize"
+
         podman build --platform "${platform}" \
             --manifest "${manifest}" \
             -f "${build_root}"/Containerfile \
@@ -101,6 +104,7 @@ function build_tools() {
             --build-arg=CAMGI_TAR="${camgi_archive}" \
             --build-arg=OC_TAR="${oc_archive}" \
             --build-arg=OC_URL="${oc_url}" \
+            --build-arg=MCO_SANITIZE_URL="${mco_sanitize_url}" \
             --build-arg=QUAY_EXPIRATION="${IMAGE_EXPIRE_TIME}" \
             --build-arg=TARGETPLATFORM="${platform}" \
             --build-arg=TARGETARCH="${TARGET_ARCH}" \

tools/Containerfile

@@ -6,7 +6,7 @@
 #
 ## Base image
 #
-FROM quay.io/fedora/fedora-minimal:41 as base
+FROM quay.io/fedora/fedora-minimal:42 as base
 RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \
     microdnf install -y curl grep tar xz gzip && \
     microdnf clean all
@@ -54,4 +54,8 @@ COPY --from=clients /clients/oc /usr/bin/oc
 COPY --from=clients /clients/jq /usr/bin/jq
 COPY --from=clients /clients/camgi /usr/bin/camgi
 
+ARG MCO_SANITIZE_URL=TBD
+ADD ${MCO_SANITIZE_URL} /usr/bin/mco-sanitize
+RUN chmod +x /usr/bin/mco-sanitize
+
 RUN ln -svf /usr/bin/oc /usr/bin/kubectl

tools/VERSION

@@ -1 +1 @@
-v0.5.0
+v0.6.0