WIP Hypershift BM - Agent based via Assisted Installer with IP echo server

ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-5.0__amd64-nightly.yaml

@@ -3,6 +3,10 @@ base_images:
     name: "5.0"
     namespace: ocp
     tag: ansible
+  assisted-service-scripts:
+    name: ocm-2.17
+    namespace: edge-infrastructure
+    tag: assisted-service-scripts
   aws-efs-csi-operator-create-efs:
     name: "5.0"
     namespace: ocp
@@ -4991,6 +4995,40 @@ tests:
     test:
     - chain: openshift-e2e-test-qe
     workflow: baremetal-lab-upi-dual-stack
+- as: metal-ds-hypershift-agent-ovn-egressip-debug
+  capabilities:
+  - intranet
+  cron: 36 15 1,17 * *
+  steps:
+    cluster_profile: equinix-ocp-hcp
+    env:
+      CLUSTERTYPE: host_384gb_el9
+      DEVSCRIPTS_CONFIG: |
+        IP_STACK=v4
+        NUM_WORKERS=3
+        WORKER_MEMORY=32768
+        WORKER_VCPU=16
+        NETWORK_TYPE=OVNKubernetes
+        VM_EXTRADISKS=true
+        VM_EXTRADISKS_LIST=vda
+        VM_EXTRADISKS_SIZE=256G
+        NUM_EXTRA_WORKERS=3
+        PROVISIONING_NETWORK_PROFILE=Disabled
+        REDFISH_EMULATOR_IGNORE_BOOT_DEVICE=True
+      HYPERSHIFT_NAMESPACE: local-cluster
+      LVM_OPERATOR_SUB_CHANNEL: stable-5.0
+      LVM_OPERATOR_SUB_SOURCE: lvm-catalogsource
+      MCE_VERSION: "2.17"
+      TIMEOUT: +18 hours
+    post:
+    - ref: hypershift-agent-ovn-ipecho-deprovision
+    - chain: cucushift-installer-rehearse-baremetalds-assisted-hypershift-agent-mce-deprovision
+    pre:
+    - chain: cucushift-installer-rehearse-baremetalds-assisted-hypershift-agent-mce-provision
+    - ref: hypershift-agent-ovn-ipecho-provision
+    test:
+    - ref: wait
+  timeout: 20h0m0s
 zz_generated_metadata:
   branch: release-5.0
   org: openshift

ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-5.0-periodics.yaml

@@ -19396,6 +19396,98 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build09
+  cron: 36 15 1,17 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+    timeout: 20h0m0s
+  extra_refs:
+  - base_ref: release-5.0
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    capability/intranet: intranet
+    ci-operator.openshift.io/cloud: equinix-ocp-metal
+    ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-hcp
+    ci-operator.openshift.io/variant: amd64-nightly
+    ci.openshift.io/generator: prowgen
+    job-release: "5.0"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-5.0-amd64-nightly-metal-ds-hypershift-agent-ovn-egressip-debug
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=metal-ds-hypershift-agent-ovn-egressip-debug
+      - --variant=amd64-nightly
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build09
   cron: 32 10 4 * *

ci-operator/step-registry/hypershift/agent/ovn/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93

ci-operator/step-registry/hypershift/agent/ovn/ipecho/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93

ci-operator/step-registry/hypershift/agent/ovn/ipecho/deprovision/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93

ci-operator/step-registry/hypershift/agent/ovn/ipecho/deprovision/hypershift-agent-ovn-ipecho-deprovision-commands.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+
+echo "Deprovisioning ipecho server from dev-scripts host"
+
+source "${SHARED_DIR}/packet-conf.sh"
+
+# SSH to host and clean up ipecho service; all errors absorbed
+ssh "${SSHOPTS[@]}" "root@${IP}" bash -s << 'EOF' || true
+systemctl stop ipecho.service || true
+systemctl disable ipecho.service || true
+rm -f /etc/systemd/system/ipecho.service || true
+rm -f /usr/local/bin/ipecho.py || true
+systemctl daemon-reload || true
+echo "ipecho server cleaned up"
+EOF
+
+echo "ipecho deprovision complete"

ci-operator/step-registry/hypershift/agent/ovn/ipecho/deprovision/hypershift-agent-ovn-ipecho-deprovision-ref.metadata.json

@@ -0,0 +1,25 @@
+{
+	"path": "hypershift/agent/ovn/ipecho/deprovision/hypershift-agent-ovn-ipecho-deprovision-ref.yaml",
+	"owners": {
+		"approvers": [
+			"tssurya",
+			"martinkennelly",
+			"jcaamano",
+			"astoycos",
+			"kyrtapz",
+			"oribon",
+			"arkadeepsen",
+			"arghosh93"
+		],
+		"reviewers": [
+			"tssurya",
+			"martinkennelly",
+			"jcaamano",
+			"astoycos",
+			"kyrtapz",
+			"oribon",
+			"arkadeepsen",
+			"arghosh93"
+		]
+	}
+}

ci-operator/step-registry/hypershift/agent/ovn/ipecho/deprovision/hypershift-agent-ovn-ipecho-deprovision-ref.yaml

@@ -0,0 +1,16 @@
+ref:
+  as: hypershift-agent-ovn-ipecho-deprovision
+  from: dev-scripts
+  commands: hypershift-agent-ovn-ipecho-deprovision-commands.sh
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  best_effort: true
+  timeout: 5m0s
+  grace_period: 5m0s
+  documentation: |-
+    Deprovisions the ipecho HTTP server on the dev-scripts provisioning host.
+    Stops and removes the systemd service and the ipecho.py script.
+
+    All errors are absorbed to avoid blocking other post steps.

ci-operator/step-registry/hypershift/agent/ovn/ipecho/provision/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93

ci-operator/step-registry/hypershift/agent/ovn/ipecho/provision/hypershift-agent-ovn-ipecho-provision-commands.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+echo "Provisioning ipecho server on dev-scripts host"
+
+source "${SHARED_DIR}/packet-conf.sh"
+
+IPECHO_HOST_IP="192.168.111.1"
+IPECHO_PORT="${IPECHO_PORT:-9095}"
+
+# Deploy ipecho server on the provisioning host via SSH
+# shellcheck disable=SC2087
+ssh "${SSHOPTS[@]}" "root@${IP}" bash -s -- "${IPECHO_PORT}" << 'EOF'
+set -euxo pipefail
+
+IPECHO_PORT="$1"
+
+# Write the ipecho Python HTTP server
+cat > /usr/local/bin/ipecho.py << 'PYEOF'
+#!/usr/bin/env python3
+"""Minimal HTTP server that returns the client source IP address."""
+import http.server
+import socketserver
+import sys
+
+class IPEchoHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        client_ip = self.client_address[0]
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        self.wfile.write(client_ip.encode("utf-8"))
+        self.wfile.write(b"\n")
+
+    def log_message(self, fmt, *args):
+        sys.stderr.write("%s - - [%s] %s\n" %
+                         (self.client_address[0],
+                          self.log_date_time_string(),
+                          fmt % args))
+
+if __name__ == "__main__":
+    port = int(sys.argv[1]) if len(sys.argv) > 1 else 9095
+    with socketserver.TCPServer(("0.0.0.0", port), IPEchoHandler) as httpd:
+        print(f"ipecho server listening on 0.0.0.0:{port}")
+        httpd.serve_forever()
+PYEOF
+chmod +x /usr/local/bin/ipecho.py
+
+# Create systemd unit
+cat > /etc/systemd/system/ipecho.service << SVCEOF
+[Unit]
+Description=ipecho HTTP server for EgressIP testing
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /usr/local/bin/ipecho.py ${IPECHO_PORT}
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+SVCEOF
+
+systemctl daemon-reload
+systemctl enable --now ipecho.service
+
+# Verify the service is running
+for i in $(seq 1 30); do
+    if curl -sf "http://192.168.111.1:${IPECHO_PORT}" >/dev/null 2>&1; then
+        echo "ipecho server is ready on port ${IPECHO_PORT}"
+        curl -s "http://192.168.111.1:${IPECHO_PORT}"
+        exit 0
+    fi
+    echo "Waiting for ipecho server... attempt ${i}/30"
+    sleep 2
+done
+
+echo "ERROR: ipecho server did not become ready"
+systemctl status ipecho.service || true
+journalctl -u ipecho.service --no-pager -n 20 || true
+exit 1
+EOF
+
+echo "ipecho server deployed successfully"
+
+# Write outputs for downstream test steps
+echo "${IPECHO_HOST_IP}" > "${SHARED_DIR}/ipecho_host_ip"
+echo "http://${IPECHO_HOST_IP}:${IPECHO_PORT}" > "${SHARED_DIR}/ipecho_url"
+
+echo "ipecho_host_ip: ${IPECHO_HOST_IP}"
+echo "ipecho_url: http://${IPECHO_HOST_IP}:${IPECHO_PORT}"