WIP Hypershift CI Serial job

ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-main.yaml

@@ -364,6 +364,13 @@ tests:
   steps:
     cluster_profile: hypershift-aws
     workflow: hypershift-aws-conformance
+- always_run: false
+  as: e2e-aws-ovn-hypershift-serial
+  optional: true
+  steps:
+    cluster_profile: hypershift-aws
+    workflow: hypershift-aws-ovn-conformance
+  timeout: 20h0m0s
 - as: unit
   commands: |
     cd go-controller

ci-operator/jobs/openshift/ovn-kubernetes/openshift-ovn-kubernetes-main-presubmits.yaml

@@ -942,6 +942,94 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-aws-ovn-hypershift,?($|\s.*)
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build06
+    context: ci/prow/e2e-aws-ovn-hypershift-serial
+    decorate: true
+    decoration_config:
+      sparse_checkout_files:
+      - .ci-operator.yaml
+      - Dockerfile
+      - Dockerfile.base
+      - Dockerfile.microshift
+      timeout: 20h0m0s
+    labels:
+      ci-operator.openshift.io/cloud: hypershift-aws
+      ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-ovn-kubernetes-main-e2e-aws-ovn-hypershift-serial
+    optional: true
+    rerun_command: /test e2e-aws-ovn-hypershift-serial
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=e2e-aws-ovn-hypershift-serial
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-aws-ovn-hypershift-serial,?($|\s.*)
   - agent: kubernetes
     always_run: false
     annotations:

ci-operator/step-registry/hypershift/aws/ovn/conformance/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93

ci-operator/step-registry/hypershift/aws/ovn/conformance/hypershift-aws-ovn-conformance-workflow.metadata.json

@@ -0,0 +1,25 @@
+{
+	"path": "hypershift/aws/ovn/conformance/hypershift-aws-ovn-conformance-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"tssurya",
+			"martinkennelly",
+			"jcaamano",
+			"astoycos",
+			"kyrtapz",
+			"oribon",
+			"arkadeepsen",
+			"arghosh93"
+		],
+		"reviewers": [
+			"tssurya",
+			"martinkennelly",
+			"jcaamano",
+			"astoycos",
+			"kyrtapz",
+			"oribon",
+			"arkadeepsen",
+			"arghosh93"
+		]
+	}
+}

ci-operator/step-registry/hypershift/aws/ovn/conformance/hypershift-aws-ovn-conformance-workflow.yaml

@@ -0,0 +1,24 @@
+workflow:
+  as: hypershift-aws-ovn-conformance
+  documentation: |-
+    The HyperShift AWS OVN conformance workflow provisions a HyperShift cluster
+    on AWS along with an ipecho bastion instance for OVN-Kubernetes EgressIP
+    and network connectivity testing.
+
+    The ipecho server returns the caller's source IP address over HTTP.
+    After cluster and bastion setup, the workflow pauses (default 18h) for
+    manual debugging, then cleans up all resources on teardown.
+  steps:
+    pre:
+    - ref: ipi-install-rbac
+    - chain: hypershift-setup-root-management-cluster
+    - chain: hypershift-aws-create
+    - ref: hypershift-aws-ovn-ipecho-provision
+    test:
+    - ref: wait
+    post:
+    - chain: hypershift-dump
+    - ref: hypershift-aws-ovn-ipecho-deprovision
+    - chain: hypershift-aws-destroy
+    env:
+      TIMEOUT: "+18 hours"

ci-operator/step-registry/hypershift/aws/ovn/ipecho/deprovision/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93

ci-operator/step-registry/hypershift/aws/ovn/ipecho/deprovision/hypershift-aws-ovn-ipecho-deprovision-commands.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -o nounset
+set -o pipefail
+# NOTE: set -e is intentionally omitted — cleanup must never fail the job.
+
+echo "--- Deprovisioning ipecho bastion ---"
+
+# Check if there is anything to clean up
+if [[ ! -f "${SHARED_DIR}/ipecho_instance_id" ]]; then
+  echo "No ipecho_instance_id found in SHARED_DIR, nothing to clean up."
+  exit 0
+fi
+
+INSTANCE_ID=$(cat "${SHARED_DIR}/ipecho_instance_id")
+REGION=$(cat "${SHARED_DIR}/ipecho_region" 2>/dev/null || echo "us-east-1")
+SG_ID=""
+if [[ -f "${SHARED_DIR}/ipecho_security_group_id" ]]; then
+  SG_ID=$(cat "${SHARED_DIR}/ipecho_security_group_id")
+fi
+
+export AWS_SHARED_CREDENTIALS_FILE="/etc/hypershift-pool-aws-credentials/.awscred"
+export AWS_DEFAULT_REGION="${REGION}"
+
+echo "Instance ID: ${INSTANCE_ID}"
+echo "Security Group: ${SG_ID:-<not set>}"
+echo "Region: ${REGION}"
+
+# --- 1. Terminate the instance ---
+echo "Terminating instance ${INSTANCE_ID}..."
+aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" || true
+
+echo "Waiting for instance to terminate..."
+aws ec2 wait instance-terminated --instance-ids "${INSTANCE_ID}" || true
+
+echo "Instance terminated."
+
+# --- 2. Delete security group with retry loop ---
+if [[ -n "${SG_ID}" ]]; then
+  echo "Deleting security group ${SG_ID}..."
+  for attempt in $(seq 1 5); do
+    if aws ec2 delete-security-group --group-id "${SG_ID}" 2>/dev/null; then
+      echo "Security group deleted."
+      break
+    fi
+    if [[ ${attempt} -eq 5 ]]; then
+      echo "WARNING: Could not delete security group ${SG_ID} after 5 attempts. It may need manual cleanup."
+      break
+    fi
+    echo "  Attempt ${attempt}/5: security group still in use (ENI detaching), retrying in 15s..."
+    sleep 15
+  done
+fi
+
+echo "--- ipecho bastion deprovisioned ---"

ci-operator/step-registry/hypershift/aws/ovn/ipecho/deprovision/hypershift-aws-ovn-ipecho-deprovision-ref.metadata.json

@@ -0,0 +1,25 @@
+{
+	"path": "hypershift/aws/ovn/ipecho/deprovision/hypershift-aws-ovn-ipecho-deprovision-ref.yaml",
+	"owners": {
+		"approvers": [
+			"tssurya",
+			"martinkennelly",
+			"jcaamano",
+			"astoycos",
+			"kyrtapz",
+			"oribon",
+			"arkadeepsen",
+			"arghosh93"
+		],
+		"reviewers": [
+			"tssurya",
+			"martinkennelly",
+			"jcaamano",
+			"astoycos",
+			"kyrtapz",
+			"oribon",
+			"arkadeepsen",
+			"arghosh93"
+		]
+	}
+}

ci-operator/step-registry/hypershift/aws/ovn/ipecho/deprovision/hypershift-aws-ovn-ipecho-deprovision-ref.yaml

@@ -0,0 +1,28 @@
+ref:
+  as: hypershift-aws-ovn-ipecho-deprovision
+  from_image:
+    namespace: ocp
+    name: "4.18"
+    tag: upi-installer
+  commands: hypershift-aws-ovn-ipecho-deprovision-commands.sh
+  credentials:
+  - mount_path: /etc/hypershift-pool-aws-credentials
+    name: hypershift-pool-aws-credentials
+    namespace: test-credentials
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  best_effort: true
+  timeout: 5m0s
+  grace_period: 5m0s
+  documentation: |-
+    Deprovisions the EC2 ipecho bastion instance and its security group
+    created by hypershift-aws-ovn-ipecho-provision.
+
+    Reads from ${SHARED_DIR}:
+    - ipecho_instance_id: EC2 instance ID
+    - ipecho_security_group_id: Security group ID
+    - ipecho_region: AWS region
+
+    All errors are absorbed to avoid blocking other post steps.

ci-operator/step-registry/hypershift/aws/ovn/ipecho/provision/OWNERS

@@ -0,0 +1,18 @@
+approvers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93
+reviewers:
+- tssurya
+- martinkennelly
+- jcaamano
+- astoycos
+- kyrtapz
+- oribon
+- arkadeepsen
+- arghosh93