hack: stage micro-VM assets without a host aws CLI or port-forward

hack/install-microvm-deps.sh

@@ -145,7 +145,7 @@ fi
 
 # --- 2. stage assets to rustfs (kind) / GCS (GKE) --------------------------
 # Upload the five assets under kata-assets/, where atelet fetches them: the
-# in-cluster rustfs (port-forwarded, S3 API) on kind, or the GCS bucket on GKE.
+# in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE.
 if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then
   log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..."
   OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh

hack/microvm-assets/stage-to-rustfs.sh

@@ -15,12 +15,22 @@
 # limitations under the License.
 
 # Stage the assembled micro-VM asset set into the kind cluster's rustfs S3 bucket
-# under kata-assets/, where atelet fetches it (per demos/counter/counter-microvm.yaml.tmpl).
+# under kata-assets/, where atelet fetches it (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl).
 # Run after the cluster is up (hack/install-ate-kind.sh) and assemble.sh has produced $OUT.
 #
-# Requires the `aws` CLI. Env: OUT (asset dir, default ./bin/microvm-assets/arm64),
-# BUCKET (default ate-snapshots), NAMESPACE (rustfs namespace, default ate-system),
-# KUBECTL_CONTEXT (optional; kube context for port-forward).
+# The S3 client runs in a throwaway container (the same pinned amazon/aws-cli
+# image as the rustfs-bucket-init Job), so developers don't need the `aws` CLI
+# installed. The container joins the kind node's network namespace, which is what
+# makes rustfs's ClusterIP routable: a container merely attached to the `kind`
+# docker network can reach the node's own IP but has no route to the service or
+# pod CIDRs. That also avoids a `kubectl port-forward`, which silently uploads
+# nothing when it targets the wrong cluster, and which a container can't reach at
+# all on macOS (the container's localhost is the Docker VM, not the host).
+#
+# Env: OUT (asset dir, default ./bin/microvm-assets/arm64), BUCKET (default
+# ate-snapshots), NAMESPACE (rustfs namespace, default ate-system),
+# KUBECTL_CONTEXT (optional; kube context), KIND_CLUSTER_NAME (default: derived
+# from KUBECTL_CONTEXT, else "kind").
 
 set -o errexit -o nounset -o pipefail
 
@@ -30,28 +40,60 @@ OUT="${OUT:-${ROOT}/bin/microvm-assets/arm64}"
 BUCKET="${BUCKET:-ate-snapshots}"
 NAMESPACE="${NAMESPACE:-ate-system}"
 KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}"
+# kind contexts are named kind-<cluster>; fall back to kind's own default.
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-${KUBECTL_CONTEXT#kind-}}"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
+
+# Keep in sync with the rustfs-bucket-init Job in
+# manifests/ate-install/kind/rustfs.yaml, which creates the bucket we upload into.
+AWS_CLI_IMAGE="amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73"
+
+ASSETS=(cloud-hypervisor virtiofsd vmlinux rootfs.img configuration-clh.toml)
 
-if ! command -v aws >/dev/null 2>&1; then
-  echo "error: the 'aws' CLI is required but was not found in PATH" >&2
+run_kubectl() {
+  kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} -n "${NAMESPACE}" "$@"
+}
+
+if ! command -v docker >/dev/null 2>&1; then
+  echo "error: 'docker' is required to run the S3 client but was not found in PATH" >&2
   exit 1
 fi
 
-export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}"
-export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}"
-export AWS_REGION="${AWS_REGION:-us-east-1}"
+for f in "${ASSETS[@]}"; do
+  if [[ ! -f "${OUT}/${f}" ]]; then
+    echo "error: missing asset ${OUT}/${f}; run hack/microvm-assets/assemble.sh first" >&2
+    exit 1
+  fi
+done
+
+# rustfs must be serving and the bucket must exist before anything is uploaded.
+echo ">> Waiting for rustfs in namespace ${NAMESPACE}..."
+run_kubectl rollout status deploy/rustfs --timeout=300s
+run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s
+
+NODE="$("${ROOT}/hack/kind.sh" get nodes --name "${KIND_CLUSTER_NAME}" | head -n1)"
+if [[ -z "${NODE}" ]]; then
+  echo "error: no nodes found for kind cluster '${KIND_CLUSTER_NAME}'" >&2
+  exit 1
+fi
+ENDPOINT="http://$(run_kubectl get svc rustfs -o jsonpath='{.spec.clusterIP}'):9000"
 
-echo ">> Port-forwarding svc/rustfs 9000 in namespace ${NAMESPACE}..."
-kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} -n "${NAMESPACE}" port-forward svc/rustfs 9000:9000 >/tmp/rustfs-pf.log 2>&1 &
-PF_PID=$!
-trap 'kill "$PF_PID" 2>/dev/null || true' EXIT
-sleep 3
+echo ">> Uploading assets to s3://${BUCKET}/kata-assets/ via ${ENDPOINT} (netns of ${NODE})..."
+aws_cli() {
+  # -i (no -t): a TTY would corrupt the binary stream piped in on stdin.
+  docker run --rm -i \
+    --network "container:${NODE}" \
+    -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}" \
+    -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}" \
+    -e AWS_REGION="${AWS_REGION:-us-east-1}" \
+    -e AWS_ENDPOINT_URL="${ENDPOINT}" \
+    "${AWS_CLI_IMAGE}" "$@"
+}
 
-ENDPOINT="http://localhost:9000"
-echo ">> Uploading assets to s3://${BUCKET}/kata-assets/ via ${ENDPOINT}..."
-for f in cloud-hypervisor virtiofsd vmlinux rootfs.img configuration-clh.toml; do
+for f in "${ASSETS[@]}"; do
   echo "   $f"
-  aws --endpoint-url "${ENDPOINT}" s3 cp "${OUT}/${f}" "s3://${BUCKET}/kata-assets/${f}"
+  aws_cli s3 cp - "s3://${BUCKET}/kata-assets/${f}" < "${OUT}/${f}"
 done
 
 echo ">> Done. Verify:"
-aws --endpoint-url "${ENDPOINT}" s3 ls "s3://${BUCKET}/kata-assets/"
+aws_cli s3 ls "s3://${BUCKET}/kata-assets/" < /dev/null