diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index 00141defa..29ae84cdf 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -145,7 +145,7 @@ fi # --- 2. stage assets to rustfs (kind) / GCS (GKE) -------------------------- # Upload the five assets under kata-assets/, where atelet fetches them: the -# in-cluster rustfs (port-forwarded, S3 API) on kind, or the GCS bucket on GKE. +# in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh index 48875b656..e0d3af4a1 100755 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ b/hack/microvm-assets/stage-to-rustfs.sh @@ -15,12 +15,22 @@ # limitations under the License. # Stage the assembled micro-VM asset set into the kind cluster's rustfs S3 bucket -# under kata-assets/, where atelet fetches it (per demos/counter/counter-microvm.yaml.tmpl). +# under kata-assets/, where atelet fetches it (per manifests/microvm/sandboxconfig-microvm.yaml.tmpl). # Run after the cluster is up (hack/install-ate-kind.sh) and assemble.sh has produced $OUT. # -# Requires the `aws` CLI. Env: OUT (asset dir, default ./bin/microvm-assets/arm64), -# BUCKET (default ate-snapshots), NAMESPACE (rustfs namespace, default ate-system), -# KUBECTL_CONTEXT (optional; kube context for port-forward). +# The S3 client runs in a throwaway container (the same pinned amazon/aws-cli +# image as the rustfs-bucket-init Job), so developers don't need the `aws` CLI +# installed. The container joins the kind node's network namespace, which is what +# makes rustfs's ClusterIP routable: a container merely attached to the `kind` +# docker network can reach the node's own IP but has no route to the service or +# pod CIDRs. That also avoids a `kubectl port-forward`, which silently uploads +# nothing when it targets the wrong cluster, and which a container can't reach at +# all on macOS (the container's localhost is the Docker VM, not the host). +# +# Env: OUT (asset dir, default ./bin/microvm-assets/arm64), BUCKET (default +# ate-snapshots), NAMESPACE (rustfs namespace, default ate-system), +# KUBECTL_CONTEXT (optional; kube context), KIND_CLUSTER_NAME (default: derived +# from KUBECTL_CONTEXT, else "kind"). set -o errexit -o nounset -o pipefail @@ -30,28 +40,60 @@ OUT="${OUT:-${ROOT}/bin/microvm-assets/arm64}" BUCKET="${BUCKET:-ate-snapshots}" NAMESPACE="${NAMESPACE:-ate-system}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" +# kind contexts are named kind-; fall back to kind's own default. +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-${KUBECTL_CONTEXT#kind-}}" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" + +# Keep in sync with the rustfs-bucket-init Job in +# manifests/ate-install/kind/rustfs.yaml, which creates the bucket we upload into. +AWS_CLI_IMAGE="amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73" + +ASSETS=(cloud-hypervisor virtiofsd vmlinux rootfs.img configuration-clh.toml) -if ! command -v aws >/dev/null 2>&1; then - echo "error: the 'aws' CLI is required but was not found in PATH" >&2 +run_kubectl() { + kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} -n "${NAMESPACE}" "$@" +} + +if ! command -v docker >/dev/null 2>&1; then + echo "error: 'docker' is required to run the S3 client but was not found in PATH" >&2 exit 1 fi -export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}" -export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}" -export AWS_REGION="${AWS_REGION:-us-east-1}" +for f in "${ASSETS[@]}"; do + if [[ ! -f "${OUT}/${f}" ]]; then + echo "error: missing asset ${OUT}/${f}; run hack/microvm-assets/assemble.sh first" >&2 + exit 1 + fi +done + +# rustfs must be serving and the bucket must exist before anything is uploaded. +echo ">> Waiting for rustfs in namespace ${NAMESPACE}..." +run_kubectl rollout status deploy/rustfs --timeout=300s +run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s + +NODE="$("${ROOT}/hack/kind.sh" get nodes --name "${KIND_CLUSTER_NAME}" | head -n1)" +if [[ -z "${NODE}" ]]; then + echo "error: no nodes found for kind cluster '${KIND_CLUSTER_NAME}'" >&2 + exit 1 +fi +ENDPOINT="http://$(run_kubectl get svc rustfs -o jsonpath='{.spec.clusterIP}'):9000" -echo ">> Port-forwarding svc/rustfs 9000 in namespace ${NAMESPACE}..." -kubectl ${KUBECTL_CONTEXT:+--context="${KUBECTL_CONTEXT}"} -n "${NAMESPACE}" port-forward svc/rustfs 9000:9000 >/tmp/rustfs-pf.log 2>&1 & -PF_PID=$! -trap 'kill "$PF_PID" 2>/dev/null || true' EXIT -sleep 3 +echo ">> Uploading assets to s3://${BUCKET}/kata-assets/ via ${ENDPOINT} (netns of ${NODE})..." +aws_cli() { + # -i (no -t): a TTY would corrupt the binary stream piped in on stdin. + docker run --rm -i \ + --network "container:${NODE}" \ + -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}" \ + -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}" \ + -e AWS_REGION="${AWS_REGION:-us-east-1}" \ + -e AWS_ENDPOINT_URL="${ENDPOINT}" \ + "${AWS_CLI_IMAGE}" "$@" +} -ENDPOINT="http://localhost:9000" -echo ">> Uploading assets to s3://${BUCKET}/kata-assets/ via ${ENDPOINT}..." -for f in cloud-hypervisor virtiofsd vmlinux rootfs.img configuration-clh.toml; do +for f in "${ASSETS[@]}"; do echo " $f" - aws --endpoint-url "${ENDPOINT}" s3 cp "${OUT}/${f}" "s3://${BUCKET}/kata-assets/${f}" + aws_cli s3 cp - "s3://${BUCKET}/kata-assets/${f}" < "${OUT}/${f}" done echo ">> Done. Verify:" -aws --endpoint-url "${ENDPOINT}" s3 ls "s3://${BUCKET}/kata-assets/" +aws_cli s3 ls "s3://${BUCKET}/kata-assets/" < /dev/null