diff --git a/cmd/build.go b/cmd/build.go index c1ce808f4..9b7b8fdd2 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -33,6 +33,7 @@ var ( buildPush bool buildPlatform string buildDryRun bool + buildBuilder string ) var buildCmd = &cobra.Command{ @@ -66,25 +67,30 @@ field is updated to reference the built image.`, tag := buildTag var settings *config.VersionedSettings - if buildBaseImage == "" || buildPush { + if buildBaseImage == "" || buildPush || buildBuilder == "cloud-build" { settings, _, err = config.LoadEffectiveSettings(projectPath) if err != nil { return fmt.Errorf("failed to load settings: %w", err) } } + imageRegistry := "" + if settings != nil { + imageRegistry = settings.ResolveImageRegistry(profile) + } + baseImage := buildBaseImage if baseImage == "" { - imageRegistry := "" - if settings != nil { - imageRegistry = settings.ResolveImageRegistry(profile) - } baseImage = "scion-base:" + tag if imageRegistry != "" { baseImage = imageRegistry + "/scion-base:" + tag } } + if buildBuilder == "cloud-build" { + return runCloudBuild(cmd, harnessConfigName, hcDir, tag, baseImage, imageRegistry, settings) + } + runtimeBin := runtime.DetectContainerRuntime() if runtimeBin == "" { return fmt.Errorf("no container runtime found (tried docker, podman)") @@ -101,10 +107,6 @@ field is updated to reference the built image.`, outputImage := imageBaseName + ":" + tag if buildPush { - imageRegistry := "" - if settings != nil { - imageRegistry = settings.ResolveImageRegistry(profile) - } if imageRegistry == "" { return fmt.Errorf("--push requires image_registry to be configured") } @@ -141,63 +143,208 @@ field is updated to reference the built image.`, } } - configPath := filepath.Join(hcDir.Path, "config.yaml") - configData, err := os.ReadFile(configPath) - if err != nil { - return fmt.Errorf("failed to read config.yaml for update: %w", err) - } - var doc yaml.Node - if err := yaml.Unmarshal(configData, &doc); err != nil { - return fmt.Errorf("failed to parse config.yaml: %w", err) - } - if len(doc.Content) > 0 && doc.Content[0].Kind == yaml.MappingNode { - mapping := doc.Content[0] - found := false - for i := 0; i < len(mapping.Content)-1; i += 2 { - if mapping.Content[i].Value == "image" { - mapping.Content[i+1].Value = outputImage - found = true - break - } - } - if !found { - mapping.Content = append(mapping.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "image"}, - &yaml.Node{Kind: yaml.ScalarNode, Value: outputImage}, - ) - } + updateBuildConfigAndSync(harnessConfigName, hcDir, outputImage) + + return nil + }, +} + +type cloudBuildConfig struct { + Steps []cloudBuildStep `yaml:"steps"` + Options cloudBuildOptions `yaml:"options"` + Timeout string `yaml:"timeout"` +} + +type cloudBuildStep struct { + Name string `yaml:"name"` + ID string `yaml:"id"` + Args []string `yaml:"args"` + Env []string `yaml:"env,omitempty"` +} + +type cloudBuildOptions struct { + DynamicSubstitutions bool `yaml:"dynamicSubstitutions"` + MachineType string `yaml:"machineType"` +} + +// runCloudBuild executes the build via gcloud builds submit. +func runCloudBuild(cmd *cobra.Command, harnessConfigName string, hcDir *config.HarnessConfigDir, tag, baseImage, imageRegistry string, settings *config.VersionedSettings) error { + if _, err := exec.LookPath("gcloud"); err != nil { + return fmt.Errorf("--builder cloud-build requires gcloud CLI to be installed and in PATH") + } + if imageRegistry == "" { + return fmt.Errorf("--builder cloud-build requires image_registry to be configured") + } + + gcpProject := resolveGCPProject(settings) + if gcpProject == "" { + return fmt.Errorf("no GCP project configured; set gcp_project_id in settings or run 'gcloud config set project '") + } + + imageBaseName := harnessConfigName + if hcDir.Config.Image != "" { + name := hcDir.Config.Image + if colonIdx := strings.LastIndex(name, ":"); colonIdx >= 0 { + name = name[:colonIdx] } - updatedData, err := yaml.Marshal(&doc) - if err != nil { - return fmt.Errorf("failed to marshal updated config.yaml: %w", err) + imageBaseName = name + } + outputImage := imageRegistry + "/" + imageBaseName + ":" + tag + + platform := buildPlatform + if platform == "" { + platform = "linux/amd64,linux/arm64" + } + + // Build the cloudbuild.yaml via structured marshaling. + cbConfig := cloudBuildConfig{ + Steps: []cloudBuildStep{ + { + Name: "gcr.io/cloud-builders/docker", + ID: "setup-buildx", + Args: []string{"buildx", "create", "--name", "builder", "--use"}, + Env: []string{"DOCKER_CLI_EXPERIMENTAL=enabled"}, + }, + { + Name: "gcr.io/cloud-builders/docker", + ID: "bootstrap-buildx", + Args: []string{"buildx", "inspect", "--bootstrap"}, + Env: []string{"DOCKER_CLI_EXPERIMENTAL=enabled"}, + }, + { + Name: "gcr.io/cloud-builders/docker", + ID: "build-image", + Args: []string{ + "buildx", "build", + "--platform", platform, + "--build-arg", "BASE_IMAGE=" + baseImage, + "-t", outputImage, + "-f", "Dockerfile", + "--push", ".", + }, + Env: []string{"DOCKER_CLI_EXPERIMENTAL=enabled"}, + }, + }, + Options: cloudBuildOptions{ + DynamicSubstitutions: true, + MachineType: "E2_HIGHCPU_8", + }, + Timeout: "1200s", + } + + cbYAML, err := yaml.Marshal(&cbConfig) + if err != nil { + return fmt.Errorf("failed to marshal cloudbuild config: %w", err) + } + + tmpFile, err := os.CreateTemp("", "scion-cloudbuild-*.yaml") + if err != nil { + return fmt.Errorf("failed to create temp cloudbuild config: %w", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.Write(cbYAML); err != nil { + tmpFile.Close() + return fmt.Errorf("failed to write cloudbuild config: %w", err) + } + tmpFile.Close() + + if buildDryRun { + fmt.Printf("gcloud builds submit --project %s --config %s %s\n", gcpProject, tmpFile.Name(), hcDir.Path) + fmt.Printf("\n# cloudbuild.yaml contents:\n%s", cbYAML) + return nil + } + + fmt.Printf("Submitting Cloud Build in project %s...\n", gcpProject) + fmt.Printf("Output image: %s\n", outputImage) + + gcloudCmd := exec.CommandContext(cmd.Context(), "gcloud", "builds", "submit", + "--project", gcpProject, + "--config", tmpFile.Name(), + hcDir.Path) + gcloudCmd.Stdout = os.Stdout + gcloudCmd.Stderr = os.Stderr + if err := gcloudCmd.Run(); err != nil { + return fmt.Errorf("Cloud Build failed: %w", err) + } + + updateBuildConfigAndSync(harnessConfigName, hcDir, outputImage) + + return nil +} + +// resolveGCPProject returns the GCP project from settings or gcloud config. +func resolveGCPProject(settings *config.VersionedSettings) string { + if settings != nil && settings.Server != nil && settings.Server.Secrets != nil && settings.Server.Secrets.GCPProjectID != "" { + return settings.Server.Secrets.GCPProjectID + } + out, err := exec.Command("gcloud", "config", "get-value", "project").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +// updateBuildConfigAndSync updates the harness config's config.yaml with the +// new image reference and syncs to Hub. +func updateBuildConfigAndSync(harnessConfigName string, hcDir *config.HarnessConfigDir, outputImage string) { + configPath := filepath.Join(hcDir.Path, "config.yaml") + configData, err := os.ReadFile(configPath) + if err != nil { + fmt.Printf("Warning: failed to read config.yaml for update: %v\n", err) + return + } + var doc yaml.Node + if err := yaml.Unmarshal(configData, &doc); err != nil { + fmt.Printf("Warning: failed to parse config.yaml: %v\n", err) + return + } + if len(doc.Content) > 0 && doc.Content[0].Kind == yaml.MappingNode { + mapping := doc.Content[0] + found := false + for i := 0; i < len(mapping.Content)-1; i += 2 { + if mapping.Content[i].Value == "image" { + mapping.Content[i+1].Value = outputImage + found = true + break + } } - if err := os.WriteFile(configPath, updatedData, 0644); err != nil { - return fmt.Errorf("failed to write updated config.yaml: %w", err) + if !found { + mapping.Content = append(mapping.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "image"}, + &yaml.Node{Kind: yaml.ScalarNode, Value: outputImage}, + ) } - fmt.Printf("Updated %s image to %s\n", configPath, outputImage) + } + updatedData, err := yaml.Marshal(&doc) + if err != nil { + fmt.Printf("Warning: failed to marshal updated config.yaml: %v\n", err) + return + } + if err := os.WriteFile(configPath, updatedData, 0644); err != nil { + fmt.Printf("Warning: failed to write updated config.yaml: %v\n", err) + return + } + fmt.Printf("Updated %s image to %s\n", configPath, outputImage) - // Sync updated config to Hub so agents pick up the new image. - var gp string - if projectPath != "" { - if resolved, err := config.GetResolvedProjectDir(projectPath); err == nil { - gp = resolved - } - } else if resolved, err := config.GetResolvedProjectDir(""); err == nil { + var gp string + if projectPath != "" { + if resolved, err := config.GetResolvedProjectDir(projectPath); err == nil { gp = resolved } - hubCtx, hubErr := CheckHubAvailabilityWithOptions(gp, true) - if hubErr != nil { - fmt.Printf("Warning: could not sync to Hub: %v\n", hubErr) + } else if resolved, err := config.GetResolvedProjectDir(""); err == nil { + gp = resolved + } + hubCtx, hubErr := CheckHubAvailabilityWithOptions(gp, true) + if hubErr != nil { + fmt.Printf("Warning: could not sync to Hub: %v\n", hubErr) + fmt.Println("Run 'scion harness-config push " + harnessConfigName + "' to sync manually.") + } else if hubCtx != nil { + if err := syncHarnessConfigToHub(hubCtx, harnessConfigName, hcDir.Path, "global", "", hcDir.Config.Harness); err != nil { + fmt.Printf("Warning: failed to sync to Hub: %v\n", err) fmt.Println("Run 'scion harness-config push " + harnessConfigName + "' to sync manually.") - } else if hubCtx != nil { - if err := syncHarnessConfigToHub(hubCtx, harnessConfigName, hcDir.Path, "global", "", hcDir.Config.Harness); err != nil { - fmt.Printf("Warning: failed to sync to Hub: %v\n", err) - fmt.Println("Run 'scion harness-config push " + harnessConfigName + "' to sync manually.") - } } - - return nil - }, + } } func init() { @@ -207,4 +354,5 @@ func init() { buildCmd.Flags().BoolVar(&buildPush, "push", false, "Push built image to image_registry after building") buildCmd.Flags().StringVar(&buildPlatform, "platform", "", "Target platform (default: current architecture)") buildCmd.Flags().BoolVar(&buildDryRun, "dry-run", false, "Show the docker build command without executing") + buildCmd.Flags().StringVar(&buildBuilder, "builder", "local", "Build backend: local or cloud-build") } diff --git a/go.mod b/go.mod index 4b6e5b719..77ba28deb 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module github.com/GoogleCloudPlatform/scion go 1.26.1 require ( + cloud.google.com/go/cloudbuild v1.25.0 cloud.google.com/go/compute/metadata v0.9.0 cloud.google.com/go/logging v1.13.2 + cloud.google.com/go/longrunning v1.0.0 cloud.google.com/go/monitoring v1.24.3 cloud.google.com/go/secretmanager v1.16.0 cloud.google.com/go/storage v1.59.1 @@ -46,16 +48,16 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 go.opentelemetry.io/proto/otlp v1.10.0 - golang.org/x/net v0.55.0 + golang.org/x/net v0.56.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sync v0.20.0 - golang.org/x/sys v0.45.0 - golang.org/x/term v0.43.0 - golang.org/x/text v0.37.0 - google.golang.org/api v0.275.0 + golang.org/x/sync v0.21.0 + golang.org/x/sys v0.46.0 + golang.org/x/term v0.44.0 + golang.org/x/text v0.38.0 + google.golang.org/api v0.285.0 google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 - google.golang.org/grpc v1.80.0 + google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 + google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.35.0 @@ -70,8 +72,7 @@ require ( cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/iam v1.5.3 // indirect - cloud.google.com/go/longrunning v0.8.0 // indirect + cloud.google.com/go/iam v1.11.0 // indirect cloud.google.com/go/trace v1.11.7 // indirect github.com/Azure/go-ntlmssp v0.1.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect @@ -87,14 +88,14 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect - github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect @@ -114,8 +115,8 @@ require ( github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect - github.com/googleapis/gax-go/v2 v2.21.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gopherjs/gopherjs v1.20.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect @@ -177,17 +178,17 @@ require ( github.com/zeebo/blake3 v0.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.52.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/mod v0.35.0 // indirect + golang.org/x/mod v0.36.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index d4c6aab27..0ace91a9e 100644 --- a/go.sum +++ b/go.sum @@ -8,14 +8,16 @@ cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/cloudbuild v1.25.0 h1:Fkg+iJdN7bfICZJzLr/XV+k9aVxXS/hakIlhjDIRIDw= +cloud.google.com/go/cloudbuild v1.25.0/go.mod h1:lCu+T6IPkobPo2Nw+vCE7wuaAl9HbXLzdPx/tcF+oWo= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= -cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= -cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY= +cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= cloud.google.com/go/secretmanager v1.16.0 h1:19QT7ZsLJ8FSP1k+4esQvuCD7npMJml6hYzilxVyT+k= @@ -174,8 +176,8 @@ github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0= github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -214,12 +216,12 @@ github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= -github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= -github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= @@ -308,10 +310,10 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= -github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= -github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= +github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gopherjs/gopherjs v1.20.1 h1:22uLWFvVcxhJ+j3dJ99NNfwGyHynxCmjhYsrcwqbY60= github.com/gopherjs/gopherjs v1.20.1/go.mod h1:h+FTmmLgbXMmmtuZFp9bUqXciN429Wx0sJEJuMnpyfM= github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= @@ -629,8 +631,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/otelslog v0.14.0 h1:eypSOd+0txRKCXPNyqLPsbSfA0jULgJcGmSAdFAnrCM= go.opentelemetry.io/contrib/bridges/otelslog v0.14.0/go.mod h1:CRGvIBL/aAxpQU34ZxyQVFlovVcp67s4cAmQu8Jh9mc= -go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= -go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= @@ -674,27 +676,27 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo= golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -709,36 +711,36 @@ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.275.0 h1:vfY5d9vFVJeWEZT65QDd9hbndr7FyZ2+6mIzGAh71NI= -google.golang.org/api v0.275.0/go.mod h1:Fnag/EWUPIcJXuIkP1pjoTgS5vdxlk3eeemL7Do6bvw= +google.golang.org/api v0.285.0 h1:B7eHHoKGAX/LrPkQvhQqnGwjgWxofbdGwCTQvpm8FkM= +google.golang.org/api v0.285.0/go.mod h1:NlOlUIr8MPoIhT9Bb/oUnRuHbJOLwxb6JSYJM8Yz+jQ= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 h1:g0RAkxK/smSu/iRwC/KIX1mwUoVJtk2OjbgaeS4DmUM= +google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad h1:45WmJvIV6C2+O/jjLkPUH+F3aOj/1miDoU2DD0+NWbg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/hub/admin_maintenance.go b/pkg/hub/admin_maintenance.go index 59cadc163..9ebc58eaa 100644 --- a/pkg/hub/admin_maintenance.go +++ b/pkg/hub/admin_maintenance.go @@ -302,6 +302,7 @@ func (s *Server) resolveMaintenanceExecutor(key string) (MaintenanceExecutor, er runtimeBin: mc.RuntimeBin, registry: mc.ImageRegistry, tag: mc.ImageTag, + gcpProject: s.config.GCPProjectID, }, nil default: return nil, fmt.Errorf("no executor registered for operation %q", key) diff --git a/pkg/hub/maintenance_executors.go b/pkg/hub/maintenance_executors.go index 8176721e3..5138dda07 100644 --- a/pkg/hub/maintenance_executors.go +++ b/pkg/hub/maintenance_executors.go @@ -15,7 +15,9 @@ package hub import ( + "archive/tar" "bytes" + "compress/gzip" "context" "encoding/json" "fmt" @@ -25,13 +27,20 @@ import ( "path/filepath" "runtime" "strings" + "time" + cloudbuild "cloud.google.com/go/cloudbuild/apiv1" + cloudbuildpb "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" + "cloud.google.com/go/longrunning/autogen/longrunningpb" + gcstorage "cloud.google.com/go/storage" scionruntime "github.com/GoogleCloudPlatform/scion/pkg/runtime" "github.com/GoogleCloudPlatform/scion/pkg/secret" "github.com/GoogleCloudPlatform/scion/pkg/storage" "github.com/GoogleCloudPlatform/scion/pkg/store" "github.com/GoogleCloudPlatform/scion/pkg/transfer" "github.com/GoogleCloudPlatform/scion/pkg/util/logging" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" "gopkg.in/yaml.v3" ) @@ -439,9 +448,21 @@ type BuildHarnessConfigImageExecutor struct { runtimeBin string registry string tag string + gcpProject string } func (e *BuildHarnessConfigImageExecutor) Run(ctx context.Context, logger io.Writer, params map[string]string) error { + if params["builder"] == "cloud-build" { + cloudExecutor := &CloudBuildHarnessConfigExecutor{ + store: e.store, + storage: e.storage, + gcpProject: e.gcpProject, + registry: e.registry, + tag: e.tag, + } + return cloudExecutor.Run(ctx, logger, params) + } + log := logging.Subsystem("hub.maintenance.build-harness-config-image") harnessConfigID := params["harness_config_id"] @@ -489,44 +510,8 @@ func (e *BuildHarnessConfigImageExecutor) Run(ctx context.Context, logger io.Wri } defer func() { _ = os.RemoveAll(tmpDir) }() - _, _ = fmt.Fprintf(logger, "Materializing %d file(s) from harness-config %q...\n", len(hc.Files), hc.Name) - for _, f := range hc.Files { - objectPath := hc.StoragePath + "/" + f.Path - reader, _, err := e.storage.Download(ctx, objectPath) - if err != nil { - return fmt.Errorf("failed to download %q from storage: %w", f.Path, err) - } - - destPath := filepath.Join(tmpDir, f.Path) - if !strings.HasPrefix(destPath, tmpDir+string(os.PathSeparator)) { - _ = reader.Close() - return fmt.Errorf("invalid file path %q: escapes build directory", f.Path) - } - if dir := filepath.Dir(destPath); dir != tmpDir { - if err := os.MkdirAll(dir, 0o755); err != nil { - _ = reader.Close() - return fmt.Errorf("failed to create directory for %q: %w", f.Path, err) - } - } - - outFile, err := os.Create(destPath) - if err != nil { - _ = reader.Close() - return fmt.Errorf("failed to create file %q: %w", f.Path, err) - } - _, err = io.Copy(outFile, reader) - _ = reader.Close() - _ = outFile.Close() - if err != nil { - return fmt.Errorf("failed to write file %q: %w", f.Path, err) - } - - if f.Mode != "" { - mode := os.FileMode(0o644) - if _, err := fmt.Sscanf(f.Mode, "%o", &mode); err == nil { - _ = os.Chmod(destPath, mode) - } - } + if err := materializeHarnessConfigFiles(ctx, e.storage, hc, tmpDir, logger); err != nil { + return err } baseImage := "scion-base:" + tag @@ -585,7 +570,7 @@ func (e *BuildHarnessConfigImageExecutor) Run(ctx context.Context, logger io.Wri // Update the harness config's image in storage and the DB so agents // pick up the newly-built image instead of the stale upstream reference. - if err := e.syncBuiltImage(ctx, logger, hc, tmpDir, outputImage); err != nil { + if err := syncBuiltImage(ctx, e.storage, e.store, logger, hc, tmpDir, outputImage); err != nil { log.Error("Failed to sync built image back to store", "error", err) _, _ = fmt.Fprintf(logger, "Warning: build succeeded but failed to update harness-config image: %v\n", err) } @@ -595,9 +580,306 @@ func (e *BuildHarnessConfigImageExecutor) Run(ctx context.Context, logger io.Wri return nil } +// materializeHarnessConfigFiles downloads all files from a harness config's +// storage path into the given directory. It is shared by both the local and +// Cloud Build executors. +func materializeHarnessConfigFiles(ctx context.Context, stor storage.Storage, hc *store.HarnessConfig, destDir string, logger io.Writer) error { + _, _ = fmt.Fprintf(logger, "Materializing %d file(s) from harness-config %q...\n", len(hc.Files), hc.Name) + for _, f := range hc.Files { + objectPath := hc.StoragePath + "/" + f.Path + reader, _, err := stor.Download(ctx, objectPath) + if err != nil { + return fmt.Errorf("failed to download %q from storage: %w", f.Path, err) + } + + destPath := filepath.Join(destDir, f.Path) + if !strings.HasPrefix(destPath, destDir+string(os.PathSeparator)) { + _ = reader.Close() + return fmt.Errorf("invalid file path %q: escapes build directory", f.Path) + } + if dir := filepath.Dir(destPath); dir != destDir { + if err := os.MkdirAll(dir, 0o755); err != nil { + _ = reader.Close() + return fmt.Errorf("failed to create directory for %q: %w", f.Path, err) + } + } + + outFile, err := os.Create(destPath) + if err != nil { + _ = reader.Close() + return fmt.Errorf("failed to create file %q: %w", f.Path, err) + } + _, err = io.Copy(outFile, reader) + _ = reader.Close() + _ = outFile.Close() + if err != nil { + return fmt.Errorf("failed to write file %q: %w", f.Path, err) + } + + if f.Mode != "" { + mode := os.FileMode(0o644) + if _, err := fmt.Sscanf(f.Mode, "%o", &mode); err == nil { + _ = os.Chmod(destPath, mode) + } + } + } + return nil +} + +// CloudBuildHarnessConfigExecutor builds a harness-config image using Google Cloud Build. +type CloudBuildHarnessConfigExecutor struct { + store store.Store + storage storage.Storage + gcpProject string + registry string + tag string +} + +func (e *CloudBuildHarnessConfigExecutor) Run(ctx context.Context, logger io.Writer, params map[string]string) error { + log := logging.Subsystem("hub.maintenance.cloud-build-harness-config-image") + + harnessConfigID := params["harness_config_id"] + if harnessConfigID == "" { + return fmt.Errorf("missing required parameter: harness_config_id") + } + + if e.gcpProject == "" { + return fmt.Errorf("Cloud Build requires a GCP project ID. Configure gcp_project_id in settings") + } + + tag := e.tag + if tag == "" { + tag = "latest" + } + if v := params["tag"]; v != "" { + tag = v + } + + registry := e.registry + if v := params["registry"]; v != "" { + registry = v + } + registry = strings.TrimSuffix(registry, "/") + if registry == "" { + return fmt.Errorf("Cloud Build requires a registry (images must be pushed). Configure image_registry first") + } + + hc, err := e.store.GetHarnessConfig(ctx, harnessConfigID) + if err != nil { + return fmt.Errorf("failed to load harness-config %q: %w", harnessConfigID, err) + } + + hasDockerfile := false + for _, f := range hc.Files { + if f.Path == "Dockerfile" { + hasDockerfile = true + break + } + } + if !hasDockerfile { + return fmt.Errorf("harness-config %q does not contain a Dockerfile", hc.Name) + } + + if e.storage == nil { + return fmt.Errorf("storage not configured") + } + + // Materialize harness-config files to a temp directory. + tmpDir, err := os.MkdirTemp("", "scion-cloudbuild-*") + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + if err := materializeHarnessConfigFiles(ctx, e.storage, hc, tmpDir, logger); err != nil { + return err + } + + // Create a tar.gz of the build context. + _, _ = fmt.Fprintln(logger, "Creating build context archive...") + var buf bytes.Buffer + if err := createTarGz(tmpDir, &buf); err != nil { + return fmt.Errorf("failed to create build context archive: %w", err) + } + + // Upload build context to the Cloud Build default staging bucket. + stagingBucket := e.gcpProject + "_cloudbuild" + objectName := fmt.Sprintf("source/scion-harness-%s-%d.tar.gz", hc.Name, time.Now().Unix()) + _, _ = fmt.Fprintf(logger, "Uploading build context to gs://%s/%s...\n", stagingBucket, objectName) + + gcsClient, err := gcstorage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create GCS client: %w", err) + } + defer gcsClient.Close() + + // Verify the staging bucket exists before attempting upload. + if _, err := gcsClient.Bucket(stagingBucket).Attrs(ctx); err != nil { + if err == gcstorage.ErrBucketNotExist { + return fmt.Errorf("Cloud Build staging bucket gs://%s does not exist. "+ + "Run 'gcloud builds submit' once manually in project %s to create it, "+ + "or create the bucket manually", stagingBucket, e.gcpProject) + } + return fmt.Errorf("failed to access staging bucket gs://%s: %w", stagingBucket, err) + } + + wc := gcsClient.Bucket(stagingBucket).Object(objectName).NewWriter(ctx) + wc.ContentType = "application/gzip" + if _, err := io.Copy(wc, &buf); err != nil { + _ = wc.Close() + return fmt.Errorf("failed to upload build context: %w", err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("failed to finalize build context upload: %w", err) + } + _, _ = fmt.Fprintln(logger, "Build context uploaded.") + + // Clean up the source archive after the build completes (success or failure). + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := gcsClient.Bucket(stagingBucket).Object(objectName).Delete(cleanupCtx); err != nil { + log.Warn("Failed to clean up Cloud Build source archive", "bucket", stagingBucket, "object", objectName, "error", err) + } + }() + + // Build the output image reference. + imageName := hc.Slug + if imageName == "" { + imageName = hc.Name + } + baseImage := registry + "/scion-base:" + tag + outputImage := registry + "/" + imageName + ":" + tag + + _, _ = fmt.Fprintf(logger, "Base image: %s\n", baseImage) + _, _ = fmt.Fprintf(logger, "Output image: %s\n", outputImage) + _, _ = fmt.Fprintf(logger, "Submitting Cloud Build job in project %q...\n", e.gcpProject) + log.Debug("Submitting Cloud Build", + "image", outputImage, "base_image", baseImage, + "project", e.gcpProject, "harness_config", hc.Name) + + // Submit Cloud Build. + cbClient, err := cloudbuild.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create Cloud Build client: %w", err) + } + defer cbClient.Close() + + build := &cloudbuildpb.Build{ + Source: &cloudbuildpb.Source{ + Source: &cloudbuildpb.Source_StorageSource{ + StorageSource: &cloudbuildpb.StorageSource{ + Bucket: stagingBucket, + Object: objectName, + }, + }, + }, + Steps: []*cloudbuildpb.BuildStep{ + { + Name: "gcr.io/cloud-builders/docker", + Args: []string{"buildx", "create", "--name", "builder", "--use"}, + Id: "setup-buildx", + Env: []string{"DOCKER_CLI_EXPERIMENTAL=enabled"}, + }, + { + Name: "gcr.io/cloud-builders/docker", + Args: []string{"buildx", "inspect", "--bootstrap"}, + Id: "bootstrap-buildx", + Env: []string{"DOCKER_CLI_EXPERIMENTAL=enabled"}, + }, + { + Name: "gcr.io/cloud-builders/docker", + Args: []string{ + "buildx", "build", + "--platform", "linux/amd64,linux/arm64", + "--build-arg", "BASE_IMAGE=" + baseImage, + "-t", outputImage, + "-f", "Dockerfile", + "--push", + ".", + }, + Id: "build-image", + Env: []string{"DOCKER_CLI_EXPERIMENTAL=enabled"}, + }, + }, + Options: &cloudbuildpb.BuildOptions{ + MachineType: cloudbuildpb.BuildOptions_E2_HIGHCPU_8, + DynamicSubstitutions: true, + }, + Timeout: durationpb.New(20 * time.Minute), + } + + op, err := cbClient.CreateBuild(ctx, &cloudbuildpb.CreateBuildRequest{ + ProjectId: e.gcpProject, + Build: build, + }) + if err != nil { + return fmt.Errorf("failed to submit Cloud Build: %w", err) + } + + buildID, err := extractBuildID(op) + if err != nil { + return fmt.Errorf("failed to get build ID from operation: %w", err) + } + _, _ = fmt.Fprintf(logger, "Cloud Build started: %s\n", buildID) + log.Info("Cloud Build started", "build_id", buildID, "project", e.gcpProject) + + // Poll for build status and stream logs. + logBucket := e.gcpProject + "_cloudbuild" + logObjectPrefix := fmt.Sprintf("log-%s", buildID) + var lastLogOffset int64 + + for { + b, err := cbClient.GetBuild(ctx, &cloudbuildpb.GetBuildRequest{ + ProjectId: e.gcpProject, + Id: buildID, + }) + if err != nil { + return fmt.Errorf("failed to poll build status: %w", err) + } + + // Stream new log content from GCS. + lastLogOffset = streamCloudBuildLogs(ctx, gcsClient, logBucket, logObjectPrefix+".txt", lastLogOffset, logger) + + switch b.Status { + case cloudbuildpb.Build_SUCCESS: + // Final log flush. + streamCloudBuildLogs(ctx, gcsClient, logBucket, logObjectPrefix+".txt", lastLogOffset, logger) + _, _ = fmt.Fprintf(logger, "\nCloud Build completed successfully: %s\n", outputImage) + log.Info("Cloud Build complete", "build_id", buildID, "image", outputImage) + + if err := syncBuiltImage(ctx, e.storage, e.store, logger, hc, tmpDir, outputImage); err != nil { + log.Error("Failed to sync built image back to store", "error", err) + _, _ = fmt.Fprintf(logger, "Warning: build succeeded but failed to update harness-config image: %v\n", err) + } + return nil + + case cloudbuildpb.Build_FAILURE: + streamCloudBuildLogs(ctx, gcsClient, logBucket, logObjectPrefix+".txt", lastLogOffset, logger) + return fmt.Errorf("Cloud Build failed: %s", b.StatusDetail) + + case cloudbuildpb.Build_TIMEOUT: + return fmt.Errorf("Cloud Build timed out") + + case cloudbuildpb.Build_CANCELLED: + return fmt.Errorf("Cloud Build was cancelled") + + case cloudbuildpb.Build_INTERNAL_ERROR: + return fmt.Errorf("Cloud Build internal error: %s", b.StatusDetail) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(5 * time.Second): + } + } +} + // syncBuiltImage updates the harness config's config.yaml in storage and the -// DB record to reference the newly-built image. -func (e *BuildHarnessConfigImageExecutor) syncBuiltImage(ctx context.Context, logger io.Writer, hc *store.HarnessConfig, tmpDir, outputImage string) error { +// DB record to reference the newly-built image. Shared by both local and Cloud +// Build executors. +func syncBuiltImage(ctx context.Context, stor storage.Storage, storeDB store.Store, logger io.Writer, hc *store.HarnessConfig, tmpDir, outputImage string) error { configPath := filepath.Join(tmpDir, "config.yaml") configData, err := os.ReadFile(configPath) if err != nil { @@ -634,16 +916,14 @@ func (e *BuildHarnessConfigImageExecutor) syncBuiltImage(ctx context.Context, lo return fmt.Errorf("failed to marshal updated config.yaml: %w", err) } - // Upload updated config.yaml to storage. - if e.storage != nil && hc.StoragePath != "" { + if stor != nil && hc.StoragePath != "" { objectPath := hc.StoragePath + "/config.yaml" - if _, err := e.storage.Upload(ctx, objectPath, bytes.NewReader(updatedData), storage.UploadOptions{}); err != nil { + if _, err := stor.Upload(ctx, objectPath, bytes.NewReader(updatedData), storage.UploadOptions{}); err != nil { return fmt.Errorf("failed to upload updated config.yaml to storage: %w", err) } _, _ = fmt.Fprintf(logger, "Updated config.yaml in storage with image %s\n", outputImage) } - // Update config.yaml entry in hc.Files manifest with new size and hash. configHash := transfer.HashBytes(updatedData) for i, f := range hc.Files { if f.Path == "config.yaml" { @@ -654,12 +934,11 @@ func (e *BuildHarnessConfigImageExecutor) syncBuiltImage(ctx context.Context, lo } hc.ContentHash = computeContentHash(hc.Files) - // Update the DB record. if hc.Config == nil { hc.Config = &store.HarnessConfigData{} } hc.Config.Image = outputImage - if err := e.store.UpdateHarnessConfig(ctx, hc); err != nil { + if err := storeDB.UpdateHarnessConfig(ctx, hc); err != nil { return fmt.Errorf("failed to update harness-config record: %w", err) } _, _ = fmt.Fprintf(logger, "Updated harness-config record image to %s\n", outputImage) @@ -667,6 +946,78 @@ func (e *BuildHarnessConfigImageExecutor) syncBuiltImage(ctx context.Context, lo return nil } +// createTarGz creates a tar.gz archive from the contents of srcDir. +func createTarGz(srcDir string, w io.Writer) error { + gzw := gzip.NewWriter(w) + defer gzw.Close() + tw := tar.NewWriter(gzw) + defer tw.Close() + + return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + if relPath == "." { + return nil + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = filepath.ToSlash(relPath) + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if info.IsDir() { + return nil + } + + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + return err + }) +} + +// extractBuildID extracts the Cloud Build ID from a CreateBuild long-running +// operation response. +func extractBuildID(op *longrunningpb.Operation) (string, error) { + if op.Metadata == nil { + return "", fmt.Errorf("operation has no metadata") + } + var meta cloudbuildpb.BuildOperationMetadata + if err := proto.Unmarshal(op.Metadata.Value, &meta); err != nil { + return "", fmt.Errorf("failed to unmarshal build operation metadata: %w", err) + } + if meta.Build == nil || meta.Build.Id == "" { + return "", fmt.Errorf("build metadata does not contain a build ID") + } + return meta.Build.Id, nil +} + +// streamCloudBuildLogs reads new log content from the Cloud Build log object +// in GCS and writes it to the logger. It returns the new offset. +func streamCloudBuildLogs(ctx context.Context, client *gcstorage.Client, bucket, object string, offset int64, logger io.Writer) int64 { + rc, err := client.Bucket(bucket).Object(object).NewRangeReader(ctx, offset, -1) + if err != nil { + return offset + } + defer rc.Close() + + n, _ := io.Copy(logger, rc) + return offset + n +} + // UpdateCheckResult contains the result of a check-for-updates operation. type UpdateCheckResult struct { UpdateAvailable bool `json:"update_available"` diff --git a/pkg/hub/system_handlers.go b/pkg/hub/system_handlers.go index 6a05e415d..0d5fd3f63 100644 --- a/pkg/hub/system_handlers.go +++ b/pkg/hub/system_handlers.go @@ -218,18 +218,20 @@ func (s *Server) handlePutRuntime(w http.ResponseWriter, r *http.Request) { // --- 2.3: Onboarding Status --- type OnboardingStatus struct { - Initialized bool `json:"initialized"` - IdentitySet bool `json:"identitySet"` - RuntimeOK bool `json:"runtimeOK"` - HarnessesSeeded bool `json:"harnessesSeeded"` - ImagesPresent bool `json:"imagesPresent"` - HasWorkspace bool `json:"hasWorkspace"` - Complete bool `json:"complete"` - EmbeddedBrokerID string `json:"embeddedBrokerID,omitempty"` - ImageRegistry string `json:"imageRegistry,omitempty"` - BuildAvailable bool `json:"buildAvailable"` - GitVersion string `json:"gitVersion,omitempty"` - GitVersionOK bool `json:"gitVersionOK"` + Initialized bool `json:"initialized"` + IdentitySet bool `json:"identitySet"` + RuntimeOK bool `json:"runtimeOK"` + HarnessesSeeded bool `json:"harnessesSeeded"` + ImagesPresent bool `json:"imagesPresent"` + HasWorkspace bool `json:"hasWorkspace"` + Complete bool `json:"complete"` + EmbeddedBrokerID string `json:"embeddedBrokerID,omitempty"` + ImageRegistry string `json:"imageRegistry,omitempty"` + BuildAvailable bool `json:"buildAvailable"` + CloudBuildAvailable bool `json:"cloudBuildAvailable"` + GCPProjectID string `json:"gcpProjectId,omitempty"` + GitVersion string `json:"gitVersion,omitempty"` + GitVersionOK bool `json:"gitVersionOK"` } func (s *Server) computeOnboardingStatus(ctx context.Context) OnboardingStatus { @@ -293,6 +295,12 @@ func (s *Server) computeOnboardingStatus(ctx context.Context) OnboardingStatus { // BuildAvailable: true only if the build script can be resolved status.BuildAvailable = resolveBuildScript() != "" + // CloudBuildAvailable: true when a GCP project is configured + status.CloudBuildAvailable = s.config.GCPProjectID != "" + if s.config.GCPProjectID != "" { + status.GCPProjectID = s.config.GCPProjectID + } + // GitVersion: report installed git version and whether it meets the worktree requirement if gitVersion, _, err := util.GetGitVersion(); err == nil { status.GitVersion = gitVersion