diff --git a/README.md b/README.md index c215baa..e1ffbaa 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ It also optionally collects each pull attempt's duration and result. This image pull secret should be usable for all images fetched by the given instance. If provided, it must be of type `kubernetes.io/dockerconfigjson` and exist in the same namespace. - `--collect-metrics`: if the image pull metrics should be collected. + - `--use-kubelet-image-credential-integration=MODE`: enables kubelet [credential provider](https://kubernetes.io/blog/2022/12/22/kubelet-credential-providers/) plugin integration. + Plugin credentials fetched dynamically and tried for the images configured in the `CredentialProviderConfig` before pull secrets. + Currently only supports mode `GKE`, which uses `/etc/srv/kubernetes/cri_auth_config.yaml` and `/home/kubernetes/bin` mounted from the host. Example: diff --git a/cmd/fetch.go b/cmd/fetch.go index 4ff806b..f4d904a 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -33,21 +33,23 @@ It talks to Container Runtime Interface API to pull images in parallel, with ret return err } imageList = append(imageList, args...) - return internal.Run(logger, criSocket, dockerConfigJSONPath, timing, metricsEndpoint, imageList...) + return internal.Run(logger, criSocket, dockerConfigJSONPath, imageCredentialProviderConfig, imageCredentialProviderBinDir, timing, metricsEndpoint, imageList...) }, } var ( - criSocket string - dockerConfigJSONPath string - imageListFile string - metricsEndpoint string - imageListTimeout = time.Minute - initialPullAttemptTimeout = 30 * time.Second - maxPullAttemptTimeout = 5 * time.Minute - overallTimeout = 20 * time.Minute - initialPullAttemptDelay = time.Second - maxPullAttemptDelay = 10 * time.Minute + criSocket string + dockerConfigJSONPath string + imageListFile string + metricsEndpoint string + imageCredentialProviderConfig string + imageCredentialProviderBinDir string + imageListTimeout = time.Minute + initialPullAttemptTimeout = 30 * time.Second + maxPullAttemptTimeout = 5 * time.Minute + overallTimeout = 20 * time.Minute + initialPullAttemptDelay = time.Second + maxPullAttemptDelay = 10 * time.Minute ) func init() { @@ -58,6 +60,8 @@ func init() { fetchCmd.Flags().StringVar(&dockerConfigJSONPath, "docker-config", "", "Path to docker config json file.") fetchCmd.Flags().StringVar(&imageListFile, "image-list-file", "", "Path to text file containing images to pull (one per line).") fetchCmd.Flags().StringVar(&metricsEndpoint, "metrics-endpoint", "", "A host:port to submit image pull metrics to.") + fetchCmd.Flags().StringVar(&imageCredentialProviderConfig, "image-credential-provider-config", "", "Path to credential provider plugin config file.") + fetchCmd.Flags().StringVar(&imageCredentialProviderBinDir, "image-credential-provider-bin-dir", "", "Path to credential provider plugin binary directory.") fetchCmd.Flags().DurationVar(&imageListTimeout, "image-list-timeout", imageListTimeout, "Timeout for image list calls (for debugging).") fetchCmd.Flags().DurationVar(&initialPullAttemptTimeout, "initial-pull-attempt-timeout", initialPullAttemptTimeout, "Timeout for initial image pull call. Each subsequent attempt doubles it until max.") diff --git a/deploy/deployment.yaml.gotpl b/deploy/deployment.yaml.gotpl index 8dd0235..e466b4d 100644 --- a/deploy/deployment.yaml.gotpl +++ b/deploy/deployment.yaml.gotpl @@ -163,6 +163,10 @@ spec: {{ if .CollectMetrics }} - "--metrics-endpoint={{ .Name }}-metrics:8443" {{ end }} + {{ if eq .UseKubeletImageCredentialIntegration "GKE" }} + - "--image-credential-provider-config=/tmp/credential-provider/cri_auth_config.yaml" + - "--image-credential-provider-bin-dir=/tmp/credential-provider-bin" + {{ end }} env: - name: NODE_NAME valueFrom: @@ -189,6 +193,14 @@ spec: name: pull-secret readOnly: true {{ end }} + {{ if eq .UseKubeletImageCredentialIntegration "GKE" }} + - mountPath: /tmp/credential-provider + name: credential-provider-config + readOnly: true + - mountPath: /tmp/credential-provider-bin + name: credential-provider-bin + readOnly: true + {{ end }} securityContext: readOnlyRootFilesystem: true {{ if .NeedsPrivileged }} @@ -225,3 +237,13 @@ spec: secret: secretName: {{ .Secret }} {{ end }} + {{ if eq .UseKubeletImageCredentialIntegration "GKE" }} + - name: credential-provider-config + hostPath: + path: /etc/srv/kubernetes + type: Directory + - name: credential-provider-bin + hostPath: + path: /home/kubernetes/bin + type: Directory + {{ end }} diff --git a/deploy/main.go b/deploy/main.go index 8fb4a95..0b36b48 100644 --- a/deploy/main.go +++ b/deploy/main.go @@ -12,14 +12,15 @@ import ( ) type settings struct { - Name string - Namespace string - Image string - Version string - Secret string - IsCRIO bool - NeedsPrivileged bool - CollectMetrics bool + Name string + Namespace string + Image string + Version string + Secret string + IsCRIO bool + NeedsPrivileged bool + CollectMetrics bool + UseKubeletImageCredentialIntegration string } const ( @@ -35,11 +36,12 @@ const imageRepo = "quay.io/stackrox-io/image-prefetcher" var deploymentTemplate string var ( - version string - namespace string - k8sFlavor k8sFlavorType - secret string - collectMetrics bool + version string + namespace string + k8sFlavor k8sFlavorType + secret string + collectMetrics bool + useKubeletImageCredentialIntegration string ) func init() { @@ -48,6 +50,7 @@ func init() { flag.TextVar(&k8sFlavor, "k8s-flavor", flavor(vanillaFlavor), fmt.Sprintf("Kubernetes flavor. Accepted values: %s", strings.Join(allFlavors, ","))) flag.StringVar(&secret, "secret", "", "Kubernetes image pull Secret to use when pulling.") flag.BoolVar(&collectMetrics, "collect-metrics", false, "Whether to collect and expose image pull metrics.") + flag.StringVar(&useKubeletImageCredentialIntegration, "use-kubelet-image-credential-integration", "", "Enable kubelet image credential provider plugin integration. Accepted values: GKE") } // processVersion processes the version string and returns the appropriate format. @@ -78,14 +81,15 @@ func main() { isOcp := k8sFlavor == ocpFlavor s := settings{ - Name: name, - Namespace: namespace, - Image: imageRepo, - Version: processVersion(version), - Secret: secret, - IsCRIO: isOcp, - NeedsPrivileged: isOcp, - CollectMetrics: collectMetrics, + Name: name, + Namespace: namespace, + Image: imageRepo, + Version: processVersion(version), + Secret: secret, + IsCRIO: isOcp, + NeedsPrivileged: isOcp, + CollectMetrics: collectMetrics, + UseKubeletImageCredentialIntegration: useKubeletImageCredentialIntegration, } tmpl := template.Must(template.New("deployment").Parse(deploymentTemplate)) if err := tmpl.Execute(os.Stdout, s); err != nil { diff --git a/go.mod b/go.mod index adeedf7..1ea9318 100644 --- a/go.mod +++ b/go.mod @@ -16,10 +16,15 @@ require ( k8s.io/client-go v0.35.3 k8s.io/cri-api v0.35.3 k8s.io/klog/v2 v2.140.0 + k8s.io/kubelet v0.35.3 + k8s.io/kubernetes v1.35.3 ) require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -27,6 +32,7 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -34,8 +40,14 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.49.0 // indirect @@ -48,6 +60,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-base v0.35.3 // indirect k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/go.sum b/go.sum index bfa437b..a76144d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -7,9 +11,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= @@ -45,6 +48,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -52,6 +57,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -68,9 +75,16 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -104,6 +118,8 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -150,12 +166,18 @@ k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg= k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c= +k8s.io/component-base v0.35.3 h1:mbKbzoIMy7JDWS/wqZobYW1JDVRn/RKRaoMQHP9c4P0= +k8s.io/component-base v0.35.3/go.mod h1:IZ8LEG30kPN4Et5NeC7vjNv5aU73ku5MS15iZyvyMYk= k8s.io/cri-api v0.35.3 h1:gONTLBvK1eBPyveXEQ39mtTqi2oANeHj1mCo1YhQosI= k8s.io/cri-api v0.35.3/go.mod h1:Cnt29u/tYl1Se1cBRL30uSZ/oJ5TaIp4sZm1xDLvcMc= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/kubelet v0.35.3 h1:Y6b9+U/aTBmou9JZ6qv18O4dpFbJOfl7cBe+ZksT7RY= +k8s.io/kubelet v0.35.3/go.mod h1:aWoMogtyUEf/mTl8VjqHbSkW5ZZkB8vTkrg9Fi6TKwE= +k8s.io/kubernetes v1.35.3 h1:J3dk2wybKFHwoH4eydDUGHJo4HAD+9CZbSlvk/YQuao= +k8s.io/kubernetes v1.35.3/go.mod h1:AaPpCpiS8oAqRbEwpY5r3RitLpwpVp5lVXKFkJril58= k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/internal/credentialprovider/doc.go b/internal/credentialprovider/doc.go index 754521b..420fb1d 100644 --- a/internal/credentialprovider/doc.go +++ b/internal/credentialprovider/doc.go @@ -8,4 +8,7 @@ // Therefore we have copy of the functionality necessary to use pull secrets the same way as kubernetes does. // The files we copied do not change often upstream, but ideally we should check for changes every kubernetes release // and update the permalink above to reflect the latest sync point. +// +// The file plugin.go contains a simplified plugin-based credential provider. +// See https://github.com/kubernetes/kubernetes/blob/master/pkg/credentialprovider/plugin/plugin.go package credentialprovider diff --git a/internal/credentialprovider/plugin.go b/internal/credentialprovider/plugin.go new file mode 100644 index 0000000..feec756 --- /dev/null +++ b/internal/credentialprovider/plugin.go @@ -0,0 +1,207 @@ +package credentialprovider + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" + "k8s.io/kubelet/pkg/apis/credentialprovider/install" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + kubeletconfigv1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1" +) + +const supportedAPIVersion = "kubelet.k8s.io/v1" + +var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme, serializer.EnableStrict) +) + +func init() { + install.Install(scheme) + utilruntime.Must(kubeletconfig.AddToScheme(scheme)) + utilruntime.Must(kubeletconfigv1.AddToScheme(scheme)) +} + +// PluginKeyring wraps the credential provider plugin functionality. +type PluginKeyring struct { + providers []pluginProviderWrapper + logger *slog.Logger +} + +type pluginProviderWrapper struct { + name string + binPath string + apiVersion string + matchImages []string + args []string +} + +// NewPluginKeyring creates a new keyring that uses credential provider plugins. +func NewPluginKeyring(logger *slog.Logger, configPath, binDir string) (*PluginKeyring, error) { + if configPath == "" || binDir == "" { + return nil, nil + } + + config, err := readCredentialProviderConfig(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read credential provider config: %w", err) + } + + kr := &PluginKeyring{ + providers: make([]pluginProviderWrapper, 0, len(config.Providers)), + logger: logger, + } + + for _, provider := range config.Providers { + // Find the plugin binary + pluginBin, err := exec.LookPath(filepath.Join(binDir, provider.Name)) + if err != nil { + return nil, fmt.Errorf("plugin binary %s not found in %s: %w", provider.Name, binDir, err) + } + + kr.providers = append(kr.providers, pluginProviderWrapper{ + name: provider.Name, + binPath: pluginBin, + apiVersion: provider.APIVersion, + matchImages: provider.MatchImages, + args: provider.Args, + }) + } + + logger.Info("initialized credential provider plugins", "count", len(kr.providers)) + return kr, nil +} + +// readCredentialProviderConfig reads and decodes the credential provider config file. +// Supports both YAML and JSON formats. +func readCredentialProviderConfig(configPath string) (*kubeletconfig.CredentialProviderConfig, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + obj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, fmt.Errorf("failed to decode config: %w", err) + } + + if gvk.Kind != "CredentialProviderConfig" { + return nil, fmt.Errorf("unexpected kind %q, expected CredentialProviderConfig", gvk.Kind) + } + + if gvk.Group != kubeletconfig.GroupName { + return nil, fmt.Errorf("unexpected group %q, expected %q", gvk.Group, kubeletconfig.GroupName) + } + + config, ok := obj.(*kubeletconfig.CredentialProviderConfig) + if !ok { + return nil, fmt.Errorf("unable to convert %T to *CredentialProviderConfig", obj) + } + + return config, nil +} + +// Lookup is like LookupWithCtx(context.Background(), ...). +func (kr *PluginKeyring) Lookup(image string) ([]AuthConfig, bool) { + return kr.LookupWithCtx(context.Background(), image) +} + +// LookupWithCtx returns credentials for the given image from matching plugins. +func (kr *PluginKeyring) LookupWithCtx(ctx context.Context, image string) ([]AuthConfig, bool) { + if kr == nil { + return nil, false + } + + var allCreds []AuthConfig + for _, provider := range kr.providers { + if !kr.matchesImage(provider.matchImages, image) { + continue + } + + kr.logger.Debug("executing credential provider plugin", "plugin", provider.name, "image", image) + creds, err := kr.execPlugin(ctx, provider, image) + if err != nil { + kr.logger.Warn("credential provider plugin failed", "plugin", provider.name, "image", image, "error", err) + continue + } + + allCreds = append(allCreds, creds...) + } + + if len(allCreds) > 0 { + return allCreds, true + } + + return nil, false +} + +// matchesImage checks if any of the match patterns match the given image. +func (kr *PluginKeyring) matchesImage(patterns []string, image string) bool { + for _, pattern := range patterns { + // Use the same matching logic as kubernetes + if matched, _ := URLsMatchStr(pattern, image); matched { + return true + } + } + return false +} + +// execPlugin executes the credential provider plugin and parses the response. +func (kr *PluginKeyring) execPlugin(ctx context.Context, provider pluginProviderWrapper, image string) ([]AuthConfig, error) { + // Prepare the request + request := credentialproviderv1.CredentialProviderRequest{ + Image: image, + } + requestJSON, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + // Execute the plugin, use the same timeout as kubelet does. + timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + cmd := exec.CommandContext(timeoutCtx, provider.binPath, provider.args...) + cmd.Stdin = bytes.NewReader(requestJSON) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("plugin execution failed: %w, stderr: %s", err, stderr.String()) + } + + // Parse the response + var response credentialproviderv1.CredentialProviderResponse + if err := json.Unmarshal(stdout.Bytes(), &response); err != nil { + return nil, fmt.Errorf("failed to parse plugin response: %w", err) + } + + if response.APIVersion != supportedAPIVersion { + return nil, fmt.Errorf("apiVersion from credential plugin response did not match expected apiVersion:%s, actual apiVersion:%s", supportedAPIVersion, response.APIVersion) + } + + // Convert to AuthConfig + var creds []AuthConfig + for registry, authConfig := range response.Auth { + creds = append(creds, AuthConfig{ + Username: authConfig.Username, + Password: authConfig.Password, + ServerAddress: registry, + }) + } + + kr.logger.Debug("received credentials from plugin", "plugin", provider.name, "count", len(creds)) + return creds, nil +} diff --git a/internal/main.go b/internal/main.go index 90c680e..c9e619d 100644 --- a/internal/main.go +++ b/internal/main.go @@ -29,7 +29,7 @@ type TimingConfig struct { MaxPullAttemptDelay time.Duration } -func Run(logger *slog.Logger, criSocketPath string, dockerConfigJSONPath string, timing TimingConfig, metricsEndpoint string, imageNames ...string) error { +func Run(logger *slog.Logger, criSocketPath string, dockerConfigJSONPath string, credentialProviderConfig string, credentialProviderBinDir string, timing TimingConfig, metricsEndpoint string, imageNames ...string) error { ctx, cancel := context.WithTimeout(context.Background(), timing.OverallTimeout) defer cancel() @@ -53,6 +53,12 @@ func Run(logger *slog.Logger, criSocketPath string, dockerConfigJSONPath string, go func() { _ = metricsSink.Run(ctx) }() // Returned error is for testing, sink already handles errors. } + // Initialize credential provider plugin keyring if configured + pluginKr, err := credentialprovider.NewPluginKeyring(logger, credentialProviderConfig, credentialProviderBinDir) + if err != nil { + return fmt.Errorf("failed to initialize credential provider plugins: %w", err) + } + kr := credentialprovider.BasicDockerKeyring{} if err := loadPullSecret(logger, &kr, dockerConfigJSONPath); err != nil { return fmt.Errorf("failed to load image pull secrets: %w", err) @@ -63,7 +69,7 @@ func Run(logger *slog.Logger, criSocketPath string, dockerConfigJSONPath string, var wg sync.WaitGroup for _, imageName := range imageNames { - auths := getAuthsForImage(ctx, logger, &kr, imageName) + auths := getAuthsForImage(ctx, logger, pluginKr, &kr, imageName) for i, auth := range auths { wg.Add(1) request := &criV1.PullImageRequest{ @@ -125,14 +131,27 @@ func loadPullSecret(logger *slog.Logger, kr *credentialprovider.BasicDockerKeyri return nil } -func getAuthsForImage(ctx context.Context, logger *slog.Logger, kr credentialprovider.DockerKeyring, imageName string) []*criV1.AuthConfig { - credsList, _ := kr.Lookup(imageName) +func getAuthsForImage(ctx context.Context, logger *slog.Logger, pluginKr *credentialprovider.PluginKeyring, kr credentialprovider.DockerKeyring, imageName string) []*criV1.AuthConfig { var auths []*criV1.AuthConfig - if len(credsList) == 0 { - logger.DebugContext(ctx, "no credentials present for image", "image", imageName) - // un-authenticated pull - auths = append(auths, nil) + + // First, try plugin credentials + if pluginKr != nil { + pluginCreds, found := pluginKr.LookupWithCtx(ctx, imageName) + if found { + logger.DebugContext(ctx, "got credentials from plugin", "image", imageName, "count", len(pluginCreds)) + for _, creds := range pluginCreds { + auth := &criV1.AuthConfig{ + Username: creds.Username, + Password: creds.Password, + ServerAddress: creds.ServerAddress, + } + auths = append(auths, auth) + } + } } + + // Then, add pull secret credentials + credsList, _ := kr.Lookup(imageName) for _, creds := range credsList { auth := &criV1.AuthConfig{ Username: creds.Username, @@ -144,6 +163,13 @@ func getAuthsForImage(ctx context.Context, logger *slog.Logger, kr credentialpro } auths = append(auths, auth) } + + // If no credentials found at all, try un-authenticated pull + if len(auths) == 0 { + logger.DebugContext(ctx, "no credentials present for image", "image", imageName) + auths = append(auths, nil) + } + return auths } @@ -207,6 +233,7 @@ func noteSuccess(sink chan<- *metricsProto.Result, name string, start time.Time, SizeBytes: sizeBytes, } } + func noteFailure(sink chan<- *metricsProto.Result, name string, start time.Time, elapsed time.Duration, err error) { if sink == nil { return