mount layers the destination registry already holds

README.md

@@ -159,6 +159,7 @@ expect - see [Known Issues](#known-issues).
       - [Flag `FF_KANIKO_EXPAND_HEREDOC`](#flag-ff_kaniko_expand_heredoc)
       - [Flag `FF_KANIKO_SKIP_CACHED_STAGES`](#flag-ff_kaniko_skip_cached_stages)
       - [Flag `FF_KANIKO_SHARED_BASE_CACHE`](#flag-ff_kaniko_shared_base_cache)
+      - [Flag `FF_KANIKO_CROSS_REPO_MOUNT`](#flag-ff_kaniko_cross_repo_mount)
     - [Assertion Overrides](#assertion-overrides)
     - [Telemetry](#telemetry)
     - [Debug Image](#debug-image)
@@ -1428,6 +1429,12 @@ Stored bases stay in `/kaniko/bases` after the build, `--cleanup` does not remov
 Defaults to `false`.
 Becomes default in `v1.29.0`.
 
+#### Flag `FF_KANIKO_CROSS_REPO_MOUNT`
+
+A registry can copy a blob between its own repositories for free, but only if it is told which repository already holds it. Kaniko loses that as soon as it copies a layer locally. Worse, with `--cache` every built layer goes up twice, once to the cache repo and once inside the image. Set this flag to `true` to remember which layers can be mounted remotely and which ones genuinely need to be pushed.
+Defaults to `false`.
+Becomes default in `v1.29.0`.
+
 ### Assertion Overrides
 
 Kaniko checks internal invariants at runtime. If one is violated the build stops with a message like:

golden/golden_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"errors"
 	"flag"
+	"io"
 	"os"
 	"path/filepath"
 	"slices"
@@ -28,9 +29,11 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	ggcrtypes "github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/osscontainertools/kaniko/cmd/executor/cmd"
 	testissuemz195 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz195"
 	testissuemz333 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz333"
@@ -43,11 +46,13 @@ import (
 	testissuemz813 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz813"
 	testissuemz822 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz822"
 	testissuemz936 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz936"
+	testissuemz989 "github.com/osscontainertools/kaniko/golden/testdata/test_issue_mz989"
 	testunittests "github.com/osscontainertools/kaniko/golden/testdata/test_unittests"
 	"github.com/osscontainertools/kaniko/golden/types"
 	"github.com/osscontainertools/kaniko/pkg/cache"
 	"github.com/osscontainertools/kaniko/pkg/config"
 	"github.com/osscontainertools/kaniko/pkg/executor"
+	"github.com/osscontainertools/kaniko/pkg/mounts"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -56,16 +61,58 @@ import (
 const cachePointerLabel = "kaniko.cache.pointer-target"
 
 type fakeLayerCache struct {
+	opts       *config.KanikoOptions
 	cachedKeys []string
 }
 
+// fakeLayer stands in for a cache entry's layer. Its content is the key that found it, so its
+// digest is derived rather than fixed by hand and stays stable across runs.
+type fakeLayer struct {
+	key string
+}
+
+func (l *fakeLayer) Digest() (v1.Hash, error) { return l.hash() }
+func (l *fakeLayer) DiffID() (v1.Hash, error) { return l.hash() }
+func (l *fakeLayer) Size() (int64, error)     { return int64(len(l.key)), nil }
+
+func (l *fakeLayer) MediaType() (ggcrtypes.MediaType, error) { return ggcrtypes.DockerLayer, nil }
+func (l *fakeLayer) Compressed() (io.ReadCloser, error)      { return l.reader(), nil }
+
+func (l *fakeLayer) Uncompressed() (io.ReadCloser, error) { return l.reader(), nil }
+
+func (l *fakeLayer) hash() (v1.Hash, error) {
+	return mounts.PlannedDigest(l.key), nil
+}
+
+func (l *fakeLayer) reader() io.ReadCloser {
+	return io.NopCloser(strings.NewReader(l.key))
+}
+
 func (f *fakeLayerCache) RetrieveLayer(key string) (v1.Image, error) {
 	if !slices.Contains(f.cachedKeys, key) {
 		return nil, errors.New("could not find layer")
 	}
 	cf := &v1.ConfigFile{}
 	cf.Config.Labels = map[string]string{cachePointerLabel: key}
-	return mutate.ConfigFile(empty.Image, cf)
+	img, err := mutate.ConfigFile(empty.Image, cf)
+	if err != nil {
+		return nil, err
+	}
+	img, err = mutate.AppendLayers(img, &fakeLayer{key: key})
+	if err != nil {
+		return nil, err
+	}
+	// The real registry cache records where it read from, and the plan reads that back.
+	if config.FF.CrossRepoMount {
+		dest, err := cache.Destination(f.opts, key)
+		if err == nil {
+			tag, err := name.NewTag(dest, name.WeakValidation)
+			if err == nil {
+				mounts.RecordImage(img, tag.Context())
+			}
+		}
+	}
+	return img, nil
 }
 
 func renderCommand(env map[string]string, args []string) string {
@@ -100,6 +147,7 @@ var allTests = map[string][]types.GoldenTests{
 	"test_issue_mz813": {testissuemz813.Tests},
 	"test_issue_mz822": {testissuemz822.Tests},
 	"test_issue_mz936": {testissuemz936.Tests},
+	"test_issue_mz989": {testissuemz989.Tests},
 	"test_unittests":   testunittests.Tests,
 }
 var update bool
@@ -132,8 +180,8 @@ func TestRun(t *testing.T) {
 
 							opts := config.KanikoOptions{}
 							origNewLayerCache := executor.NewLayerCache
-							executor.NewLayerCache = func(_ *config.KanikoOptions) cache.LayerCache {
-								return &fakeLayerCache{cachedKeys: test.CachedKeys}
+							executor.NewLayerCache = func(opts *config.KanikoOptions) cache.LayerCache {
+								return &fakeLayerCache{opts: opts, cachedKeys: test.CachedKeys}
 							}
 							t.Cleanup(func() { executor.NewLayerCache = origNewLayerCache })
 							exec := &cobra.Command{

golden/testdata/test_issue_mz195/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:12.10 AS first-stage
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS first-stage
 
 FROM first-stage AS second-stage
 
@@ -14,5 +14,5 @@ COPY --from=third-stage test test
 
 # When we optimize stages out this should not
 # impact squashing logic at all
-FROM debian:12.10 as noise
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 as noise
 FROM fourth-stage AS fifth-stage

golden/testdata/test_issue_mz195/plans/fourth

@@ -1,14 +1,14 @@
-FROM debian:12.10 AS first-stage
-STREAM debian:12.10
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS first-stage
+  STREAM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56
 SAVE STAGE /kaniko/stages/0
 CLEAN
 
 FROM first-stage AS third-stage
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 RUN touch test
 SAVE FILES [test] /kaniko/deps/2
 CLEAN
 
 FROM first-stage AS fourth-stage
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 COPY --from=third-stage test test

golden/testdata/test_issue_mz195/plans/noise

@@ -1,2 +1,2 @@
-FROM debian:12.10 AS noise
-STREAM debian:12.10
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS noise
+  STREAM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56

golden/testdata/test_issue_mz195/plans/normal

@@ -1,14 +1,14 @@
-FROM debian:12.10 AS first-stage
-STREAM debian:12.10
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS first-stage
+  STREAM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56
 SAVE STAGE /kaniko/stages/0
 CLEAN
 
 FROM first-stage AS third-stage
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 RUN touch test
 SAVE FILES [test] /kaniko/deps/2
 CLEAN
 
 FROM first-stage AS fifth-stage
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 COPY --from=third-stage test test

golden/testdata/test_issue_mz195/plans/push

@@ -1,15 +1,17 @@
-FROM debian:12.10 AS first-stage
-STREAM debian:12.10
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS first-stage
+  STREAM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56
 SAVE STAGE /kaniko/stages/0
 CLEAN
 
 FROM first-stage AS third-stage
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 RUN touch test
 SAVE FILES [test] /kaniko/deps/2
 CLEAN
 
 FROM first-stage AS fifth-stage
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 COPY --from=third-stage test test
 PUSH [registry]
+  UPLOAD sha256:cf05a52c02353f0b2b6f9be0549ac916c3fb1dc8d4bacd405eac7f28562ec9f2
+  UPLOAD COPY --from=third-stage test test

golden/testdata/test_issue_mz333/plans/plan

@@ -1,9 +1,9 @@
 FROM busybox AS base
-STREAM busybox
+  STREAM busybox
 RUN touch blubb
 SAVE FILES [blubb] /kaniko/deps/0
 CLEAN
 
 FROM scratch AS final
-STREAM scratch
+  STREAM scratch
 COPY --from=base blubb .

golden/testdata/test_issue_mz334/plans/cached

@@ -1,27 +1,27 @@
 FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d AS first
-STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
-CACHE HIT: 72e9e0e54e4522d381e54427f5ac6f24dd09910e1ff8d4bc7f60d02f54e2cdc3
+  STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
 RUN touch /blubb
+  CACHE HIT: 72e9e0e54e4522d381e54427f5ac6f24dd09910e1ff8d4bc7f60d02f54e2cdc3
 SAVE STAGE /kaniko/stages/0
 SAVE FILES [/blubb] /kaniko/deps/0
 CLEAN
 
 FROM first AS second
-UNPACK /kaniko/stages/0
-CACHE HIT: 3829b10dc17cc7bafd22450e05b7f265b73e94d46b08817d0502848b21dd69aa
+  UNPACK /kaniko/stages/0
 RUN touch /bla
+  CACHE HIT: 3829b10dc17cc7bafd22450e05b7f265b73e94d46b08817d0502848b21dd69aa
 SAVE STAGE /kaniko/stages/1
 CLEAN
 
 FROM second AS third
-UNPACK /kaniko/stages/1
-CACHE HIT: 256455fba386c671b4808e621379712ca6dfecce4d4e9ed2d6edab8b5e415b75
+  UNPACK /kaniko/stages/1
 RUN touch /bli
+  CACHE HIT: 256455fba386c671b4808e621379712ca6dfecce4d4e9ed2d6edab8b5e415b75
 SAVE FILES [/bli] /kaniko/deps/2
 CLEAN
 
 FROM second AS final
-UNPACK /kaniko/stages/1
+  UNPACK /kaniko/stages/1
 COPY --from=first /blubb /blubb
 COPY --from=third /bli /bli
 RUN ls -lah /blubb

golden/testdata/test_issue_mz334/plans/eliminated

@@ -1,14 +1,14 @@
 FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d AS final
-STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
-CACHE HIT: ec50b204a07f169d1beec66434b673ca44caf8b98ee6c98e886320969926d029
+  STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
 RUN touch /blubb
-CACHE HIT: 9097bbf817837a54a5ba9b91d9d2a771a145d25f5389c96a4c3315119aa482a5
+  CACHE HIT: ec50b204a07f169d1beec66434b673ca44caf8b98ee6c98e886320969926d029
 RUN touch /bla
-CACHE REDIRECT HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
-CACHE HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
+  CACHE HIT: 9097bbf817837a54a5ba9b91d9d2a771a145d25f5389c96a4c3315119aa482a5
 COPY --from=first /blubb /blubb
-CACHE REDIRECT HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
-CACHE HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
+  CACHE REDIRECT HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
+  CACHE HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
 COPY --from=third /bli /bli
-CACHE HIT: e3d0a39d0f55303c063b93635b40f56d535f5bd2b48b770af66bf0fe61b7debc
+  CACHE REDIRECT HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
+  CACHE HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
 RUN ls -lah /blubb
+  CACHE HIT: e3d0a39d0f55303c063b93635b40f56d535f5bd2b48b770af66bf0fe61b7debc

golden/testdata/test_issue_mz334/plans/inferred

@@ -1,32 +1,32 @@
 FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d AS first
-STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
-CACHE HIT: ec50b204a07f169d1beec66434b673ca44caf8b98ee6c98e886320969926d029
+  STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
 RUN touch /blubb
+  CACHE HIT: ec50b204a07f169d1beec66434b673ca44caf8b98ee6c98e886320969926d029
 SAVE STAGE /kaniko/stages/0
 SAVE FILES [/blubb] /kaniko/deps/0
 CLEAN
 
 FROM first AS second
-UNPACK /kaniko/stages/0
-CACHE HIT: 9097bbf817837a54a5ba9b91d9d2a771a145d25f5389c96a4c3315119aa482a5
+  UNPACK /kaniko/stages/0
 RUN touch /bla
+  CACHE HIT: 9097bbf817837a54a5ba9b91d9d2a771a145d25f5389c96a4c3315119aa482a5
 SAVE STAGE /kaniko/stages/1
 CLEAN
 
 FROM second AS third
-UNPACK /kaniko/stages/1
-CACHE HIT: dd8070993f952ce8287efcf5c6b32d3d2399905c6fc8dc9b4036d4196b80e10f
+  UNPACK /kaniko/stages/1
 RUN touch /bli
+  CACHE HIT: dd8070993f952ce8287efcf5c6b32d3d2399905c6fc8dc9b4036d4196b80e10f
 SAVE FILES [/bli] /kaniko/deps/2
 CLEAN
 
 FROM second AS final
-UNPACK /kaniko/stages/1
-CACHE REDIRECT HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
-CACHE HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
+  UNPACK /kaniko/stages/1
 COPY --from=first /blubb /blubb
-CACHE REDIRECT HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
-CACHE HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
+  CACHE REDIRECT HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
+  CACHE HIT: 8ef1283833b79b08b093a99de08ffc3fed1fbbf8fb328f81e9d7561af0524e95
 COPY --from=third /bli /bli
-CACHE HIT: e3d0a39d0f55303c063b93635b40f56d535f5bd2b48b770af66bf0fe61b7debc
+  CACHE REDIRECT HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
+  CACHE HIT: c59fad0bd865d2ed209d0f7a29d55161a8332681a1b68abc9e98da2dca1254cb
 RUN ls -lah /blubb
+  CACHE HIT: e3d0a39d0f55303c063b93635b40f56d535f5bd2b48b770af66bf0fe61b7debc

golden/testdata/test_issue_mz334/plans/plan

@@ -1,24 +1,24 @@
 FROM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d AS first
-STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
+  STREAM busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d
 RUN touch /blubb
 SAVE STAGE /kaniko/stages/0
 SAVE FILES [/blubb] /kaniko/deps/0
 CLEAN
 
 FROM first AS second
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 RUN touch /bla
 SAVE STAGE /kaniko/stages/1
 CLEAN
 
 FROM second AS third
-UNPACK /kaniko/stages/1
+  UNPACK /kaniko/stages/1
 RUN touch /bli
 SAVE FILES [/bli] /kaniko/deps/2
 CLEAN
 
 FROM second AS final
-UNPACK /kaniko/stages/1
+  UNPACK /kaniko/stages/1
 COPY --from=first /blubb /blubb
 COPY --from=third /bli /bli
 RUN ls -lah /blubb

golden/testdata/test_issue_mz338/plans/plan

@@ -1,5 +1,5 @@
 FROM alpine
-STREAM alpine
+  STREAM alpine
 RUN rm -rf /blubb
 RUN touch /blubb
 RUN ls -lah /blubb

golden/testdata/test_issue_mz480/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:12.10 AS base
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS base
 RUN install
 
 FROM base AS build

golden/testdata/test_issue_mz480/plans/final

@@ -1,16 +1,19 @@
-FROM debian:12.10 AS base
-STREAM debian:12.10
+FROM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56 AS base
+  STREAM debian@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56
 RUN install
 SAVE STAGE /kaniko/stages/0
 CLEAN
 
 FROM base AS build
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 RUN compile
 SAVE FILES [output] /kaniko/deps/1
 CLEAN
 
 FROM base AS final
-UNPACK /kaniko/stages/0
+  UNPACK /kaniko/stages/0
 COPY --from=build output output
 PUSH [registry]
+  UPLOAD sha256:cf05a52c02353f0b2b6f9be0549ac916c3fb1dc8d4bacd405eac7f28562ec9f2
+  UPLOAD RUN install
+  UPLOAD COPY --from=build output output