diff --git a/dev-tools/packaging/testing/package_test.go b/dev-tools/packaging/testing/package_test.go index 05101d7f35c..9a327ba3525 100644 --- a/dev-tools/packaging/testing/package_test.go +++ b/dev-tools/packaging/testing/package_test.go @@ -17,7 +17,6 @@ import ( "debug/buildinfo" "debug/elf" "encoding/hex" - "encoding/json" "errors" "flag" "fmt" @@ -25,6 +24,7 @@ import ( "io" "maps" "os" + "path" "path/filepath" "regexp" "slices" @@ -34,6 +34,7 @@ import ( "github.com/blakesmith/ar" "github.com/cavaliergopher/rpm" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1082,60 +1083,78 @@ func openZip(zipFile string) (*zip.ReadCloser, error) { func readDocker(t *testing.T, dockerFile string, filterWorkingDir bool) (*packageFile, *dockerInfo, error) { t.Helper() - // Read the manifest file first so that the config file and layer - // names are known in advance. - manifest, err := getDockerManifest(dockerFile) + // Decompress the .tar.gz file, go-containerregistry wants uncompressed here + f, err := os.Open(dockerFile) require.NoError(t, err) - file, err := os.Open(dockerFile) + gz, err := gzip.NewReader(f) require.NoError(t, err) - defer file.Close() - var info *dockerInfo + _, dockerFileName := path.Split(dockerFile) - stat, err := file.Stat() + tempDir := t.TempDir() + uncompressed, err := os.CreateTemp(tempDir, dockerFileName) require.NoError(t, err) - layers := make(map[string]*packageFile) - - gzipReader, err := gzip.NewReader(file) + _, err = io.Copy(uncompressed, gz) require.NoError(t, err) - defer gzipReader.Close() + require.NoError(t, uncompressed.Close()) - tarReader := tar.NewReader(gzipReader) - for { - header, err := tarReader.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - require.NoError(t, err) - } + // Load the Docker image tarball using go-containerregistry + img, err := tarball.ImageFromPath(uncompressed.Name(), nil) + require.NoError(t, err, "failed to load Docker image from %s", dockerFile) - switch { - case header.Name == manifest.Config: - info, err = readDockerInfo(tarReader) - require.NoError(t, err) - case slices.Contains(manifest.Layers, header.Name): - layer, err := readTarContents(header.Name, tarReader) - require.NoError(t, err) - layers[header.Name] = layer - } + // Get the config file which contains entrypoint, labels, user, workingDir + configFile, err := img.ConfigFile() + require.NoError(t, err, "failed to get config file from image") + + imgSize, err := img.Size() + require.NoError(t, err, "failed to get image size") + info := &dockerInfo{ + Size: imgSize, } + info.Config.Entrypoint = configFile.Config.Entrypoint + info.Config.Labels = configFile.Config.Labels + info.Config.User = configFile.Config.User + info.Config.WorkingDir = configFile.Config.WorkingDir require.NotZero(t, len(info.Config.Entrypoint), "no entrypoint") workingDir := info.Config.WorkingDir entrypoint := info.Config.Entrypoint[0] + // Get all layers and read their contents + layers, err := img.Layers() + require.NoError(t, err, "failed to get layers from image") + // Read layers in order and for each file keep only the entry seen in the later layer p := &packageFile{Name: filepath.Base(dockerFile), Contents: map[string]packageEntry{}} - for _, layer := range manifest.Layers { - layerFile, found := layers[layer] - require.True(t, found, fmt.Sprintf("layer not found: %s", layer)) - for name, entry := range layerFile.Contents { + for _, layer := range layers { + rc, err := layer.Uncompressed() + require.NoError(t, err, "failed to get uncompressed layer") + + tarReader := tar.NewReader(rc) + for { + header, err := tarReader.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + assert.NoError(t, rc.Close()) + require.NoError(t, err) + } + + name := header.Name if excludedPathsPattern.MatchString(name) { continue } + + entry := packageEntry{ + File: name, + UID: header.Uid, + GID: header.Gid, + Mode: header.FileInfo().Mode(), + } + // Check only files in working dir and entrypoint if !filterWorkingDir || strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { p.Contents[name] = entry @@ -1147,19 +1166,13 @@ func readDocker(t *testing.T, dockerFile string, filterWorkingDir bool) (*packag } } } + assert.NoError(t, rc.Close()) } require.NotZero(t, len(p.Contents), fmt.Sprintf("no files found in docker working directory (%s)", info.Config.WorkingDir)) - info.Size = stat.Size() return p, info, nil } -type dockerManifest struct { - Config string - RepoTags []string - Layers []string -} - type dockerInfo struct { Config struct { Entrypoint []string @@ -1170,78 +1183,6 @@ type dockerInfo struct { Size int64 } -func readDockerInfo(r io.Reader) (*dockerInfo, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - var info dockerInfo - err = json.Unmarshal(data, &info) - if err != nil { - return nil, err - } - - return &info, nil -} - -// getDockerManifest opens a gzipped tar file to read the Docker manifest.json -// that it is expected to contain. -func getDockerManifest(file string) (*dockerManifest, error) { - f, err := os.Open(file) - if err != nil { - return nil, err - } - defer f.Close() - - gzipReader, err := gzip.NewReader(f) - if err != nil { - return nil, err - } - defer gzipReader.Close() - - var manifest *dockerManifest - tarReader := tar.NewReader(gzipReader) - for { - header, err := tarReader.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return nil, err - } - - if header.Name == "manifest.json" { - manifest, err = readDockerManifest(tarReader) - if err != nil { - return nil, err - } - break - } - } - - return manifest, nil -} - -func readDockerManifest(r io.Reader) (*dockerManifest, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - var manifests []*dockerManifest - err = json.Unmarshal(data, &manifests) - if err != nil { - return nil, err - } - - if len(manifests) != 1 { - return nil, fmt.Errorf("one and only one manifest expected, %d found", len(manifests)) - } - - return manifests[0], nil -} - func checkSha512PackageHash(t *testing.T, packageFile string) { t.Run("check hash file", func(t *testing.T) { expectedHashFile := packageFile + ".sha512" diff --git a/go.mod b/go.mod index d8769e30bad..c672a5b558a 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/gofrs/flock v0.12.1 github.com/gofrs/uuid/v5 v5.4.0 github.com/google/go-cmp v0.7.0 + github.com/google/go-containerregistry v0.20.3 github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8 github.com/gorilla/mux v1.8.1 github.com/jaypipes/ghw v0.12.0 @@ -261,6 +262,7 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/cyphar/filepath-securejoin v0.6.0 // indirect @@ -634,6 +636,7 @@ require ( github.com/ugorji/go/codec v1.2.7 // indirect github.com/urfave/cli/v2 v2.27.4 // indirect github.com/valyala/fastjson v1.6.4 // indirect + github.com/vbatts/tar-split v0.11.6 // indirect github.com/vmware/govmomi v0.52.0 // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index 52bcbe260ad..7aa74016a0b 100644 --- a/go.sum +++ b/go.sum @@ -396,6 +396,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= @@ -772,6 +774,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= +github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA= @@ -1581,6 +1585,8 @@ github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8= github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs= +github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY= github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw=