From ffad8ada79445462c855d7fd75dde592e72d8d5b Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Thu, 28 Mar 2019 17:26:34 +0800 Subject: [PATCH] exporter: support unpack opt for image exporter It is enhancement which allows to unpack image into containerd snapshotter storage by `--output type=image,<.>=<.>,unpack=true`. In order to support this feature, we needs to extend the Snapshotter witwh `Name() string` function. Because we needs to set gc label for snapshotter which need snapshotter name. fix: #908 Signed-off-by: Wei Fu --- cache/contenthash/checksum_test.go | 32 +++++----- cache/manager_test.go | 14 ++--- cmd/buildctl/build_test.go | 18 +++++- exporter/containerimage/export.go | 95 ++++++++++++++++++++++++++++++ exporter/containerimage/writer.go | 9 +++ snapshot/containerd/snapshotter.go | 4 +- snapshot/snapshotter.go | 10 +++- source/git/gitsource_test.go | 2 +- source/http/httpsource_test.go | 2 +- vendor/modules.txt | 2 +- worker/base/worker.go | 1 + worker/containerd/containerd.go | 2 +- worker/runc/runc.go | 2 +- 13 files changed, 159 insertions(+), 34 deletions(-) diff --git a/cache/contenthash/checksum_test.go b/cache/contenthash/checksum_test.go index dd5ec946..f07e2c64 100644 --- a/cache/contenthash/checksum_test.go +++ b/cache/contenthash/checksum_test.go @@ -37,7 +37,7 @@ func TestChecksumHardlinks(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -120,7 +120,7 @@ func TestChecksumWildcard(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -175,7 +175,7 @@ func TestSymlinksNoFollow(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -234,7 +234,7 @@ func TestChecksumBasicFile(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -384,7 +384,7 @@ func TestHandleChange(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -462,7 +462,7 @@ func TestHandleRecursiveDir(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -511,7 +511,7 @@ func TestChecksumUnorderedFiles(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -564,7 +564,7 @@ func TestSymlinkInPathScan(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -595,7 +595,7 @@ func TestSymlinkNeedsScan(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -628,7 +628,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -655,7 +655,7 @@ func TestSymlinkThroughParent(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -710,7 +710,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -773,7 +773,7 @@ func TestPersistence(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -810,7 +810,7 @@ func TestPersistence(t *testing.T) { // we can't close snapshotter and open it twice (especially, its internal bbolt store) cm.Close() getDefaultManager().lru.Purge() - cm = setupCacheManager(t, tmpdir, snapshotter) + cm = setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ref, err = cm.Get(context.TODO(), id) @@ -843,12 +843,12 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe return ref } -func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) cache.Manager { +func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) cache.Manager { md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) require.NoError(t, err) cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil), + Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil), MetadataStore: md, }) require.NoError(t, err) diff --git a/cache/manager_test.go b/cache/manager_test.go index bd90ad30..6b47cc40 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -28,7 +28,7 @@ func TestManager(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := getCacheManager(t, tmpdir, snapshotter) + cm := getCacheManager(t, tmpdir, "native", snapshotter) _, err = cm.Get(ctx, "foobar") require.Error(t, err) @@ -152,7 +152,7 @@ func TestPrune(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := getCacheManager(t, tmpdir, snapshotter) + cm := getCacheManager(t, tmpdir, "native", snapshotter) active, err := cm.New(ctx, nil) require.NoError(t, err) @@ -250,7 +250,7 @@ func TestLazyCommit(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := getCacheManager(t, tmpdir, snapshotter) + cm := getCacheManager(t, tmpdir, "native", snapshotter) active, err := cm.New(ctx, nil, CachePolicyRetain) require.NoError(t, err) @@ -332,7 +332,7 @@ func TestLazyCommit(t *testing.T) { require.NoError(t, err) // we can't close snapshotter and open it twice (especially, its internal bbolt store) - cm = getCacheManager(t, tmpdir, snapshotter) + cm = getCacheManager(t, tmpdir, "native", snapshotter) snap2, err = cm.Get(ctx, snap.ID()) require.NoError(t, err) @@ -353,7 +353,7 @@ func TestLazyCommit(t *testing.T) { err = cm.Close() require.NoError(t, err) - cm = getCacheManager(t, tmpdir, snapshotter) + cm = getCacheManager(t, tmpdir, "native", snapshotter) snap2, err = cm.Get(ctx, snap.ID()) require.NoError(t, err) @@ -369,12 +369,12 @@ func TestLazyCommit(t *testing.T) { require.Equal(t, errNotFound, errors.Cause(err)) } -func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) Manager { +func getCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) Manager { md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) require.NoError(t, err) cm, err := NewManager(ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil), + Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil), MetadataStore: md, }) require.NoError(t, err, fmt.Sprintf("error: %+v", err)) diff --git a/cmd/buildctl/build_test.go b/cmd/buildctl/build_test.go index f54312bd..86bbfbf0 100644 --- a/cmd/buildctl/build_test.go +++ b/cmd/buildctl/build_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "testing" "time" @@ -82,7 +83,15 @@ func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) { rdr, err := marshal(st.Root()) require.NoError(t, err) - cmd := sb.Cmd("build --progress=plain --exporter=image --exporter-opt name=example.com/moby/imageexporter:test") + imageName := "example.com/moby/imageexporter:test" + + buildCmd := []string{ + "build", "--progress=plain", + "--exporter=image", "--exporter-opt", "unpack=true", + "--exporter-opt", "name=" + imageName, + } + + cmd := sb.Cmd(strings.Join(buildCmd, " ")) cmd.Stdin = rdr err = cmd.Run() require.NoError(t, err) @@ -93,8 +102,13 @@ func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) { ctx := namespaces.WithNamespace(context.Background(), "buildkit") - _, err = client.ImageService().Get(ctx, "example.com/moby/imageexporter:test") + img, err := client.GetImage(ctx, imageName) require.NoError(t, err) + + // NOTE: by default, it is overlayfs + ok, err := img.IsUnpacked(ctx, "overlayfs") + require.NoError(t, err) + require.Equal(t, ok, true) } func marshal(st llb.State) (io.Reader, error) { diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 6e8f9525..e2dd2eee 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -2,16 +2,24 @@ package containerimage import ( "context" + "fmt" "strconv" "strings" "time" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/rootfs" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/util/push" "github.com/moby/buildkit/util/resolver" + digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -20,6 +28,7 @@ const ( keyPush = "push" keyPushByDigest = "push-by-digest" keyInsecure = "registry.insecure" + keyUnpack = "unpack" ociTypes = "oci-mediatypes" ) @@ -79,6 +88,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } i.insecure = b + case keyUnpack: + if v == "" { + i.unpack = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.unpack = b case ociTypes: if v == "" { i.ociTypes = true @@ -104,6 +123,7 @@ type imageExporterInstance struct { targetName string push bool pushByDigest bool + unpack bool insecure bool ociTypes bool meta map[string][]byte @@ -156,6 +176,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) } } tagDone(nil) + + if e.unpack { + if err := e.unpackImage(ctx, img); err != nil { + return nil, err + } + } } if e.push { if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil { @@ -169,3 +195,72 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) resp["containerimage.digest"] = desc.Digest.String() return resp, nil } + +func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) { + unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name) + defer func() { + unpackDone(err0) + }() + + var ( + contentStore = e.opt.ImageWriter.ContentStore() + applier = e.opt.ImageWriter.Applier() + snapshotter = e.opt.ImageWriter.Snapshotter() + ) + + // fetch manifest by default platform + manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default()) + if err != nil { + return err + } + + layers, err := getLayers(ctx, contentStore, manifest) + if err != nil { + return err + } + + // get containerd snapshotter + ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter) + defer release() + + var chain []digest.Digest + for _, layer := range layers { + if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil { + return err + } + chain = append(chain, layer.Diff.Digest) + } + + var ( + keyGCLabel = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name()) + valueGCLabel = identity.ChainID(chain).String() + ) + + cinfo := content.Info{ + Digest: manifest.Config.Digest, + Labels: map[string]string{keyGCLabel: valueGCLabel}, + } + _, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel)) + return err +} + +func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) { + diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config) + if err != nil { + return nil, errors.Wrap(err, "failed to resolve rootfs") + } + + if len(diffIDs) != len(manifest.Layers) { + return nil, errors.Errorf("mismatched image rootfs and manifest layers") + } + + layers := make([]rootfs.Layer, len(diffIDs)) + for i := range diffIDs { + layers[i].Diff = ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Digest: diffIDs[i], + } + layers[i].Blob = manifest.Layers[i] + } + return layers, nil +} diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index f980afdc..0efc2d6f 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -33,6 +33,7 @@ const ( type WriterOpt struct { Snapshotter snapshot.Snapshotter ContentStore content.Store + Applier diff.Applier Differ diff.Comparer } @@ -290,6 +291,14 @@ func (ic *ImageWriter) ContentStore() content.Store { return ic.opt.ContentStore } +func (ic *ImageWriter) Snapshotter() snapshot.Snapshotter { + return ic.opt.Snapshotter +} + +func (ic *ImageWriter) Applier() diff.Applier { + return ic.opt.Applier +} + func emptyImageConfig() ([]byte, error) { img := ocispec.Image{ Architecture: runtime.GOARCH, diff --git a/snapshot/containerd/snapshotter.go b/snapshot/containerd/snapshotter.go index cef856c7..62f53f66 100644 --- a/snapshot/containerd/snapshotter.go +++ b/snapshot/containerd/snapshotter.go @@ -14,10 +14,10 @@ import ( "github.com/moby/buildkit/snapshot/blobmapping" ) -func NewSnapshotter(snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error, idmap *idtools.IdentityMapping) snapshot.Snapshotter { +func NewSnapshotter(name string, snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error, idmap *idtools.IdentityMapping) snapshot.Snapshotter { return blobmapping.NewSnapshotter(blobmapping.Opt{ Content: store, - Snapshotter: snapshot.FromContainerdSnapshotter(&nsSnapshotter{ns, snapshotter, gc}, idmap), + Snapshotter: snapshot.FromContainerdSnapshotter(name, &nsSnapshotter{ns, snapshotter, gc}, idmap), MetadataStore: mdstore, }) } diff --git a/snapshot/snapshotter.go b/snapshot/snapshotter.go index 19155dc6..b712f18c 100644 --- a/snapshot/snapshotter.go +++ b/snapshot/snapshotter.go @@ -18,6 +18,7 @@ type Mountable interface { } type SnapshotterBase interface { + Name() string Mounts(ctx context.Context, key string) (Mountable, error) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (Mountable, error) @@ -43,15 +44,20 @@ type Blobmapper interface { SetBlob(ctx context.Context, key string, diffID, blob digest.Digest) error } -func FromContainerdSnapshotter(s snapshots.Snapshotter, idmap *idtools.IdentityMapping) SnapshotterBase { - return &fromContainerd{Snapshotter: s, idmap: idmap} +func FromContainerdSnapshotter(name string, s snapshots.Snapshotter, idmap *idtools.IdentityMapping) SnapshotterBase { + return &fromContainerd{name: name, Snapshotter: s, idmap: idmap} } type fromContainerd struct { + name string snapshots.Snapshotter idmap *idtools.IdentityMapping } +func (s *fromContainerd) Name() string { + return s.name +} + func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, error) { mounts, err := s.Snapshotter.Mounts(ctx, key) if err != nil { diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 1a1b8aa7..5bcaac72 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -297,7 +297,7 @@ func setupGitSource(t *testing.T, tmpdir string) source.Source { assert.NoError(t, err) cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil), + Snapshotter: snapshot.FromContainerdSnapshotter("native", snapshotter, nil), MetadataStore: md, }) assert.NoError(t, err) diff --git a/source/http/httpsource_test.go b/source/http/httpsource_test.go index f9fce971..159c4e5a 100644 --- a/source/http/httpsource_test.go +++ b/source/http/httpsource_test.go @@ -315,7 +315,7 @@ func newHTTPSource(tmpdir string) (source.Source, error) { } cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil), + Snapshotter: snapshot.FromContainerdSnapshotter("native", snapshotter, nil), MetadataStore: md, }) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index d772a417..3e609eb0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -46,13 +46,13 @@ github.com/containerd/containerd/containers github.com/containerd/containerd/contrib/seccomp github.com/containerd/containerd/namespaces github.com/containerd/containerd/errdefs +github.com/containerd/containerd/rootfs github.com/containerd/containerd/images/oci github.com/containerd/containerd/api/services/content/v1 github.com/containerd/containerd/content/proxy github.com/containerd/containerd/services/content/contentserver github.com/containerd/containerd/reference github.com/containerd/containerd/remotes/docker/schema1 -github.com/containerd/containerd/rootfs github.com/containerd/containerd/images/archive github.com/containerd/containerd/archive github.com/containerd/containerd/archive/compression diff --git a/worker/base/worker.go b/worker/base/worker.go index bf682923..f154ddc1 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -155,6 +155,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) { iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{ Snapshotter: opt.Snapshotter, ContentStore: opt.ContentStore, + Applier: opt.Applier, Differ: opt.Differ, }) if err != nil { diff --git a/worker/containerd/containerd.go b/worker/containerd/containerd.go index 5e04d0e0..d18e4f9b 100644 --- a/worker/containerd/containerd.go +++ b/worker/containerd/containerd.go @@ -106,7 +106,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s Labels: xlabels, MetadataStore: md, Executor: containerdexecutor.New(client, root, "", network.Default()), - Snapshotter: containerdsnapshot.NewSnapshotter(client.SnapshotService(snapshotterName), cs, md, ns, gc, nil), + Snapshotter: containerdsnapshot.NewSnapshotter(snapshotterName, client.SnapshotService(snapshotterName), cs, md, ns, gc, nil), ContentStore: cs, Applier: winlayers.NewFileSystemApplierWithWindows(cs, df), Differ: winlayers.NewWalkingDiffWithWindows(cs, df), diff --git a/worker/runc/runc.go b/worker/runc/runc.go index 7d203384..13168a69 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -103,7 +103,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc Labels: xlabels, MetadataStore: md, Executor: exe, - Snapshotter: containerdsnapshot.NewSnapshotter(mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc, idmap), + Snapshotter: containerdsnapshot.NewSnapshotter(snFactory.Name, mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc, idmap), ContentStore: c, Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)), Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)),