JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

exporter: support unpack opt for image exporter 

It is enhancement which allows to unpack image into containerd
snapshotter storage by `--output type=image,<.>=<.>,unpack=true`.

In order to support this feature, we needs to extend the Snapshotter
witwh `Name() string` function. Because we needs to set gc label for
snapshotter which need snapshotter name.

fix: #908

Signed-off-by: Wei Fu <fuweid89@gmail.com>
docker-19.03
Wei Fu 2019-03-28 17:26:34 +08:00
parent 6085a25a2f
commit ffad8ada79
13 changed files with 159 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cache/contenthash/checksum_test.go vendored
View File

@ -37,7 +37,7 @@ func TestChecksumHardlinks(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -120,7 +120,7 @@ func TestChecksumWildcard(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -175,7 +175,7 @@ func TestSymlinksNoFollow(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -234,7 +234,7 @@ func TestChecksumBasicFile(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -384,7 +384,7 @@ func TestHandleChange(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -462,7 +462,7 @@ func TestHandleRecursiveDir(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -511,7 +511,7 @@ func TestChecksumUnorderedFiles(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -564,7 +564,7 @@ func TestSymlinkInPathScan(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -595,7 +595,7 @@ func TestSymlinkNeedsScan(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -628,7 +628,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -655,7 +655,7 @@ func TestSymlinkThroughParent(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -710,7 +710,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -773,7 +773,7 @@ func TestPersistence(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ch := []string{
@ -810,7 +810,7 @@ func TestPersistence(t *testing.T) {
// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm.Close()
getDefaultManager().lru.Purge()
cm = setupCacheManager(t, tmpdir, snapshotter)
cm = setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()
ref, err = cm.Get(context.TODO(), id)
@ -843,12 +843,12 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe
return ref
}
func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) cache.Manager {
func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) cache.Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil),
MetadataStore: md,
})
require.NoError(t, err)

14
cache/manager_test.go vendored
View File

@ -28,7 +28,7 @@ func TestManager(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)
_, err = cm.Get(ctx, "foobar")
require.Error(t, err)
@ -152,7 +152,7 @@ func TestPrune(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)
active, err := cm.New(ctx, nil)
require.NoError(t, err)
@ -250,7 +250,7 @@ func TestLazyCommit(t *testing.T) {
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)
active, err := cm.New(ctx, nil, CachePolicyRetain)
require.NoError(t, err)
@ -332,7 +332,7 @@ func TestLazyCommit(t *testing.T) {
require.NoError(t, err)
// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm = getCacheManager(t, tmpdir, snapshotter)
cm = getCacheManager(t, tmpdir, "native", snapshotter)
snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
@ -353,7 +353,7 @@ func TestLazyCommit(t *testing.T) {
err = cm.Close()
require.NoError(t, err)
cm = getCacheManager(t, tmpdir, snapshotter)
cm = getCacheManager(t, tmpdir, "native", snapshotter)
snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
@ -369,12 +369,12 @@ func TestLazyCommit(t *testing.T) {
require.Equal(t, errNotFound, errors.Cause(err))
}
func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) Manager {
func getCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)
cm, err := NewManager(ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil),
MetadataStore: md,
})
require.NoError(t, err, fmt.Sprintf("error: %+v", err))

18
cmd/buildctl/build_test.go
View File

@ -8,6 +8,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
@ -82,7 +83,15 @@ func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) {
rdr, err := marshal(st.Root())
require.NoError(t, err)
cmd := sb.Cmd("build --progress=plain --exporter=image --exporter-opt name=example.com/moby/imageexporter:test")
imageName := "example.com/moby/imageexporter:test"
buildCmd := []string{
"build", "--progress=plain",
"--exporter=image", "--exporter-opt", "unpack=true",
"--exporter-opt", "name=" + imageName,
}
cmd := sb.Cmd(strings.Join(buildCmd, " "))
cmd.Stdin = rdr
err = cmd.Run()
require.NoError(t, err)
@ -93,8 +102,13 @@ func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) {
ctx := namespaces.WithNamespace(context.Background(), "buildkit")
_, err = client.ImageService().Get(ctx, "example.com/moby/imageexporter:test")
img, err := client.GetImage(ctx, imageName)
require.NoError(t, err)
// NOTE: by default, it is overlayfs
ok, err := img.IsUnpacked(ctx, "overlayfs")
require.NoError(t, err)
require.Equal(t, ok, true)
}
func marshal(st llb.State) (io.Reader, error) {

95
exporter/containerimage/export.go
View File

@ -2,16 +2,24 @@ package containerimage
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/push"
"github.com/moby/buildkit/util/resolver"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -20,6 +28,7 @@ const (
keyPush = "push"
keyPushByDigest = "push-by-digest"
keyInsecure = "registry.insecure"
keyUnpack = "unpack"
ociTypes = "oci-mediatypes"
)
@ -79,6 +88,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.insecure = b
case keyUnpack:
if v == "" {
i.unpack = true
continue
}
b, err := strconv.ParseBool(v)
if err != nil {
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.unpack = b
case ociTypes:
if v == "" {
i.ociTypes = true
@ -104,6 +123,7 @@ type imageExporterInstance struct {
targetName string
push bool
pushByDigest bool
unpack bool
insecure bool
ociTypes bool
meta map[string][]byte
@ -156,6 +176,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
}
}
tagDone(nil)
if e.unpack {
if err := e.unpackImage(ctx, img); err != nil {
return nil, err
}
}
}
if e.push {
if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil {
@ -169,3 +195,72 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
resp["containerimage.digest"] = desc.Digest.String()
return resp, nil
}
func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) {
unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name)
defer func() {
unpackDone(err0)
}()
var (
contentStore = e.opt.ImageWriter.ContentStore()
applier = e.opt.ImageWriter.Applier()
snapshotter = e.opt.ImageWriter.Snapshotter()
)
// fetch manifest by default platform
manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default())
if err != nil {
return err
}
layers, err := getLayers(ctx, contentStore, manifest)
if err != nil {
return err
}
// get containerd snapshotter
ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter)
defer release()
var chain []digest.Digest
for _, layer := range layers {
if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil {
return err
}
chain = append(chain, layer.Diff.Digest)
}
var (
keyGCLabel = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name())
valueGCLabel = identity.ChainID(chain).String()
)
cinfo := content.Info{
Digest: manifest.Config.Digest,
Labels: map[string]string{keyGCLabel: valueGCLabel},
}
_, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel))
return err
}
func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) {
diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config)
if err != nil {
return nil, errors.Wrap(err, "failed to resolve rootfs")
}
if len(diffIDs) != len(manifest.Layers) {
return nil, errors.Errorf("mismatched image rootfs and manifest layers")
}
layers := make([]rootfs.Layer, len(diffIDs))
for i := range diffIDs {
layers[i].Diff = ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayer,
Digest: diffIDs[i],
}
layers[i].Blob = manifest.Layers[i]
}
return layers, nil
}

9
exporter/containerimage/writer.go
View File

@ -33,6 +33,7 @@ const (
type WriterOpt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Applier diff.Applier
Differ diff.Comparer
}
@ -290,6 +291,14 @@ func (ic *ImageWriter) ContentStore() content.Store {
return ic.opt.ContentStore
}
func (ic *ImageWriter) Snapshotter() snapshot.Snapshotter {
return ic.opt.Snapshotter
}
func (ic *ImageWriter) Applier() diff.Applier {
return ic.opt.Applier
}
func emptyImageConfig() ([]byte, error) {
img := ocispec.Image{
Architecture: runtime.GOARCH,

4
snapshot/containerd/snapshotter.go
View File

@ -14,10 +14,10 @@ import (
"github.com/moby/buildkit/snapshot/blobmapping"
)
func NewSnapshotter(snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error, idmap *idtools.IdentityMapping) snapshot.Snapshotter {
func NewSnapshotter(name string, snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error, idmap *idtools.IdentityMapping) snapshot.Snapshotter {
return blobmapping.NewSnapshotter(blobmapping.Opt{
Content: store,
Snapshotter: snapshot.FromContainerdSnapshotter(&nsSnapshotter{ns, snapshotter, gc}, idmap),
Snapshotter: snapshot.FromContainerdSnapshotter(name, &nsSnapshotter{ns, snapshotter, gc}, idmap),
MetadataStore: mdstore,
})
}

10
snapshot/snapshotter.go
View File

@ -18,6 +18,7 @@ type Mountable interface {
}
type SnapshotterBase interface {
Name() string
Mounts(ctx context.Context, key string) (Mountable, error)
Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error
View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (Mountable, error)
@ -43,15 +44,20 @@ type Blobmapper interface {
SetBlob(ctx context.Context, key string, diffID, blob digest.Digest) error
}
func FromContainerdSnapshotter(s snapshots.Snapshotter, idmap *idtools.IdentityMapping) SnapshotterBase {
return &fromContainerd{Snapshotter: s, idmap: idmap}
func FromContainerdSnapshotter(name string, s snapshots.Snapshotter, idmap *idtools.IdentityMapping) SnapshotterBase {
return &fromContainerd{name: name, Snapshotter: s, idmap: idmap}
}
type fromContainerd struct {
name string
snapshots.Snapshotter
idmap *idtools.IdentityMapping
}
func (s *fromContainerd) Name() string {
return s.name
}
func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, error) {
mounts, err := s.Snapshotter.Mounts(ctx, key)
if err != nil {

2
source/git/gitsource_test.go
View File

@ -297,7 +297,7 @@ func setupGitSource(t *testing.T, tmpdir string) source.Source {
assert.NoError(t, err)
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
Snapshotter: snapshot.FromContainerdSnapshotter("native", snapshotter, nil),
MetadataStore: md,
})
assert.NoError(t, err)

2
source/http/httpsource_test.go
View File

@ -315,7 +315,7 @@ func newHTTPSource(tmpdir string) (source.Source, error) {
}
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
Snapshotter: snapshot.FromContainerdSnapshotter("native", snapshotter, nil),
MetadataStore: md,
})
if err != nil {

2
vendor/modules.txt vendored
View File

@ -46,13 +46,13 @@ github.com/containerd/containerd/containers
github.com/containerd/containerd/contrib/seccomp
github.com/containerd/containerd/namespaces
github.com/containerd/containerd/errdefs
github.com/containerd/containerd/rootfs
github.com/containerd/containerd/images/oci
github.com/containerd/containerd/api/services/content/v1
github.com/containerd/containerd/content/proxy
github.com/containerd/containerd/services/content/contentserver
github.com/containerd/containerd/reference
github.com/containerd/containerd/remotes/docker/schema1
github.com/containerd/containerd/rootfs
github.com/containerd/containerd/images/archive
github.com/containerd/containerd/archive
github.com/containerd/containerd/archive/compression

1
worker/base/worker.go
View File

@ -155,6 +155,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{
Snapshotter: opt.Snapshotter,
ContentStore: opt.ContentStore,
Applier: opt.Applier,
Differ: opt.Differ,
})
if err != nil {

2
worker/containerd/containerd.go
View File

@ -106,7 +106,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s
Labels: xlabels,
MetadataStore: md,
Executor: containerdexecutor.New(client, root, "", network.Default()),
Snapshotter: containerdsnapshot.NewSnapshotter(client.SnapshotService(snapshotterName), cs, md, ns, gc, nil),
Snapshotter: containerdsnapshot.NewSnapshotter(snapshotterName, client.SnapshotService(snapshotterName), cs, md, ns, gc, nil),
ContentStore: cs,
Applier: winlayers.NewFileSystemApplierWithWindows(cs, df),
Differ: winlayers.NewWalkingDiffWithWindows(cs, df),

2
worker/runc/runc.go
View File

@ -103,7 +103,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc
Labels: xlabels,
MetadataStore: md,
Executor: exe,
Snapshotter: containerdsnapshot.NewSnapshotter(mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc, idmap),
Snapshotter: containerdsnapshot.NewSnapshotter(snFactory.Name, mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc, idmap),
ContentStore: c,
Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)),
Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)),