JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

blobs: move blobs differ to common function 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2017-10-12 11:24:15 -07:00
parent 0fd670fb73
commit 897291de2a
3 changed files with 114 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
cache/blobs/blobs.go vendored Normal file
View File

@ -0,0 +1,98 @@
package blobs
import (
gocontext "context"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/flightcontrol"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
var g flightcontrol.Group
type DiffPair struct {
DiffID digest.Digest
Blobsum digest.Digest
}
type blobmapper interface {
GetBlob(ctx gocontext.Context, key string) (digest.Digest, error)
SetBlob(ctx gocontext.Context, key string, blob digest.Digest) error
}
func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ rootfs.MountDiffer, ref cache.ImmutableRef) ([]DiffPair, error) {
blobmap, ok := snapshotter.(blobmapper)
if !ok {
return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support")
}
eg, ctx := errgroup.WithContext(ctx)
var diffPairs []DiffPair
var currentPair DiffPair
parent := ref.Parent()
if parent != nil {
defer parent.Release(context.TODO())
eg.Go(func() error {
dp, err := GetDiffPairs(ctx, snapshotter, differ, parent)
if err != nil {
return err
}
diffPairs = dp
return nil
})
}
eg.Go(func() error {
dp, err := g.Do(ctx, ref.ID(), func(ctx context.Context) (interface{}, error) {
blob, err := blobmap.GetBlob(ctx, ref.ID())
if err != nil {
return nil, err
}
if blob != "" {
diffID, err := digest.Parse(ref.ID())
if err != nil {
diffID = blob
}
return DiffPair{DiffID: diffID, Blobsum: blob}, nil
}
// reference needs to be committed
parent := ref.Parent()
var lower []mount.Mount
if parent != nil {
defer parent.Release(context.TODO())
lower, err = parent.Mount(ctx, true)
if err != nil {
return nil, err
}
}
upper, err := ref.Mount(ctx, true)
if err != nil {
return nil, err
}
descr, err := differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID())
if err != nil {
return nil, err
}
if err := blobmap.SetBlob(ctx, ref.ID(), descr.Digest); err != nil {
return nil, err
}
return DiffPair{DiffID: descr.Digest, Blobsum: descr.Digest}, nil
})
if err != nil {
return err
}
currentPair = dp.(DiffPair)
return nil
})
err := eg.Wait()
if err != nil {
return nil, err
}
return append(diffPairs, currentPair), nil
}

9
control/control_default.go
View File

@ -108,11 +108,10 @@ func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) {
exporters := map[string]exporter.Exporter{}
imageExporter, err := imageexporter.New(imageexporter.Opt{
Snapshotter: snapshotter,
ContentStore: pd.ContentStore,
Differ: pd.Differ,
CacheAccessor: cm,
Images: pd.Images,
Snapshotter: snapshotter,
ContentStore: pd.ContentStore,
Differ: pd.Differ,
Images: pd.Images,
})
if err != nil {
return nil, err

112
exporter/containerimage/export.go
View File

@ -2,7 +2,6 @@ package containerimage
import (
"bytes"
gocontext "context"
"encoding/json"
"runtime"
"time"
@ -10,13 +9,11 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/cache/blobs"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/system"
digest "github.com/opencontainers/go-digest"
@ -24,7 +21,6 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
const (
@ -33,37 +29,18 @@ const (
)
type Opt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Differ rootfs.MountDiffer
CacheAccessor cache.Accessor
MetadataStore metadata.Store
Images images.Store
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Differ rootfs.MountDiffer
Images images.Store
}
type imageExporter struct {
blobmap blobmapper
opt Opt
g flightcontrol.Group
}
type diffPair struct {
diffID digest.Digest
blobsum digest.Digest
}
type blobmapper interface {
GetBlob(ctx gocontext.Context, key string) (digest.Digest, error)
SetBlob(ctx gocontext.Context, key string, blob digest.Digest) error
opt Opt
}
func New(opt Opt) (exporter.Exporter, error) {
blobmap, ok := opt.Snapshotter.(blobmapper)
if !ok {
return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support")
}
im := &imageExporter{opt: opt, blobmap: blobmap}
im := &imageExporter{opt: opt}
return im, nil
}
@ -80,71 +57,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
return i, nil
}
func (e *imageExporter) getBlobs(ctx context.Context, ref cache.ImmutableRef) ([]diffPair, error) {
eg, ctx := errgroup.WithContext(ctx)
var diffPairs []diffPair
var currentPair diffPair
parent := ref.Parent()
if parent != nil {
defer parent.Release(context.TODO())
eg.Go(func() error {
dp, err := e.getBlobs(ctx, parent)
if err != nil {
return err
}
diffPairs = dp
return nil
})
}
eg.Go(func() error {
dp, err := e.g.Do(ctx, ref.ID(), func(ctx context.Context) (interface{}, error) {
blob, err := e.blobmap.GetBlob(ctx, ref.ID())
if err != nil {
return nil, err
}
if blob != "" {
diffID, err := digest.Parse(ref.ID())
if err != nil {
diffID = blob
}
return diffPair{diffID: diffID, blobsum: blob}, nil
}
// reference needs to be committed
parent := ref.Parent()
var lower []mount.Mount
if parent != nil {
defer parent.Release(context.TODO())
lower, err = parent.Mount(ctx, true)
if err != nil {
return nil, err
}
}
upper, err := ref.Mount(ctx, true)
if err != nil {
return nil, err
}
descr, err := e.opt.Differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID())
if err != nil {
return nil, err
}
if err := e.blobmap.SetBlob(ctx, ref.ID(), descr.Digest); err != nil {
return nil, err
}
return diffPair{diffID: descr.Digest, blobsum: descr.Digest}, nil
})
if err != nil {
return err
}
currentPair = dp.(diffPair)
return nil
})
err := eg.Wait()
if err != nil {
return nil, err
}
return append(diffPairs, currentPair), nil
}
type imageExporterInstance struct {
*imageExporter
targetName string
@ -156,7 +68,7 @@ func (e *imageExporterInstance) Name() string {
func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableRef, opt map[string][]byte) error {
layersDone := oneOffProgress(ctx, "exporting layers")
diffPairs, err := e.getBlobs(ctx, ref)
diffPairs, err := blobs.GetDiffPairs(ctx, e.opt.Snapshotter, e.opt.Differ, ref)
if err != nil {
return err
}
@ -164,7 +76,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
diffIDs := make([]digest.Digest, 0, len(diffPairs))
for _, dp := range diffPairs {
diffIDs = append(diffIDs, dp.diffID)
diffIDs = append(diffIDs, dp.DiffID)
}
var dt []byte
@ -198,12 +110,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
mfst.SchemaVersion = 2
for _, dp := range diffPairs {
info, err := e.opt.ContentStore.Info(ctx, dp.blobsum)
info, err := e.opt.ContentStore.Info(ctx, dp.Blobsum)
if err != nil {
return configDone(errors.Wrapf(err, "could not get blob %s", dp.blobsum))
return configDone(errors.Wrapf(err, "could not get blob %s", dp.Blobsum))
}
mfst.Layers = append(mfst.Layers, ocispec.Descriptor{
Digest: dp.blobsum,
Digest: dp.Blobsum,
Size: info.Size,
MediaType: ocispec.MediaTypeImageLayerGzip,
})