diff --git a/cache/blobs/blobs.go b/cache/blobs/blobs.go new file mode 100644 index 00000000..5986f028 --- /dev/null +++ b/cache/blobs/blobs.go @@ -0,0 +1,98 @@ +package blobs + +import ( + gocontext "context" + + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/rootfs" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/flightcontrol" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/net/context" + "golang.org/x/sync/errgroup" +) + +var g flightcontrol.Group + +type DiffPair struct { + DiffID digest.Digest + Blobsum digest.Digest +} + +type blobmapper interface { + GetBlob(ctx gocontext.Context, key string) (digest.Digest, error) + SetBlob(ctx gocontext.Context, key string, blob digest.Digest) error +} + +func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ rootfs.MountDiffer, ref cache.ImmutableRef) ([]DiffPair, error) { + blobmap, ok := snapshotter.(blobmapper) + if !ok { + return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support") + } + + eg, ctx := errgroup.WithContext(ctx) + var diffPairs []DiffPair + var currentPair DiffPair + parent := ref.Parent() + if parent != nil { + defer parent.Release(context.TODO()) + eg.Go(func() error { + dp, err := GetDiffPairs(ctx, snapshotter, differ, parent) + if err != nil { + return err + } + diffPairs = dp + return nil + }) + } + eg.Go(func() error { + dp, err := g.Do(ctx, ref.ID(), func(ctx context.Context) (interface{}, error) { + blob, err := blobmap.GetBlob(ctx, ref.ID()) + if err != nil { + return nil, err + } + if blob != "" { + diffID, err := digest.Parse(ref.ID()) + if err != nil { + diffID = blob + } + return DiffPair{DiffID: diffID, Blobsum: blob}, nil + } + // reference needs to be committed + parent := ref.Parent() + var lower []mount.Mount + if parent != nil { + defer parent.Release(context.TODO()) + lower, err = parent.Mount(ctx, true) + if err != nil { + return nil, err + } + } + upper, err := ref.Mount(ctx, true) + if err != nil { + return nil, err + } + descr, err := differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID()) + if err != nil { + return nil, err + } + if err := blobmap.SetBlob(ctx, ref.ID(), descr.Digest); err != nil { + return nil, err + } + return DiffPair{DiffID: descr.Digest, Blobsum: descr.Digest}, nil + }) + if err != nil { + return err + } + currentPair = dp.(DiffPair) + return nil + }) + err := eg.Wait() + if err != nil { + return nil, err + } + return append(diffPairs, currentPair), nil +} diff --git a/control/control_default.go b/control/control_default.go index 727f79ee..4879eb08 100644 --- a/control/control_default.go +++ b/control/control_default.go @@ -108,11 +108,10 @@ func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) { exporters := map[string]exporter.Exporter{} imageExporter, err := imageexporter.New(imageexporter.Opt{ - Snapshotter: snapshotter, - ContentStore: pd.ContentStore, - Differ: pd.Differ, - CacheAccessor: cm, - Images: pd.Images, + Snapshotter: snapshotter, + ContentStore: pd.ContentStore, + Differ: pd.Differ, + Images: pd.Images, }) if err != nil { return nil, err diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index eef5e340..59567720 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -2,7 +2,6 @@ package containerimage import ( "bytes" - gocontext "context" "encoding/json" "runtime" "time" @@ -10,13 +9,11 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/mount" "github.com/containerd/containerd/rootfs" "github.com/moby/buildkit/cache" - "github.com/moby/buildkit/cache/metadata" + "github.com/moby/buildkit/cache/blobs" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/snapshot" - "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/system" digest "github.com/opencontainers/go-digest" @@ -24,7 +21,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "golang.org/x/sync/errgroup" ) const ( @@ -33,37 +29,18 @@ const ( ) type Opt struct { - Snapshotter snapshot.Snapshotter - ContentStore content.Store - Differ rootfs.MountDiffer - CacheAccessor cache.Accessor - MetadataStore metadata.Store - Images images.Store + Snapshotter snapshot.Snapshotter + ContentStore content.Store + Differ rootfs.MountDiffer + Images images.Store } type imageExporter struct { - blobmap blobmapper - opt Opt - g flightcontrol.Group -} - -type diffPair struct { - diffID digest.Digest - blobsum digest.Digest -} - -type blobmapper interface { - GetBlob(ctx gocontext.Context, key string) (digest.Digest, error) - SetBlob(ctx gocontext.Context, key string, blob digest.Digest) error + opt Opt } func New(opt Opt) (exporter.Exporter, error) { - blobmap, ok := opt.Snapshotter.(blobmapper) - if !ok { - return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support") - } - - im := &imageExporter{opt: opt, blobmap: blobmap} + im := &imageExporter{opt: opt} return im, nil } @@ -80,71 +57,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp return i, nil } -func (e *imageExporter) getBlobs(ctx context.Context, ref cache.ImmutableRef) ([]diffPair, error) { - eg, ctx := errgroup.WithContext(ctx) - var diffPairs []diffPair - var currentPair diffPair - parent := ref.Parent() - if parent != nil { - defer parent.Release(context.TODO()) - eg.Go(func() error { - dp, err := e.getBlobs(ctx, parent) - if err != nil { - return err - } - diffPairs = dp - return nil - }) - } - eg.Go(func() error { - dp, err := e.g.Do(ctx, ref.ID(), func(ctx context.Context) (interface{}, error) { - blob, err := e.blobmap.GetBlob(ctx, ref.ID()) - if err != nil { - return nil, err - } - if blob != "" { - diffID, err := digest.Parse(ref.ID()) - if err != nil { - diffID = blob - } - return diffPair{diffID: diffID, blobsum: blob}, nil - } - // reference needs to be committed - parent := ref.Parent() - var lower []mount.Mount - if parent != nil { - defer parent.Release(context.TODO()) - lower, err = parent.Mount(ctx, true) - if err != nil { - return nil, err - } - } - upper, err := ref.Mount(ctx, true) - if err != nil { - return nil, err - } - descr, err := e.opt.Differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID()) - if err != nil { - return nil, err - } - if err := e.blobmap.SetBlob(ctx, ref.ID(), descr.Digest); err != nil { - return nil, err - } - return diffPair{diffID: descr.Digest, blobsum: descr.Digest}, nil - }) - if err != nil { - return err - } - currentPair = dp.(diffPair) - return nil - }) - err := eg.Wait() - if err != nil { - return nil, err - } - return append(diffPairs, currentPair), nil -} - type imageExporterInstance struct { *imageExporter targetName string @@ -156,7 +68,7 @@ func (e *imageExporterInstance) Name() string { func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableRef, opt map[string][]byte) error { layersDone := oneOffProgress(ctx, "exporting layers") - diffPairs, err := e.getBlobs(ctx, ref) + diffPairs, err := blobs.GetDiffPairs(ctx, e.opt.Snapshotter, e.opt.Differ, ref) if err != nil { return err } @@ -164,7 +76,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR diffIDs := make([]digest.Digest, 0, len(diffPairs)) for _, dp := range diffPairs { - diffIDs = append(diffIDs, dp.diffID) + diffIDs = append(diffIDs, dp.DiffID) } var dt []byte @@ -198,12 +110,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR mfst.SchemaVersion = 2 for _, dp := range diffPairs { - info, err := e.opt.ContentStore.Info(ctx, dp.blobsum) + info, err := e.opt.ContentStore.Info(ctx, dp.Blobsum) if err != nil { - return configDone(errors.Wrapf(err, "could not get blob %s", dp.blobsum)) + return configDone(errors.Wrapf(err, "could not get blob %s", dp.Blobsum)) } mfst.Layers = append(mfst.Layers, ocispec.Descriptor{ - Digest: dp.blobsum, + Digest: dp.Blobsum, Size: info.Size, MediaType: ocispec.MediaTypeImageLayerGzip, })