2017-10-12 22:04:33 +00:00
|
|
|
package cacheimport
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
|
|
|
|
"github.com/containerd/containerd/content"
|
|
|
|
"github.com/containerd/containerd/rootfs"
|
|
|
|
"github.com/moby/buildkit/cache"
|
|
|
|
"github.com/moby/buildkit/cache/blobs"
|
|
|
|
"github.com/moby/buildkit/snapshot"
|
|
|
|
digest "github.com/opencontainers/go-digest"
|
2017-10-13 00:53:28 +00:00
|
|
|
"github.com/opencontainers/image-spec/identity"
|
2017-10-12 22:04:33 +00:00
|
|
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
)
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
type ImportOpt struct {
|
|
|
|
ContentStore content.Store
|
|
|
|
Snapshotter snapshot.Snapshotter
|
|
|
|
Applier rootfs.Applier
|
|
|
|
CacheAccessor cache.Accessor
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func NewCacheImporter(opt ImportOpt) *CacheImporter {
|
|
|
|
return &CacheImporter{opt: opt}
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
type CacheImporter struct {
|
|
|
|
opt ImportOpt
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func (ci *CacheImporter) Import(ctx context.Context, dgst digest.Digest) (InstructionCache, error) {
|
|
|
|
dt, err := content.ReadBlob(ctx, ci.opt.ContentStore, dgst)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
var mfst ocispec.Index
|
|
|
|
if err := json.Unmarshal(dt, &mfst); err != nil {
|
|
|
|
return nil, err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
allBlobs := map[digest.Digest]ocispec.Descriptor{}
|
|
|
|
byCacheKey := map[digest.Digest]ocispec.Descriptor{}
|
|
|
|
byContentKey := map[digest.Digest][]digest.Digest{}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
for _, m := range mfst.Manifests {
|
|
|
|
if m.Digest != "" {
|
|
|
|
allBlobs[m.Digest] = m
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
2017-10-13 00:53:28 +00:00
|
|
|
if m.Annotations != nil {
|
|
|
|
if cacheKey := m.Annotations["buildkit.cachekey"]; cacheKey != "" {
|
|
|
|
cacheKeyDigest, err := digest.Parse(cacheKey)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
byCacheKey[cacheKeyDigest] = m
|
|
|
|
if contentKey := m.Annotations["buildkit.contentkey"]; contentKey != "" {
|
|
|
|
contentKeyDigest, err := digest.Parse(contentKey)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
byContentKey[contentKeyDigest] = append(byContentKey[contentKeyDigest], cacheKeyDigest)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return &importInfo{
|
|
|
|
CacheImporter: ci,
|
|
|
|
byCacheKey: byCacheKey,
|
|
|
|
byContentKey: byContentKey,
|
|
|
|
allBlobs: allBlobs,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type importInfo struct {
|
|
|
|
*CacheImporter
|
|
|
|
byCacheKey map[digest.Digest]ocispec.Descriptor
|
|
|
|
byContentKey map[digest.Digest][]digest.Digest
|
|
|
|
allBlobs map[digest.Digest]ocispec.Descriptor
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func (ii *importInfo) Probe(ctx context.Context, key digest.Digest) (bool, error) {
|
|
|
|
_, ok := ii.byCacheKey[key]
|
|
|
|
return ok, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ii *importInfo) getChain(dgst digest.Digest) ([]blobs.DiffPair, error) {
|
|
|
|
desc, ok := ii.allBlobs[dgst]
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf("blob %s not found in cache", dgst)
|
|
|
|
}
|
|
|
|
var parentDigest digest.Digest
|
|
|
|
if desc.Annotations == nil {
|
|
|
|
return nil, errors.Errorf("missing annotations")
|
|
|
|
}
|
|
|
|
parent := desc.Annotations["buildkit.parent"]
|
|
|
|
if parent != "" {
|
|
|
|
dgst, err := digest.Parse(parent)
|
2017-10-12 22:04:33 +00:00
|
|
|
if err != nil {
|
2017-10-13 00:53:28 +00:00
|
|
|
return nil, err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
2017-10-13 00:53:28 +00:00
|
|
|
parentDigest = dgst
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
var out []blobs.DiffPair
|
|
|
|
if parentDigest != "" {
|
|
|
|
parentChain, err := ii.getChain(parentDigest)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
2017-10-13 00:53:28 +00:00
|
|
|
out = parentChain
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
diffIDStr := desc.Annotations["buildkit.diffid"]
|
|
|
|
diffID, err := digest.Parse(diffIDStr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
return append(out, blobs.DiffPair{Blobsum: dgst, DiffID: diffID}), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ii *importInfo) Lookup(ctx context.Context, key digest.Digest) (interface{}, error) {
|
|
|
|
desc, ok := ii.byCacheKey[key]
|
|
|
|
if !ok || desc.Digest == "" {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
ch, err := ii.getChain(desc.Digest)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
2017-10-13 00:53:28 +00:00
|
|
|
return ii.fetch(ctx, ch)
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func (ii *importInfo) Set(key digest.Digest, ref interface{}) error {
|
|
|
|
return nil
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func (ii *importInfo) SetContentMapping(contentKey, key digest.Digest) error {
|
|
|
|
return nil
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func (ii *importInfo) GetContentMapping(dgst digest.Digest) ([]digest.Digest, error) {
|
|
|
|
dgsts, ok := ii.byContentKey[dgst]
|
|
|
|
if !ok {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
return dgsts, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ii *importInfo) fetch(ctx context.Context, chain []blobs.DiffPair) (cache.ImmutableRef, error) {
|
|
|
|
chainid, err := ii.unpack(ctx, chain)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
return ii.opt.CacheAccessor.Get(ctx, chainid, cache.WithDescription("imported cache")) // TODO: more descriptive name
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ii *importInfo) unpack(ctx context.Context, dpairs []blobs.DiffPair) (string, error) {
|
|
|
|
layers, err := ii.getLayers(ctx, dpairs)
|
2017-10-12 22:04:33 +00:00
|
|
|
if err != nil {
|
2017-10-13 00:53:28 +00:00
|
|
|
return "", err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
chainID, err := rootfs.ApplyLayers(ctx, layers, ii.opt.Snapshotter, ii.opt.Applier)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
if err := ii.fillBlobMapping(ctx, layers); err != nil {
|
|
|
|
return "", err
|
2017-10-12 22:04:33 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
return string(chainID), nil
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
|
2017-10-13 00:53:28 +00:00
|
|
|
func (ii *importInfo) fillBlobMapping(ctx context.Context, layers []rootfs.Layer) error {
|
|
|
|
var chain []digest.Digest
|
|
|
|
for _, l := range layers {
|
|
|
|
chain = append(chain, l.Diff.Digest)
|
|
|
|
chainID := identity.ChainID(chain)
|
|
|
|
if err := ii.opt.Snapshotter.(blobmapper).SetBlob(ctx, string(chainID), l.Blob.Digest); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2017-10-12 22:04:33 +00:00
|
|
|
return nil
|
|
|
|
}
|
2017-10-13 00:53:28 +00:00
|
|
|
|
|
|
|
func (ii *importInfo) getLayers(ctx context.Context, dpairs []blobs.DiffPair) ([]rootfs.Layer, error) {
|
|
|
|
layers := make([]rootfs.Layer, len(dpairs))
|
|
|
|
for i := range dpairs {
|
|
|
|
layers[i].Diff = ocispec.Descriptor{
|
|
|
|
// TODO: derive media type from compressed type
|
|
|
|
MediaType: ocispec.MediaTypeImageLayer,
|
|
|
|
Digest: dpairs[i].DiffID,
|
|
|
|
}
|
|
|
|
info, err := ii.opt.ContentStore.Info(ctx, dpairs[i].Blobsum)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
layers[i].Blob = ocispec.Descriptor{
|
|
|
|
// TODO: derive media type from compressed type
|
|
|
|
MediaType: ocispec.MediaTypeImageLayerGzip,
|
|
|
|
Digest: dpairs[i].Blobsum,
|
|
|
|
Size: info.Size,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return layers, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type InstructionCache interface {
|
|
|
|
Probe(ctx context.Context, key digest.Digest) (bool, error)
|
|
|
|
Lookup(ctx context.Context, key digest.Digest) (interface{}, error) // TODO: regular ref
|
|
|
|
Set(key digest.Digest, ref interface{}) error
|
|
|
|
SetContentMapping(contentKey, key digest.Digest) error
|
|
|
|
GetContentMapping(dgst digest.Digest) ([]digest.Digest, error)
|
|
|
|
}
|