JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cache: export cache to content store 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2017-10-12 15:04:33 -07:00
parent 897291de2a
commit 8e611643f7
4 changed files with 163 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
cache/cacheimport/import.go vendored Normal file
View File

@ -0,0 +1,120 @@
package cacheimport
import (
"bytes"
"context"
"encoding/json"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/blobs"
"github.com/moby/buildkit/snapshot"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type CacheRecord struct {
CacheKey digest.Digest
Reference cache.ImmutableRef
ContentKey digest.Digest
}
type Opt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Differ rootfs.MountDiffer
}
func NewCacheExporter(opt Opt) *CacheExporter {
return &CacheExporter{opt: opt}
}
type CacheExporter struct {
opt Opt
}
func (ce *CacheExporter) Export(ctx context.Context, rec []CacheRecord) error {
allBlobs := map[digest.Digest][]blobs.DiffPair{}
currentBlobs := map[digest.Digest]struct{}{}
type cr struct {
CacheRecord
dgst digest.Digest
}
list := make([]cr, 0, len(rec))
for _, r := range rec {
ref := r.Reference
if ref == nil {
list = append(list, cr{CacheRecord: r})
continue
}
dpairs, err := blobs.GetDiffPairs(ctx, ce.opt.Snapshotter, ce.opt.Differ, ref)
if err != nil {
return err
}
for i, dp := range dpairs {
allBlobs[dp.Blobsum] = dpairs[:i]
}
dgst := dpairs[len(dpairs)-1].Blobsum
list = append(list, cr{CacheRecord: r, dgst: dgst})
currentBlobs[dgst] = struct{}{}
}
for b := range allBlobs {
if _, ok := currentBlobs[b]; !ok {
list = append(list, cr{dgst: b})
}
}
mfst := ocispec.Index{}
mfst.SchemaVersion = 2
for _, l := range list {
var size int64
parent := ""
if l.dgst != "" {
info, err := ce.opt.ContentStore.Info(ctx, l.dgst)
if err != nil {
return err
}
size = info.Size
parents := allBlobs[l.dgst]
if len(parents) > 0 {
parent = parents[len(parents)-1].Blobsum.String()
}
}
mfst.Manifests = append(mfst.Manifests, ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayerGzip,
Size: size,
Digest: l.dgst,
Annotations: map[string]string{
"buildkit.cachekey": l.CacheKey.String(),
"buildkit.contentkey": l.ContentKey.String(),
"buildkit.parent": parent,
},
})
}
dt, err := json.Marshal(mfst)
if err != nil {
return errors.Wrap(err, "failed to marshal manifest")
}
dgst := digest.FromBytes(dt)
if err := content.WriteBlob(ctx, ce.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst); err != nil {
return errors.Wrap(err, "error writing manifest blob")
}
logrus.Debugf("cache-manifest: %s", dgst)
return nil
}

3
control/control.go
View File

@ -4,6 +4,7 @@ import (
"github.com/containerd/containerd/snapshot"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/cacheimport"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/frontend"
@ -29,6 +30,7 @@ type Opt struct {
SessionManager *session.Manager
Frontends map[string]frontend.Frontend
ImageSource source.Source
CacheExporter *cacheimport.CacheExporter
}
type Controller struct { // TODO: ControlService
@ -46,6 +48,7 @@ func NewController(opt Opt) (*Controller, error) {
InstructionCache: opt.InstructionCache,
ImageSource: opt.ImageSource,
Frontends: opt.Frontends,
CacheExporter: opt.CacheExporter,
}),
}
return c, nil

8
control/control_default.go
View File

@ -10,6 +10,7 @@ import (
"github.com/containerd/containerd/rootfs"
ctdsnapshot "github.com/containerd/containerd/snapshot"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/cacheimport"
"github.com/moby/buildkit/cache/instructioncache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/client"
@ -130,6 +131,12 @@ func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) {
frontends["dockerfile.v0"] = dockerfile.NewDockerfileFrontend()
frontends["gateway.v0"] = gateway.NewGatewayFrontend()
ce := cacheimport.NewCacheExporter(cacheimport.Opt{
Snapshotter: snapshotter,
ContentStore: pd.ContentStore,
Differ: pd.Differ,
})
return &Opt{
Snapshotter: snapshotter,
CacheManager: cm,
@ -139,5 +146,6 @@ func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) {
SessionManager: sessm,
Frontends: frontends,
ImageSource: is,
CacheExporter: ce,
}, nil
}

63
solver/solver.go
View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/cacheimport"
"github.com/moby/buildkit/cache/contenthash"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter"
@ -31,6 +32,7 @@ type LLBOpt struct {
InstructionCache InstructionCache
ImageSource source.Source
Frontends map[string]frontend.Frontend // used by nested invocations
CacheExporter *cacheimport.CacheExporter
}
func NewLLBSolver(opt LLBOpt) *Solver {
@ -46,7 +48,7 @@ func NewLLBSolver(opt LLBOpt) *Solver {
default:
return nil, nil
}
}, opt.InstructionCache, opt.ImageSource, opt.Worker, opt.CacheManager, opt.Frontends)
}, opt.InstructionCache, opt.ImageSource, opt.Worker, opt.CacheManager, opt.Frontends, opt.CacheExporter)
return s
}
@ -86,10 +88,11 @@ type Solver struct {
worker worker.Worker
cm cache.Manager // TODO: remove with immutableRef.New()
frontends map[string]frontend.Frontend
ce *cacheimport.CacheExporter
}
func New(resolve ResolveOpFunc, cache InstructionCache, imageSource source.Source, worker worker.Worker, cm cache.Manager, f map[string]frontend.Frontend) *Solver {
return &Solver{resolve: resolve, jobs: newJobList(), cache: cache, imageSource: imageSource, worker: worker, cm: cm, frontends: f}
func New(resolve ResolveOpFunc, cache InstructionCache, imageSource source.Source, worker worker.Worker, cm cache.Manager, f map[string]frontend.Frontend, ce *cacheimport.CacheExporter) *Solver {
return &Solver{resolve: resolve, jobs: newJobList(), cache: cache, imageSource: imageSource, worker: worker, cm: cm, frontends: f, ce: ce}
}
type SolveRequest struct {
@ -151,25 +154,29 @@ func (s *Solver) Solve(ctx context.Context, id string, req SolveRequest) error {
}
if exp := req.Exporter; exp != nil {
return inVertexContext(ctx, exp.Name(), func(ctx context.Context) error {
if err := inVertexContext(ctx, exp.Name(), func(ctx context.Context) error {
return exp.Export(ctx, immutable, exporterOpt)
})
}); err != nil {
return err
}
}
cache, err := j.cacheExporter(ref)
if err != nil {
if err := inVertexContext(ctx, "exporting build cache", func(ctx context.Context) error {
cache, err := j.cacheExporter(ref)
if err != nil {
return err
}
records, err := cache.Export(ctx)
if err != nil {
return err
}
return s.ce.Export(ctx, records)
}); err != nil {
return err
}
records, err := cache.Export(ctx)
if err != nil {
return err
}
for _, r := range records {
logrus.Debugf("cache-record: %#v", r)
}
return err
}
@ -283,13 +290,7 @@ func markCached(ctx context.Context, cv client.Vertex) {
}
type CacheExporter interface {
Export(context.Context) ([]CacheRecord, error)
}
type CacheRecord struct {
CacheKey digest.Digest
Reference cache.ImmutableRef
ContentKey digest.Digest
Export(context.Context) ([]cacheimport.CacheRecord, error)
}
func (vs *vertexSolver) Cache(index Index) CacheExporter {
@ -301,23 +302,23 @@ type cacheExporter struct {
index Index
}
func (ce *cacheExporter) Export(ctx context.Context) ([]CacheRecord, error) {
func (ce *cacheExporter) Export(ctx context.Context) ([]cacheimport.CacheRecord, error) {
return ce.vertexSolver.Export(ctx, ce.index)
}
func (vs *vertexSolver) Export(ctx context.Context, index Index) ([]CacheRecord, error) {
mp := map[digest.Digest]CacheRecord{}
func (vs *vertexSolver) Export(ctx context.Context, index Index) ([]cacheimport.CacheRecord, error) {
mp := map[digest.Digest]cacheimport.CacheRecord{}
if err := vs.appendInputCache(ctx, mp); err != nil {
return nil, err
}
out := make([]CacheRecord, 0, len(mp))
out := make([]cacheimport.CacheRecord, 0, len(mp))
for _, cr := range mp {
out = append(out, cr)
}
return out, nil
}
func (vs *vertexSolver) appendInputCache(ctx context.Context, mp map[digest.Digest]CacheRecord) error {
func (vs *vertexSolver) appendInputCache(ctx context.Context, mp map[digest.Digest]cacheimport.CacheRecord) error {
for i, inp := range vs.inputs {
mainDgst, err := inp.solver.(*vertexSolver).mainCacheKey()
if err != nil {
@ -333,9 +334,9 @@ func (vs *vertexSolver) appendInputCache(ctx context.Context, mp map[digest.Dige
if !ok {
return errors.Errorf("invalid reference")
}
mp[dgst] = CacheRecord{CacheKey: dgst, Reference: ref}
mp[dgst] = cacheimport.CacheRecord{CacheKey: dgst, Reference: ref}
} else {
mp[dgst] = CacheRecord{CacheKey: dgst}
mp[dgst] = cacheimport.CacheRecord{CacheKey: dgst}
}
}
}
@ -344,7 +345,7 @@ func (vs *vertexSolver) appendInputCache(ctx context.Context, mp map[digest.Dige
if err != nil {
return err
}
mp[ck] = CacheRecord{CacheKey: mainDgst, ContentKey: ck}
mp[ck] = cacheimport.CacheRecord{CacheKey: mainDgst, ContentKey: ck}
}
return nil
}