JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

solver: optimize cache storage interface 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2018-03-16 13:34:06 -07:00
parent 513018806b
commit 5593bb9f48
5 changed files with 333 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

421
solver-next/cache.go
View File

@ -10,7 +10,6 @@ import (
"github.com/moby/buildkit/identity" "github.com/moby/buildkit/identity"
digest "github.com/opencontainers/go-digest" digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -52,21 +51,6 @@ type inMemoryCacheKey struct {
} }
func (ck *inMemoryCacheKey) Deps() []CacheKeyWithSelector { func (ck *inMemoryCacheKey) Deps() []CacheKeyWithSelector {
if len(ck.deps) == 0 || len(ck.CacheKeyInfo.Deps) > 0 {
deps := make([]CacheKeyWithSelector, len(ck.CacheKeyInfo.Deps))
for i, dep := range ck.CacheKeyInfo.Deps {
k, err := ck.manager.backend.Get(dep.ID)
if err != nil {
logrus.Errorf("dependency %s not found", dep.ID)
} else {
deps[i] = CacheKeyWithSelector{
CacheKey: withExporter(ck.manager.toInMemoryCacheKey(k), nil, nil),
Selector: dep.Selector,
}
}
}
ck.deps = deps
}
return ck.deps return ck.deps
} }
@ -77,18 +61,18 @@ func (ck *inMemoryCacheKey) Output() Index {
return Index(ck.CacheKeyInfo.Output) return Index(ck.CacheKeyInfo.Output)
} }
func withExporter(ck *inMemoryCacheKey, cacheResult *CacheResult, dep *CacheKeyWithSelector) ExportableCacheKey { func withExporter(ck *inMemoryCacheKey, cacheResult *CacheResult, deps []CacheKeyWithSelector) ExportableCacheKey {
return ExportableCacheKey{ck, &cacheExporter{ return ExportableCacheKey{ck, &cacheExporter{
inMemoryCacheKey: ck, inMemoryCacheKey: ck,
cacheResult: cacheResult, cacheResult: cacheResult,
dep: dep, deps: deps,
}} }}
} }
type cacheExporter struct { type cacheExporter struct {
*inMemoryCacheKey *inMemoryCacheKey
cacheResult *CacheResult cacheResult *CacheResult
dep *CacheKeyWithSelector deps []CacheKeyWithSelector
} }
func (ce *cacheExporter) Export(ctx context.Context, m map[digest.Digest]*ExportRecord, converter func(context.Context, Result) (*Remote, error)) (*ExportRecord, error) { func (ce *cacheExporter) Export(ctx context.Context, m map[digest.Digest]*ExportRecord, converter func(context.Context, Result) (*Remote, error)) (*ExportRecord, error) {
@ -130,8 +114,6 @@ func (ce *cacheExporter) Export(ctx context.Context, m map[digest.Digest]*Export
cacheID = remote.Descriptors[0].Digest cacheID = remote.Descriptors[0].Digest
} }
deps := ce.deps
rec, ok := m[cacheID] rec, ok := m[cacheID]
if !ok { if !ok {
rec = &ExportRecord{ rec = &ExportRecord{
@ -142,20 +124,17 @@ func (ce *cacheExporter) Export(ctx context.Context, m map[digest.Digest]*Export
m[cacheID] = rec m[cacheID] = rec
} }
if len(deps) == 0 { if len(ce.Deps()) == 0 {
rec.Links[CacheLink{ rec.Links[CacheLink{
Output: ce.Output(), Output: ce.Output(),
Base: ce.Digest(), Base: ce.Digest(),
}] = struct{}{} }] = struct{}{}
} }
allDeps := ce.Deps() for i, dep := range ce.deps {
if dep.CacheKey.Exporter == nil {
if ce.dep != nil { continue
allDeps = []CacheKeyWithSelector{*ce.dep} }
}
for i, dep := range allDeps {
r, err := dep.CacheKey.Export(ctx, m, converter) r, err := dep.CacheKey.Export(ctx, m, converter)
if err != nil { if err != nil {
return nil, err return nil, err
@ -185,12 +164,15 @@ func (c *inMemoryCacheManager) ID() string {
return c.id return c.id
} }
func (c *inMemoryCacheManager) toInMemoryCacheKey(cki CacheKeyInfo) *inMemoryCacheKey { func (c *inMemoryCacheManager) toInMemoryCacheKey(cki CacheKeyInfo, deps []CacheKeyWithSelector) *inMemoryCacheKey {
return &inMemoryCacheKey{ ck := &inMemoryCacheKey{
CacheKeyInfo: cki, CacheKeyInfo: cki,
manager: c, manager: c,
CacheKey: NewCacheKey("", 0, nil), CacheKey: NewCacheKey("", 0, nil),
deps: deps,
} }
ck.SetValue(internalMemoryKey, cki.ID)
return ck
} }
func (c *inMemoryCacheManager) getBestResult(cki CacheKeyInfo) (*CacheResult, error) { func (c *inMemoryCacheManager) getBestResult(cki CacheKeyInfo) (*CacheResult, error) {
@ -213,72 +195,55 @@ func (c *inMemoryCacheManager) getBestResult(cki CacheKeyInfo) (*CacheResult, er
return nil, nil return nil, nil
} }
func (c *inMemoryCacheManager) Query(deps []ExportableCacheKey, input Index, dgst digest.Digest, output Index, selector digest.Digest) ([]*CacheRecord, error) { func (c *inMemoryCacheManager) Query(deps []CacheKeyWithSelector, input Index, dgst digest.Digest, output Index) ([]*CacheRecord, error) {
c.mu.RLock() c.mu.RLock()
defer c.mu.RUnlock() defer c.mu.RUnlock()
refs := map[string]*CacheKeyWithSelector{} type dep struct {
sublinks := map[string]*CacheKeyWithSelector{} results map[string]struct{}
key CacheKeyWithSelector
internalKey *inMemoryCacheKey
}
formatDeps := func(deps []dep, index int) []CacheKeyWithSelector {
keys := make([]CacheKeyWithSelector, index+1)
if len(deps) == 1 {
keys[index] = deps[0].key
} else {
k2 := make([]CacheKeyWithSelector, 0, len(deps))
for _, d := range deps {
k2 = append(k2, d.key)
}
keys[index] = CacheKeyWithSelector{CacheKey: ExportableCacheKey{CacheKey: NewCacheKey("", 0, k2)}}
}
return keys
}
allDeps := make([]dep, 0, len(deps))
for _, d := range deps { for _, d := range deps {
var dd []CacheKeyWithSelector for _, k := range c.getAllKeys(d) {
if d.Digest() == "" && d.Output() == 0 { d := dep{key: k, results: map[string]struct{}{}}
for _, dep := range d.Deps() { internalKey, err := c.getInternalKey(k.CacheKey, false)
dd = append(dd, dep) if err != nil {
} if errors.Cause(err) == ErrNotFound {
} else { allDeps = append(allDeps, d)
dd = append(dd, CacheKeyWithSelector{Selector: selector, CacheKey: d}) } else {
} return nil, err
}
for _, dep := range dd { } else {
ck, err := c.getInternalKey(dep.CacheKey, false) d.internalKey = internalKey
if err == nil {
if err := c.backend.WalkLinks(ck.CacheKeyInfo.ID, CacheInfoLink{input, output, dgst, selector}, func(id string) error {
refs[id] = &CacheKeyWithSelector{Selector: selector, CacheKey: d}
return nil
}); err != nil {
return nil, err
}
if err := c.backend.WalkLinks(ck.CacheKeyInfo.ID, CacheInfoLink{Index(-1), Index(0), "", selector}, func(id string) error {
sublinks[id] = &CacheKeyWithSelector{Selector: selector, CacheKey: d}
return nil
}); err != nil {
return nil, err
}
if err := c.backend.WalkLinks(ck.CacheKeyInfo.ID, CacheInfoLink{Index(-1), Index(0), "", NoSelector}, func(id string) error {
sublinks[id] = &CacheKeyWithSelector{Selector: selector, CacheKey: d}
return nil
}); err != nil {
return nil, err
}
} }
allDeps = append(allDeps, d)
} }
} }
for id, mainKey := range sublinks { allRes := map[string]struct{}{}
ck, err := c.backend.Get(id) for _, d := range allDeps {
if err == nil { if d.internalKey != nil {
mainCk, err := c.getInternalKey(mainKey.CacheKey, false) if err := c.backend.WalkLinks(d.internalKey.CacheKeyInfo.ID, CacheInfoLink{input, output, dgst, d.key.Selector}, func(id string) error {
addNewKey := mainKey.CacheKey.Digest() == "" && (err != nil || mainCk.CacheKeyInfo.ID != ck.ID) d.results[id] = struct{}{}
if err := c.backend.WalkLinks(ck.ID, CacheInfoLink{input, output, dgst, ""}, func(id string) error { allRes[id] = struct{}{}
if addNewKey {
ck, err := c.getInternalKey(mainKey.CacheKey, true)
if err != nil {
return err
}
err = c.backend.AddLink(ck.CacheKeyInfo.ID, CacheInfoLink{
Input: input,
Output: output,
Digest: dgst,
Selector: "",
}, id)
if err != nil {
return err
}
}
refs[id] = mainKey
return nil return nil
}); err != nil { }); err != nil {
return nil, err return nil, err
@ -287,51 +252,77 @@ func (c *inMemoryCacheManager) Query(deps []ExportableCacheKey, input Index, dgs
} }
if len(deps) == 0 { if len(deps) == 0 {
ck, err := c.getInternalKey(NewCacheKey(dgst, 0, nil), false) allRes[digest.FromBytes([]byte(fmt.Sprintf("%s@%d", dgst, output))).String()] = struct{}{}
if err != nil {
return nil, nil
}
refs[ck.CacheKeyInfo.ID] = nil
} }
keys := make(map[string]*CacheKeyWithSelector) outs := make([]*CacheRecord, 0, len(allRes))
outs := make([]*CacheRecord, 0, len(refs))
for id, dep := range refs { for res := range allRes {
cki, err := c.backend.Get(id) for _, d := range allDeps {
if err == nil { if d.internalKey == nil {
k := c.toInMemoryCacheKey(cki) internalKey, err := c.getInternalKey(d.key.CacheKey, true)
keys[cki.ID] = dep if err != nil {
if err := c.backend.WalkResults(id, func(r CacheResult) error { return nil, err
if c.results.Exists(r.ID) { }
outs = append(outs, &CacheRecord{ d.internalKey = internalKey
ID: id + "@" + r.ID, }
CacheKey: withExporter(k, &r, dep), if _, ok := d.results[res]; !ok {
CacheManager: c, if err := c.backend.AddLink(d.internalKey.CacheKeyInfo.ID, CacheInfoLink{
Loadable: true, Input: input,
CreatedAt: r.CreatedAt, Output: output,
}) Digest: dgst,
} else { Selector: d.key.Selector,
c.backend.Release(r.ID) }, res); err != nil {
return nil, err
} }
return nil
}); err != nil {
return nil, err
} }
} }
} hadResults := false
if len(outs) == 0 { cki, err := c.backend.Get(res)
for id, dep := range keys { if err != nil {
cki, err := c.backend.Get(id) if errors.Cause(err) == ErrNotFound {
if err == nil { continue
k := c.toInMemoryCacheKey(cki)
outs = append(outs, &CacheRecord{
ID: k.CacheKeyInfo.ID,
CacheKey: withExporter(k, nil, dep),
CacheManager: c,
Loadable: false,
})
} }
return nil, errors.Wrapf(err, "failed lookup by internal ID %s", res)
}
deps := formatDeps(allDeps, int(input))
k := c.toInMemoryCacheKey(cki, deps)
// TODO: invoke this only once per input
if err := c.backend.WalkResults(res, func(r CacheResult) error {
if c.results.Exists(r.ID) {
outs = append(outs, &CacheRecord{
ID: res + "@" + r.ID,
CacheKey: withExporter(k, &r, deps),
CacheManager: c,
Loadable: true,
CreatedAt: r.CreatedAt,
})
hadResults = true
} else {
c.backend.Release(r.ID)
}
return nil
}); err != nil {
return nil, err
}
if !hadResults {
cki, err := c.backend.Get(res)
if err != nil {
if errors.Cause(err) == ErrNotFound {
continue
}
return nil, errors.Wrapf(err, "failed lookup by internal ID %s", res)
}
deps := formatDeps(allDeps, int(input))
k := c.toInMemoryCacheKey(cki, deps)
outs = append(outs, &CacheRecord{
ID: res,
CacheKey: withExporter(k, nil, deps),
CacheManager: c,
Loadable: false,
})
} }
} }
@ -379,7 +370,42 @@ func (c *inMemoryCacheManager) Save(k CacheKey, r Result) (ExportableCacheKey, e
return empty, err return empty, err
} }
return withExporter(ck, &res, nil), nil return withExporter(ck, &res, ck.Deps()), nil
}
func (c *inMemoryCacheManager) getInternalKeys(d CacheKeyWithSelector, createIfNotExist bool) ([]CacheKeyWithSelector, error) {
keys := make([]CacheKeyWithSelector, 0, 1)
if d.CacheKey.Digest() == "" {
for _, d := range d.CacheKey.Deps() {
k, err := c.getInternalKey(d.CacheKey, createIfNotExist)
if err != nil {
if !createIfNotExist && errors.Cause(err) == ErrNotFound {
continue
}
return nil, err
}
keys = append(keys, CacheKeyWithSelector{Selector: d.Selector, CacheKey: ExportableCacheKey{CacheKey: k, Exporter: d.CacheKey.Exporter}})
}
} else {
k, err := c.getInternalKey(d.CacheKey, createIfNotExist)
if err != nil {
return nil, err
}
keys = append(keys, CacheKeyWithSelector{Selector: d.Selector, CacheKey: ExportableCacheKey{CacheKey: k, Exporter: d.CacheKey.Exporter}})
}
return keys, nil
}
func (c *inMemoryCacheManager) getAllKeys(d CacheKeyWithSelector) []CacheKeyWithSelector {
keys := make([]CacheKeyWithSelector, 0, 1)
if d.CacheKey.Digest() == "" {
for _, d := range d.CacheKey.Deps() {
keys = append(keys, d)
}
} else {
keys = append(keys, d)
}
return keys
} }
func (c *inMemoryCacheManager) getInternalKey(k CacheKey, createIfNotExist bool) (*inMemoryCacheKey, error) { func (c *inMemoryCacheManager) getInternalKey(k CacheKey, createIfNotExist bool) (*inMemoryCacheKey, error) {
@ -391,66 +417,107 @@ func (c *inMemoryCacheManager) getInternalKey(k CacheKey, createIfNotExist bool)
} }
internalV := k.GetValue(internalMemoryKey) internalV := k.GetValue(internalMemoryKey)
if internalV != nil { if internalV != nil {
ck, err := c.backend.Get(internalV.(string)) cki, err := c.backend.Get(internalV.(string))
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed lookup by internal ID %s", internalV.(string)) return nil, errors.Wrapf(err, "failed lookup by internal ID %s", internalV.(string))
} }
return c.toInMemoryCacheKey(ck), nil return c.toInMemoryCacheKey(cki, k.Deps()), nil
} }
inputs := make([]CacheKeyInfoWithSelector, len(k.Deps())) matches := make(map[string]struct{})
dgstr := digest.SHA256.Digester() deps := make([][]CacheKeyWithSelector, 0, len(k.Deps()))
for i, inp := range k.Deps() { for i, inp := range k.Deps() {
ck, err := c.getInternalKey(inp.CacheKey, createIfNotExist) allKeys := c.getAllKeys(inp)
cks := make([]CacheKeyWithSelector, 0, len(allKeys))
for _, k := range allKeys {
internalKey, err := c.getInternalKey(k.CacheKey, createIfNotExist)
if err == nil {
cks = append(cks, CacheKeyWithSelector{Selector: k.Selector, CacheKey: ExportableCacheKey{CacheKey: internalKey, Exporter: k.CacheKey.Exporter}})
}
}
if len(cks) == 0 {
return nil, errors.WithStack(ErrNotFound)
}
if i == 0 || len(matches) > 0 {
for _, ck := range cks {
internalCk := ck.CacheKey.CacheKey.(*inMemoryCacheKey)
m2 := make(map[string]struct{})
if err := c.backend.WalkLinks(internalCk.CacheKeyInfo.ID, CacheInfoLink{
Input: Index(i),
Output: Index(k.Output()),
Digest: k.Digest(),
Selector: ck.Selector,
}, func(id string) error {
if i == 0 {
matches[id] = struct{}{}
} else {
m2[id] = struct{}{}
}
return nil
}); err != nil {
return nil, err
}
if i != 0 {
for id := range matches {
if _, ok := m2[id]; !ok {
delete(matches, id)
}
}
}
}
}
deps = append(deps, cks)
}
var internalKey string
if len(matches) == 0 && len(k.Deps()) > 0 {
if createIfNotExist {
internalKey = identity.NewID()
} else {
return nil, errors.WithStack(ErrNotFound)
}
} else {
for k := range matches {
internalKey = k
break
}
if len(k.Deps()) == 0 {
internalKey = digest.FromBytes([]byte(fmt.Sprintf("%s@%d", k.Digest(), k.Output()))).String()
}
cki, err := c.backend.Get(internalKey)
if err != nil { if err != nil {
return nil, err if errors.Cause(err) == ErrNotFound {
} if !createIfNotExist {
inputs[i] = CacheKeyInfoWithSelector{ID: ck.CacheKeyInfo.ID, Selector: inp.Selector} return nil, err
if _, err := dgstr.Hash().Write([]byte(fmt.Sprintf("%s:%s,", ck.CacheKeyInfo.ID, inp.Selector))); err != nil { }
return nil, err } else {
}
}
if _, err := dgstr.Hash().Write([]byte(k.Digest())); err != nil {
return nil, err
}
if _, err := dgstr.Hash().Write([]byte(fmt.Sprintf("%d", k.Output()))); err != nil {
return nil, err
}
internalKey := string(dgstr.Digest())
cki, err := c.backend.Get(internalKey)
if err != nil {
if errors.Cause(err) == ErrNotFound {
if !createIfNotExist {
return nil, err return nil, err
} }
} else { } else {
return nil, err return c.toInMemoryCacheKey(cki, k.Deps()), nil
} }
}
cki = CacheKeyInfo{ cki := CacheKeyInfo{
ID: internalKey, ID: internalKey,
Base: k.Digest(), Base: k.Digest(),
Output: int(k.Output()), Output: int(k.Output()),
Deps: inputs, }
}
if err := c.backend.Set(cki); err != nil { if err := c.backend.Set(cki); err != nil {
return nil, err return nil, err
} }
for i, inp := range inputs { for i, dep := range deps {
if cki.Base == "" { for _, ck := range dep {
i = -1 internalCk := ck.CacheKey.CacheKey.(*inMemoryCacheKey)
} err := c.backend.AddLink(internalCk.CacheKeyInfo.ID, CacheInfoLink{
err := c.backend.AddLink(inp.ID, CacheInfoLink{
Input: Index(i), Input: Index(i),
Output: Index(cki.Output), Output: Index(cki.Output),
Digest: cki.Base, Digest: cki.Base,
Selector: inp.Selector, Selector: ck.Selector,
}, cki.ID) }, cki.ID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -458,15 +525,7 @@ func (c *inMemoryCacheManager) getInternalKey(k CacheKey, createIfNotExist bool)
} }
} }
ck := &inMemoryCacheKey{ return c.toInMemoryCacheKey(cki, k.Deps()), nil
CacheKey: k,
CacheKeyInfo: cki,
manager: c,
deps: k.Deps(),
}
ck.SetValue(internalMemoryKey, internalKey)
return ck, nil
} }
func newCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager { func newCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager {
@ -491,14 +550,14 @@ func (cm *combinedCacheManager) ID() string {
return cm.id return cm.id
} }
func (cm *combinedCacheManager) Query(inp []ExportableCacheKey, inputIndex Index, dgst digest.Digest, outputIndex Index, selector digest.Digest) ([]*CacheRecord, error) { func (cm *combinedCacheManager) Query(inp []CacheKeyWithSelector, inputIndex Index, dgst digest.Digest, outputIndex Index) ([]*CacheRecord, error) {
eg, _ := errgroup.WithContext(context.TODO()) eg, _ := errgroup.WithContext(context.TODO())
res := make(map[string]*CacheRecord, len(cm.cms)) res := make(map[string]*CacheRecord, len(cm.cms))
var mu sync.Mutex var mu sync.Mutex
for i, c := range cm.cms { for i, c := range cm.cms {
func(i int, c CacheManager) { func(i int, c CacheManager) {
eg.Go(func() error { eg.Go(func() error {
recs, err := c.Query(inp, inputIndex, dgst, outputIndex, selector) recs, err := c.Query(inp, inputIndex, dgst, outputIndex)
if err != nil { if err != nil {
return err return err
} }

71
solver-next/cache_test.go
View File

@ -9,6 +9,14 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func depKeys(cks ...CacheKey) []CacheKeyWithSelector {
var keys []CacheKeyWithSelector
for _, ck := range cks {
keys = append(keys, CacheKeyWithSelector{CacheKey: ExportableCacheKey{CacheKey: ck}})
}
return keys
}
func TestInMemoryCache(t *testing.T) { func TestInMemoryCache(t *testing.T) {
ctx := context.TODO() ctx := context.TODO()
@ -17,7 +25,7 @@ func TestInMemoryCache(t *testing.T) {
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0, nil), testResult("result0")) cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0, nil), testResult("result0"))
require.NoError(t, err) require.NoError(t, err)
matches, err := m.Query(nil, 0, dgst("foo"), 0, "") matches, err := m.Query(nil, 0, dgst("foo"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -29,7 +37,7 @@ func TestInMemoryCache(t *testing.T) {
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0, nil), testResult("result1")) cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0, nil), testResult("result1"))
require.NoError(t, err) require.NoError(t, err)
matches, err = m.Query(nil, 0, dgst("bar"), 0, "") matches, err = m.Query(nil, 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -38,7 +46,7 @@ func TestInMemoryCache(t *testing.T) {
require.Equal(t, "result1", unwrap(res)) require.Equal(t, "result1", unwrap(res))
// invalid request // invalid request
matches, err = m.Query(nil, 0, dgst("baz"), 0, "") matches, err = m.Query(nil, 0, dgst("baz"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
@ -49,19 +57,19 @@ func TestInMemoryCache(t *testing.T) {
cacheBaz, err := m.Save(k, testResult("result2")) cacheBaz, err := m.Save(k, testResult("result2"))
require.NoError(t, err) require.NoError(t, err)
matches, err = m.Query(nil, 0, dgst("baz"), 0, "") matches, err = m.Query(nil, 0, dgst("baz"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("baz"), 0, "") matches, err = m.Query(depKeys(cacheFoo), 0, dgst("baz"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 1, dgst("baz"), Index(1), "") matches, err = m.Query(depKeys(cacheFoo), 1, dgst("baz"), Index(1))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("baz"), Index(1), "") matches, err = m.Query(depKeys(cacheFoo), 0, dgst("baz"), Index(1))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -69,7 +77,7 @@ func TestInMemoryCache(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "result2", unwrap(res)) require.Equal(t, "result2", unwrap(res))
matches2, err := m.Query([]ExportableCacheKey{cacheBar}, 1, dgst("baz"), Index(1), "") matches2, err := m.Query(depKeys(cacheBar), 1, dgst("baz"), Index(1))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches2), 1) require.Equal(t, len(matches2), 1)
@ -81,7 +89,7 @@ func TestInMemoryCache(t *testing.T) {
_, err = m.Save(k, testResult("result3")) _, err = m.Save(k, testResult("result3"))
require.NoError(t, err) require.NoError(t, err)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("baz"), Index(1), "") matches, err = m.Query(depKeys(cacheFoo), 0, dgst("baz"), Index(1))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 2) require.Equal(t, len(matches), 2)
@ -97,18 +105,18 @@ func TestInMemoryCache(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// foo, bar, baz should all point to result4 // foo, bar, baz should all point to result4
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bax"), 0, "") matches, err = m.Query(depKeys(cacheFoo), 0, dgst("bax"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
id := matches[0].ID id := matches[0].ID
matches, err = m.Query([]ExportableCacheKey{cacheBar}, 1, dgst("bax"), 0, "") matches, err = m.Query(depKeys(cacheBar), 1, dgst("bax"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
require.Equal(t, matches[0].ID, id) require.Equal(t, matches[0].ID, id)
matches, err = m.Query([]ExportableCacheKey{cacheBaz}, 0, dgst("bax"), 0, "") matches, err = m.Query(depKeys(cacheBaz), 0, dgst("bax"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
require.Equal(t, matches[0].ID, id) require.Equal(t, matches[0].ID, id)
@ -127,15 +135,15 @@ func TestInMemoryCacheSelector(t *testing.T) {
}), testResult("result1")) }), testResult("result1"))
require.NoError(t, err) require.NoError(t, err)
matches, err := m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bar"), 0, "") matches, err := m.Query(depKeys(cacheFoo), 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bar"), 0, "sel-invalid") matches, err = m.Query([]CacheKeyWithSelector{{Selector: "sel-invalid", CacheKey: ExportableCacheKey{CacheKey: cacheFoo}}}, 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bar"), 0, dgst("sel0")) matches, err = m.Query([]CacheKeyWithSelector{{Selector: dgst("sel0"), CacheKey: ExportableCacheKey{CacheKey: cacheFoo}}}, 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -154,7 +162,7 @@ func TestInMemoryCacheSelectorNested(t *testing.T) {
k2 := NewCacheKey("", 0, []CacheKeyWithSelector{ k2 := NewCacheKey("", 0, []CacheKeyWithSelector{
{CacheKey: cacheFoo, Selector: dgst("sel0")}, {CacheKey: cacheFoo, Selector: dgst("sel0")},
{CacheKey: ExportableCacheKey{CacheKey: NewCacheKey(dgst("second"), 0, nil)}, Selector: NoSelector}, {CacheKey: ExportableCacheKey{CacheKey: NewCacheKey(dgst("second"), 0, nil)}},
}) })
_, err = m.Save(NewCacheKey(dgst("bar"), 0, []CacheKeyWithSelector{ _, err = m.Save(NewCacheKey(dgst("bar"), 0, []CacheKeyWithSelector{
@ -162,34 +170,29 @@ func TestInMemoryCacheSelectorNested(t *testing.T) {
}), testResult("result1")) }), testResult("result1"))
require.NoError(t, err) require.NoError(t, err)
matches, err := m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bar"), 0, dgst("sel0")) matches, err := m.Query([]CacheKeyWithSelector{{Selector: dgst("sel0"), CacheKey: ExportableCacheKey{CacheKey: cacheFoo}}}, 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
res, err := m.Load(ctx, matches[0]) res, err := m.Load(ctx, matches[0])
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "result1", unwrap(res)) require.Equal(t, "result1", unwrap(res))
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bar"), 0, "") matches, err = m.Query(depKeys(cacheFoo), 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
matches, err = m.Query([]ExportableCacheKey{cacheFoo}, 0, dgst("bar"), 0, dgst("bar")) matches, err = m.Query([]CacheKeyWithSelector{{Selector: dgst("bar"), CacheKey: ExportableCacheKey{CacheKey: cacheFoo}}}, 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
// keys with NoSelector always match matches, err = m.Query(depKeys(NewCacheKey(dgst("second"), 0, nil)), 0, dgst("bar"), 0)
matches, err = m.Query([]ExportableCacheKey{{CacheKey: NewCacheKey(dgst("second"), 0, nil)}}, 0, dgst("bar"), 0, dgst("sel0"))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
res, err = m.Load(ctx, matches[0]) res, err = m.Load(ctx, matches[0])
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "result1", unwrap(res)) require.Equal(t, "result1", unwrap(res))
matches, err = m.Query([]ExportableCacheKey{{CacheKey: NewCacheKey(dgst("second"), 0, nil)}}, 0, dgst("bar"), 0, "") matches, err = m.Query(depKeys(NewCacheKey(dgst("second"), 0, nil)), 0, dgst("bar"), 0)
require.NoError(t, err)
require.Equal(t, len(matches), 1)
matches, err = m.Query([]ExportableCacheKey{{CacheKey: NewCacheKey(dgst("second"), 0, nil)}}, 0, dgst("bar"), 0, dgst("bar"))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
} }
@ -209,7 +212,7 @@ func TestInMemoryCacheReleaseParent(t *testing.T) {
}), res1) }), res1)
require.NoError(t, err) require.NoError(t, err)
matches, err := m.Query(nil, 0, dgst("foo"), 0, "") matches, err := m.Query(nil, 0, dgst("foo"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -219,13 +222,13 @@ func TestInMemoryCacheReleaseParent(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// foo becomes unloadable // foo becomes unloadable
matches, err = m.Query(nil, 0, dgst("foo"), 0, "") matches, err = m.Query(nil, 0, dgst("foo"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
require.False(t, matches[0].Loadable) require.False(t, matches[0].Loadable)
matches, err = m.Query([]ExportableCacheKey{matches[0].CacheKey}, 0, dgst("bar"), 0, "") matches, err = m.Query(depKeys(matches[0].CacheKey), 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -235,7 +238,7 @@ func TestInMemoryCacheReleaseParent(t *testing.T) {
err = storage.Release(res1.ID()) err = storage.Release(res1.ID())
require.NoError(t, err) require.NoError(t, err)
matches, err = m.Query(nil, 0, dgst("foo"), 0, "") matches, err = m.Query(nil, 0, dgst("foo"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 0) require.Equal(t, len(matches), 0)
} }
@ -263,12 +266,12 @@ func TestInMemoryCacheRestoreOfflineDeletion(t *testing.T) {
m = NewCacheManager(identity.NewID(), storage, results2) m = NewCacheManager(identity.NewID(), storage, results2)
matches, err := m.Query(nil, 0, dgst("foo"), 0, "") matches, err := m.Query(nil, 0, dgst("foo"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
require.False(t, matches[0].Loadable) require.False(t, matches[0].Loadable)
matches, err = m.Query([]ExportableCacheKey{matches[0].CacheKey}, 0, dgst("bar"), 0, "") matches, err = m.Query(depKeys(matches[0].CacheKey), 0, dgst("bar"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
@ -299,11 +302,11 @@ func TestCarryOverFromSublink(t *testing.T) {
{CacheKey: ExportableCacheKey{CacheKey: NewCacheKey(dgst("content0"), 0, nil)}, Selector: NoSelector}, {CacheKey: ExportableCacheKey{CacheKey: NewCacheKey(dgst("content0"), 0, nil)}, Selector: NoSelector},
}) })
matches, err := m.Query([]ExportableCacheKey{{CacheKey: k3}}, 0, dgst("res"), 0, "") matches, err := m.Query(depKeys(k3), 0, dgst("res"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
matches, err = m.Query([]ExportableCacheKey{{CacheKey: cacheBar}}, 0, dgst("res"), 0, dgst("sel0")) matches, err = m.Query([]CacheKeyWithSelector{{Selector: dgst("sel0"), CacheKey: ExportableCacheKey{CacheKey: cacheBar}}}, 0, dgst("res"), 0)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, len(matches), 1) require.Equal(t, len(matches), 1)
} }

2
solver-next/cachestorage.go
View File

@ -30,7 +30,7 @@ type CacheKeyInfo struct {
ID string ID string
Base digest.Digest Base digest.Digest
Output int Output int
Deps []CacheKeyInfoWithSelector // Deps []CacheKeyInfoWithSelector
} }
// CacheKeyInfoWithSelector is CacheKeyInfo combined with a selector // CacheKeyInfoWithSelector is CacheKeyInfo combined with a selector

90
solver-next/edge.go
View File

@ -164,14 +164,14 @@ func (e *edge) updateIncoming(req pipe.Sender) {
// probeCache is called with unprocessed cache keys for dependency // probeCache is called with unprocessed cache keys for dependency
// if the key could match the edge, the cacheRecords for dependency are filled // if the key could match the edge, the cacheRecords for dependency are filled
func (e *edge) probeCache(d *dep, keys []ExportableCacheKey) bool { func (e *edge) probeCache(d *dep, keys []CacheKeyWithSelector) bool {
if len(keys) == 0 { if len(keys) == 0 {
return false return false
} }
if e.op.IgnoreCache() { if e.op.IgnoreCache() {
return false return false
} }
records, err := e.op.Cache().Query(keys, d.index, e.cacheMap.Digest, e.edge.Index, e.cacheMap.Deps[d.index].Selector) records, err := e.op.Cache().Query(keys, d.index, e.cacheMap.Digest, e.edge.Index)
if err != nil { if err != nil {
e.err = errors.Wrap(err, "error on cache query") e.err = errors.Wrap(err, "error on cache query")
} }
@ -309,6 +309,14 @@ func (e *edge) unpark(incoming []pipe.Sender, updates, allPipes []pipe.Receiver,
} }
func withSelector(keys []ExportableCacheKey, selector digest.Digest) []CacheKeyWithSelector {
out := make([]CacheKeyWithSelector, len(keys))
for i, k := range keys {
out[i] = CacheKeyWithSelector{Selector: selector, CacheKey: k}
}
return out
}
// processUpdate is called by unpark for every updated pipe request // processUpdate is called by unpark for every updated pipe request
func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) { func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
// response for cachemap request // response for cachemap request
@ -322,7 +330,7 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
e.cacheMap = upt.Status().Value.(*CacheMap) e.cacheMap = upt.Status().Value.(*CacheMap)
if len(e.deps) == 0 { if len(e.deps) == 0 {
if !e.op.IgnoreCache() { if !e.op.IgnoreCache() {
records, err := e.op.Cache().Query(nil, 0, e.cacheMap.Digest, e.edge.Index, "") records, err := e.op.Cache().Query(nil, 0, e.cacheMap.Digest, e.edge.Index)
if err != nil { if err != nil {
logrus.Error(errors.Wrap(err, "invalid query response")) // make the build fail for this error logrus.Error(errors.Wrap(err, "invalid query response")) // make the build fail for this error
} else { } else {
@ -342,8 +350,8 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
e.state = edgeStatusCacheSlow e.state = edgeStatusCacheSlow
} }
// probe keys that were loaded before cache map // probe keys that were loaded before cache map
for _, dep := range e.deps { for i, dep := range e.deps {
e.probeCache(dep, dep.keys) e.probeCache(dep, withSelector(dep.keys, e.cacheMap.Deps[i].Selector))
e.checkDepMatchPossible(dep) e.checkDepMatchPossible(dep)
} }
} }
@ -377,9 +385,8 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
if len(dep.keys) < len(state.keys) { if len(dep.keys) < len(state.keys) {
newKeys := state.keys[len(dep.keys):] newKeys := state.keys[len(dep.keys):]
if e.cacheMap != nil { if e.cacheMap != nil {
e.probeCache(dep, newKeys) e.probeCache(dep, withSelector(newKeys, e.cacheMap.Deps[dep.index].Selector))
if e.allDepsHaveKeys() { if e.allDepsHaveKeys() {
e.keysDidChange = true e.keysDidChange = true
} }
@ -417,7 +424,7 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
{CacheKey: ExportableCacheKey{CacheKey: k, Exporter: &emptyExporter{k}}, Selector: NoSelector}, {CacheKey: ExportableCacheKey{CacheKey: k, Exporter: &emptyExporter{k}}, Selector: NoSelector},
}) })
dep.slowCacheKey = &ExportableCacheKey{ck, &emptyExporter{ck}} dep.slowCacheKey = &ExportableCacheKey{ck, &emptyExporter{ck}}
dep.slowCacheFoundKey = e.probeCache(dep, []ExportableCacheKey{*dep.slowCacheKey}) dep.slowCacheFoundKey = e.probeCache(dep, []CacheKeyWithSelector{{CacheKey: ExportableCacheKey{CacheKey: *dep.slowCacheKey}}})
dep.slowCacheComplete = true dep.slowCacheComplete = true
e.keysDidChange = true e.keysDidChange = true
e.checkDepMatchPossible(dep) // not matching key here doesn't set nocachematch possible to true e.checkDepMatchPossible(dep) // not matching key here doesn't set nocachematch possible to true
@ -456,9 +463,15 @@ func (e *edge) recalcCurrentState() {
} }
for k, r := range newRecords { for k, r := range newRecords {
e.keys = append(e.keys, r.CacheKey) mergedKey := r.CacheKey
if len(e.deps) > 0 {
mergedKey = toMergedCacheKey(e.deps, k)
}
e.keys = append(e.keys, mergedKey)
if r.Loadable { if r.Loadable {
e.cacheRecords[k] = r r2 := r
r2.CacheKey = mergedKey
e.cacheRecords[k] = r2
} }
} }
@ -714,24 +727,12 @@ func (e *edge) loadCache(ctx context.Context) (interface{}, error) {
rec = r rec = r
} }
} }
var exp Exporter
if len(e.deps) > 0 {
for _, dep := range e.deps {
if exp == nil {
exp = dep.cacheRecords[rec.ID].CacheKey
} else {
exp = &mergedExporter{exp, dep.cacheRecords[rec.ID].CacheKey}
}
}
} else {
exp = rec.CacheKey
}
logrus.Debugf("load cache for %s with %s", e.edge.Vertex.Name(), rec.ID) logrus.Debugf("load cache for %s with %s", e.edge.Vertex.Name(), rec.ID)
res, err := e.op.Cache().Load(ctx, rec) res, err := e.op.Cache().Load(ctx, rec)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return NewCachedResult(res, rec.CacheKey, exp), nil return NewCachedResult(res, rec.CacheKey, rec.CacheKey), nil
} }
// execOp creates a request to execute the vertex operation // execOp creates a request to execute the vertex operation
@ -810,20 +811,37 @@ func (e *emptyExporter) Export(ctx context.Context, m map[digest.Digest]*ExportR
} }
type mergedExporter struct { type mergedExporter struct {
e1, e2 Exporter exporters []Exporter
} }
func (e *mergedExporter) Export(ctx context.Context, m map[digest.Digest]*ExportRecord, fn func(context.Context, Result) (*Remote, error)) (*ExportRecord, error) { func (e *mergedExporter) Export(ctx context.Context, m map[digest.Digest]*ExportRecord, fn func(context.Context, Result) (*Remote, error)) (er *ExportRecord, err error) {
for _, e := range e.exporters {
_, err := e.e1.Export(ctx, m, fn) er, err = e.Export(ctx, m, fn)
if err != nil { if err != nil {
return nil, err return nil, err
}
} }
return
r, err := e.e2.Export(ctx, m, fn) }
if err != nil {
return nil, err func toMergedCacheKey(deps []*dep, id string) ExportableCacheKey {
} depKeys := make([]CacheKeyWithSelector, len(deps))
exporters := make([]Exporter, len(deps))
return r, nil for i, d := range deps {
depKeys[i] = d.cacheRecords[id].CacheKey.Deps()[i]
exporters[i] = d.cacheRecords[id].CacheKey
}
return ExportableCacheKey{
CacheKey: &mergedCacheKey{CacheKey: deps[0].cacheRecords[id].CacheKey, deps: depKeys},
Exporter: &mergedExporter{exporters: exporters},
}
}
type mergedCacheKey struct {
CacheKey
deps []CacheKeyWithSelector
}
func (ck *mergedCacheKey) Deps() []CacheKeyWithSelector {
return ck.deps
} }

2
solver-next/types.go
View File

@ -143,7 +143,7 @@ type CacheManager interface {
ID() string ID() string
// Query searches for cache paths from one cache key to the output of a // Query searches for cache paths from one cache key to the output of a
// possible match. // possible match.
Query(inp []ExportableCacheKey, inputIndex Index, dgst digest.Digest, outputIndex Index, selector digest.Digest) ([]*CacheRecord, error) Query(inp []CacheKeyWithSelector, inputIndex Index, dgst digest.Digest, outputIndex Index) ([]*CacheRecord, error)
// Load pulls and returns the cached result // Load pulls and returns the cached result
Load(ctx context.Context, rec *CacheRecord) (Result, error) Load(ctx context.Context, rec *CacheRecord) (Result, error)
// Save saves a result based on a cache key // Save saves a result based on a cache key