JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cache: maintain creation time with remote cache 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2018-05-04 22:18:11 -07:00
parent 439877f59c
commit a7bc9b9fd2
4 changed files with 152 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cache/cacheimport/v1/doc.go vendored
View File

@ -7,7 +7,8 @@ package cacheimport
// Manifests array contains descriptors to the cache layers and one instance of
// build cache config with media type application/vnd.buildkit.cacheconfig.v0 .
// The cache layer descripts need to have an annotation with uncompressed digest
// to allow deduplication on extraction.
// to allow deduplication on extraction and optionally "buildkit/createdat"
// annotation to support maintaining original timestamps.
//
// Cache config file layout:
//

6
cache/manager.go vendored
View File

@ -547,6 +547,12 @@ func WithDescription(descr string) RefOption {
}
}
func WithCreationTime(tm time.Time) RefOption {
return func(m withMetadata) error {
return queueCreatedAt(m.Metadata(), tm)
}
}
func initializeMetadata(m withMetadata, opts ...RefOption) error {
md := m.Metadata()
if tm := GetCreatedAt(md); !tm.IsZero() {

95
frontend/dockerfile/dockerfile_test.go
View File

@ -54,6 +54,7 @@ func TestIntegration(t *testing.T) {
testLabels,
testCacheImportExport,
testReproducibleIDs,
testImportExportReproducibleIDs,
})
}
@ -1309,7 +1310,7 @@ COPY --from=base unique /
target := registry + "/buildkit/testexportdf:latest"
err = c.Solve(context.TODO(), nil, client.SolveOpt{
_, err = c.Solve(context.TODO(), nil, client.SolveOpt{
Frontend: "dockerfile.v0",
Exporter: client.ExporterLocal,
ExporterOutputDir: destDir,
@ -1337,7 +1338,7 @@ COPY --from=base unique /
require.NoError(t, err)
defer os.RemoveAll(destDir)
err = c.Solve(context.TODO(), nil, client.SolveOpt{
_, err = c.Solve(context.TODO(), nil, client.SolveOpt{
Frontend: "dockerfile.v0",
FrontendAttrs: map[string]string{
"cache-from": target,
@ -1434,6 +1435,96 @@ RUN echo bar > bar
require.Equal(t, img.Target, img2.Target)
}
func testImportExportReproducibleIDs(t *testing.T, sb integration.Sandbox) {
var cdAddress string
if cd, ok := sb.(interface {
ContainerdAddress() string
}); !ok {
t.Skip("only for containerd worker")
} else {
cdAddress = cd.ContainerdAddress()
}
t.Parallel()
registry, err := sb.NewRegistry()
if errors.Cause(err) == integration.ErrorRequirements {
t.Skip(err.Error())
}
require.NoError(t, err)
dockerfile := []byte(`
FROM busybox
ENV foo=bar
COPY foo /
RUN echo bar > bar
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateFile("foo", []byte("foobar"), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
target := "example.com/moby/dockerfileexpids:test"
cacheTarget := registry + "/test/dockerfileexpids:cache"
opt := client.SolveOpt{
Frontend: "dockerfile.v0",
FrontendAttrs: map[string]string{},
Exporter: client.ExporterImage,
ExportCache: cacheTarget,
ExporterAttrs: map[string]string{
"name": target,
},
LocalDirs: map[string]string{
builder.LocalNameDockerfile: dir,
builder.LocalNameContext: dir,
},
}
client, err := containerd.New(cdAddress)
require.NoError(t, err)
defer client.Close()
ctx := namespaces.WithNamespace(context.Background(), "buildkit")
_, err = c.Solve(context.TODO(), nil, opt, nil)
require.NoError(t, err)
img, err := client.ImageService().Get(ctx, target)
require.NoError(t, err)
err = client.ImageService().Delete(ctx, target)
require.NoError(t, err)
err = c.Prune(context.TODO(), nil)
require.NoError(t, err)
checkAllRemoved(t, c, sb)
target2 := "example.com/moby/dockerfileexpids2:test"
opt.ExporterAttrs["name"] = target2
opt.FrontendAttrs["cache-from"] = cacheTarget
_, err = c.Solve(context.TODO(), nil, opt, nil)
require.NoError(t, err)
img2, err := client.ImageService().Get(ctx, target2)
require.NoError(t, err)
require.Equal(t, img.Target, img2.Target)
}
func tmpdir(appliers ...fstest.Applier) (string, error) {
tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile")
if err != nil {

58
worker/base/worker.go
View File

@ -47,6 +47,8 @@ import (
"golang.org/x/sync/errgroup"
)
const labelCreatedAt = "buildkit/createdat"
// TODO: this file should be removed. containerd defines ContainerdWorker, oci defines OCIWorker. There is no base worker.
// WorkerOpt is specific to a worker.
@ -263,6 +265,11 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver
return nil, nil
}
createdTimes := getCreatedTimes(ref)
if len(createdTimes) != len(diffPairs) {
return nil, errors.Errorf("invalid createdtimes/diffpairs")
}
descs := make([]ocispec.Descriptor, len(diffPairs))
for i, dp := range diffPairs {
@ -270,12 +277,19 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver
if err != nil {
return nil, err
}
tm, err := createdTimes[i].MarshalText()
if err != nil {
return nil, err
}
descs[i] = ocispec.Descriptor{
Digest: dp.Blobsum,
Size: info.Size,
MediaType: schema2.MediaTypeLayer,
Annotations: map[string]string{
"containerd.io/uncompressed": dp.DiffID.String(),
labelCreatedAt: string(tm),
},
}
}
@ -286,6 +300,15 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver
}, nil
}
func getCreatedTimes(ref cache.ImmutableRef) (out []time.Time) {
parent := ref.Parent()
if parent != nil {
defer parent.Release(context.TODO())
out = getCreatedTimes(parent)
}
return append(out, cache.GetCreatedAt(ref.Metadata()))
}
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
eg, gctx := errgroup.WithContext(ctx)
for _, desc := range remote.Descriptors {
@ -305,19 +328,35 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
defer release()
unpackProgressDone := oneOffProgress(ctx, "unpacking")
chainID, err := w.unpack(ctx, remote.Descriptors, cs)
chainIDs, err := w.unpack(ctx, remote.Descriptors, cs)
if err != nil {
return nil, unpackProgressDone(err)
}
unpackProgressDone(nil)
return w.CacheManager.Get(ctx, chainID, cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
for i, chainID := range chainIDs {
tm := time.Now()
if tmstr, ok := remote.Descriptors[i].Annotations[labelCreatedAt]; ok {
if err := (&tm).UnmarshalText([]byte(tmstr)); err != nil {
return nil, err
}
}
ref, err := w.CacheManager.Get(ctx, chainID, cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[i].Digest)), cache.WithCreationTime(tm))
if err != nil {
return nil, err
}
if i == len(remote.Descriptors)-1 {
return ref, nil
}
ref.Release(context.TODO())
}
return nil, errors.Errorf("unreachable")
}
func (w *Worker) unpack(ctx context.Context, descs []ocispec.Descriptor, s cdsnapshot.Snapshotter) (string, error) {
func (w *Worker) unpack(ctx context.Context, descs []ocispec.Descriptor, s cdsnapshot.Snapshotter) ([]string, error) {
layers, err := getLayers(ctx, descs)
if err != nil {
return "", err
return nil, err
}
var chain []digest.Digest
@ -326,17 +365,22 @@ func (w *Worker) unpack(ctx context.Context, descs []ocispec.Descriptor, s cdsna
"containerd.io/uncompressed": layer.Diff.Digest.String(),
}
if _, err := rootfs.ApplyLayer(ctx, layer, chain, s, w.Applier, cdsnapshot.WithLabels(labels)); err != nil {
return "", err
return nil, err
}
chain = append(chain, layer.Diff.Digest)
chainID := ociidentity.ChainID(chain)
if err := w.Snapshotter.SetBlob(ctx, string(chainID), layer.Diff.Digest, layer.Blob.Digest); err != nil {
return "", err
return nil, err
}
}
return string(ociidentity.ChainID(chain)), nil
ids := make([]string, len(chain))
for i := range chain {
ids[i] = string(ociidentity.ChainID(chain[:i+1]))
}
return ids, nil
}
// utility function. could be moved to the constructor logic?