From b8b55a8c22daef10eb1dbc8b56d4ad649ca0f325 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 4 May 2018 13:29:04 -0700 Subject: [PATCH] image: export reproducible timestamps Signed-off-by: Tonis Tiigi --- cache/manager.go | 14 ++--- cache/metadata.go | 6 +-- exporter/containerimage/writer.go | 54 +++++++++++++------ frontend/dockerfile/dockerfile2llb/convert.go | 4 +- 4 files changed, 48 insertions(+), 30 deletions(-) diff --git a/cache/manager.go b/cache/manager.go index 2e2dff57..869345fd 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -340,7 +340,7 @@ func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo) err Mutable: cr.mutable, InUse: len(cr.refs) > 0, Size: getSize(cr.md), - CreatedAt: getCreatedAt(cr.md), + CreatedAt: GetCreatedAt(cr.md), Description: GetDescription(cr.md), LastUsedAt: lastUsedAt, UsageCount: usageCount, @@ -417,7 +417,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) refs: len(cr.refs), mutable: cr.mutable, size: getSize(cr.md), - createdAt: getCreatedAt(cr.md), + createdAt: GetCreatedAt(cr.md), usageCount: usageCount, lastUsedAt: lastUsedAt, description: GetDescription(cr.md), @@ -538,19 +538,19 @@ func WithDescription(descr string) RefOption { func initializeMetadata(m withMetadata, opts ...RefOption) error { md := m.Metadata() - if tm := getCreatedAt(md); !tm.IsZero() { + if tm := GetCreatedAt(md); !tm.IsZero() { return nil } + if err := queueCreatedAt(md, time.Now()); err != nil { + return err + } + for _, opt := range opts { if err := opt(m); err != nil { return err } } - if err := queueCreatedAt(md); err != nil { - return err - } - return md.Commit() } diff --git a/cache/metadata.go b/cache/metadata.go index 2bbdb4f2..787b5a5b 100644 --- a/cache/metadata.go +++ b/cache/metadata.go @@ -141,8 +141,8 @@ func GetDescription(si *metadata.StorageItem) string { return str } -func queueCreatedAt(si *metadata.StorageItem) error { - v, err := metadata.NewValue(time.Now().UnixNano()) +func queueCreatedAt(si *metadata.StorageItem, tm time.Time) error { + v, err := metadata.NewValue(tm.UnixNano()) if err != nil { return errors.Wrap(err, "failed to create createdAt value") } @@ -152,7 +152,7 @@ func queueCreatedAt(si *metadata.StorageItem) error { return nil } -func getCreatedAt(si *metadata.StorageItem) time.Time { +func GetCreatedAt(si *metadata.StorageItem) time.Time { v := si.Get(keyCreatedAt) if v == nil { return time.Time{} diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index e946c8e2..e2bf4307 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "runtime" - "strings" "time" "github.com/containerd/containerd/content" @@ -179,24 +178,35 @@ func patchImageConfig(dt []byte, dps []blobs.DiffPair, history []ocispec.History } m["history"] = dt - now := time.Now() - dt, err = json.Marshal(&now) - if err != nil { - return nil, errors.Wrap(err, "failed to marshal creation time") + if _, ok := m["created"]; !ok { + var tm *time.Time + for _, h := range history { + if h.Created != nil { + tm = h.Created + } + } + dt, err = json.Marshal(&tm) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal creation time") + } + m["created"] = dt } - m["created"] = dt dt, err = json.Marshal(m) return dt, errors.Wrap(err, "failed to marshal config after patch") } func normalizeLayersAndHistory(diffs []blobs.DiffPair, history []ocispec.History, ref cache.ImmutableRef) ([]blobs.DiffPair, []ocispec.History) { + + refMeta := getRefMetadata(ref, len(diffs)) + var historyLayers int for _, h := range history { if !h.EmptyLayer { historyLayers += 1 } } + if historyLayers > len(diffs) { // this case shouldn't happen but if it does force set history layers empty // from the bottom @@ -217,11 +227,10 @@ func normalizeLayersAndHistory(diffs []blobs.DiffPair, history []ocispec.History if len(diffs) > historyLayers { // some history items are missing. add them based on the ref metadata - for _, msg := range getRefDesciptions(ref, len(diffs)-historyLayers) { - tm := time.Now().UTC() + for _, md := range refMeta[historyLayers:] { history = append(history, ocispec.History{ - Created: &tm, - CreatedBy: msg, + Created: &md.createdAt, + CreatedBy: md.description, Comment: "buildkit.exporter.image.v0", }) } @@ -230,6 +239,9 @@ func normalizeLayersAndHistory(diffs []blobs.DiffPair, history []ocispec.History var layerIndex int for i, h := range history { if !h.EmptyLayer { + if h.Created == nil { + h.Created = &refMeta[layerIndex].createdAt + } if diffs[layerIndex].Blobsum == emptyGZLayer { h.EmptyLayer = true diffs = append(diffs[:layerIndex], diffs[layerIndex+1:]...) @@ -243,23 +255,31 @@ func normalizeLayersAndHistory(diffs []blobs.DiffPair, history []ocispec.History return diffs, history } -func getRefDesciptions(ref cache.ImmutableRef, limit int) []string { +type refMetadata struct { + description string + createdAt time.Time +} + +func getRefMetadata(ref cache.ImmutableRef, limit int) []refMetadata { if limit <= 0 { return nil } - defaultMsg := "created by buildkit" // shouldn't happen but don't fail build + meta := refMetadata{ + description: "created by buildkit", // shouldn't be shown but don't fail build + createdAt: time.Now(), + } if ref == nil { - strings.Repeat(defaultMsg, limit) + return append(getRefMetadata(nil, limit-1), meta) } - descr := cache.GetDescription(ref.Metadata()) - if descr == "" { - descr = defaultMsg + if descr := cache.GetDescription(ref.Metadata()); descr != "" { + meta.description = descr } + meta.createdAt = cache.GetCreatedAt(ref.Metadata()) p := ref.Parent() if p != nil { defer p.Release(context.TODO()) } - return append(getRefDesciptions(p, limit-1), descr) + return append(getRefMetadata(p, limit-1), meta) } func oneOffProgress(ctx context.Context, id string) func(err error) error { diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 7b177125..8dfe9fd3 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -11,7 +11,6 @@ import ( "sort" "strconv" "strings" - "time" "github.com/docker/distribution/reference" "github.com/docker/docker/builder/dockerfile/instructions" @@ -146,6 +145,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, if err := json.Unmarshal(dt, &img); err != nil { return err } + img.Created = nil d.image = img if dgst != "" { ref, err = reference.WithDigest(ref, dgst) @@ -729,9 +729,7 @@ func commitToHistory(img *Image, msg string, withLayer bool, st *llb.State) erro msg += " # buildkit" } - tm := time.Now().UTC() img.History = append(img.History, ocispec.History{ - Created: &tm, CreatedBy: msg, Comment: historyComment, EmptyLayer: !withLayer,