JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

image: set up history properly for images 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2017-12-08 10:53:17 -08:00
parent 1f93a77f78
commit 6ffb48395b
4 changed files with 142 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cache/manager.go vendored
View File

@ -288,7 +288,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo)
createdAt: getCreatedAt(cr.md),
usageCount: usageCount,
lastUsedAt: lastUsedAt,
description: getDescription(cr.md),
description: GetDescription(cr.md),
}
if cr.parent != nil {
c.parent = cr.parent.ID()

2
cache/metadata.go vendored
View File

@ -111,7 +111,7 @@ func queueDescription(si *metadata.StorageItem, descr string) error {
return nil
}
func getDescription(si *metadata.StorageItem) string {
func GetDescription(si *metadata.StorageItem) string {
v := si.Get(keyDescription)
if v == nil {
return ""

2
cache/refs.go vendored
View File

@ -256,7 +256,7 @@ func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
md: md,
}
if descr := getDescription(sr.md); descr != "" {
if descr := GetDescription(sr.md); descr != "" {
if err := queueDescription(md, descr); err != nil {
return nil, err
}

187
exporter/containerimage/export.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"runtime"
"strings"
"time"
"github.com/containerd/containerd/content"
@ -32,6 +33,8 @@ const (
keyPush = "push"
keyInsecure = "registry.insecure"
exporterImageConfig = "containerimage.config"
emptyGZLayer = digest.Digest("sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1")
)
type Opt struct {
@ -62,7 +65,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
case keyInsecure:
i.insecure = true
default:
logrus.Warnf("unknown exporter option %s", k)
logrus.Warnf("image exporter: unknown option %s", k)
}
}
return i, nil
@ -83,44 +86,46 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
layersDone := oneOffProgress(ctx, "exporting layers")
diffPairs, err := blobs.GetDiffPairs(ctx, e.opt.ContentStore, e.opt.Snapshotter, e.opt.Differ, ref)
if err != nil {
return err
return errors.Wrap(err, "failed calculaing diff pairs for exported snapshot")
}
layersDone(nil)
diffIDs := make([]digest.Digest, 0, len(diffPairs))
for _, dp := range diffPairs {
diffIDs = append(diffIDs, dp.DiffID)
}
var dt []byte
if config, ok := opt[exporterImageConfig]; ok {
dt, err = setDiffIDs(config, diffIDs)
config, ok := opt[exporterImageConfig]
if !ok {
config, err = emptyImageConfig()
if err != nil {
return err
}
} else {
dt, err = json.Marshal(imageConfig(diffIDs))
if err != nil {
return errors.Wrap(err, "failed to marshal image config")
}
}
history, err := parseHistoryFromConfig(config)
if err != nil {
return err
}
diffPairs, history = normalizeLayersAndHistory(diffPairs, history, ref)
config, err = patchImageConfig(config, diffPairs, history)
if err != nil {
return err
}
addAsRoot := content.WithLabels(map[string]string{
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
})
dgst := digest.FromBytes(dt)
configDone := oneOffProgress(ctx, "exporting config "+dgst.String())
configDigest := digest.FromBytes(config)
configDone := oneOffProgress(ctx, "exporting config "+configDigest.String())
if err := content.WriteBlob(ctx, e.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst, addAsRoot); err != nil {
if err := content.WriteBlob(ctx, e.opt.ContentStore, configDigest.String(), bytes.NewReader(config), int64(len(config)), configDigest, addAsRoot); err != nil {
return configDone(errors.Wrap(err, "error writing config blob"))
}
configDone(nil)
mfst := schema2.Manifest{
Config: distribution.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
Digest: configDigest,
Size: int64(len(config)),
MediaType: schema2.MediaTypeImageConfig,
},
}
@ -130,7 +135,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
for _, dp := range diffPairs {
info, err := e.opt.ContentStore.Info(ctx, dp.Blobsum)
if err != nil {
return configDone(errors.Wrapf(err, "could not get blob %s", dp.Blobsum))
return configDone(errors.Wrapf(err, "could not find blob %s from contentstore", dp.Blobsum))
}
mfst.Layers = append(mfst.Layers, distribution.Descriptor{
Digest: dp.Blobsum,
@ -139,80 +144,166 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
})
}
dt, err = json.Marshal(mfst)
mfstJSON, err := json.Marshal(mfst)
if err != nil {
return errors.Wrap(err, "failed to marshal manifest")
}
dgst = digest.FromBytes(dt)
mfstDone := oneOffProgress(ctx, "exporting manifest "+dgst.String())
mfstDigest := digest.FromBytes(mfstJSON)
mfstDone := oneOffProgress(ctx, "exporting manifest "+mfstDigest.String())
if err := content.WriteBlob(ctx, e.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst, addAsRoot); err != nil {
return mfstDone(errors.Wrap(err, "error writing manifest blob"))
if err := content.WriteBlob(ctx, e.opt.ContentStore, mfstDigest.String(), bytes.NewReader(mfstJSON), int64(len(mfstJSON)), mfstDigest, addAsRoot); err != nil {
return mfstDone(errors.Wrapf(err, "error writing manifest blob %s", mfstDigest))
}
mfstDone(nil)
if e.targetName != "" {
if e.opt.Images != nil {
tagDone := oneOffProgress(ctx, "naming to "+e.targetName)
imgrec := images.Image{
img := images.Image{
Name: e.targetName,
Target: ocispec.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
Digest: mfstDigest,
Size: int64(len(mfstJSON)),
MediaType: ocispec.MediaTypeImageManifest,
},
CreatedAt: time.Now(),
}
_, err := e.opt.Images.Update(ctx, imgrec)
if err != nil {
if _, err := e.opt.Images.Update(ctx, img); err != nil {
if !errdefs.IsNotFound(err) {
return tagDone(err)
}
_, err := e.opt.Images.Create(ctx, imgrec)
if err != nil {
if _, err := e.opt.Images.Create(ctx, img); err != nil {
return tagDone(err)
}
}
tagDone(nil)
}
if e.push {
return push.Push(ctx, e.opt.SessionManager, e.opt.ContentStore, dgst, e.targetName, e.insecure)
return push.Push(ctx, e.opt.SessionManager, e.opt.ContentStore, mfstDigest, e.targetName, e.insecure)
}
}
return err
return nil
}
// this is temporary: should move to dockerfile frontend
func imageConfig(diffIDs []digest.Digest) ocispec.Image {
func emptyImageConfig() ([]byte, error) {
img := ocispec.Image{
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
}
img.RootFS.Type = "layers"
img.RootFS.DiffIDs = diffIDs
img.Config.WorkingDir = "/"
img.Config.Env = []string{"PATH=" + system.DefaultPathEnv}
return img
dt, err := json.Marshal(img)
return dt, errors.Wrap(err, "failed to create empty image config")
}
func setDiffIDs(config []byte, diffIDs []digest.Digest) ([]byte, error) {
mp := map[string]json.RawMessage{}
if err := json.Unmarshal(config, &mp); err != nil {
return nil, err
func parseHistoryFromConfig(dt []byte) ([]ocispec.History, error) {
var config struct {
History []ocispec.History
}
if err := json.Unmarshal(dt, &config); err != nil {
return nil, errors.Wrap(err, "failed to unmarshal history from config")
}
return config.History, nil
}
func patchImageConfig(dt []byte, dps []blobs.DiffPair, history []ocispec.History) ([]byte, error) {
m := map[string]json.RawMessage{}
if err := json.Unmarshal(dt, &m); err != nil {
return nil, errors.Wrap(err, "failed to parse image config for patch")
}
var rootFS ocispec.RootFS
rootFS.Type = "layers"
rootFS.DiffIDs = diffIDs
for _, dp := range dps {
rootFS.DiffIDs = append(rootFS.DiffIDs, dp.DiffID)
}
dt, err := json.Marshal(rootFS)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "failed to marshal rootfs")
}
mp["rootfs"] = dt
return json.Marshal(mp)
m["rootfs"] = dt
dt, err = json.Marshal(history)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal history")
}
m["history"] = dt
dt, err = json.Marshal(m)
return dt, errors.Wrap(err, "failed to marshal config after patch")
}
func normalizeLayersAndHistory(diffs []blobs.DiffPair, history []ocispec.History, ref cache.ImmutableRef) ([]blobs.DiffPair, []ocispec.History) {
var historyLayers int
for _, h := range history {
if !h.EmptyLayer {
historyLayers += 1
}
}
if historyLayers > len(diffs) {
// this case shouldn't happen but if it does force set history layers empty
// from the bottom
logrus.Warn("invalid image config with unaccounted layers")
historyCopy := make([]ocispec.History, 0, len(history))
var l int
for _, h := range history {
if l >= len(diffs) {
h.EmptyLayer = true
}
if !h.EmptyLayer {
l++
}
historyCopy = append(historyCopy, h)
}
history = historyCopy
}
if len(diffs) > historyLayers {
// some history items are missing. add them based on the ref metadata
for _, msg := range getRefDesciptions(ref, len(diffs)-historyLayers) {
tm := time.Now().UTC()
history = append(history, ocispec.History{
Created: &tm,
CreatedBy: msg,
Comment: "buildkit.exporter.image.v0",
})
}
}
var layerIndex int
for i, h := range history {
if !h.EmptyLayer {
if diffs[layerIndex].Blobsum == emptyGZLayer {
h.EmptyLayer = true
diffs = append(diffs[:layerIndex], diffs[layerIndex+1:]...)
} else {
layerIndex++
}
}
history[i] = h
}
return diffs, history
}
func getRefDesciptions(ref cache.ImmutableRef, limit int) []string {
if limit <= 0 {
return nil
}
defaultMsg := "created by buildkit" // shouldn't happen but don't fail build
if ref == nil {
strings.Repeat(defaultMsg, limit)
}
descr := cache.GetDescription(ref.Metadata())
if descr == "" {
descr = defaultMsg
}
return append(getRefDesciptions(ref.Parent(), limit-1), descr)
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {