JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

diff: implement windows layer support for linux 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2018-07-14 17:07:39 -07:00
parent 98f1604134
commit bc765861be
8 changed files with 533 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
cache/blobs/blobs.go vendored
View File

@ -10,6 +10,7 @@ import (
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/winlayers"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@ -36,6 +37,18 @@ func GetDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
return nil, err
}
if isTypeWindows(ref) {
ctx = winlayers.UseWindowsLayerMode(ctx)
}
return getDiffPairs(ctx, contentStore, snapshotter, differ, ref, createBlobs)
}
func getDiffPairs(ctx context.Context, contentStore content.Store, snapshotter snapshot.Snapshotter, differ diff.Comparer, ref cache.ImmutableRef, createBlobs bool) ([]DiffPair, error) {
if ref == nil {
return nil, nil
}
eg, ctx := errgroup.WithContext(ctx)
var diffPairs []DiffPair
var currentPair DiffPair
@ -43,7 +56,7 @@ func GetDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
if parent != nil {
defer parent.Release(context.TODO())
eg.Go(func() error {
dp, err := GetDiffPairs(ctx, contentStore, snapshotter, differ, parent, createBlobs)
dp, err := getDiffPairs(ctx, contentStore, snapshotter, differ, parent, createBlobs)
if err != nil {
return err
}
@ -125,3 +138,14 @@ func GetDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
}
return append(diffPairs, currentPair), nil
}
func isTypeWindows(ref cache.ImmutableRef) bool {
if cache.GetLayerType(ref) == "windows" {
return true
}
if parent := ref.Parent(); parent != nil {
defer parent.Release(context.TODO())
return isTypeWindows(parent)
}
return false
}

24
cache/metadata.go vendored
View File

@ -16,6 +16,7 @@ const keyDescription = "cache.description"
const keyCreatedAt = "cache.createdAt"
const keyLastUsedAt = "cache.lastUsedAt"
const keyUsageCount = "cache.usageCount"
const keyLayerType = "cache.layerType"
const keyDeleted = "cache.deleted"
@ -204,3 +205,26 @@ func updateLastUsed(si *metadata.StorageItem) error {
return si.SetValue(b, keyLastUsedAt, v2)
})
}
func SetLayerType(m withMetadata, value string) error {
v, err := metadata.NewValue(value)
if err != nil {
return errors.Wrap(err, "failed to create layertype value")
}
m.Metadata().Queue(func(b *bolt.Bucket) error {
return m.Metadata().SetValue(b, keyLayerType, v)
})
return m.Metadata().Commit()
}
func GetLayerType(m withMetadata) string {
v := m.Metadata().Get(keyLayerType)
if v == nil {
return ""
}
var str string
if err := v.Unmarshal(&str); err != nil {
return ""
}
return str
}

34
source/containerimage/pull.go
View File

@ -3,7 +3,7 @@ package containerimage
import (
"context"
"encoding/json"
"fmt"
"runtime"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
@ -17,6 +17,7 @@ import (
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/pull"
"github.com/moby/buildkit/util/winlayers"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
specs "github.com/opencontainers/image-spec/specs-go/v1"
@ -155,6 +156,14 @@ func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error)
}
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
layerNeedsTypeWindows := false
if platform := p.Puller.Platform; platform != nil {
if platform.OS == "windows" && runtime.GOOS != "windows" {
ctx = winlayers.UseWindowsLayerMode(ctx)
layerNeedsTypeWindows = true
}
}
pulled, err := p.Puller.Pull(ctx)
if err != nil {
return nil, err
@ -162,7 +171,28 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
if pulled.ChainID == "" {
return nil, nil
}
return p.CacheAccessor.GetFromSnapshotter(ctx, string(pulled.ChainID), cache.WithDescription(fmt.Sprintf("pulled from %s", pulled.Ref)))
ref, err := p.CacheAccessor.GetFromSnapshotter(ctx, string(pulled.ChainID), cache.WithDescription("pulled from "+pulled.Ref))
if err != nil {
return nil, err
}
if layerNeedsTypeWindows && ref != nil {
if err := markRefLayerTypeWindows(ref); err != nil {
ref.Release(context.TODO())
return nil, err
}
}
return ref, nil
}
func markRefLayerTypeWindows(ref cache.ImmutableRef) error {
if parent := ref.Parent(); parent != nil {
defer parent.Release(context.TODO())
if err := markRefLayerTypeWindows(parent); err != nil {
return err
}
}
return cache.SetLayerType(ref, "windows")
}
// cacheKeyFromConfig returns a stable digest from image config. If image config

153
util/winlayers/applier.go Normal file
View File

@ -0,0 +1,153 @@
package winlayers
import (
"archive/tar"
"context"
"io"
"io/ioutil"
"runtime"
"strings"
"github.com/containerd/containerd/archive"
"github.com/containerd/containerd/archive/compression"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/mount"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func NewFileSystemApplierWithWindows(cs content.Provider, a diff.Applier) diff.Applier {
if runtime.GOOS == "windows" {
return a
}
return &winApplier{
cs: cs,
a: a,
}
}
type winApplier struct {
cs content.Provider
a diff.Applier
}
func (s *winApplier) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount) (d ocispec.Descriptor, err error) {
if !hasWindowsLayerMode(ctx) {
return s.a.Apply(ctx, desc, mounts)
}
isCompressed, err := images.IsCompressedDiff(ctx, desc.MediaType)
if err != nil {
return ocispec.Descriptor{}, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
}
var ocidesc ocispec.Descriptor
if err := mount.WithTempMount(ctx, mounts, func(root string) error {
ra, err := s.cs.ReaderAt(ctx, desc)
if err != nil {
return errors.Wrap(err, "failed to get reader from content store")
}
defer ra.Close()
r := content.NewReader(ra)
if isCompressed {
ds, err := compression.DecompressStream(r)
if err != nil {
return err
}
defer ds.Close()
r = ds
}
digester := digest.Canonical.Digester()
rc := &readCounter{
r: io.TeeReader(r, digester.Hash()),
}
rc2 := filter(rc, func(hdr *tar.Header) bool {
if strings.HasPrefix(hdr.Name, "Files/") {
hdr.Name = strings.TrimPrefix(hdr.Name, "Files/")
hdr.Linkname = strings.TrimPrefix(hdr.Linkname, "Files/")
// TODO: could convert the windows PAX headers to xattr here to reuse
// the original ones in diff for parent directories and file modifications
return true
}
return false
})
if _, err := archive.Apply(ctx, root, rc2); err != nil {
return err
}
// Read any trailing data
if _, err := io.Copy(ioutil.Discard, rc); err != nil {
return err
}
ocidesc = ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayer,
Size: rc.c,
Digest: digester.Digest(),
}
return nil
}); err != nil {
return ocispec.Descriptor{}, err
}
return ocidesc, nil
}
type readCounter struct {
r io.Reader
c int64
}
func (rc *readCounter) Read(p []byte) (n int, err error) {
n, err = rc.r.Read(p)
rc.c += int64(n)
return
}
func filter(in io.Reader, f func(*tar.Header) bool) io.Reader {
pr, pw := io.Pipe()
go func() {
tarReader := tar.NewReader(in)
tarWriter := tar.NewWriter(pw)
pw.CloseWithError(func() error {
for {
h, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if f(h) {
if err := tarWriter.WriteHeader(h); err != nil {
return err
}
if h.Size > 0 {
if _, err := io.Copy(tarWriter, tarReader); err != nil {
return err
}
}
} else {
if h.Size > 0 {
if _, err := io.Copy(ioutil.Discard, tarReader); err != nil {
return err
}
}
}
}
return tarWriter.Close()
}())
}()
return pr
}

19
util/winlayers/context.go Normal file
View File

@ -0,0 +1,19 @@
package winlayers
import "context"
type contextKeyT string
var contextKey = contextKeyT("buildkit/winlayers-on")
func UseWindowsLayerMode(ctx context.Context) context.Context {
return context.WithValue(ctx, contextKey, true)
}
func hasWindowsLayerMode(ctx context.Context) bool {
v := ctx.Value(contextKey)
if v == nil {
return false
}
return true
}

274
util/winlayers/differ.go Normal file
View File

@ -0,0 +1,274 @@
package winlayers
import (
"archive/tar"
"context"
"crypto/rand"
"encoding/base64"
"fmt"
"io"
"time"
"github.com/containerd/containerd/archive"
"github.com/containerd/containerd/archive/compression"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/mount"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
keyFileAttr = "MSWINDOWS.fileattr"
keySDRaw = "MSWINDOWS.rawsd"
keyCreationTime = "LIBARCHIVE.creationtime"
)
func NewWalkingDiffWithWindows(store content.Store, d diff.Comparer) diff.Comparer {
return &winDiffer{
store: store,
d: d,
}
}
var emptyDesc = ocispec.Descriptor{}
type winDiffer struct {
store content.Store
d diff.Comparer
}
// Compare creates a diff between the given mounts and uploads the result
// to the content store.
func (s *winDiffer) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
if !hasWindowsLayerMode(ctx) {
return s.d.Compare(ctx, lower, upper, opts...)
}
var config diff.Config
for _, opt := range opts {
if err := opt(&config); err != nil {
return emptyDesc, err
}
}
if config.MediaType == "" {
config.MediaType = ocispec.MediaTypeImageLayerGzip
}
var isCompressed bool
switch config.MediaType {
case ocispec.MediaTypeImageLayer:
case ocispec.MediaTypeImageLayerGzip:
isCompressed = true
default:
return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType)
}
var ocidesc ocispec.Descriptor
if err := mount.WithTempMount(ctx, lower, func(lowerRoot string) error {
return mount.WithTempMount(ctx, upper, func(upperRoot string) error {
var newReference bool
if config.Reference == "" {
newReference = true
config.Reference = uniqueRef()
}
cw, err := s.store.Writer(ctx,
content.WithRef(config.Reference),
content.WithDescriptor(ocispec.Descriptor{
MediaType: config.MediaType, // most contentstore implementations just ignore this
}))
if err != nil {
return errors.Wrap(err, "failed to open writer")
}
defer func() {
if err != nil {
cw.Close()
if newReference {
if err := s.store.Abort(ctx, config.Reference); err != nil {
log.G(ctx).WithField("ref", config.Reference).Warnf("failed to delete diff upload")
}
}
}
}()
if !newReference {
if err := cw.Truncate(0); err != nil {
return err
}
}
if isCompressed {
dgstr := digest.SHA256.Digester()
compressed, err := compression.CompressStream(cw, compression.Gzip)
if err != nil {
return errors.Wrap(err, "failed to get compressed stream")
}
var w io.Writer = io.MultiWriter(compressed, dgstr.Hash())
w, discard, done := makeWindowsLayer(w)
err = archive.WriteDiff(ctx, w, lowerRoot, upperRoot)
if err != nil {
discard(err)
}
<-done
compressed.Close()
if err != nil {
return errors.Wrap(err, "failed to write compressed diff")
}
if config.Labels == nil {
config.Labels = map[string]string{}
}
config.Labels["containerd.io/uncompressed"] = dgstr.Digest().String()
} else {
w, discard, done := makeWindowsLayer(cw)
if err = archive.WriteDiff(ctx, w, lowerRoot, upperRoot); err != nil {
discard(err)
return errors.Wrap(err, "failed to write diff")
}
<-done
}
var commitopts []content.Opt
if config.Labels != nil {
commitopts = append(commitopts, content.WithLabels(config.Labels))
}
dgst := cw.Digest()
if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {
return errors.Wrap(err, "failed to commit")
}
info, err := s.store.Info(ctx, dgst)
if err != nil {
return errors.Wrap(err, "failed to get info from content store")
}
ocidesc = ocispec.Descriptor{
MediaType: config.MediaType,
Size: info.Size,
Digest: info.Digest,
}
return nil
})
}); err != nil {
return emptyDesc, err
}
return ocidesc, nil
}
func uniqueRef() string {
t := time.Now()
var b [3]byte
// Ignore read failures, just decreases uniqueness
rand.Read(b[:])
return fmt.Sprintf("%d-%s", t.UnixNano(), base64.URLEncoding.EncodeToString(b[:]))
}
func prepareWinHeader(h *tar.Header) {
if h.PAXRecords == nil {
h.PAXRecords = map[string]string{}
}
if h.Typeflag == tar.TypeDir {
h.Mode |= 1 << 14
h.PAXRecords[keyFileAttr] = "16"
}
if h.Typeflag == tar.TypeReg {
h.Mode |= 1 << 15
h.PAXRecords[keyFileAttr] = "32"
}
if !h.ModTime.IsZero() {
h.PAXRecords[keyCreationTime] = fmt.Sprintf("%d.%d", h.ModTime.Unix(), h.ModTime.Nanosecond())
}
h.Format = tar.FormatPAX
}
func addSecurityDescriptor(h *tar.Header) {
if h.Typeflag == tar.TypeDir {
// O:BAG:SYD:(A;OICI;FA;;;BA)(A;OICI;FA;;;SY)(A;;FA;;;BA)(A;OICIIO;GA;;;CO)(A;OICI;0x1200a9;;;BU)(A;CI;LC;;;BU)(A;CI;DC;;;BU)
h.PAXRecords[keySDRaw] = "AQAEgBQAAAAkAAAAAAAAADAAAAABAgAAAAAABSAAAAAgAgAAAQEAAAAAAAUSAAAAAgCoAAcAAAAAAxgA/wEfAAECAAAAAAAFIAAAACACAAAAAxQA/wEfAAEBAAAAAAAFEgAAAAAAGAD/AR8AAQIAAAAAAAUgAAAAIAIAAAALFAAAAAAQAQEAAAAAAAMAAAAAAAMYAKkAEgABAgAAAAAABSAAAAAhAgAAAAIYAAQAAAABAgAAAAAABSAAAAAhAgAAAAIYAAIAAAABAgAAAAAABSAAAAAhAgAA"
}
if h.Typeflag == tar.TypeReg {
// O:BAG:SYD:(A;;FA;;;BA)(A;;FA;;;SY)(A;;0x1200a9;;;BU)
h.PAXRecords[keySDRaw] = "AQAEgBQAAAAkAAAAAAAAADAAAAABAgAAAAAABSAAAAAgAgAAAQEAAAAAAAUSAAAAAgBMAAMAAAAAABgA/wEfAAECAAAAAAAFIAAAACACAAAAABQA/wEfAAEBAAAAAAAFEgAAAAAAGACpABIAAQIAAAAAAAUgAAAAIQIAAA=="
}
}
func makeWindowsLayer(w io.Writer) (io.Writer, func(error), chan error) {
pr, pw := io.Pipe()
done := make(chan error)
go func() {
tarReader := tar.NewReader(pr)
tarWriter := tar.NewWriter(w)
err := func() error {
h := &tar.Header{
Name: "Hives",
Typeflag: tar.TypeDir,
ModTime: time.Now(),
}
prepareWinHeader(h)
if err := tarWriter.WriteHeader(h); err != nil {
return err
}
h = &tar.Header{
Name: "Files",
Typeflag: tar.TypeDir,
ModTime: time.Now(),
}
prepareWinHeader(h)
if err := tarWriter.WriteHeader(h); err != nil {
return err
}
for {
h, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
h.Name = "Files/" + h.Name
if h.Linkname != "" {
h.Linkname = "Files/" + h.Linkname
}
prepareWinHeader(h)
addSecurityDescriptor(h)
if err := tarWriter.WriteHeader(h); err != nil {
return err
}
if h.Size > 0 {
if _, err := io.Copy(tarWriter, tarReader); err != nil {
return err
}
}
}
return tarWriter.Close()
}()
if err != nil {
logrus.Errorf("makeWindowsLayer %+v", err)
}
pw.CloseWithError(err)
done <- err
return
}()
discard := func(err error) {
pw.CloseWithError(err)
}
return pw, discard, done
}

5
worker/containerd/containerd.go
View File

@ -16,6 +16,7 @@ import (
"github.com/moby/buildkit/identity"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/util/throttle"
"github.com/moby/buildkit/util/winlayers"
"github.com/moby/buildkit/worker/base"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@ -109,8 +110,8 @@ func newContainerd(root string, client *containerd.Client, snapshotterName strin
Executor: containerdexecutor.New(client, root),
Snapshotter: containerdsnapshot.NewSnapshotter(client.SnapshotService(snapshotterName), cs, md, "buildkit", gc),
ContentStore: cs,
Applier: df,
Differ: df,
Applier: winlayers.NewFileSystemApplierWithWindows(cs, df),
Differ: winlayers.NewWalkingDiffWithWindows(cs, df),
ImageStore: client.ImageService(),
Platforms: platforms,
}

5
worker/runc/runc.go
View File

@ -17,6 +17,7 @@ import (
"github.com/moby/buildkit/executor/runcexecutor"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/util/throttle"
"github.com/moby/buildkit/util/winlayers"
"github.com/moby/buildkit/worker/base"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
@ -101,8 +102,8 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, labe
Executor: exe,
Snapshotter: containerdsnapshot.NewSnapshotter(mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc),
ContentStore: c,
Applier: apply.NewFileSystemApplier(c),
Differ: walking.NewWalkingDiff(c),
Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)),
Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)),
ImageStore: nil, // explicitly
Platforms: []specs.Platform{platforms.Normalize(platforms.DefaultSpec())},
}