JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2181 from ktock/overlaydiff 

Compute diff from the upper directory of overlayfs-based snapshotter
master
Tõnis Tiigi 2021-09-02 16:04:52 -07:00 committed by GitHub
parent ebd5157fe5 d586efd5db
commit cf80727e5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 948 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
cache/blobs.go vendored
View File

@ -4,6 +4,8 @@ import (
"context"
"fmt"
"io"
"os"
"strconv"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
@ -17,6 +19,7 @@ import (
imagespecidentity "github.com/opencontainers/image-spec/identity"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
)
@ -106,13 +109,52 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
if release != nil {
defer release()
}
desc, err := sr.cm.Differ.Compare(ctx, lower, upper,
diff.WithMediaType(mediaType),
diff.WithReference(sr.ID()),
diff.WithCompressor(compressorFunc),
)
if err != nil {
return nil, err
var desc ocispecs.Descriptor
// Determine differ and error/log handling according to the platform, envvar and the snapshotter.
var enableOverlay, fallback, logWarnOnErr bool
if forceOvlStr := os.Getenv("BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF"); forceOvlStr != "" {
enableOverlay, err = strconv.ParseBool(forceOvlStr)
if err != nil {
return nil, errors.Wrapf(err, "invalid boolean in BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF")
}
fallback = false // prohibit fallback on debug
} else if !isTypeWindows(sr) {
enableOverlay, fallback = true, true
switch sr.cm.ManagerOpt.Snapshotter.Name() {
case "overlayfs", "fuse-overlayfs", "stargz":
logWarnOnErr = true // snapshotter should support overlay diff. so print warn log on failure
}
}
if enableOverlay {
computed, ok, err := sr.tryComputeOverlayBlob(ctx, lower, upper, mediaType, sr.ID(), compressorFunc)
if !ok || err != nil {
if !fallback {
if !ok {
return nil, errors.Errorf("overlay mounts not detected (lower=%+v,upper=%+v)", lower, upper)
}
if err != nil {
return nil, errors.Wrapf(err, "failed to compute overlay diff")
}
}
if logWarnOnErr {
logrus.Warnf("failed to compute blob by overlay differ (ok=%v): %v", ok, err)
}
}
if ok {
desc = computed
}
}
if desc.Digest == "" {
desc, err = sr.cm.Differ.Compare(ctx, lower, upper,
diff.WithMediaType(mediaType),
diff.WithReference(sr.ID()),
diff.WithCompressor(compressorFunc),
)
if err != nil {
return nil, err
}
}
if desc.Annotations == nil {

423
cache/blobs_linux.go vendored Normal file
View File

@ -0,0 +1,423 @@
// +build linux
package cache
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/containerd/containerd/archive"
ctdcompression "github.com/containerd/containerd/archive/compression"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/mount"
"github.com/containerd/continuity/devices"
"github.com/containerd/continuity/fs"
"github.com/containerd/continuity/sysx"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
var emptyDesc = ocispecs.Descriptor{}
// computeOverlayBlob provides overlayfs-specialized method to compute
// diff between lower and upper snapshot. If the passed mounts cannot
// be computed (e.g. because the mounts aren't overlayfs), it returns
// an error.
func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string, compressorFunc compressor) (_ ocispecs.Descriptor, ok bool, err error) {
// Get upperdir location if mounts are overlayfs that can be processed by this differ.
upperdir, err := getOverlayUpperdir(lower, upper)
if err != nil {
// This is not an overlayfs snapshot. This is not an error so don't return error here
// and let the caller fallback to another differ.
return emptyDesc, false, nil
}
if compressorFunc == nil {
switch mediaType {
case ocispecs.MediaTypeImageLayer:
case ocispecs.MediaTypeImageLayerGzip:
compressorFunc = func(dest io.Writer, requiredMediaType string) (io.WriteCloser, error) {
return ctdcompression.CompressStream(dest, ctdcompression.Gzip)
}
default:
return emptyDesc, false, errors.Errorf("unsupported diff media type: %v", mediaType)
}
}
cw, err := sr.cm.ContentStore.Writer(ctx,
content.WithRef(ref),
content.WithDescriptor(ocispecs.Descriptor{
MediaType: mediaType, // most contentstore implementations just ignore this
}))
if err != nil {
return emptyDesc, false, errors.Wrap(err, "failed to open writer")
}
defer func() {
if err != nil {
cw.Close()
}
}()
var labels map[string]string
if compressorFunc != nil {
dgstr := digest.SHA256.Digester()
compressed, err := compressorFunc(cw, mediaType)
if err != nil {
return emptyDesc, false, errors.Wrap(err, "failed to get compressed stream")
}
err = writeOverlayUpperdir(ctx, io.MultiWriter(compressed, dgstr.Hash()), upperdir, lower)
compressed.Close()
if err != nil {
return emptyDesc, false, errors.Wrap(err, "failed to write compressed diff")
}
if labels == nil {
labels = map[string]string{}
}
labels[containerdUncompressed] = dgstr.Digest().String()
} else {
if err = writeOverlayUpperdir(ctx, cw, upperdir, lower); err != nil {
return emptyDesc, false, errors.Wrap(err, "failed to write diff")
}
}
var commitopts []content.Opt
if labels != nil {
commitopts = append(commitopts, content.WithLabels(labels))
}
dgst := cw.Digest()
if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {
if !errdefs.IsAlreadyExists(err) {
return emptyDesc, false, errors.Wrap(err, "failed to commit")
}
}
cinfo, err := sr.cm.ContentStore.Info(ctx, dgst)
if err != nil {
return emptyDesc, false, errors.Wrap(err, "failed to get info from content store")
}
if cinfo.Labels == nil {
cinfo.Labels = make(map[string]string)
}
// Set uncompressed label if digest already existed without label
if _, ok := cinfo.Labels[containerdUncompressed]; !ok {
cinfo.Labels[containerdUncompressed] = labels[containerdUncompressed]
if _, err := sr.cm.ContentStore.Update(ctx, cinfo, "labels."+containerdUncompressed); err != nil {
return emptyDesc, false, errors.Wrap(err, "error setting uncompressed label")
}
}
return ocispecs.Descriptor{
MediaType: mediaType,
Size: cinfo.Size,
Digest: cinfo.Digest,
}, true, nil
}
// getOverlayUpperdir parses the passed mounts and identifies the directory
// that contains diff between upper and lower.
func getOverlayUpperdir(lower, upper []mount.Mount) (string, error) {
var upperdir string
if len(lower) == 0 && len(upper) == 1 { // upper is the bottommost snpashot
// Get layer directories of upper snapshot
upperM := upper[0]
if upperM.Type != "bind" {
return "", errors.Errorf("bottommost upper must be bind mount but %q", upperM.Type)
}
upperdir = upperM.Source
} else if len(lower) == 1 && len(upper) == 1 {
// Get layer directories of lower snapshot
var lowerlayers []string
lowerM := lower[0]
switch lowerM.Type {
case "bind":
// lower snapshot is a bind mount of one layer
lowerlayers = []string{lowerM.Source}
case "overlay":
// lower snapshot is an overlay mount of multiple layers
var err error
lowerlayers, err = getOverlayLayers(lowerM)
if err != nil {
return "", err
}
default:
return "", errors.Errorf("cannot get layer information from mount option (type = %q)", lowerM.Type)
}
// Get layer directories of upper snapshot
upperM := upper[0]
if upperM.Type != "overlay" {
return "", errors.Errorf("upper snapshot isn't overlay mounted (type = %q)", upperM.Type)
}
upperlayers, err := getOverlayLayers(upperM)
if err != nil {
return "", err
}
// Check if the diff directory can be determined
if len(upperlayers) != len(lowerlayers)+1 {
return "", errors.Errorf("cannot determine diff of more than one upper directories")
}
for i := 0; i < len(lowerlayers); i++ {
if upperlayers[i] != lowerlayers[i] {
return "", errors.Errorf("layer %d must be common between upper and lower snapshots", i)
}
}
upperdir = upperlayers[len(upperlayers)-1] // get the topmost layer that indicates diff
} else {
return "", errors.Errorf("multiple mount configurations are not supported")
}
if upperdir == "" {
return "", errors.Errorf("cannot determine upperdir from mount option")
}
return upperdir, nil
}
// getOverlayLayers returns all layer directories of an overlayfs mount.
func getOverlayLayers(m mount.Mount) ([]string, error) {
var u string
var uFound bool
var l []string // l[0] = bottommost
for _, o := range m.Options {
if strings.HasPrefix(o, "upperdir=") {
u, uFound = strings.TrimPrefix(o, "upperdir="), true
} else if strings.HasPrefix(o, "lowerdir=") {
l = strings.Split(strings.TrimPrefix(o, "lowerdir="), ":")
for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
l[i], l[j] = l[j], l[i] // make l[0] = bottommost
}
} else if strings.HasPrefix(o, "workdir=") || o == "index=off" || o == "userxattr" {
// these options are possible to specfied by the snapshotter but not indicate dir locations.
continue
} else {
// encountering an unknown option. return error and fallback to walking differ
// to avoid unexpected diff.
return nil, errors.Errorf("unknown option %q specified by snapshotter", o)
}
}
if uFound {
return append(l, u), nil
}
return l, nil
}
// writeOverlayUpperdir writes a layer tar archive into the specified writer, based on
// the diff information stored in the upperdir.
func writeOverlayUpperdir(ctx context.Context, w io.Writer, upperdir string, lower []mount.Mount) error {
emptyLower, err := ioutil.TempDir("", "buildkit") // empty directory used for the lower of diff view
if err != nil {
return errors.Wrapf(err, "failed to create temp dir")
}
defer os.Remove(emptyLower)
upperView := []mount.Mount{
{
Type: "overlay",
Source: "overlay",
Options: []string{fmt.Sprintf("lowerdir=%s", strings.Join([]string{upperdir, emptyLower}, ":"))},
},
}
return mount.WithTempMount(ctx, lower, func(lowerRoot string) error {
return mount.WithTempMount(ctx, upperView, func(upperViewRoot string) error {
cw := archive.NewChangeWriter(w, upperViewRoot)
if err := overlayChanges(ctx, cw.HandleChange, upperdir, upperViewRoot, lowerRoot); err != nil {
if err2 := cw.Close(); err2 != nil {
return errors.Wrapf(err, "failed torecord upperdir changes (close error: %v)", err2)
}
return errors.Wrapf(err, "failed torecord upperdir changes")
}
return cw.Close()
})
})
}
// overlayChanges is continuty's `fs.Change`-like method but leverages overlayfs's
// "upperdir" for computing the diff. "upperdirView" is overlayfs mounted view of
// the upperdir that doesn't contain whiteouts. This is used for computing
// changes under opaque directories.
func overlayChanges(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView, base string) error {
return filepath.Walk(upperdir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
// Rebase path
path, err = filepath.Rel(upperdir, path)
if err != nil {
return err
}
path = filepath.Join(string(os.PathSeparator), path)
// Skip root
if path == string(os.PathSeparator) {
return nil
}
// Check if this is a deleted entry
isDelete, skip, err := checkDelete(upperdir, path, base, f)
if err != nil {
return err
} else if skip {
return nil
}
var kind fs.ChangeKind
var skipRecord bool
if isDelete {
// This is a deleted entry.
kind = fs.ChangeKindDelete
f = nil
} else if baseF, err := os.Lstat(filepath.Join(base, path)); err == nil {
// File exists in the base layer. Thus this is modified.
kind = fs.ChangeKindModify
// Avoid including directory that hasn't been modified. If /foo/bar/baz is modified,
// then /foo will apper here even if it's not been modified because it's the parent of bar.
if same, err := sameDir(baseF, f, filepath.Join(base, path), filepath.Join(upperdirView, path)); same {
skipRecord = true // Both are the same, don't record the change
} else if err != nil {
return err
}
} else if os.IsNotExist(err) {
// File doesn't exist in the base layer. Thus this is added.
kind = fs.ChangeKindAdd
} else if err != nil {
return err
}
if !skipRecord {
if err := changeFn(kind, path, f, nil); err != nil {
return err
}
}
if f != nil {
if isOpaque, err := checkOpaque(upperdir, path, base, f); err != nil {
return err
} else if isOpaque {
// This is an opaque directory. Start a new walking differ to get adds/deletes of
// this directory. We use "upperdirView" directory which doesn't contain whiteouts.
if err := fs.Changes(ctx, filepath.Join(base, path), filepath.Join(upperdirView, path),
func(k fs.ChangeKind, p string, f os.FileInfo, err error) error {
return changeFn(k, filepath.Join(path, p), f, err) // rebase path to be based on the opaque dir
},
); err != nil {
return err
}
return filepath.SkipDir // We completed this directory. Do not walk files under this directory anymore.
}
}
return nil
})
}
// checkDelete checks if the specified file is a whiteout
func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) {
if f.Mode()&os.ModeCharDevice != 0 {
if _, ok := f.Sys().(*syscall.Stat_t); ok {
maj, min, err := devices.DeviceInfo(f)
if err != nil {
return false, false, errors.Wrapf(err, "failed to get device info")
}
if maj == 0 && min == 0 {
// This file is a whiteout (char 0/0) that indicates this is deleted from the base
if _, err := os.Lstat(filepath.Join(base, path)); err != nil {
if !os.IsNotExist(err) {
return false, false, errors.Wrapf(err, "failed to lstat")
}
// This file doesn't exist even in the base dir.
// We don't need whiteout. Just skip this file.
return false, true, nil
}
return true, false, nil
}
}
}
return false, false, nil
}
// checkDelete checks if the specified file is an opaque directory
func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) {
if f.IsDir() {
for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} {
opaque, err := sysx.LGetxattr(filepath.Join(upperdir, path), oKey)
if err != nil && err != unix.ENODATA {
return false, errors.Wrapf(err, "failed to retrieve %s attr", oKey)
} else if len(opaque) == 1 && opaque[0] == 'y' {
// This is an opaque whiteout directory.
if _, err := os.Lstat(filepath.Join(base, path)); err != nil {
if !os.IsNotExist(err) {
return false, errors.Wrapf(err, "failed to lstat")
}
// This file doesn't exist even in the base dir. We don't need treat this as an opaque.
return false, nil
}
return true, nil
}
}
}
return false, nil
}
// sameDir performs continity-compatible comparison of directories.
// https://github.com/containerd/continuity/blob/v0.1.0/fs/path.go#L91-L133
// This doesn't compare files because it requires to compare their contents.
// This is what we want to avoid by this overlayfs-specialized differ.
func sameDir(f1, f2 os.FileInfo, f1fullPath, f2fullPath string) (bool, error) {
if !f1.IsDir() || !f2.IsDir() {
return false, nil
}
if os.SameFile(f1, f2) {
return true, nil
}
equalStat, err := compareSysStat(f1.Sys(), f2.Sys())
if err != nil || !equalStat {
return equalStat, err
}
if eq, err := compareCapabilities(f1fullPath, f2fullPath); err != nil || !eq {
return eq, err
}
return true, nil
}
// Ported from continuity project
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L43-L54
// Copyright The containerd Authors.
func compareSysStat(s1, s2 interface{}) (bool, error) {
ls1, ok := s1.(*syscall.Stat_t)
if !ok {
return false, nil
}
ls2, ok := s2.(*syscall.Stat_t)
if !ok {
return false, nil
}
return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil
}
// Ported from continuity project
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L56-L66
// Copyright The containerd Authors.
func compareCapabilities(p1, p2 string) (bool, error) {
c1, err := sysx.LGetxattr(p1, "security.capability")
if err != nil && err != sysx.ENODATA {
return false, errors.Wrapf(err, "failed to get xattr for %s", p1)
}
c2, err := sysx.LGetxattr(p2, "security.capability")
if err != nil && err != sysx.ENODATA {
return false, errors.Wrapf(err, "failed to get xattr for %s", p2)
}
return bytes.Equal(c1, c2), nil
}

458
cache/blobs_linux_test.go vendored Normal file
View File

@ -0,0 +1,458 @@
// +build linux
package cache
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/containerd/containerd/mount"
"github.com/containerd/continuity/fs"
"github.com/containerd/continuity/fs/fstest"
"github.com/pkg/errors"
)
// This test file contains tests that are required in continuity project.
// (https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go)
// Most of them are ported from that project and patched to test our
// overlayfs-optimized differ.
// TestSimpleDiff is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L46-L73
// Copyright The containerd Authors.
func TestSimpleDiff(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateDir("/etc", 0755),
fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0644),
fstest.CreateFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644),
fstest.CreateFile("/etc/unexpected", []byte("#!/bin/sh"), 0644),
)
l2 := fstest.Apply(
fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644),
fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0666),
fstest.CreateDir("/root", 0700),
fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644),
fstest.Remove("/etc/unexpected"),
)
diff := []TestChange{
Modify("/etc/hosts"),
Modify("/etc/profile"),
Delete("/etc/unexpected"),
Add("/root"),
Add("/root/.bashrc"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestEmptyFileDiff is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L75-L89
// Copyright The containerd Authors.
func TestEmptyFileDiff(t *testing.T) {
tt := time.Now().Truncate(time.Second)
l1 := fstest.Apply(
fstest.CreateDir("/etc", 0755),
fstest.CreateFile("/etc/empty", []byte(""), 0644),
fstest.Chtimes("/etc/empty", tt, tt),
)
l2 := fstest.Apply()
diff := []TestChange{}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestNestedDeletion is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L91-L111
// Copyright The containerd Authors.
func TestNestedDeletion(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateDir("/d0", 0755),
fstest.CreateDir("/d1", 0755),
fstest.CreateDir("/d1/d2", 0755),
fstest.CreateFile("/d1/d2/f1", []byte("mydomain 10.0.0.1"), 0644),
)
l2 := fstest.Apply(
fstest.RemoveAll("/d0"),
fstest.RemoveAll("/d1"),
)
diff := []TestChange{
Delete("/d0"),
Delete("/d1"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestDirectoryReplace is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L113-L134
// Copyright The containerd Authors.
func TestDirectoryReplace(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateDir("/dir1", 0755),
fstest.CreateFile("/dir1/f1", []byte("#####"), 0644),
fstest.CreateDir("/dir1/f2", 0755),
fstest.CreateFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644),
)
l2 := fstest.Apply(
fstest.CreateFile("/dir1/f11", []byte("#New file here"), 0644),
fstest.RemoveAll("/dir1/f2"),
fstest.CreateFile("/dir1/f2", []byte("Now file"), 0666),
)
diff := []TestChange{
Add("/dir1/f11"),
Modify("/dir1/f2"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestRemoveDirectoryTree is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L136-L152
// Copyright The containerd Authors.
func TestRemoveDirectoryTree(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateDir("/dir1/dir2/dir3", 0755),
fstest.CreateFile("/dir1/f1", []byte("f1"), 0644),
fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0644),
)
l2 := fstest.Apply(
fstest.RemoveAll("/dir1"),
)
diff := []TestChange{
Delete("/dir1"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestRemoveDirectoryTreeWithDash is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L154-L172
// Copyright The containerd Authors.
func TestRemoveDirectoryTreeWithDash(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateDir("/dir1/dir2/dir3", 0755),
fstest.CreateFile("/dir1/f1", []byte("f1"), 0644),
fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0644),
fstest.CreateDir("/dir1-before", 0755),
fstest.CreateFile("/dir1-before/f2", []byte("f2"), 0644),
)
l2 := fstest.Apply(
fstest.RemoveAll("/dir1"),
)
diff := []TestChange{
Delete("/dir1"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestFileReplace is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L174-L192
// Copyright The containerd Authors.
func TestFileReplace(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateFile("/dir1", []byte("a file, not a directory"), 0644),
)
l2 := fstest.Apply(
fstest.Remove("/dir1"),
fstest.CreateDir("/dir1/dir2", 0755),
fstest.CreateFile("/dir1/dir2/f1", []byte("also a file"), 0644),
)
diff := []TestChange{
Modify("/dir1"),
Add("/dir1/dir2"),
Add("/dir1/dir2/f1"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestParentDirectoryPermission is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L194-L219
// Copyright The containerd Authors.
func TestParentDirectoryPermission(t *testing.T) {
l1 := fstest.Apply(
fstest.CreateDir("/dir1", 0700),
fstest.CreateDir("/dir2", 0751),
fstest.CreateDir("/dir3", 0777),
)
l2 := fstest.Apply(
fstest.CreateDir("/dir1/d", 0700),
fstest.CreateFile("/dir1/d/f", []byte("irrelevant"), 0644),
fstest.CreateFile("/dir1/f", []byte("irrelevant"), 0644),
fstest.CreateFile("/dir2/f", []byte("irrelevant"), 0644),
fstest.CreateFile("/dir3/f", []byte("irrelevant"), 0644),
)
diff := []TestChange{
Add("/dir1/d"),
Add("/dir1/d/f"),
Add("/dir1/f"),
Add("/dir2/f"),
Add("/dir3/f"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestUpdateWithSameTime is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L221-L269
// Copyright The containerd Authors.
//
// NOTE: This test is patched for our differ. See the following NOTE for details.
func TestUpdateWithSameTime(t *testing.T) {
tt := time.Now().Truncate(time.Second)
t1 := tt.Add(5 * time.Nanosecond)
t2 := tt.Add(6 * time.Nanosecond)
l1 := fstest.Apply(
fstest.CreateFile("/file-modified-time", []byte("1"), 0644),
fstest.Chtimes("/file-modified-time", t1, t1),
fstest.CreateFile("/file-no-change", []byte("1"), 0644),
fstest.Chtimes("/file-no-change", t1, t1),
fstest.CreateFile("/file-same-time", []byte("1"), 0644),
fstest.Chtimes("/file-same-time", t1, t1),
fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0644),
fstest.Chtimes("/file-truncated-time-1", tt, tt),
fstest.CreateFile("/file-truncated-time-2", []byte("1"), 0644),
fstest.Chtimes("/file-truncated-time-2", tt, tt),
fstest.CreateFile("/file-truncated-time-3", []byte("1"), 0644),
fstest.Chtimes("/file-truncated-time-3", t1, t1),
)
l2 := fstest.Apply(
fstest.CreateFile("/file-modified-time", []byte("2"), 0644),
fstest.Chtimes("/file-modified-time", t2, t2),
fstest.CreateFile("/file-no-change", []byte("1"), 0644),
fstest.Chtimes("/file-no-change", t1, t1),
fstest.CreateFile("/file-same-time", []byte("2"), 0644),
fstest.Chtimes("/file-same-time", t1, t1),
fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0644),
fstest.Chtimes("/file-truncated-time-1", t1, t1),
fstest.CreateFile("/file-truncated-time-2", []byte("2"), 0644),
fstest.Chtimes("/file-truncated-time-2", tt, tt),
fstest.CreateFile("/file-truncated-time-3", []byte("1"), 0644),
fstest.Chtimes("/file-truncated-time-3", tt, tt),
)
diff := []TestChange{
Modify("/file-modified-time"),
// NOTE: Even if the file is identical, overlayfs copies it to
// the upper layer when the modification occurred. continuity's differ avoids counting
// this as "modify" by comparing the time and the file contents between upper and lower
// but here we want to avoid comparing bits which makes the differ slower.
// TODO: we need a way to effectively determine two files are identical
// without copmaring bits.
Modify("/file-no-change"),
Modify("/file-same-time"),
// Include changes with truncated timestamps. Comparing newly
// extracted tars which have truncated timestamps will be
// expected to produce changes. The expectation is that diff
// archives are generated once and kept, newly generated diffs
// will not consider cases where only one side is truncated.
Modify("/file-truncated-time-1"),
Modify("/file-truncated-time-2"),
Modify("/file-truncated-time-3"),
}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
// TestLchtimes is a test ported from
// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L271-L291
// Copyright The containerd Authors.
// buildkit#172
func TestLchtimes(t *testing.T) {
mtimes := []time.Time{
time.Unix(0, 0), // nsec is 0
time.Unix(0, 42), // nsec > 0
}
for _, mtime := range mtimes {
atime := time.Unix(424242, 42)
l1 := fstest.Apply(
fstest.CreateFile("/foo", []byte("foo"), 0644),
fstest.Symlink("/foo", "/lnk0"),
fstest.Lchtimes("/lnk0", atime, mtime),
)
l2 := fstest.Apply() // empty
diff := []TestChange{}
if err := testDiffWithBase(l1, l2, diff); err != nil {
t.Fatalf("Failed diff with base: %+v", err)
}
}
}
func testDiffWithBase(base, diff fstest.Applier, expected []TestChange) error {
t1, err := ioutil.TempDir("", "diff-with-base-lower-")
if err != nil {
return errors.Wrap(err, "failed to create temp dir")
}
defer os.RemoveAll(t1)
if err := base.Apply(t1); err != nil {
return errors.Wrap(err, "failed to apply base filesystem")
}
tupper, err := ioutil.TempDir("", "diff-with-base-upperdir-")
if err != nil {
return errors.Wrap(err, "failed to create temp dir")
}
defer os.RemoveAll(tupper)
workdir, err := ioutil.TempDir("", "diff-with-base-workdir-")
if err != nil {
return errors.Wrap(err, "failed to create temp dir")
}
defer os.RemoveAll(workdir)
return mount.WithTempMount(context.Background(), []mount.Mount{
{
Type: "overlay",
Source: "overlay",
Options: []string{fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", t1, tupper, workdir)},
},
}, func(overlayRoot string) error {
if err := diff.Apply(overlayRoot); err != nil {
return errors.Wrapf(err, "failed to apply diff to overlayRoot")
}
if err := collectAndCheckChanges(t1, tupper, expected); err != nil {
return errors.Wrap(err, "failed to collect changes")
}
return nil
})
}
func checkChanges(root string, changes, expected []TestChange) error {
if len(changes) != len(expected) {
return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected))
}
for i := range changes {
if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind {
return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected))
}
if changes[i].Kind != fs.ChangeKindDelete {
filename := filepath.Join(root, changes[i].Path)
efi, err := os.Stat(filename)
if err != nil {
return errors.Wrapf(err, "failed to stat %q", filename)
}
afi := changes[i].FileInfo
if afi.Size() != efi.Size() {
return errors.Errorf("Unexpected change size %d, %q has size %d", afi.Size(), filename, efi.Size())
}
if afi.Mode() != efi.Mode() {
return errors.Errorf("Unexpected change mode %s, %q has mode %s", afi.Mode(), filename, efi.Mode())
}
if afi.ModTime() != efi.ModTime() {
return errors.Errorf("Unexpected change modtime %s, %q has modtime %s", afi.ModTime(), filename, efi.ModTime())
}
if expected := filepath.Join(root, changes[i].Path); changes[i].Source != expected {
return errors.Errorf("Unexpected source path %s, expected %s", changes[i].Source, expected)
}
}
}
return nil
}
type TestChange struct {
Kind fs.ChangeKind
Path string
FileInfo os.FileInfo
Source string
}
func collectAndCheckChanges(base, upperdir string, expected []TestChange) error {
ctx := context.Background()
changes := []TestChange{}
emptyLower, err := ioutil.TempDir("", "buildkit-test-emptylower") // empty directory used for the lower of diff view
if err != nil {
return errors.Wrapf(err, "failed to create temp dir")
}
defer os.Remove(emptyLower)
upperView := []mount.Mount{
{
Type: "overlay",
Source: "overlay",
Options: []string{fmt.Sprintf("lowerdir=%s", strings.Join([]string{upperdir, emptyLower}, ":"))},
},
}
return mount.WithTempMount(ctx, upperView, func(upperViewRoot string) error {
if err := overlayChanges(ctx, func(k fs.ChangeKind, p string, f os.FileInfo, err error) error {
if err != nil {
return err
}
changes = append(changes, TestChange{
Kind: k,
Path: p,
FileInfo: f,
Source: filepath.Join(upperViewRoot, p),
})
return nil
}, upperdir, upperViewRoot, base); err != nil {
return err
}
if err := checkChanges(upperViewRoot, changes, expected); err != nil {
return errors.Wrapf(err, "change check falied")
}
return nil
})
}
func diffString(c1, c2 []TestChange) string {
return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2))
}
func changesString(c []TestChange) string {
strs := make([]string, len(c))
for i := range c {
strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path)
}
return strings.Join(strs, "\n")
}
func Add(p string) TestChange {
return TestChange{
Kind: fs.ChangeKindAdd,
Path: filepath.FromSlash(p),
}
}
func Delete(p string) TestChange {
return TestChange{
Kind: fs.ChangeKindDelete,
Path: filepath.FromSlash(p),
}
}
func Modify(p string) TestChange {
return TestChange{
Kind: fs.ChangeKindModify,
Path: filepath.FromSlash(p),
}
}

15
cache/blobs_nolinux.go vendored Normal file
View File

@ -0,0 +1,15 @@
// +build !linux
package cache
import (
"context"
"github.com/containerd/containerd/mount"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string, compressorFunc compressor) (_ ocispecs.Descriptor, ok bool, err error) {
return ocispecs.Descriptor{}, true, errors.Errorf("overlayfs-based diff computing is unsupported")
}

3
util/testutil/integration/sandbox.go
View File

@ -182,6 +182,9 @@ func runBuildkitd(ctx context.Context, conf *BackendConfig, args []string, logs
args = append(args, "--root", tmpdir, "--addr", address, "--debug")
cmd := exec.Command(args[0], args[1:]...)
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
if runtime.GOOS != "windows" {
cmd.Env = append(cmd.Env, "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=true")
}
cmd.Env = append(cmd.Env, extraEnv...)
cmd.SysProcAttr = getSysProcAttr()