From bcc98bf29d36035815e1d58ef21eac2d4a4292ed Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Nov 2017 14:50:03 -0800 Subject: [PATCH 1/4] frontend: add some dockerfile integration tests Signed-off-by: Tonis Tiigi --- frontend/dockerfile/dockerfile_test.go | 129 ++++ util/testutil/integration/standalone.go | 7 + .../containerd/fs/fstest/compare.go | 53 ++ .../containerd/fs/fstest/continuity_util.go | 189 ++++++ .../containerd/containerd/fs/fstest/file.go | 120 ++++ .../containerd/fs/fstest/file_unix.go | 29 + .../containerd/fs/fstest/file_windows.go | 14 + .../containerd/fs/fstest/testsuite.go | 197 ++++++ .../containerd/continuity/context.go | 641 ++++++++++++++++++ .../containerd/continuity/devices/devices.go | 5 + .../continuity/devices/devices_darwin.go | 15 + .../continuity/devices/devices_dummy.go | 23 + .../continuity/devices/devices_freebsd.go | 15 + .../continuity/devices/devices_linux.go | 15 + .../continuity/devices/devices_solaris.go | 18 + .../continuity/devices/devices_unix.go | 55 ++ .../continuity/devices/devices_windows.go | 11 + .../containerd/continuity/digests.go | 88 +++ .../containerd/continuity/driver/driver.go | 162 +++++ .../continuity/driver/driver_unix.go | 122 ++++ .../continuity/driver/driver_windows.go | 21 + .../containerd/continuity/groups_unix.go | 113 +++ .../containerd/continuity/hardlinks.go | 57 ++ .../containerd/continuity/hardlinks_unix.go | 36 + .../continuity/hardlinks_windows.go | 12 + .../containerd/continuity/ioutils.go | 39 ++ .../containerd/continuity/manifest.go | 144 ++++ .../continuity/manifest_test_darwin.go | 23 + .../continuity/pathdriver/path_driver.go | 85 +++ .../containerd/continuity/proto/gen.go | 3 + .../continuity/proto/manifest.pb.go | 181 +++++ .../continuity/proto/manifest.proto | 97 +++ .../containerd/continuity/resource.go | 574 ++++++++++++++++ .../containerd/continuity/resource_unix.go | 37 + .../containerd/continuity/resource_windows.go | 12 + 35 files changed, 3342 insertions(+) create mode 100644 frontend/dockerfile/dockerfile_test.go create mode 100644 vendor/github.com/containerd/containerd/fs/fstest/compare.go create mode 100644 vendor/github.com/containerd/containerd/fs/fstest/continuity_util.go create mode 100644 vendor/github.com/containerd/containerd/fs/fstest/file.go create mode 100644 vendor/github.com/containerd/containerd/fs/fstest/file_unix.go create mode 100644 vendor/github.com/containerd/containerd/fs/fstest/file_windows.go create mode 100644 vendor/github.com/containerd/containerd/fs/fstest/testsuite.go create mode 100644 vendor/github.com/containerd/continuity/context.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_darwin.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_dummy.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_freebsd.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_linux.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_solaris.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_unix.go create mode 100644 vendor/github.com/containerd/continuity/devices/devices_windows.go create mode 100644 vendor/github.com/containerd/continuity/digests.go create mode 100644 vendor/github.com/containerd/continuity/driver/driver.go create mode 100644 vendor/github.com/containerd/continuity/driver/driver_unix.go create mode 100644 vendor/github.com/containerd/continuity/driver/driver_windows.go create mode 100644 vendor/github.com/containerd/continuity/groups_unix.go create mode 100644 vendor/github.com/containerd/continuity/hardlinks.go create mode 100644 vendor/github.com/containerd/continuity/hardlinks_unix.go create mode 100644 vendor/github.com/containerd/continuity/hardlinks_windows.go create mode 100644 vendor/github.com/containerd/continuity/ioutils.go create mode 100644 vendor/github.com/containerd/continuity/manifest.go create mode 100644 vendor/github.com/containerd/continuity/manifest_test_darwin.go create mode 100644 vendor/github.com/containerd/continuity/pathdriver/path_driver.go create mode 100644 vendor/github.com/containerd/continuity/proto/gen.go create mode 100644 vendor/github.com/containerd/continuity/proto/manifest.pb.go create mode 100644 vendor/github.com/containerd/continuity/proto/manifest.proto create mode 100644 vendor/github.com/containerd/continuity/resource.go create mode 100644 vendor/github.com/containerd/continuity/resource_unix.go create mode 100644 vendor/github.com/containerd/continuity/resource_windows.go diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go new file mode 100644 index 00000000..65c081fa --- /dev/null +++ b/frontend/dockerfile/dockerfile_test.go @@ -0,0 +1,129 @@ +package dockerfile + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/containerd/containerd/fs/fstest" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/stretchr/testify/require" +) + +func TestIntegration(t *testing.T) { + integration.Run(t, []integration.Test{ + testDockerfileDirs, + testDockerfileInvalidCommand, + }) +} + +func testDockerfileDirs(t *testing.T, sb integration.Sandbox) { + t.Parallel() + dockerfile := []byte(` + FROM busybox + COPY foo /foo2 + COPY foo / + RUN echo -n bar > foo3 + RUN test -f foo + RUN cmp -s foo foo2 + RUN cmp -s foo foo3 +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("foo", []byte("bar"), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + args, trace := dfCmdArgs(dir, dir) + defer os.RemoveAll(trace) + + cmd := sb.Cmd(args) + require.NoError(t, cmd.Run()) + + _, err = os.Stat(trace) + require.NoError(t, err) + + // relative urls + args, trace = dfCmdArgs(".", ".") + defer os.RemoveAll(trace) + + cmd = sb.Cmd(args) + cmd.Dir = dir + require.NoError(t, cmd.Run()) + + _, err = os.Stat(trace) + require.NoError(t, err) + + // different context and dockerfile directories + dir1, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir1) + + dir2, err := tmpdir( + fstest.CreateFile("foo", []byte("bar"), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir2) + + args, trace = dfCmdArgs(dir2, dir1) + defer os.RemoveAll(trace) + + cmd = sb.Cmd(args) + cmd.Dir = dir + require.NoError(t, cmd.Run()) + + _, err = os.Stat(trace) + require.NoError(t, err) + + // TODO: test trace file output, cache hits, logs etc. + // TODO: output metadata about original dockerfile command in trace +} + +func testDockerfileInvalidCommand(t *testing.T, sb integration.Sandbox) { + t.Parallel() + dockerfile := []byte(` + FROM busybox + RUN invalidcmd +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + args, trace := dfCmdArgs(dir, dir) + defer os.RemoveAll(trace) + + cmd := sb.Cmd(args) + stdout := new(bytes.Buffer) + cmd.Stderr = stdout + err = cmd.Run() + require.Error(t, err) + require.Contains(t, stdout.String(), "/bin/sh -c invalidcmd") + require.Contains(t, stdout.String(), "worker failed running") +} + +func tmpdir(appliers ...fstest.Applier) (string, error) { + tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") + if err != nil { + return "", err + } + if err := fstest.Apply(appliers...).Apply(tmpdir); err != nil { + return "", err + } + return tmpdir, nil +} + +func dfCmdArgs(ctx, dockerfile string) (string, string) { + traceFile := filepath.Join(os.TempDir(), "trace"+identity.NewID()) + return fmt.Sprintf("build --no-progress --frontend dockerfile.v0 --local context=%s --local dockerfile=%s --trace=%s", ctx, dockerfile, traceFile), traceFile +} diff --git a/util/testutil/integration/standalone.go b/util/testutil/integration/standalone.go index e3f19d89..913e9639 100644 --- a/util/testutil/integration/standalone.go +++ b/util/testutil/integration/standalone.go @@ -10,6 +10,8 @@ import ( "runtime" "testing" "time" + + "github.com/google/shlex" ) func init() { @@ -59,6 +61,11 @@ func (sb *sandbox) PrintLogs(t *testing.T) { } func (sb *sandbox) Cmd(args ...string) *exec.Cmd { + if len(args) == 1 { + if split, err := shlex.Split(args[0]); err == nil { + args = split + } + } cmd := exec.Command("buildctl", args...) cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address()) diff --git a/vendor/github.com/containerd/containerd/fs/fstest/compare.go b/vendor/github.com/containerd/containerd/fs/fstest/compare.go new file mode 100644 index 00000000..4f55f1c6 --- /dev/null +++ b/vendor/github.com/containerd/containerd/fs/fstest/compare.go @@ -0,0 +1,53 @@ +package fstest + +import ( + "io/ioutil" + "os" + + "github.com/containerd/continuity" + "github.com/pkg/errors" +) + +// CheckDirectoryEqual compares two directory paths to make sure that +// the content of the directories is the same. +func CheckDirectoryEqual(d1, d2 string) error { + c1, err := continuity.NewContext(d1) + if err != nil { + return errors.Wrap(err, "failed to build context") + } + + c2, err := continuity.NewContext(d2) + if err != nil { + return errors.Wrap(err, "failed to build context") + } + + m1, err := continuity.BuildManifest(c1) + if err != nil { + return errors.Wrap(err, "failed to build manifest") + } + + m2, err := continuity.BuildManifest(c2) + if err != nil { + return errors.Wrap(err, "failed to build manifest") + } + + diff := diffResourceList(m1.Resources, m2.Resources) + if diff.HasDiff() { + return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) + } + + return nil +} + +// CheckDirectoryEqualWithApplier compares directory against applier +func CheckDirectoryEqualWithApplier(root string, a Applier) error { + applied, err := ioutil.TempDir("", "fstest") + if err != nil { + return err + } + defer os.RemoveAll(applied) + if err := a.Apply(applied); err != nil { + return err + } + return CheckDirectoryEqual(applied, root) +} diff --git a/vendor/github.com/containerd/containerd/fs/fstest/continuity_util.go b/vendor/github.com/containerd/containerd/fs/fstest/continuity_util.go new file mode 100644 index 00000000..448eda5e --- /dev/null +++ b/vendor/github.com/containerd/containerd/fs/fstest/continuity_util.go @@ -0,0 +1,189 @@ +package fstest + +import ( + "bytes" + "fmt" + + "github.com/containerd/continuity" +) + +type resourceUpdate struct { + Original continuity.Resource + Updated continuity.Resource +} + +func (u resourceUpdate) String() string { + return fmt.Sprintf("%s(mode: %o, uid: %d, gid: %d) -> %s(mode: %o, uid: %d, gid: %d)", + u.Original.Path(), u.Original.Mode(), u.Original.UID(), u.Original.GID(), + u.Updated.Path(), u.Updated.Mode(), u.Updated.UID(), u.Updated.GID(), + ) +} + +type resourceListDifference struct { + Additions []continuity.Resource + Deletions []continuity.Resource + Updates []resourceUpdate +} + +func (l resourceListDifference) HasDiff() bool { + return len(l.Additions) > 0 || len(l.Deletions) > 0 || len(l.Updates) > 0 +} + +func (l resourceListDifference) String() string { + buf := bytes.NewBuffer(nil) + for _, add := range l.Additions { + fmt.Fprintf(buf, "+ %s\n", add.Path()) + } + for _, del := range l.Deletions { + fmt.Fprintf(buf, "- %s\n", del.Path()) + } + for _, upt := range l.Updates { + fmt.Fprintf(buf, "~ %s\n", upt.String()) + } + return string(buf.Bytes()) +} + +// diffManifest compares two resource lists and returns the list +// of adds updates and deletes, resource lists are not reordered +// before doing difference. +func diffResourceList(r1, r2 []continuity.Resource) resourceListDifference { + i1 := 0 + i2 := 0 + var d resourceListDifference + + for i1 < len(r1) && i2 < len(r2) { + p1 := r1[i1].Path() + p2 := r2[i2].Path() + switch { + case p1 < p2: + d.Deletions = append(d.Deletions, r1[i1]) + i1++ + case p1 == p2: + if !compareResource(r1[i1], r2[i2]) { + d.Updates = append(d.Updates, resourceUpdate{ + Original: r1[i1], + Updated: r2[i2], + }) + } + i1++ + i2++ + case p1 > p2: + d.Additions = append(d.Additions, r2[i2]) + i2++ + } + } + + for i1 < len(r1) { + d.Deletions = append(d.Deletions, r1[i1]) + i1++ + + } + for i2 < len(r2) { + d.Additions = append(d.Additions, r2[i2]) + i2++ + } + + return d +} + +func compareResource(r1, r2 continuity.Resource) bool { + if r1.Path() != r2.Path() { + return false + } + if r1.Mode() != r2.Mode() { + return false + } + if r1.UID() != r2.UID() { + return false + } + if r1.GID() != r2.GID() { + return false + } + + // TODO(dmcgowan): Check if is XAttrer + + return compareResourceTypes(r1, r2) + +} + +func compareResourceTypes(r1, r2 continuity.Resource) bool { + switch t1 := r1.(type) { + case continuity.RegularFile: + t2, ok := r2.(continuity.RegularFile) + if !ok { + return false + } + return compareRegularFile(t1, t2) + case continuity.Directory: + t2, ok := r2.(continuity.Directory) + if !ok { + return false + } + return compareDirectory(t1, t2) + case continuity.SymLink: + t2, ok := r2.(continuity.SymLink) + if !ok { + return false + } + return compareSymLink(t1, t2) + case continuity.NamedPipe: + t2, ok := r2.(continuity.NamedPipe) + if !ok { + return false + } + return compareNamedPipe(t1, t2) + case continuity.Device: + t2, ok := r2.(continuity.Device) + if !ok { + return false + } + return compareDevice(t1, t2) + default: + // TODO(dmcgowan): Should this panic? + return r1 == r2 + } +} + +func compareRegularFile(r1, r2 continuity.RegularFile) bool { + if r1.Size() != r2.Size() { + return false + } + p1 := r1.Paths() + p2 := r2.Paths() + if len(p1) != len(p2) { + return false + } + for i := range p1 { + if p1[i] != p2[i] { + return false + } + } + d1 := r1.Digests() + d2 := r2.Digests() + if len(d1) != len(d2) { + return false + } + for i := range d1 { + if d1[i] != d2[i] { + return false + } + } + + return true +} + +func compareSymLink(r1, r2 continuity.SymLink) bool { + return r1.Target() == r2.Target() +} + +func compareDirectory(r1, r2 continuity.Directory) bool { + return true +} + +func compareNamedPipe(r1, r2 continuity.NamedPipe) bool { + return true +} + +func compareDevice(r1, r2 continuity.Device) bool { + return r1.Major() == r2.Major() && r1.Minor() == r2.Minor() +} diff --git a/vendor/github.com/containerd/containerd/fs/fstest/file.go b/vendor/github.com/containerd/containerd/fs/fstest/file.go new file mode 100644 index 00000000..9d614ee9 --- /dev/null +++ b/vendor/github.com/containerd/containerd/fs/fstest/file.go @@ -0,0 +1,120 @@ +package fstest + +import ( + "io/ioutil" + "os" + "path/filepath" + "time" +) + +// Applier applies single file changes +type Applier interface { + Apply(root string) error +} + +type applyFn func(root string) error + +func (a applyFn) Apply(root string) error { + return a(root) +} + +// CreateFile returns a file applier which creates a file as the +// provided name with the given content and permission. +func CreateFile(name string, content []byte, perm os.FileMode) Applier { + return applyFn(func(root string) error { + fullPath := filepath.Join(root, name) + if err := ioutil.WriteFile(fullPath, content, perm); err != nil { + return err + } + return os.Chmod(fullPath, perm) + }) +} + +// Remove returns a file applier which removes the provided file name +func Remove(name string) Applier { + return applyFn(func(root string) error { + return os.Remove(filepath.Join(root, name)) + }) +} + +// RemoveAll returns a file applier which removes the provided file name +// as in os.RemoveAll +func RemoveAll(name string) Applier { + return applyFn(func(root string) error { + return os.RemoveAll(filepath.Join(root, name)) + }) +} + +// CreateDir returns a file applier to create the directory with +// the provided name and permission +func CreateDir(name string, perm os.FileMode) Applier { + return applyFn(func(root string) error { + fullPath := filepath.Join(root, name) + if err := os.MkdirAll(fullPath, perm); err != nil { + return err + } + return os.Chmod(fullPath, perm) + }) +} + +// Rename returns a file applier which renames a file +func Rename(old, new string) Applier { + return applyFn(func(root string) error { + return os.Rename(filepath.Join(root, old), filepath.Join(root, new)) + }) +} + +// Chown returns a file applier which changes the ownership of a file +func Chown(name string, uid, gid int) Applier { + return applyFn(func(root string) error { + return os.Chown(filepath.Join(root, name), uid, gid) + }) +} + +// Chtimes changes access and mod time of file. +// Use Lchtimes for symbolic links. +func Chtimes(name string, atime, mtime time.Time) Applier { + return applyFn(func(root string) error { + return os.Chtimes(filepath.Join(root, name), atime, mtime) + }) +} + +// Chmod returns a file applier which changes the file permission +func Chmod(name string, perm os.FileMode) Applier { + return applyFn(func(root string) error { + return os.Chmod(filepath.Join(root, name), perm) + }) +} + +// Symlink returns a file applier which creates a symbolic link +func Symlink(oldname, newname string) Applier { + return applyFn(func(root string) error { + return os.Symlink(oldname, filepath.Join(root, newname)) + }) +} + +// Link returns a file applier which creates a hard link +func Link(oldname, newname string) Applier { + return applyFn(func(root string) error { + return os.Link(filepath.Join(root, oldname), filepath.Join(root, newname)) + }) +} + +// TODO: Make platform specific, windows applier is always no-op +//func Mknod(name string, mode int32, dev int) Applier { +// return func(root string) error { +// return return syscall.Mknod(path, mode, dev) +// } +//} + +// Apply returns a new applier from the given appliers +func Apply(appliers ...Applier) Applier { + return applyFn(func(root string) error { + for _, a := range appliers { + if err := a.Apply(root); err != nil { + return err + } + } + return nil + }) +} diff --git a/vendor/github.com/containerd/containerd/fs/fstest/file_unix.go b/vendor/github.com/containerd/containerd/fs/fstest/file_unix.go new file mode 100644 index 00000000..af223b94 --- /dev/null +++ b/vendor/github.com/containerd/containerd/fs/fstest/file_unix.go @@ -0,0 +1,29 @@ +// +build !windows + +package fstest + +import ( + "path/filepath" + "time" + + "github.com/containerd/continuity/sysx" + "golang.org/x/sys/unix" +) + +// SetXAttr sets the xatter for the file +func SetXAttr(name, key, value string) Applier { + return applyFn(func(root string) error { + return sysx.LSetxattr(name, key, []byte(value), 0) + }) +} + +// Lchtimes changes access and mod time of file without following symlink +func Lchtimes(name string, atime, mtime time.Time) Applier { + return applyFn(func(root string) error { + path := filepath.Join(root, name) + at := unix.NsecToTimespec(atime.UnixNano()) + mt := unix.NsecToTimespec(mtime.UnixNano()) + utimes := [2]unix.Timespec{at, mt} + return unix.UtimesNanoAt(unix.AT_FDCWD, path, utimes[0:], unix.AT_SYMLINK_NOFOLLOW) + }) +} diff --git a/vendor/github.com/containerd/containerd/fs/fstest/file_windows.go b/vendor/github.com/containerd/containerd/fs/fstest/file_windows.go new file mode 100644 index 00000000..022a09f1 --- /dev/null +++ b/vendor/github.com/containerd/containerd/fs/fstest/file_windows.go @@ -0,0 +1,14 @@ +package fstest + +import ( + "time" + + "github.com/containerd/containerd/errdefs" +) + +// Lchtimes changes access and mod time of file without following symlink +func Lchtimes(name string, atime, mtime time.Time) Applier { + return applyFn(func(root string) error { + return errdefs.ErrNotImplemented + }) +} diff --git a/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go b/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go new file mode 100644 index 00000000..546be89d --- /dev/null +++ b/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go @@ -0,0 +1,197 @@ +package fstest + +import ( + "context" + "io/ioutil" + "os" + "testing" +) + +// TestApplier applies the test context +type TestApplier interface { + TestContext(context.Context) (context.Context, func(), error) + Apply(context.Context, Applier) (string, func(), error) +} + +// FSSuite runs the path test suite +func FSSuite(t *testing.T, a TestApplier) { + t.Run("Basic", makeTest(t, a, basicTest)) + t.Run("Deletion", makeTest(t, a, deletionTest)) + t.Run("Update", makeTest(t, a, updateTest)) + t.Run("DirectoryPermission", makeTest(t, a, directoryPermissionsTest)) + t.Run("HardlinkUnmodified", makeTest(t, a, hardlinkUnmodified)) + t.Run("HardlinkBeforeUnmodified", makeTest(t, a, hardlinkBeforeUnmodified)) + t.Run("HardlinkBeforeModified", makeTest(t, a, hardlinkBeforeModified)) +} + +func makeTest(t *testing.T, ta TestApplier, as []Applier) func(t *testing.T) { + return func(t *testing.T) { + ctx, cleanup, err := ta.TestContext(context.Background()) + if err != nil { + t.Fatalf("Unable to get test context: %+v", err) + } + defer cleanup() + + applyDir, err := ioutil.TempDir("", "test-expected-") + if err != nil { + t.Fatalf("Unable to make temp directory: %+v", err) + } + defer os.RemoveAll(applyDir) + + for i, a := range as { + testDir, c, err := ta.Apply(ctx, a) + if err != nil { + t.Fatalf("Apply failed at %d: %+v", i, err) + } + if err := a.Apply(applyDir); err != nil { + if c != nil { + c() + } + t.Fatalf("Error applying change to apply directory: %+v", err) + } + + err = CheckDirectoryEqual(applyDir, testDir) + if c != nil { + c() + } + if err != nil { + t.Fatalf("Directories not equal at %d (expected <> tested): %+v", i, err) + } + } + } +} + +var ( + // baseApplier creates a basic filesystem layout + // with multiple types of files for basic tests. + baseApplier = Apply( + CreateDir("/etc/", 0755), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644), + Link("/etc/hosts", "/etc/hosts.allow"), + CreateDir("/usr/local/lib", 0755), + CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), + Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"), + CreateDir("/home", 0755), + CreateDir("/home/derek", 0700), + ) + + // basicTest covers basic operations + basicTest = []Applier{ + baseApplier, + Apply( + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + CreateFile("/etc/fstab", []byte("/dev/sda1\t/\text4\tdefaults 1 1\n"), 0600), + CreateFile("/etc/badfile", []byte(""), 0666), + CreateFile("/home/derek/.zshrc", []byte("#ZSH is just better\n"), 0640), + ), + Apply( + Remove("/etc/badfile"), + Rename("/home/derek", "/home/notderek"), + ), + Apply( + RemoveAll("/usr"), + Remove("/etc/hosts.allow"), + ), + Apply( + RemoveAll("/home"), + CreateDir("/home/derek", 0700), + CreateFile("/home/derek/.bashrc", []byte("#not going away\n"), 0640), + Link("/etc/hosts", "/etc/hosts.allow"), + ), + } + + // deletionTest covers various deletion scenarios to ensure + // deletions are properly picked up and applied + deletionTest = []Applier{ + Apply( + CreateDir("/test/somedir", 0755), + CreateDir("/lib", 0700), + CreateFile("/lib/hidden", []byte{}, 0644), + ), + Apply( + CreateFile("/test/a", []byte{}, 0644), + CreateFile("/test/b", []byte{}, 0644), + CreateDir("/test/otherdir", 0755), + CreateFile("/test/otherdir/.empty", []byte{}, 0644), + RemoveAll("/lib"), + CreateDir("/lib", 0700), + CreateFile("/lib/not-hidden", []byte{}, 0644), + ), + Apply( + Remove("/test/a"), + Remove("/test/b"), + RemoveAll("/test/otherdir"), + CreateFile("/lib/newfile", []byte{}, 0644), + ), + } + + // updateTest covers file updates for content and permission + updateTest = []Applier{ + Apply( + CreateDir("/d1", 0755), + CreateDir("/d2", 0700), + CreateFile("/d1/f1", []byte("something..."), 0644), + CreateFile("/d1/f2", []byte("else..."), 0644), + CreateFile("/d1/f3", []byte("entirely..."), 0644), + ), + Apply( + CreateFile("/d1/f1", []byte("file content of a different length"), 0664), + Remove("/d1/f3"), + CreateFile("/d1/f3", []byte("updated content"), 0664), + Chmod("/d1/f2", 0766), + Chmod("/d2", 0777), + ), + } + + // directoryPermissionsTest covers directory permissions on update + directoryPermissionsTest = []Applier{ + Apply( + CreateDir("/d1", 0700), + CreateDir("/d2", 0751), + CreateDir("/d3", 0777), + ), + Apply( + CreateFile("/d1/f", []byte("irrelevant"), 0644), + CreateDir("/d1/d", 0700), + CreateFile("/d1/d/f", []byte("irrelevant"), 0644), + CreateFile("/d2/f", []byte("irrelevant"), 0644), + CreateFile("/d3/f", []byte("irrelevant"), 0644), + ), + } + + hardlinkUnmodified = []Applier{ + baseApplier, + Apply( + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + ), + Apply( + Link("/etc/hosts", "/etc/hosts.deny"), + ), + } + + // Hardlink name before with modification + // Tests link is created for unmodified files when new hardlinked file is seen first + hardlinkBeforeUnmodified = []Applier{ + baseApplier, + Apply( + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + ), + Apply( + Link("/etc/hosts", "/etc/before-hosts"), + ), + } + + // Hardlink name after without modification + // tests link is created for modified file with new hardlink + hardlinkBeforeModified = []Applier{ + baseApplier, + Apply( + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + ), + Apply( + Remove("/etc/hosts"), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644), + Link("/etc/hosts", "/etc/before-hosts"), + ), + } +) diff --git a/vendor/github.com/containerd/continuity/context.go b/vendor/github.com/containerd/continuity/context.go new file mode 100644 index 00000000..2667a97d --- /dev/null +++ b/vendor/github.com/containerd/continuity/context.go @@ -0,0 +1,641 @@ +package continuity + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/containerd/continuity/devices" + driverpkg "github.com/containerd/continuity/driver" + "github.com/containerd/continuity/pathdriver" + "github.com/opencontainers/go-digest" +) + +var ( + ErrNotFound = fmt.Errorf("not found") + ErrNotSupported = fmt.Errorf("not supported") +) + +// Context represents a file system context for accessing resources. The +// responsibility of the context is to convert system specific resources to +// generic Resource objects. Most of this is safe path manipulation, as well +// as extraction of resource details. +type Context interface { + Apply(Resource) error + Verify(Resource) error + Resource(string, os.FileInfo) (Resource, error) + Walk(filepath.WalkFunc) error +} + +// SymlinkPath is intended to give the symlink target value +// in a root context. Target and linkname are absolute paths +// not under the given root. +type SymlinkPath func(root, linkname, target string) (string, error) + +type ContextOptions struct { + Digester Digester + Driver driverpkg.Driver + PathDriver pathdriver.PathDriver + Provider ContentProvider +} + +// context represents a file system context for accessing resources. +// Generally, all path qualified access and system considerations should land +// here. +type context struct { + driver driverpkg.Driver + pathDriver pathdriver.PathDriver + root string + digester Digester + provider ContentProvider +} + +// NewContext returns a Context associated with root. The default driver will +// be used, as returned by NewDriver. +func NewContext(root string) (Context, error) { + return NewContextWithOptions(root, ContextOptions{}) +} + +// NewContextWithOptions returns a Context associate with the root. +func NewContextWithOptions(root string, options ContextOptions) (Context, error) { + // normalize to absolute path + pathDriver := options.PathDriver + if pathDriver == nil { + pathDriver = pathdriver.LocalPathDriver + } + + root = pathDriver.FromSlash(root) + root, err := pathDriver.Abs(pathDriver.Clean(root)) + if err != nil { + return nil, err + } + + driver := options.Driver + if driver == nil { + driver, err = driverpkg.NewSystemDriver() + if err != nil { + return nil, err + } + } + + digester := options.Digester + if digester == nil { + digester = simpleDigester{digest.Canonical} + } + + // Check the root directory. Need to be a little careful here. We are + // allowing a link for now, but this may have odd behavior when + // canonicalizing paths. As long as all files are opened through the link + // path, this should be okay. + fi, err := driver.Stat(root) + if err != nil { + return nil, err + } + + if !fi.IsDir() { + return nil, &os.PathError{Op: "NewContext", Path: root, Err: os.ErrInvalid} + } + + return &context{ + root: root, + driver: driver, + pathDriver: pathDriver, + digester: digester, + provider: options.Provider, + }, nil +} + +// Resource returns the resource as path p, populating the entry with info +// from fi. The path p should be the path of the resource in the context, +// typically obtained through Walk or from the value of Resource.Path(). If fi +// is nil, it will be resolved. +func (c *context) Resource(p string, fi os.FileInfo) (Resource, error) { + fp, err := c.fullpath(p) + if err != nil { + return nil, err + } + + if fi == nil { + fi, err = c.driver.Lstat(fp) + if err != nil { + return nil, err + } + } + + base, err := newBaseResource(p, fi) + if err != nil { + return nil, err + } + + base.xattrs, err = c.resolveXAttrs(fp, fi, base) + if err == ErrNotSupported { + log.Printf("resolving xattrs on %s not supported", fp) + } else if err != nil { + return nil, err + } + + // TODO(stevvooe): Handle windows alternate data streams. + + if fi.Mode().IsRegular() { + dgst, err := c.digest(p) + if err != nil { + return nil, err + } + + return newRegularFile(*base, base.paths, fi.Size(), dgst) + } + + if fi.Mode().IsDir() { + return newDirectory(*base) + } + + if fi.Mode()&os.ModeSymlink != 0 { + // We handle relative links vs absolute links by including a + // beginning slash for absolute links. Effectively, the bundle's + // root is treated as the absolute link anchor. + target, err := c.driver.Readlink(fp) + if err != nil { + return nil, err + } + + return newSymLink(*base, target) + } + + if fi.Mode()&os.ModeNamedPipe != 0 { + return newNamedPipe(*base, base.paths) + } + + if fi.Mode()&os.ModeDevice != 0 { + deviceDriver, ok := c.driver.(driverpkg.DeviceInfoDriver) + if !ok { + log.Printf("device extraction not supported %s", fp) + return nil, ErrNotSupported + } + + // character and block devices merely need to recover the + // major/minor device number. + major, minor, err := deviceDriver.DeviceInfo(fi) + if err != nil { + return nil, err + } + + return newDevice(*base, base.paths, major, minor) + } + + log.Printf("%q (%v) is not supported", fp, fi.Mode()) + return nil, ErrNotFound +} + +func (c *context) verifyMetadata(resource, target Resource) error { + if target.Mode() != resource.Mode() { + return fmt.Errorf("resource %q has incorrect mode: %v != %v", target.Path(), target.Mode(), resource.Mode()) + } + + if target.UID() != resource.UID() { + return fmt.Errorf("unexpected uid for %q: %v != %v", target.Path(), target.UID(), resource.GID()) + } + + if target.GID() != resource.GID() { + return fmt.Errorf("unexpected gid for %q: %v != %v", target.Path(), target.GID(), target.GID()) + } + + if xattrer, ok := resource.(XAttrer); ok { + txattrer, tok := target.(XAttrer) + if !tok { + return fmt.Errorf("resource %q has xattrs but target does not support them", resource.Path()) + } + + // For xattrs, only ensure that we have those defined in the resource + // and their values match. We can ignore other xattrs. In other words, + // we only verify that target has the subset defined by resource. + txattrs := txattrer.XAttrs() + for attr, value := range xattrer.XAttrs() { + tvalue, ok := txattrs[attr] + if !ok { + return fmt.Errorf("resource %q target missing xattr %q", resource.Path(), attr) + } + + if !bytes.Equal(value, tvalue) { + return fmt.Errorf("xattr %q value differs for resource %q", attr, resource.Path()) + } + } + } + + switch r := resource.(type) { + case RegularFile: + // TODO(stevvooe): Another reason to use a record-based approach. We + // have to do another type switch to get this to work. This could be + // fixed with an Equal function, but let's study this a little more to + // be sure. + t, ok := target.(RegularFile) + if !ok { + return fmt.Errorf("resource %q target not a regular file", r.Path()) + } + + if t.Size() != r.Size() { + return fmt.Errorf("resource %q target has incorrect size: %v != %v", t.Path(), t.Size(), r.Size()) + } + case Directory: + t, ok := target.(Directory) + if !ok { + return fmt.Errorf("resource %q target not a directory", t.Path()) + } + case SymLink: + t, ok := target.(SymLink) + if !ok { + return fmt.Errorf("resource %q target not a symlink", t.Path()) + } + + if t.Target() != r.Target() { + return fmt.Errorf("resource %q target has mismatched target: %q != %q", t.Path(), t.Target(), r.Target()) + } + case Device: + t, ok := target.(Device) + if !ok { + return fmt.Errorf("resource %q is not a device", t.Path()) + } + + if t.Major() != r.Major() || t.Minor() != r.Minor() { + return fmt.Errorf("resource %q has mismatched major/minor numbers: %d,%d != %d,%d", t.Path(), t.Major(), t.Minor(), r.Major(), r.Minor()) + } + case NamedPipe: + t, ok := target.(NamedPipe) + if !ok { + return fmt.Errorf("resource %q is not a named pipe", t.Path()) + } + default: + return fmt.Errorf("cannot verify resource: %v", resource) + } + + return nil +} + +// Verify the resource in the context. An error will be returned a discrepancy +// is found. +func (c *context) Verify(resource Resource) error { + fp, err := c.fullpath(resource.Path()) + if err != nil { + return err + } + + fi, err := c.driver.Lstat(fp) + if err != nil { + return err + } + + target, err := c.Resource(resource.Path(), fi) + if err != nil { + return err + } + + if target.Path() != resource.Path() { + return fmt.Errorf("resource paths do not match: %q != %q", target.Path(), resource.Path()) + } + + if err := c.verifyMetadata(resource, target); err != nil { + return err + } + + if h, isHardlinkable := resource.(Hardlinkable); isHardlinkable { + hardlinkKey, err := newHardlinkKey(fi) + if err == errNotAHardLink { + if len(h.Paths()) > 1 { + return fmt.Errorf("%q is not a hardlink to %q", h.Paths()[1], resource.Path()) + } + } else if err != nil { + return err + } + + for _, path := range h.Paths()[1:] { + fpLink, err := c.fullpath(path) + if err != nil { + return err + } + + fiLink, err := c.driver.Lstat(fpLink) + if err != nil { + return err + } + + targetLink, err := c.Resource(path, fiLink) + if err != nil { + return err + } + + hardlinkKeyLink, err := newHardlinkKey(fiLink) + if err != nil { + return err + } + + if hardlinkKeyLink != hardlinkKey { + return fmt.Errorf("%q is not a hardlink to %q", path, resource.Path()) + } + + if err := c.verifyMetadata(resource, targetLink); err != nil { + return err + } + } + } + + switch r := resource.(type) { + case RegularFile: + t, ok := target.(RegularFile) + if !ok { + return fmt.Errorf("resource %q target not a regular file", r.Path()) + } + + // TODO(stevvooe): This may need to get a little more sophisticated + // for digest comparison. We may want to actually calculate the + // provided digests, rather than the implementations having an + // overlap. + if !digestsMatch(t.Digests(), r.Digests()) { + return fmt.Errorf("digests for resource %q do not match: %v != %v", t.Path(), t.Digests(), r.Digests()) + } + } + + return nil +} + +func (c *context) checkoutFile(fp string, rf RegularFile) error { + if c.provider == nil { + return fmt.Errorf("no file provider") + } + var ( + r io.ReadCloser + err error + ) + for _, dgst := range rf.Digests() { + r, err = c.provider.Reader(dgst) + if err == nil { + break + } + } + if err != nil { + return fmt.Errorf("file content could not be provided: %v", err) + } + defer r.Close() + + return atomicWriteFile(fp, r, rf) +} + +// Apply the resource to the contexts. An error will be returned if the +// operation fails. Depending on the resource type, the resource may be +// created. For resource that cannot be resolved, an error will be returned. +func (c *context) Apply(resource Resource) error { + fp, err := c.fullpath(resource.Path()) + if err != nil { + return err + } + + if !strings.HasPrefix(fp, c.root) { + return fmt.Errorf("resource %v escapes root", resource) + } + + var chmod = true + fi, err := c.driver.Lstat(fp) + if err != nil { + if !os.IsNotExist(err) { + return err + } + } + + switch r := resource.(type) { + case RegularFile: + if fi == nil { + if err := c.checkoutFile(fp, r); err != nil { + return fmt.Errorf("error checking out file %q: %v", resource.Path(), err) + } + chmod = false + } else { + if !fi.Mode().IsRegular() { + return fmt.Errorf("file %q should be a regular file, but is not", resource.Path()) + } + if fi.Size() != r.Size() { + if err := c.checkoutFile(fp, r); err != nil { + return fmt.Errorf("error checking out file %q: %v", resource.Path(), err) + } + } else { + for _, dgst := range r.Digests() { + f, err := os.Open(fp) + if err != nil { + return fmt.Errorf("failure opening file for read %q: %v", resource.Path(), err) + } + compared, err := dgst.Algorithm().FromReader(f) + if err == nil && dgst != compared { + if err := c.checkoutFile(fp, r); err != nil { + return fmt.Errorf("error checking out file %q: %v", resource.Path(), err) + } + break + } + if err1 := f.Close(); err == nil { + err = err1 + } + if err != nil { + return fmt.Errorf("error checking digest for %q: %v", resource.Path(), err) + } + } + } + } + case Directory: + if fi == nil { + if err := c.driver.Mkdir(fp, resource.Mode()); err != nil { + return err + } + } else if !fi.Mode().IsDir() { + return fmt.Errorf("%q should be a directory, but is not", resource.Path()) + } + + case SymLink: + var target string // only possibly set if target resource is a symlink + + if fi != nil { + if fi.Mode()&os.ModeSymlink != 0 { + target, err = c.driver.Readlink(fp) + if err != nil { + return err + } + } + } + + if target != r.Target() { + if fi != nil { + if err := c.driver.Remove(fp); err != nil { // RemoveAll in case of directory? + return err + } + } + + if err := c.driver.Symlink(r.Target(), fp); err != nil { + return err + } + } + + // NOTE(stevvooe): Chmod on symlink is not supported on linux. We + // may want to maintain support for other platforms that have it. + chmod = false + + case Device: + if fi == nil { + if err := c.driver.Mknod(fp, resource.Mode(), int(r.Major()), int(r.Minor())); err != nil { + return err + } + } else if (fi.Mode() & os.ModeDevice) == 0 { + return fmt.Errorf("%q should be a device, but is not", resource.Path()) + } else { + major, minor, err := devices.DeviceInfo(fi) + if err != nil { + return err + } + if major != r.Major() || minor != r.Minor() { + if err := c.driver.Remove(fp); err != nil { + return err + } + + if err := c.driver.Mknod(fp, resource.Mode(), int(r.Major()), int(r.Minor())); err != nil { + return err + } + } + } + + case NamedPipe: + if fi == nil { + if err := c.driver.Mkfifo(fp, resource.Mode()); err != nil { + return err + } + } else if (fi.Mode() & os.ModeNamedPipe) == 0 { + return fmt.Errorf("%q should be a named pipe, but is not", resource.Path()) + } + } + + if h, isHardlinkable := resource.(Hardlinkable); isHardlinkable { + for _, path := range h.Paths() { + if path == resource.Path() { + continue + } + + lp, err := c.fullpath(path) + if err != nil { + return err + } + + if _, fi := c.driver.Lstat(lp); fi == nil { + c.driver.Remove(lp) + } + if err := c.driver.Link(fp, lp); err != nil { + return err + } + } + } + + // Update filemode if file was not created + if chmod { + if err := c.driver.Lchmod(fp, resource.Mode()); err != nil { + return err + } + } + + if err := c.driver.Lchown(fp, resource.UID(), resource.GID()); err != nil { + return err + } + + if xattrer, ok := resource.(XAttrer); ok { + // For xattrs, only ensure that we have those defined in the resource + // and their values are set. We can ignore other xattrs. In other words, + // we only set xattres defined by resource but never remove. + + if _, ok := resource.(SymLink); ok { + lxattrDriver, ok := c.driver.(driverpkg.LXAttrDriver) + if !ok { + return fmt.Errorf("unsupported symlink xattr for resource %q", resource.Path()) + } + if err := lxattrDriver.LSetxattr(fp, xattrer.XAttrs()); err != nil { + return err + } + } else { + xattrDriver, ok := c.driver.(driverpkg.XAttrDriver) + if !ok { + return fmt.Errorf("unsupported xattr for resource %q", resource.Path()) + } + if err := xattrDriver.Setxattr(fp, xattrer.XAttrs()); err != nil { + return err + } + } + } + + return nil +} + +// Walk provides a convenience function to call filepath.Walk correctly for +// the context. Otherwise identical to filepath.Walk, the path argument is +// corrected to be contained within the context. +func (c *context) Walk(fn filepath.WalkFunc) error { + return c.pathDriver.Walk(c.root, func(p string, fi os.FileInfo, err error) error { + contained, err := c.contain(p) + return fn(contained, fi, err) + }) +} + +// fullpath returns the system path for the resource, joined with the context +// root. The path p must be a part of the context. +func (c *context) fullpath(p string) (string, error) { + p = c.pathDriver.Join(c.root, p) + if !strings.HasPrefix(p, c.root) { + return "", fmt.Errorf("invalid context path") + } + + return p, nil +} + +// contain cleans and santizes the filesystem path p to be an absolute path, +// effectively relative to the context root. +func (c *context) contain(p string) (string, error) { + sanitized, err := c.pathDriver.Rel(c.root, p) + if err != nil { + return "", err + } + + // ZOMBIES(stevvooe): In certain cases, we may want to remap these to a + // "containment error", so the caller can decide what to do. + return c.pathDriver.Join("/", c.pathDriver.Clean(sanitized)), nil +} + +// digest returns the digest of the file at path p, relative to the root. +func (c *context) digest(p string) (digest.Digest, error) { + f, err := c.driver.Open(c.pathDriver.Join(c.root, p)) + if err != nil { + return "", err + } + defer f.Close() + + return c.digester.Digest(f) +} + +// resolveXAttrs attempts to resolve the extended attributes for the resource +// at the path fp, which is the full path to the resource. If the resource +// cannot have xattrs, nil will be returned. +func (c *context) resolveXAttrs(fp string, fi os.FileInfo, base *resource) (map[string][]byte, error) { + if fi.Mode().IsRegular() || fi.Mode().IsDir() { + xattrDriver, ok := c.driver.(driverpkg.XAttrDriver) + if !ok { + log.Println("xattr extraction not supported") + return nil, ErrNotSupported + } + + return xattrDriver.Getxattr(fp) + } + + if fi.Mode()&os.ModeSymlink != 0 { + lxattrDriver, ok := c.driver.(driverpkg.LXAttrDriver) + if !ok { + log.Println("xattr extraction for symlinks not supported") + return nil, ErrNotSupported + } + + return lxattrDriver.LGetxattr(fp) + } + + return nil, nil +} diff --git a/vendor/github.com/containerd/continuity/devices/devices.go b/vendor/github.com/containerd/continuity/devices/devices.go new file mode 100644 index 00000000..70864070 --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices.go @@ -0,0 +1,5 @@ +package devices + +import "fmt" + +var ErrNotSupported = fmt.Errorf("not supported") diff --git a/vendor/github.com/containerd/continuity/devices/devices_darwin.go b/vendor/github.com/containerd/continuity/devices/devices_darwin.go new file mode 100644 index 00000000..5041e666 --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_darwin.go @@ -0,0 +1,15 @@ +package devices + +// from /usr/include/sys/types.h + +func getmajor(dev int32) uint64 { + return (uint64(dev) >> 24) & 0xff +} + +func getminor(dev int32) uint64 { + return uint64(dev) & 0xffffff +} + +func makedev(major int, minor int) int { + return ((major << 24) | minor) +} diff --git a/vendor/github.com/containerd/continuity/devices/devices_dummy.go b/vendor/github.com/containerd/continuity/devices/devices_dummy.go new file mode 100644 index 00000000..9a48330a --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_dummy.go @@ -0,0 +1,23 @@ +// +build solaris,!cgo + +// +// Implementing the functions below requires cgo support. Non-cgo stubs +// versions are defined below to enable cross-compilation of source code +// that depends on these functions, but the resultant cross-compiled +// binaries cannot actually be used. If the stub function(s) below are +// actually invoked they will cause the calling process to exit. +// + +package devices + +func getmajor(dev uint64) uint64 { + panic("getmajor() support requires cgo.") +} + +func getminor(dev uint64) uint64 { + panic("getminor() support requires cgo.") +} + +func makedev(major int, minor int) int { + panic("makedev() support requires cgo.") +} diff --git a/vendor/github.com/containerd/continuity/devices/devices_freebsd.go b/vendor/github.com/containerd/continuity/devices/devices_freebsd.go new file mode 100644 index 00000000..a5c7b931 --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_freebsd.go @@ -0,0 +1,15 @@ +package devices + +// from /usr/include/sys/types.h + +func getmajor(dev uint32) uint64 { + return (uint64(dev) >> 24) & 0xff +} + +func getminor(dev uint32) uint64 { + return uint64(dev) & 0xffffff +} + +func makedev(major int, minor int) int { + return ((major << 24) | minor) +} diff --git a/vendor/github.com/containerd/continuity/devices/devices_linux.go b/vendor/github.com/containerd/continuity/devices/devices_linux.go new file mode 100644 index 00000000..454cf668 --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_linux.go @@ -0,0 +1,15 @@ +package devices + +// from /usr/include/linux/kdev_t.h + +func getmajor(dev uint64) uint64 { + return dev >> 8 +} + +func getminor(dev uint64) uint64 { + return dev & 0xff +} + +func makedev(major int, minor int) int { + return ((major << 8) | minor) +} diff --git a/vendor/github.com/containerd/continuity/devices/devices_solaris.go b/vendor/github.com/containerd/continuity/devices/devices_solaris.go new file mode 100644 index 00000000..8819ac82 --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_solaris.go @@ -0,0 +1,18 @@ +// +build cgo + +package devices + +//#include +import "C" + +func getmajor(dev uint64) uint64 { + return uint64(C.major(C.dev_t(dev))) +} + +func getminor(dev uint64) uint64 { + return uint64(C.minor(C.dev_t(dev))) +} + +func makedev(major int, minor int) int { + return int(C.makedev(C.major_t(major), C.minor_t(minor))) +} diff --git a/vendor/github.com/containerd/continuity/devices/devices_unix.go b/vendor/github.com/containerd/continuity/devices/devices_unix.go new file mode 100644 index 00000000..85e9a68c --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_unix.go @@ -0,0 +1,55 @@ +// +build linux darwin freebsd solaris + +package devices + +import ( + "fmt" + "os" + "syscall" +) + +func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo") + } + + return getmajor(sys.Rdev), getminor(sys.Rdev), nil +} + +// mknod provides a shortcut for syscall.Mknod +func Mknod(p string, mode os.FileMode, maj, min int) error { + var ( + m = syscallMode(mode.Perm()) + dev int + ) + + if mode&os.ModeDevice != 0 { + dev = makedev(maj, min) + + if mode&os.ModeCharDevice != 0 { + m |= syscall.S_IFCHR + } else { + m |= syscall.S_IFBLK + } + } else if mode&os.ModeNamedPipe != 0 { + m |= syscall.S_IFIFO + } + + return syscall.Mknod(p, m, dev) +} + +// syscallMode returns the syscall-specific mode bits from Go's portable mode bits. +func syscallMode(i os.FileMode) (o uint32) { + o |= uint32(i.Perm()) + if i&os.ModeSetuid != 0 { + o |= syscall.S_ISUID + } + if i&os.ModeSetgid != 0 { + o |= syscall.S_ISGID + } + if i&os.ModeSticky != 0 { + o |= syscall.S_ISVTX + } + return +} diff --git a/vendor/github.com/containerd/continuity/devices/devices_windows.go b/vendor/github.com/containerd/continuity/devices/devices_windows.go new file mode 100644 index 00000000..6099d1d7 --- /dev/null +++ b/vendor/github.com/containerd/continuity/devices/devices_windows.go @@ -0,0 +1,11 @@ +package devices + +import ( + "os" + + "github.com/pkg/errors" +) + +func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) { + return 0, 0, errors.Wrap(ErrNotSupported, "cannot get device info on windows") +} diff --git a/vendor/github.com/containerd/continuity/digests.go b/vendor/github.com/containerd/continuity/digests.go new file mode 100644 index 00000000..355b0803 --- /dev/null +++ b/vendor/github.com/containerd/continuity/digests.go @@ -0,0 +1,88 @@ +package continuity + +import ( + "fmt" + "io" + "sort" + + "github.com/opencontainers/go-digest" +) + +// Digester produces a digest for a given read stream +type Digester interface { + Digest(io.Reader) (digest.Digest, error) +} + +// ContentProvider produces a read stream for a given digest +type ContentProvider interface { + Reader(digest.Digest) (io.ReadCloser, error) +} + +type simpleDigester struct { + algorithm digest.Algorithm +} + +func (sd simpleDigester) Digest(r io.Reader) (digest.Digest, error) { + digester := sd.algorithm.Digester() + + if _, err := io.Copy(digester.Hash(), r); err != nil { + return "", err + } + + return digester.Digest(), nil +} + +// uniqifyDigests sorts and uniqifies the provided digest, ensuring that the +// digests are not repeated and no two digests with the same algorithm have +// different values. Because a stable sort is used, this has the effect of +// "zipping" digest collections from multiple resources. +func uniqifyDigests(digests ...digest.Digest) ([]digest.Digest, error) { + sort.Stable(digestSlice(digests)) // stable sort is important for the behavior here. + seen := map[digest.Digest]struct{}{} + algs := map[digest.Algorithm][]digest.Digest{} // detect different digests. + + var out []digest.Digest + // uniqify the digests + for _, d := range digests { + if _, ok := seen[d]; ok { + continue + } + + seen[d] = struct{}{} + algs[d.Algorithm()] = append(algs[d.Algorithm()], d) + + if len(algs[d.Algorithm()]) > 1 { + return nil, fmt.Errorf("conflicting digests for %v found", d.Algorithm()) + } + + out = append(out, d) + } + + return out, nil +} + +// digestsMatch compares the two sets of digests to see if they match. +func digestsMatch(as, bs []digest.Digest) bool { + all := append(as, bs...) + + uniqified, err := uniqifyDigests(all...) + if err != nil { + // the only error uniqifyDigests returns is when the digests disagree. + return false + } + + disjoint := len(as) + len(bs) + if len(uniqified) == disjoint { + // if these two sets have the same cardinality, we know both sides + // didn't share any digests. + return false + } + + return true +} + +type digestSlice []digest.Digest + +func (p digestSlice) Len() int { return len(p) } +func (p digestSlice) Less(i, j int) bool { return p[i] < p[j] } +func (p digestSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/containerd/continuity/driver/driver.go b/vendor/github.com/containerd/continuity/driver/driver.go new file mode 100644 index 00000000..aa1dd7d2 --- /dev/null +++ b/vendor/github.com/containerd/continuity/driver/driver.go @@ -0,0 +1,162 @@ +package driver + +import ( + "fmt" + "io" + "os" +) + +var ErrNotSupported = fmt.Errorf("not supported") + +// Driver provides all of the system-level functions in a common interface. +// The context should call these with full paths and should never use the `os` +// package or any other package to access resources on the filesystem. This +// mechanism let's us carefully control access to the context and maintain +// path and resource integrity. It also gives us an interface to reason about +// direct resource access. +// +// Implementations don't need to do much other than meet the interface. For +// example, it is not required to wrap os.FileInfo to return correct paths for +// the call to Name(). +type Driver interface { + // Note that Open() returns a File interface instead of *os.File. This + // is because os.File is a struct, so if Open was to return *os.File, + // the only way to fulfill the interface would be to call os.Open() + Open(path string) (File, error) + OpenFile(path string, flag int, perm os.FileMode) (File, error) + + Stat(path string) (os.FileInfo, error) + Lstat(path string) (os.FileInfo, error) + Readlink(p string) (string, error) + Mkdir(path string, mode os.FileMode) error + Remove(path string) error + + Link(oldname, newname string) error + Lchmod(path string, mode os.FileMode) error + Lchown(path string, uid, gid int64) error + Symlink(oldname, newname string) error + + MkdirAll(path string, perm os.FileMode) error + RemoveAll(path string) error + + // TODO(aaronl): These methods might move outside the main Driver + // interface in the future as more platforms are added. + Mknod(path string, mode os.FileMode, major int, minor int) error + Mkfifo(path string, mode os.FileMode) error +} + +// File is the interface for interacting with files returned by continuity's Open +// This is needed since os.File is a struct, instead of an interface, so it can't +// be used. +type File interface { + io.ReadWriteCloser + io.Seeker + Readdir(n int) ([]os.FileInfo, error) +} + +func NewSystemDriver() (Driver, error) { + // TODO(stevvooe): Consider having this take a "hint" path argument, which + // would be the context root. The hint could be used to resolve required + // filesystem support when assembling the driver to use. + return &driver{}, nil +} + +// XAttrDriver should be implemented on operation systems and filesystems that +// have xattr support for regular files and directories. +type XAttrDriver interface { + // Getxattr returns all of the extended attributes for the file at path. + // Typically, this takes a syscall call to Listxattr and Getxattr. + Getxattr(path string) (map[string][]byte, error) + + // Setxattr sets all of the extended attributes on file at path, following + // any symbolic links, if necessary. All attributes on the target are + // replaced by the values from attr. If the operation fails to set any + // attribute, those already applied will not be rolled back. + Setxattr(path string, attr map[string][]byte) error +} + +// LXAttrDriver should be implemented by drivers on operating systems and +// filesystems that support setting and getting extended attributes on +// symbolic links. If this is not implemented, extended attributes will be +// ignored on symbolic links. +type LXAttrDriver interface { + // LGetxattr returns all of the extended attributes for the file at path + // and does not follow symlinks. Typically, this takes a syscall call to + // Llistxattr and Lgetxattr. + LGetxattr(path string) (map[string][]byte, error) + + // LSetxattr sets all of the extended attributes on file at path, without + // following symbolic links. All attributes on the target are replaced by + // the values from attr. If the operation fails to set any attribute, + // those already applied will not be rolled back. + LSetxattr(path string, attr map[string][]byte) error +} + +type DeviceInfoDriver interface { + DeviceInfo(fi os.FileInfo) (maj uint64, min uint64, err error) +} + +// driver is a simple default implementation that sends calls out to the "os" +// package. Extend the "driver" type in system-specific files to add support, +// such as xattrs, which can add support at compile time. +type driver struct{} + +var _ File = &os.File{} + +// LocalDriver is the exported Driver struct for convenience. +var LocalDriver Driver = &driver{} + +func (d *driver) Open(p string) (File, error) { + return os.Open(p) +} + +func (d *driver) OpenFile(path string, flag int, perm os.FileMode) (File, error) { + return os.OpenFile(path, flag, perm) +} + +func (d *driver) Stat(p string) (os.FileInfo, error) { + return os.Stat(p) +} + +func (d *driver) Lstat(p string) (os.FileInfo, error) { + return os.Lstat(p) +} + +func (d *driver) Readlink(p string) (string, error) { + return os.Readlink(p) +} + +func (d *driver) Mkdir(p string, mode os.FileMode) error { + return os.Mkdir(p, mode) +} + +// Remove is used to unlink files and remove directories. +// This is following the golang os package api which +// combines the operations into a higher level Remove +// function. If explicit unlinking or directory removal +// to mirror system call is required, they should be +// split up at that time. +func (d *driver) Remove(path string) error { + return os.Remove(path) +} + +func (d *driver) Link(oldname, newname string) error { + return os.Link(oldname, newname) +} + +func (d *driver) Lchown(name string, uid, gid int64) error { + // TODO: error out if uid excesses int bit width? + return os.Lchown(name, int(uid), int(gid)) +} + +func (d *driver) Symlink(oldname, newname string) error { + return os.Symlink(oldname, newname) +} + +func (d *driver) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (d *driver) RemoveAll(path string) error { + return os.RemoveAll(path) +} diff --git a/vendor/github.com/containerd/continuity/driver/driver_unix.go b/vendor/github.com/containerd/continuity/driver/driver_unix.go new file mode 100644 index 00000000..d9ab1656 --- /dev/null +++ b/vendor/github.com/containerd/continuity/driver/driver_unix.go @@ -0,0 +1,122 @@ +// +build linux darwin freebsd solaris + +package driver + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/containerd/continuity/devices" + "github.com/containerd/continuity/sysx" +) + +func (d *driver) Mknod(path string, mode os.FileMode, major, minor int) error { + return devices.Mknod(path, mode, major, minor) +} + +func (d *driver) Mkfifo(path string, mode os.FileMode) error { + if mode&os.ModeNamedPipe == 0 { + return errors.New("mode passed to Mkfifo does not have the named pipe bit set") + } + // mknod with a mode that has ModeNamedPipe set creates a fifo, not a + // device. + return devices.Mknod(path, mode, 0, 0) +} + +// Lchmod changes the mode of an file not following symlinks. +func (d *driver) Lchmod(path string, mode os.FileMode) (err error) { + if !filepath.IsAbs(path) { + path, err = filepath.Abs(path) + if err != nil { + return + } + } + + return sysx.Fchmodat(0, path, uint32(mode), sysx.AtSymlinkNofollow) +} + +// Getxattr returns all of the extended attributes for the file at path p. +func (d *driver) Getxattr(p string) (map[string][]byte, error) { + xattrs, err := sysx.Listxattr(p) + if err != nil { + return nil, fmt.Errorf("listing %s xattrs: %v", p, err) + } + + sort.Strings(xattrs) + m := make(map[string][]byte, len(xattrs)) + + for _, attr := range xattrs { + value, err := sysx.Getxattr(p, attr) + if err != nil { + return nil, fmt.Errorf("getting %q xattr on %s: %v", attr, p, err) + } + + // NOTE(stevvooe): This append/copy tricky relies on unique + // xattrs. Break this out into an alloc/copy if xattrs are no + // longer unique. + m[attr] = append(m[attr], value...) + } + + return m, nil +} + +// Setxattr sets all of the extended attributes on file at path, following +// any symbolic links, if necessary. All attributes on the target are +// replaced by the values from attr. If the operation fails to set any +// attribute, those already applied will not be rolled back. +func (d *driver) Setxattr(path string, attrMap map[string][]byte) error { + for attr, value := range attrMap { + if err := sysx.Setxattr(path, attr, value, 0); err != nil { + return fmt.Errorf("error setting xattr %q on %s: %v", attr, path, err) + } + } + + return nil +} + +// LGetxattr returns all of the extended attributes for the file at path p +// not following symbolic links. +func (d *driver) LGetxattr(p string) (map[string][]byte, error) { + xattrs, err := sysx.LListxattr(p) + if err != nil { + return nil, fmt.Errorf("listing %s xattrs: %v", p, err) + } + + sort.Strings(xattrs) + m := make(map[string][]byte, len(xattrs)) + + for _, attr := range xattrs { + value, err := sysx.LGetxattr(p, attr) + if err != nil { + return nil, fmt.Errorf("getting %q xattr on %s: %v", attr, p, err) + } + + // NOTE(stevvooe): This append/copy tricky relies on unique + // xattrs. Break this out into an alloc/copy if xattrs are no + // longer unique. + m[attr] = append(m[attr], value...) + } + + return m, nil +} + +// LSetxattr sets all of the extended attributes on file at path, not +// following any symbolic links. All attributes on the target are +// replaced by the values from attr. If the operation fails to set any +// attribute, those already applied will not be rolled back. +func (d *driver) LSetxattr(path string, attrMap map[string][]byte) error { + for attr, value := range attrMap { + if err := sysx.LSetxattr(path, attr, value, 0); err != nil { + return fmt.Errorf("error setting xattr %q on %s: %v", attr, path, err) + } + } + + return nil +} + +func (d *driver) DeviceInfo(fi os.FileInfo) (maj uint64, min uint64, err error) { + return devices.DeviceInfo(fi) +} diff --git a/vendor/github.com/containerd/continuity/driver/driver_windows.go b/vendor/github.com/containerd/continuity/driver/driver_windows.go new file mode 100644 index 00000000..e4cfa64f --- /dev/null +++ b/vendor/github.com/containerd/continuity/driver/driver_windows.go @@ -0,0 +1,21 @@ +package driver + +import ( + "os" + + "github.com/pkg/errors" +) + +func (d *driver) Mknod(path string, mode os.FileMode, major, minor int) error { + return errors.Wrap(ErrNotSupported, "cannot create device node on Windows") +} + +func (d *driver) Mkfifo(path string, mode os.FileMode) error { + return errors.Wrap(ErrNotSupported, "cannot create fifo on Windows") +} + +// Lchmod changes the mode of an file not following symlinks. +func (d *driver) Lchmod(path string, mode os.FileMode) (err error) { + // TODO: Use Window's equivalent + return os.Chmod(path, mode) +} diff --git a/vendor/github.com/containerd/continuity/groups_unix.go b/vendor/github.com/containerd/continuity/groups_unix.go new file mode 100644 index 00000000..e15c14ff --- /dev/null +++ b/vendor/github.com/containerd/continuity/groups_unix.go @@ -0,0 +1,113 @@ +package continuity + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +// TODO(stevvooe): This needs a lot of work before we can call it useful. + +type groupIndex struct { + byName map[string]*group + byGID map[int]*group +} + +func getGroupIndex() (*groupIndex, error) { + f, err := os.Open("/etc/group") + if err != nil { + return nil, err + } + defer f.Close() + + groups, err := parseGroups(f) + if err != nil { + return nil, err + } + + return newGroupIndex(groups), nil +} + +func newGroupIndex(groups []group) *groupIndex { + gi := &groupIndex{ + byName: make(map[string]*group), + byGID: make(map[int]*group), + } + + for i, group := range groups { + gi.byGID[group.gid] = &groups[i] + gi.byName[group.name] = &groups[i] + } + + return gi +} + +type group struct { + name string + gid int + members []string +} + +func getGroupName(gid int) (string, error) { + f, err := os.Open("/etc/group") + if err != nil { + return "", err + } + defer f.Close() + + groups, err := parseGroups(f) + if err != nil { + return "", err + } + + for _, group := range groups { + if group.gid == gid { + return group.name, nil + } + } + + return "", fmt.Errorf("no group for gid") +} + +// parseGroups parses an /etc/group file for group names, ids and membership. +// This is unix specific. +func parseGroups(rd io.Reader) ([]group, error) { + var groups []group + scanner := bufio.NewScanner(rd) + + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "#") { + continue // skip comment + } + + parts := strings.SplitN(scanner.Text(), ":", 4) + + if len(parts) != 4 { + return nil, fmt.Errorf("bad entry: %q", scanner.Text()) + } + + name, _, sgid, smembers := parts[0], parts[1], parts[2], parts[3] + + gid, err := strconv.Atoi(sgid) + if err != nil { + return nil, fmt.Errorf("bad gid: %q", gid) + } + + members := strings.Split(smembers, ",") + + groups = append(groups, group{ + name: name, + gid: gid, + members: members, + }) + } + + if scanner.Err() != nil { + return nil, scanner.Err() + } + + return groups, nil +} diff --git a/vendor/github.com/containerd/continuity/hardlinks.go b/vendor/github.com/containerd/continuity/hardlinks.go new file mode 100644 index 00000000..8b39bd06 --- /dev/null +++ b/vendor/github.com/containerd/continuity/hardlinks.go @@ -0,0 +1,57 @@ +package continuity + +import ( + "fmt" + "os" +) + +var ( + errNotAHardLink = fmt.Errorf("invalid hardlink") +) + +type hardlinkManager struct { + hardlinks map[hardlinkKey][]Resource +} + +func newHardlinkManager() *hardlinkManager { + return &hardlinkManager{ + hardlinks: map[hardlinkKey][]Resource{}, + } +} + +// Add attempts to add the resource to the hardlink manager. If the resource +// cannot be considered as a hardlink candidate, errNotAHardLink is returned. +func (hlm *hardlinkManager) Add(fi os.FileInfo, resource Resource) error { + if _, ok := resource.(Hardlinkable); !ok { + return errNotAHardLink + } + + key, err := newHardlinkKey(fi) + if err != nil { + return err + } + + hlm.hardlinks[key] = append(hlm.hardlinks[key], resource) + + return nil +} + +// Merge processes the current state of the hardlink manager and merges any +// shared nodes into hardlinked resources. +func (hlm *hardlinkManager) Merge() ([]Resource, error) { + var resources []Resource + for key, linked := range hlm.hardlinks { + if len(linked) < 1 { + return nil, fmt.Errorf("no hardlink entrys for dev, inode pair: %#v", key) + } + + merged, err := Merge(linked...) + if err != nil { + return nil, fmt.Errorf("error merging hardlink: %v", err) + } + + resources = append(resources, merged) + } + + return resources, nil +} diff --git a/vendor/github.com/containerd/continuity/hardlinks_unix.go b/vendor/github.com/containerd/continuity/hardlinks_unix.go new file mode 100644 index 00000000..1d81a3f9 --- /dev/null +++ b/vendor/github.com/containerd/continuity/hardlinks_unix.go @@ -0,0 +1,36 @@ +// +build linux darwin freebsd solaris + +package continuity + +import ( + "fmt" + "os" + "syscall" +) + +// hardlinkKey provides a tuple-key for managing hardlinks. This is system- +// specific. +type hardlinkKey struct { + dev uint64 + inode uint64 +} + +// newHardlinkKey returns a hardlink key for the provided file info. If the +// resource does not represent a possible hardlink, errNotAHardLink will be +// returned. +func newHardlinkKey(fi os.FileInfo) (hardlinkKey, error) { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return hardlinkKey{}, fmt.Errorf("cannot resolve (*syscall.Stat_t) from os.FileInfo") + } + + if sys.Nlink < 2 { + // NOTE(stevvooe): This is not always true for all filesystems. We + // should somehow detect this and provided a slow "polyfill" that + // leverages os.SameFile if we detect a filesystem where link counts + // is not really supported. + return hardlinkKey{}, errNotAHardLink + } + + return hardlinkKey{dev: uint64(sys.Dev), inode: uint64(sys.Ino)}, nil +} diff --git a/vendor/github.com/containerd/continuity/hardlinks_windows.go b/vendor/github.com/containerd/continuity/hardlinks_windows.go new file mode 100644 index 00000000..be516c56 --- /dev/null +++ b/vendor/github.com/containerd/continuity/hardlinks_windows.go @@ -0,0 +1,12 @@ +package continuity + +import "os" + +type hardlinkKey struct{} + +func newHardlinkKey(fi os.FileInfo) (hardlinkKey, error) { + // NOTE(stevvooe): Obviously, this is not yet implemented. However, the + // makings of an implementation are available in src/os/types_windows.go. More + // investigation needs to be done to figure out exactly how to do this. + return hardlinkKey{}, errNotAHardLink +} diff --git a/vendor/github.com/containerd/continuity/ioutils.go b/vendor/github.com/containerd/continuity/ioutils.go new file mode 100644 index 00000000..0e2cc5fb --- /dev/null +++ b/vendor/github.com/containerd/continuity/ioutils.go @@ -0,0 +1,39 @@ +package continuity + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// atomicWriteFile writes data to a file by first writing to a temp +// file and calling rename. +func atomicWriteFile(filename string, r io.Reader, rf RegularFile) error { + f, err := ioutil.TempFile(filepath.Dir(filename), ".tmp-"+filepath.Base(filename)) + if err != nil { + return err + } + err = os.Chmod(f.Name(), rf.Mode()) + if err != nil { + f.Close() + return err + } + n, err := io.Copy(f, r) + if err == nil && n < rf.Size() { + f.Close() + return io.ErrShortWrite + } + if err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), filename) +} diff --git a/vendor/github.com/containerd/continuity/manifest.go b/vendor/github.com/containerd/continuity/manifest.go new file mode 100644 index 00000000..20706f35 --- /dev/null +++ b/vendor/github.com/containerd/continuity/manifest.go @@ -0,0 +1,144 @@ +package continuity + +import ( + "fmt" + "io" + "log" + "os" + "sort" + + pb "github.com/containerd/continuity/proto" + "github.com/golang/protobuf/proto" +) + +// Manifest provides the contents of a manifest. Users of this struct should +// not typically modify any fields directly. +type Manifest struct { + // Resources specifies all the resources for a manifest in order by path. + Resources []Resource +} + +func Unmarshal(p []byte) (*Manifest, error) { + var bm pb.Manifest + + if err := proto.Unmarshal(p, &bm); err != nil { + return nil, err + } + + var m Manifest + for _, b := range bm.Resource { + r, err := fromProto(b) + if err != nil { + return nil, err + } + + m.Resources = append(m.Resources, r) + } + + return &m, nil +} + +func Marshal(m *Manifest) ([]byte, error) { + var bm pb.Manifest + for _, resource := range m.Resources { + bm.Resource = append(bm.Resource, toProto(resource)) + } + + return proto.Marshal(&bm) +} + +func MarshalText(w io.Writer, m *Manifest) error { + var bm pb.Manifest + for _, resource := range m.Resources { + bm.Resource = append(bm.Resource, toProto(resource)) + } + + return proto.MarshalText(w, &bm) +} + +// BuildManifest creates the manifest for the given context +func BuildManifest(ctx Context) (*Manifest, error) { + resourcesByPath := map[string]Resource{} + hardlinks := newHardlinkManager() + + if err := ctx.Walk(func(p string, fi os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error walking %s: %v", p, err) + } + + if p == "/" { + // skip root + return nil + } + + resource, err := ctx.Resource(p, fi) + if err != nil { + if err == ErrNotFound { + return nil + } + log.Printf("error getting resource %q: %v", p, err) + return err + } + + // add to the hardlink manager + if err := hardlinks.Add(fi, resource); err == nil { + // Resource has been accepted by hardlink manager so we don't add + // it to the resourcesByPath until we merge at the end. + return nil + } else if err != errNotAHardLink { + // handle any other case where we have a proper error. + return fmt.Errorf("adding hardlink %s: %v", p, err) + } + + resourcesByPath[p] = resource + + return nil + }); err != nil { + return nil, err + } + + // merge and post-process the hardlinks. + hardlinked, err := hardlinks.Merge() + if err != nil { + return nil, err + } + + for _, resource := range hardlinked { + resourcesByPath[resource.Path()] = resource + } + + var resources []Resource + for _, resource := range resourcesByPath { + resources = append(resources, resource) + } + + sort.Stable(ByPath(resources)) + + return &Manifest{ + Resources: resources, + }, nil +} + +// VerifyManifest verifies all the resources in a manifest +// against files from the given context. +func VerifyManifest(ctx Context, manifest *Manifest) error { + for _, resource := range manifest.Resources { + if err := ctx.Verify(resource); err != nil { + return err + } + } + + return nil +} + +// ApplyManifest applies on the resources in a manifest to +// the given context. +func ApplyManifest(ctx Context, manifest *Manifest) error { + for _, resource := range manifest.Resources { + if err := ctx.Apply(resource); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/containerd/continuity/manifest_test_darwin.go b/vendor/github.com/containerd/continuity/manifest_test_darwin.go new file mode 100644 index 00000000..873ec318 --- /dev/null +++ b/vendor/github.com/containerd/continuity/manifest_test_darwin.go @@ -0,0 +1,23 @@ +// +build ignore + +package continuity + +import "os" + +var ( + devNullResource = resource{ + kind: chardev, + path: "/dev/null", + major: 3, + minor: 2, + mode: 0666 | os.ModeDevice | os.ModeCharDevice, + } + + devZeroResource = resource{ + kind: chardev, + path: "/dev/zero", + major: 3, + minor: 3, + mode: 0666 | os.ModeDevice | os.ModeCharDevice, + } +) diff --git a/vendor/github.com/containerd/continuity/pathdriver/path_driver.go b/vendor/github.com/containerd/continuity/pathdriver/path_driver.go new file mode 100644 index 00000000..b43d55fe --- /dev/null +++ b/vendor/github.com/containerd/continuity/pathdriver/path_driver.go @@ -0,0 +1,85 @@ +package pathdriver + +import ( + "path/filepath" +) + +// PathDriver provides all of the path manipulation functions in a common +// interface. The context should call these and never use the `filepath` +// package or any other package to manipulate paths. +type PathDriver interface { + Join(paths ...string) string + IsAbs(path string) bool + Rel(base, target string) (string, error) + Base(path string) string + Dir(path string) string + Clean(path string) string + Split(path string) (dir, file string) + Separator() byte + Abs(path string) (string, error) + Walk(string, filepath.WalkFunc) error + FromSlash(path string) string + ToSlash(path string) string + Match(pattern, name string) (matched bool, err error) +} + +// pathDriver is a simple default implementation calls the filepath package. +type pathDriver struct{} + +// LocalPathDriver is the exported pathDriver struct for convenience. +var LocalPathDriver PathDriver = &pathDriver{} + +func (*pathDriver) Join(paths ...string) string { + return filepath.Join(paths...) +} + +func (*pathDriver) IsAbs(path string) bool { + return filepath.IsAbs(path) +} + +func (*pathDriver) Rel(base, target string) (string, error) { + return filepath.Rel(base, target) +} + +func (*pathDriver) Base(path string) string { + return filepath.Base(path) +} + +func (*pathDriver) Dir(path string) string { + return filepath.Dir(path) +} + +func (*pathDriver) Clean(path string) string { + return filepath.Clean(path) +} + +func (*pathDriver) Split(path string) (dir, file string) { + return filepath.Split(path) +} + +func (*pathDriver) Separator() byte { + return filepath.Separator +} + +func (*pathDriver) Abs(path string) (string, error) { + return filepath.Abs(path) +} + +// Note that filepath.Walk calls os.Stat, so if the context wants to +// to call Driver.Stat() for Walk, they need to create a new struct that +// overrides this method. +func (*pathDriver) Walk(root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, walkFn) +} + +func (*pathDriver) FromSlash(path string) string { + return filepath.FromSlash(path) +} + +func (*pathDriver) ToSlash(path string) string { + return filepath.ToSlash(path) +} + +func (*pathDriver) Match(pattern, name string) (bool, error) { + return filepath.Match(pattern, name) +} diff --git a/vendor/github.com/containerd/continuity/proto/gen.go b/vendor/github.com/containerd/continuity/proto/gen.go new file mode 100644 index 00000000..8f26ff50 --- /dev/null +++ b/vendor/github.com/containerd/continuity/proto/gen.go @@ -0,0 +1,3 @@ +package proto + +//go:generate protoc --go_out=. manifest.proto diff --git a/vendor/github.com/containerd/continuity/proto/manifest.pb.go b/vendor/github.com/containerd/continuity/proto/manifest.pb.go new file mode 100644 index 00000000..3e575107 --- /dev/null +++ b/vendor/github.com/containerd/continuity/proto/manifest.pb.go @@ -0,0 +1,181 @@ +// Code generated by protoc-gen-go. +// source: manifest.proto +// DO NOT EDIT! + +/* +Package proto is a generated protocol buffer package. + +It is generated from these files: + manifest.proto + +It has these top-level messages: + Manifest + Resource + XAttr + ADSEntry +*/ +package proto + +import proto1 "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto1.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto1.ProtoPackageIsVersion2 // please upgrade the proto package + +// Manifest specifies the entries in a container bundle, keyed and sorted by +// path. +type Manifest struct { + Resource []*Resource `protobuf:"bytes,1,rep,name=resource" json:"resource,omitempty"` +} + +func (m *Manifest) Reset() { *m = Manifest{} } +func (m *Manifest) String() string { return proto1.CompactTextString(m) } +func (*Manifest) ProtoMessage() {} +func (*Manifest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *Manifest) GetResource() []*Resource { + if m != nil { + return m.Resource + } + return nil +} + +type Resource struct { + // Path specifies the path from the bundle root. If more than one + // path is present, the entry may represent a hardlink, rather than using + // a link target. The path format is operating system specific. + Path []string `protobuf:"bytes,1,rep,name=path" json:"path,omitempty"` + // Uid specifies the user id for the resource. + Uid int64 `protobuf:"varint,2,opt,name=uid" json:"uid,omitempty"` + // Gid specifies the group id for the resource. + Gid int64 `protobuf:"varint,3,opt,name=gid" json:"gid,omitempty"` + // user and group are not currently used but their field numbers have been + // reserved for future use. As such, they are marked as deprecated. + User string `protobuf:"bytes,4,opt,name=user" json:"user,omitempty"` + Group string `protobuf:"bytes,5,opt,name=group" json:"group,omitempty"` + // Mode defines the file mode and permissions. We've used the same + // bit-packing from Go's os package, + // http://golang.org/pkg/os/#FileMode, since they've done the work of + // creating a cross-platform layout. + Mode uint32 `protobuf:"varint,6,opt,name=mode" json:"mode,omitempty"` + // Size specifies the size in bytes of the resource. This is only valid + // for regular files. + Size uint64 `protobuf:"varint,7,opt,name=size" json:"size,omitempty"` + // Digest specifies the content digest of the target file. Only valid for + // regular files. The strings are formatted in OCI style, i.e. :. + // For detailed information about the format, please refer to OCI Image Spec: + // https://github.com/opencontainers/image-spec/blob/master/descriptor.md#digests-and-verification + // The digests are sorted in lexical order and implementations may choose + // which algorithms they prefer. + Digest []string `protobuf:"bytes,8,rep,name=digest" json:"digest,omitempty"` + // Target defines the target of a hard or soft link. Absolute links start + // with a slash and specify the resource relative to the bundle root. + // Relative links do not start with a slash and are relative to the + // resource path. + Target string `protobuf:"bytes,9,opt,name=target" json:"target,omitempty"` + // Major specifies the major device number for charactor and block devices. + Major uint64 `protobuf:"varint,10,opt,name=major" json:"major,omitempty"` + // Minor specifies the minor device number for charactor and block devices. + Minor uint64 `protobuf:"varint,11,opt,name=minor" json:"minor,omitempty"` + // Xattr provides storage for extended attributes for the target resource. + Xattr []*XAttr `protobuf:"bytes,12,rep,name=xattr" json:"xattr,omitempty"` + // Ads stores one or more alternate data streams for the target resource. + Ads []*ADSEntry `protobuf:"bytes,13,rep,name=ads" json:"ads,omitempty"` +} + +func (m *Resource) Reset() { *m = Resource{} } +func (m *Resource) String() string { return proto1.CompactTextString(m) } +func (*Resource) ProtoMessage() {} +func (*Resource) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *Resource) GetXattr() []*XAttr { + if m != nil { + return m.Xattr + } + return nil +} + +func (m *Resource) GetAds() []*ADSEntry { + if m != nil { + return m.Ads + } + return nil +} + +// XAttr encodes extended attributes for a resource. +type XAttr struct { + // Name specifies the attribute name. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Data specifies the associated data for the attribute. + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *XAttr) Reset() { *m = XAttr{} } +func (m *XAttr) String() string { return proto1.CompactTextString(m) } +func (*XAttr) ProtoMessage() {} +func (*XAttr) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +// ADSEntry encodes information for a Windows Alternate Data Stream. +type ADSEntry struct { + // Name specifices the stream name. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Data specifies the stream data. + // See also the description about the digest below. + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + // Digest is a CAS representation of the stream data. + // + // At least one of data or digest MUST be specified, and either one of them + // SHOULD be specified. + // + // How to access the actual data using the digest is implementation-specific, + // and implementations can choose not to implement digest. + // So, digest SHOULD be used only when the stream data is large. + Digest string `protobuf:"bytes,3,opt,name=digest" json:"digest,omitempty"` +} + +func (m *ADSEntry) Reset() { *m = ADSEntry{} } +func (m *ADSEntry) String() string { return proto1.CompactTextString(m) } +func (*ADSEntry) ProtoMessage() {} +func (*ADSEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } + +func init() { + proto1.RegisterType((*Manifest)(nil), "proto.Manifest") + proto1.RegisterType((*Resource)(nil), "proto.Resource") + proto1.RegisterType((*XAttr)(nil), "proto.XAttr") + proto1.RegisterType((*ADSEntry)(nil), "proto.ADSEntry") +} + +func init() { proto1.RegisterFile("manifest.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 317 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x90, 0x4f, 0x4b, 0xf3, 0x40, + 0x10, 0xc6, 0x49, 0x93, 0xf4, 0x4d, 0xa7, 0xed, 0xab, 0x2c, 0x52, 0xe6, 0x18, 0x73, 0x0a, 0x08, + 0x15, 0xf4, 0xe0, 0xb9, 0xa2, 0x17, 0xc1, 0xcb, 0x7a, 0xf1, 0xba, 0xba, 0x6b, 0x5c, 0x21, 0xd9, + 0xb0, 0xd9, 0x80, 0xfa, 0xe5, 0xfc, 0x6a, 0x32, 0xb3, 0x69, 0xd1, 0x9b, 0xa7, 0x3c, 0xcf, 0x6f, + 0xfe, 0x64, 0xf6, 0x81, 0xff, 0xad, 0xea, 0xec, 0x8b, 0x19, 0xc2, 0xb6, 0xf7, 0x2e, 0x38, 0x91, + 0xf3, 0xa7, 0xba, 0x82, 0xe2, 0x7e, 0x2a, 0x88, 0x33, 0x28, 0xbc, 0x19, 0xdc, 0xe8, 0x9f, 0x0d, + 0x26, 0x65, 0x5a, 0x2f, 0x2f, 0x8e, 0x62, 0xf3, 0x56, 0x4e, 0x58, 0x1e, 0x1a, 0xaa, 0xaf, 0x19, + 0x14, 0x7b, 0x2c, 0x04, 0x64, 0xbd, 0x0a, 0xaf, 0x3c, 0xb5, 0x90, 0xac, 0xc5, 0x31, 0xa4, 0xa3, + 0xd5, 0x38, 0x2b, 0x93, 0x3a, 0x95, 0x24, 0x89, 0x34, 0x56, 0x63, 0x1a, 0x49, 0x63, 0xb5, 0xd8, + 0x40, 0x36, 0x0e, 0xc6, 0x63, 0x56, 0x26, 0xf5, 0xe2, 0x7a, 0x86, 0x89, 0x64, 0x2f, 0x10, 0xf2, + 0xc6, 0xbb, 0xb1, 0xc7, 0xfc, 0x50, 0x88, 0x80, 0xfe, 0xd4, 0x3a, 0x6d, 0x70, 0x5e, 0x26, 0xf5, + 0x5a, 0xb2, 0x26, 0x36, 0xd8, 0x4f, 0x83, 0xff, 0xca, 0xa4, 0xce, 0x24, 0x6b, 0xb1, 0x81, 0xb9, + 0xb6, 0x8d, 0x19, 0x02, 0x16, 0x7c, 0xd3, 0xe4, 0x88, 0x07, 0xe5, 0x1b, 0x13, 0x70, 0x41, 0xab, + 0xe5, 0xe4, 0xc4, 0x09, 0xe4, 0xad, 0x7a, 0x73, 0x1e, 0x81, 0x97, 0x44, 0xc3, 0xd4, 0x76, 0xce, + 0xe3, 0x72, 0xa2, 0x64, 0x44, 0x05, 0xf9, 0xbb, 0x0a, 0xc1, 0xe3, 0x8a, 0x43, 0x5a, 0x4d, 0x21, + 0x3d, 0xee, 0x42, 0xf0, 0x32, 0x96, 0xc4, 0x29, 0xa4, 0x4a, 0x0f, 0xb8, 0xfe, 0x15, 0xe3, 0xee, + 0xe6, 0xe1, 0xb6, 0x0b, 0xfe, 0x43, 0x52, 0xad, 0x3a, 0x87, 0x9c, 0x47, 0xe8, 0xfe, 0x4e, 0xb5, + 0x94, 0x39, 0x5d, 0xc4, 0x9a, 0x98, 0x56, 0x41, 0x71, 0x7c, 0x2b, 0xc9, 0xba, 0xba, 0x83, 0x62, + 0xbf, 0xe1, 0xaf, 0x33, 0x3f, 0x72, 0x48, 0xe3, 0x7b, 0xa3, 0x7b, 0x9a, 0xf3, 0x45, 0x97, 0xdf, + 0x01, 0x00, 0x00, 0xff, 0xff, 0xef, 0x27, 0x99, 0xf7, 0x17, 0x02, 0x00, 0x00, +} diff --git a/vendor/github.com/containerd/continuity/proto/manifest.proto b/vendor/github.com/containerd/continuity/proto/manifest.proto new file mode 100644 index 00000000..e2e110f1 --- /dev/null +++ b/vendor/github.com/containerd/continuity/proto/manifest.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package proto; + +// Manifest specifies the entries in a container bundle, keyed and sorted by +// path. +message Manifest { + repeated Resource resource = 1; +} + +message Resource { + // Path specifies the path from the bundle root. If more than one + // path is present, the entry may represent a hardlink, rather than using + // a link target. The path format is operating system specific. + repeated string path = 1; + + // NOTE(stevvooe): Need to define clear precedence for user/group/uid/gid precedence. + + // Uid specifies the user id for the resource. + int64 uid = 2; + + // Gid specifies the group id for the resource. + int64 gid = 3; + + // user and group are not currently used but their field numbers have been + // reserved for future use. As such, they are marked as deprecated. + string user = 4 [deprecated=true]; // "deprecated" stands for "reserved" here + string group = 5 [deprecated=true]; // "deprecated" stands for "reserved" here + + // Mode defines the file mode and permissions. We've used the same + // bit-packing from Go's os package, + // http://golang.org/pkg/os/#FileMode, since they've done the work of + // creating a cross-platform layout. + uint32 mode = 6; + + // NOTE(stevvooe): Beyond here, we start defining type specific fields. + + // Size specifies the size in bytes of the resource. This is only valid + // for regular files. + uint64 size = 7; + + // Digest specifies the content digest of the target file. Only valid for + // regular files. The strings are formatted in OCI style, i.e. :. + // For detailed information about the format, please refer to OCI Image Spec: + // https://github.com/opencontainers/image-spec/blob/master/descriptor.md#digests-and-verification + // The digests are sorted in lexical order and implementations may choose + // which algorithms they prefer. + repeated string digest = 8; + + // Target defines the target of a hard or soft link. Absolute links start + // with a slash and specify the resource relative to the bundle root. + // Relative links do not start with a slash and are relative to the + // resource path. + string target = 9; + + // Major specifies the major device number for charactor and block devices. + uint64 major = 10; + + // Minor specifies the minor device number for charactor and block devices. + uint64 minor = 11; + + // Xattr provides storage for extended attributes for the target resource. + repeated XAttr xattr = 12; + + // Ads stores one or more alternate data streams for the target resource. + repeated ADSEntry ads = 13; + +} + +// XAttr encodes extended attributes for a resource. +message XAttr { + // Name specifies the attribute name. + string name = 1; + + // Data specifies the associated data for the attribute. + bytes data = 2; +} + +// ADSEntry encodes information for a Windows Alternate Data Stream. +message ADSEntry { + // Name specifices the stream name. + string name = 1; + + // Data specifies the stream data. + // See also the description about the digest below. + bytes data = 2; + + // Digest is a CAS representation of the stream data. + // + // At least one of data or digest MUST be specified, and either one of them + // SHOULD be specified. + // + // How to access the actual data using the digest is implementation-specific, + // and implementations can choose not to implement digest. + // So, digest SHOULD be used only when the stream data is large. + string digest = 3; +} diff --git a/vendor/github.com/containerd/continuity/resource.go b/vendor/github.com/containerd/continuity/resource.go new file mode 100644 index 00000000..3643effb --- /dev/null +++ b/vendor/github.com/containerd/continuity/resource.go @@ -0,0 +1,574 @@ +package continuity + +import ( + "errors" + "fmt" + "os" + "reflect" + "sort" + + pb "github.com/containerd/continuity/proto" + "github.com/opencontainers/go-digest" +) + +// TODO(stevvooe): A record based model, somewhat sketched out at the bottom +// of this file, will be more flexible. Another possibly is to tie the package +// interface directly to the protobuf type. This will have efficiency +// advantages at the cost coupling the nasty codegen types to the exported +// interface. + +type Resource interface { + // Path provides the primary resource path relative to the bundle root. In + // cases where resources have more than one path, such as with hard links, + // this will return the primary path, which is often just the first entry. + Path() string + + // Mode returns the + Mode() os.FileMode + + UID() int64 + GID() int64 +} + +// ByPath provides the canonical sort order for a set of resources. Use with +// sort.Stable for deterministic sorting. +type ByPath []Resource + +func (bp ByPath) Len() int { return len(bp) } +func (bp ByPath) Swap(i, j int) { bp[i], bp[j] = bp[j], bp[i] } +func (bp ByPath) Less(i, j int) bool { return bp[i].Path() < bp[j].Path() } + +type XAttrer interface { + XAttrs() map[string][]byte +} + +// Hardlinkable is an interface that a resource type satisfies if it can be a +// hardlink target. +type Hardlinkable interface { + // Paths returns all paths of the resource, including the primary path + // returned by Resource.Path. If len(Paths()) > 1, the resource is a hard + // link. + Paths() []string +} + +type RegularFile interface { + Resource + XAttrer + Hardlinkable + + Size() int64 + Digests() []digest.Digest +} + +// Merge two or more Resources into new file. Typically, this should be +// used to merge regular files as hardlinks. If the files are not identical, +// other than Paths and Digests, the merge will fail and an error will be +// returned. +func Merge(fs ...Resource) (Resource, error) { + if len(fs) < 1 { + return nil, fmt.Errorf("please provide a resource to merge") + } + + if len(fs) == 1 { + return fs[0], nil + } + + var paths []string + var digests []digest.Digest + bypath := map[string][]Resource{} + + // The attributes are all compared against the first to make sure they + // agree before adding to the above collections. If any of these don't + // correctly validate, the merge fails. + prototype := fs[0] + xattrs := make(map[string][]byte) + + // initialize xattrs for use below. All files must have same xattrs. + if prototypeXAttrer, ok := prototype.(XAttrer); ok { + for attr, value := range prototypeXAttrer.XAttrs() { + xattrs[attr] = value + } + } + + for _, f := range fs { + h, isHardlinkable := f.(Hardlinkable) + if !isHardlinkable { + return nil, errNotAHardLink + } + + if f.Mode() != prototype.Mode() { + return nil, fmt.Errorf("modes do not match: %v != %v", f.Mode(), prototype.Mode()) + } + + if f.UID() != prototype.UID() { + return nil, fmt.Errorf("uid does not match: %v != %v", f.UID(), prototype.UID()) + } + + if f.GID() != prototype.GID() { + return nil, fmt.Errorf("gid does not match: %v != %v", f.GID(), prototype.GID()) + } + + if xattrer, ok := f.(XAttrer); ok { + fxattrs := xattrer.XAttrs() + if !reflect.DeepEqual(fxattrs, xattrs) { + return nil, fmt.Errorf("resource %q xattrs do not match: %v != %v", f, fxattrs, xattrs) + } + } + + for _, p := range h.Paths() { + pfs, ok := bypath[p] + if !ok { + // ensure paths are unique by only appending on a new path. + paths = append(paths, p) + } + + bypath[p] = append(pfs, f) + } + + if regFile, isRegFile := f.(RegularFile); isRegFile { + prototypeRegFile, prototypeIsRegFile := prototype.(RegularFile) + if !prototypeIsRegFile { + return nil, errors.New("prototype is not a regular file") + } + + if regFile.Size() != prototypeRegFile.Size() { + return nil, fmt.Errorf("size does not match: %v != %v", regFile.Size(), prototypeRegFile.Size()) + } + + digests = append(digests, regFile.Digests()...) + } else if device, isDevice := f.(Device); isDevice { + prototypeDevice, prototypeIsDevice := prototype.(Device) + if !prototypeIsDevice { + return nil, errors.New("prototype is not a device") + } + + if device.Major() != prototypeDevice.Major() { + return nil, fmt.Errorf("major number does not match: %v != %v", device.Major(), prototypeDevice.Major()) + } + if device.Minor() != prototypeDevice.Minor() { + return nil, fmt.Errorf("minor number does not match: %v != %v", device.Minor(), prototypeDevice.Minor()) + } + } else if _, isNamedPipe := f.(NamedPipe); isNamedPipe { + _, prototypeIsNamedPipe := prototype.(NamedPipe) + if !prototypeIsNamedPipe { + return nil, errors.New("prototype is not a named pipe") + } + } else { + return nil, errNotAHardLink + } + } + + sort.Stable(sort.StringSlice(paths)) + + // Choose a "canonical" file. Really, it is just the first file to sort + // against. We also effectively select the very first digest as the + // "canonical" one for this file. + first := bypath[paths[0]][0] + + resource := resource{ + paths: paths, + mode: first.Mode(), + uid: first.UID(), + gid: first.GID(), + xattrs: xattrs, + } + + switch typedF := first.(type) { + case RegularFile: + var err error + digests, err = uniqifyDigests(digests...) + if err != nil { + return nil, err + } + + return ®ularFile{ + resource: resource, + size: typedF.Size(), + digests: digests, + }, nil + case Device: + return &device{ + resource: resource, + major: typedF.Major(), + minor: typedF.Minor(), + }, nil + + case NamedPipe: + return &namedPipe{ + resource: resource, + }, nil + + default: + return nil, errNotAHardLink + } +} + +type Directory interface { + Resource + XAttrer + + // Directory is a no-op method to identify directory objects by interface. + Directory() +} + +type SymLink interface { + Resource + + // Target returns the target of the symlink contained in the . + Target() string +} + +type NamedPipe interface { + Resource + Hardlinkable + XAttrer + + // Pipe is a no-op method to allow consistent resolution of NamedPipe + // interface. + Pipe() +} + +type Device interface { + Resource + Hardlinkable + XAttrer + + Major() uint64 + Minor() uint64 +} + +type resource struct { + paths []string + mode os.FileMode + uid, gid int64 + xattrs map[string][]byte +} + +var _ Resource = &resource{} + +func (r *resource) Path() string { + if len(r.paths) < 1 { + return "" + } + + return r.paths[0] +} + +func (r *resource) Mode() os.FileMode { + return r.mode +} + +func (r *resource) UID() int64 { + return r.uid +} + +func (r *resource) GID() int64 { + return r.gid +} + +type regularFile struct { + resource + size int64 + digests []digest.Digest +} + +var _ RegularFile = ®ularFile{} + +// newRegularFile returns the RegularFile, using the populated base resource +// and one or more digests of the content. +func newRegularFile(base resource, paths []string, size int64, dgsts ...digest.Digest) (RegularFile, error) { + if !base.Mode().IsRegular() { + return nil, fmt.Errorf("not a regular file") + } + + base.paths = make([]string, len(paths)) + copy(base.paths, paths) + + // make our own copy of digests + ds := make([]digest.Digest, len(dgsts)) + copy(ds, dgsts) + + return ®ularFile{ + resource: base, + size: size, + digests: ds, + }, nil +} + +func (rf *regularFile) Paths() []string { + paths := make([]string, len(rf.paths)) + copy(paths, rf.paths) + return paths +} + +func (rf *regularFile) Size() int64 { + return rf.size +} + +func (rf *regularFile) Digests() []digest.Digest { + digests := make([]digest.Digest, len(rf.digests)) + copy(digests, rf.digests) + return digests +} + +func (rf *regularFile) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(rf.xattrs)) + + for attr, value := range rf.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +type directory struct { + resource +} + +var _ Directory = &directory{} + +func newDirectory(base resource) (Directory, error) { + if !base.Mode().IsDir() { + return nil, fmt.Errorf("not a directory") + } + + return &directory{ + resource: base, + }, nil +} + +func (d *directory) Directory() {} + +func (d *directory) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(d.xattrs)) + + for attr, value := range d.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +type symLink struct { + resource + target string +} + +var _ SymLink = &symLink{} + +func newSymLink(base resource, target string) (SymLink, error) { + if base.Mode()&os.ModeSymlink == 0 { + return nil, fmt.Errorf("not a symlink") + } + + return &symLink{ + resource: base, + target: target, + }, nil +} + +func (l *symLink) Target() string { + return l.target +} + +type namedPipe struct { + resource +} + +var _ NamedPipe = &namedPipe{} + +func newNamedPipe(base resource, paths []string) (NamedPipe, error) { + if base.Mode()&os.ModeNamedPipe == 0 { + return nil, fmt.Errorf("not a namedpipe") + } + + base.paths = make([]string, len(paths)) + copy(base.paths, paths) + + return &namedPipe{ + resource: base, + }, nil +} + +func (np *namedPipe) Pipe() {} + +func (np *namedPipe) Paths() []string { + paths := make([]string, len(np.paths)) + copy(paths, np.paths) + return paths +} + +func (np *namedPipe) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(np.xattrs)) + + for attr, value := range np.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +type device struct { + resource + major, minor uint64 +} + +var _ Device = &device{} + +func newDevice(base resource, paths []string, major, minor uint64) (Device, error) { + if base.Mode()&os.ModeDevice == 0 { + return nil, fmt.Errorf("not a device") + } + + base.paths = make([]string, len(paths)) + copy(base.paths, paths) + + return &device{ + resource: base, + major: major, + minor: minor, + }, nil +} + +func (d *device) Paths() []string { + paths := make([]string, len(d.paths)) + copy(paths, d.paths) + return paths +} + +func (d *device) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(d.xattrs)) + + for attr, value := range d.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +func (d device) Major() uint64 { + return d.major +} + +func (d device) Minor() uint64 { + return d.minor +} + +// toProto converts a resource to a protobuf record. We'd like to push this +// the individual types but we want to keep this all together during +// prototyping. +func toProto(resource Resource) *pb.Resource { + b := &pb.Resource{ + Path: []string{resource.Path()}, + Mode: uint32(resource.Mode()), + Uid: resource.UID(), + Gid: resource.GID(), + } + + if xattrer, ok := resource.(XAttrer); ok { + // Sorts the XAttrs by name for consistent ordering. + keys := []string{} + xattrs := xattrer.XAttrs() + for k := range xattrs { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + b.Xattr = append(b.Xattr, &pb.XAttr{Name: k, Data: xattrs[k]}) + } + } + + switch r := resource.(type) { + case RegularFile: + b.Path = r.Paths() + b.Size = uint64(r.Size()) + + for _, dgst := range r.Digests() { + b.Digest = append(b.Digest, dgst.String()) + } + case SymLink: + b.Target = r.Target() + case Device: + b.Major, b.Minor = r.Major(), r.Minor() + b.Path = r.Paths() + case NamedPipe: + b.Path = r.Paths() + } + + // enforce a few stability guarantees that may not be provided by the + // resource implementation. + sort.Strings(b.Path) + + return b +} + +// fromProto converts from a protobuf Resource to a Resource interface. +func fromProto(b *pb.Resource) (Resource, error) { + base := &resource{ + paths: b.Path, + mode: os.FileMode(b.Mode), + uid: b.Uid, + gid: b.Gid, + } + + base.xattrs = make(map[string][]byte, len(b.Xattr)) + + for _, attr := range b.Xattr { + base.xattrs[attr.Name] = attr.Data + } + + switch { + case base.Mode().IsRegular(): + dgsts := make([]digest.Digest, len(b.Digest)) + for i, dgst := range b.Digest { + // TODO(stevvooe): Should we be validating at this point? + dgsts[i] = digest.Digest(dgst) + } + + return newRegularFile(*base, b.Path, int64(b.Size), dgsts...) + case base.Mode().IsDir(): + return newDirectory(*base) + case base.Mode()&os.ModeSymlink != 0: + return newSymLink(*base, b.Target) + case base.Mode()&os.ModeNamedPipe != 0: + return newNamedPipe(*base, b.Path) + case base.Mode()&os.ModeDevice != 0: + return newDevice(*base, b.Path, b.Major, b.Minor) + } + + return nil, fmt.Errorf("unknown resource record (%#v): %s", b, base.Mode()) +} + +// NOTE(stevvooe): An alternative model that supports inline declaration. +// Convenient for unit testing where inline declarations may be desirable but +// creates an awkward API for the standard use case. + +// type ResourceKind int + +// const ( +// ResourceRegularFile = iota + 1 +// ResourceDirectory +// ResourceSymLink +// Resource +// ) + +// type Resource struct { +// Kind ResourceKind +// Paths []string +// Mode os.FileMode +// UID string +// GID string +// Size int64 +// Digests []digest.Digest +// Target string +// Major, Minor int +// XAttrs map[string][]byte +// } + +// type RegularFile struct { +// Paths []string +// Size int64 +// Digests []digest.Digest +// Perm os.FileMode // os.ModePerm + sticky, setuid, setgid +// } diff --git a/vendor/github.com/containerd/continuity/resource_unix.go b/vendor/github.com/containerd/continuity/resource_unix.go new file mode 100644 index 00000000..4144643e --- /dev/null +++ b/vendor/github.com/containerd/continuity/resource_unix.go @@ -0,0 +1,37 @@ +// +build linux darwin freebsd solaris + +package continuity + +import ( + "fmt" + "os" + "syscall" +) + +// newBaseResource returns a *resource, populated with data from p and fi, +// where p will be populated directly. +func newBaseResource(p string, fi os.FileInfo) (*resource, error) { + // TODO(stevvooe): This need to be resolved for the container's root, + // where here we are really getting the host OS's value. We need to allow + // this be passed in and fixed up to make these uid/gid mappings portable. + // Either this can be part of the driver or we can achieve it through some + // other mechanism. + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + // TODO(stevvooe): This may not be a hard error for all platforms. We + // may want to move this to the driver. + return nil, fmt.Errorf("unable to resolve syscall.Stat_t from (os.FileInfo).Sys(): %#v", fi) + } + + return &resource{ + paths: []string{p}, + mode: fi.Mode(), + + uid: int64(sys.Uid), + gid: int64(sys.Gid), + + // NOTE(stevvooe): Population of shared xattrs field is deferred to + // the resource types that populate it. Since they are a property of + // the context, they must set there. + }, nil +} diff --git a/vendor/github.com/containerd/continuity/resource_windows.go b/vendor/github.com/containerd/continuity/resource_windows.go new file mode 100644 index 00000000..7b44414a --- /dev/null +++ b/vendor/github.com/containerd/continuity/resource_windows.go @@ -0,0 +1,12 @@ +package continuity + +import "os" + +// newBaseResource returns a *resource, populated with data from p and fi, +// where p will be populated directly. +func newBaseResource(p string, fi os.FileInfo) (*resource, error) { + return &resource{ + paths: []string{p}, + mode: fi.Mode(), + }, nil +} From 3d91404c072cedca789d7c67460f599d127f9130 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Nov 2017 16:06:09 -0800 Subject: [PATCH 2/4] buildctl: add integration test with local files Signed-off-by: Tonis Tiigi --- cmd/buildctl/build_test.go | 61 +++++++++++++++++++++++++++++++++++ cmd/buildctl/buildctl_test.go | 1 + 2 files changed, 62 insertions(+) create mode 100644 cmd/buildctl/build_test.go diff --git a/cmd/buildctl/build_test.go b/cmd/buildctl/build_test.go new file mode 100644 index 00000000..de54a8cf --- /dev/null +++ b/cmd/buildctl/build_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "testing" + + "github.com/containerd/containerd/fs/fstest" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/stretchr/testify/require" +) + +func testBuildWithLocalFiles(t *testing.T, sb integration.Sandbox) { + dir, err := tmpdir( + fstest.CreateFile("foo", []byte("bar"), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + st := llb.Image("busybox"). + Run(llb.Shlex("sh -c 'echo -n bar > foo2'")). + Run(llb.Shlex("cmp -s /mnt/foo foo2")) + + st.AddMount("/mnt", llb.Local("src"), llb.Readonly) + + rdr, err := marshal(st.Root()) + require.NoError(t, err) + + cmd := sb.Cmd(fmt.Sprintf("build --no-progress --local src=%s", dir)) + cmd.Stdin = rdr + + err = cmd.Run() + require.NoError(t, err) +} + +func marshal(st llb.State) (io.Reader, error) { + def, err := st.Marshal() + if err != nil { + return nil, err + } + dt, err := def.ToPB().Marshal() + if err != nil { + return nil, err + } + return bytes.NewBuffer(dt), nil +} + +func tmpdir(appliers ...fstest.Applier) (string, error) { + tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") + if err != nil { + return "", err + } + if err := fstest.Apply(appliers...).Apply(tmpdir); err != nil { + return "", err + } + return tmpdir, nil +} diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go index b789c1d0..97a4730b 100644 --- a/cmd/buildctl/buildctl_test.go +++ b/cmd/buildctl/buildctl_test.go @@ -9,5 +9,6 @@ import ( func TestCLIIntegration(t *testing.T) { integration.Run(t, []integration.Test{ testDiskUsage, + testBuildWithLocalFiles, }) } From 02d01c035027d23a00d89b6e7d002c704c708062 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Nov 2017 16:28:35 -0800 Subject: [PATCH 3/4] buildctl: add integration test for local exporter Signed-off-by: Tonis Tiigi --- cmd/buildctl/build_test.go | 29 ++++++++++++++++++++++++++++- cmd/buildctl/buildctl_test.go | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/buildctl/build_test.go b/cmd/buildctl/build_test.go index de54a8cf..aaf21ef9 100644 --- a/cmd/buildctl/build_test.go +++ b/cmd/buildctl/build_test.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "testing" "github.com/containerd/containerd/fs/fstest" @@ -15,6 +16,7 @@ import ( ) func testBuildWithLocalFiles(t *testing.T, sb integration.Sandbox) { + t.Parallel() dir, err := tmpdir( fstest.CreateFile("foo", []byte("bar"), 0600), ) @@ -37,6 +39,31 @@ func testBuildWithLocalFiles(t *testing.T, sb integration.Sandbox) { require.NoError(t, err) } +func testBuildLocalExporter(t *testing.T, sb integration.Sandbox) { + t.Parallel() + st := llb.Image("busybox"). + Run(llb.Shlex("sh -c 'echo -n bar > /out/foo'")) + + out := st.AddMount("/out", llb.Scratch()) + + rdr, err := marshal(out) + require.NoError(t, err) + + tmpdir, err := ioutil.TempDir("", "buildkit-buildctl") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + cmd := sb.Cmd(fmt.Sprintf("build --no-progress --exporter=local --exporter-opt output=%s", tmpdir)) + cmd.Stdin = rdr + err = cmd.Run() + + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo")) + require.NoError(t, err) + require.Equal(t, string(dt), "bar") +} + func marshal(st llb.State) (io.Reader, error) { def, err := st.Marshal() if err != nil { @@ -50,7 +77,7 @@ func marshal(st llb.State) (io.Reader, error) { } func tmpdir(appliers ...fstest.Applier) (string, error) { - tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") + tmpdir, err := ioutil.TempDir("", "buildkit-buildctl") if err != nil { return "", err } diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go index 97a4730b..e781384f 100644 --- a/cmd/buildctl/buildctl_test.go +++ b/cmd/buildctl/buildctl_test.go @@ -10,5 +10,6 @@ func TestCLIIntegration(t *testing.T) { integration.Run(t, []integration.Test{ testDiskUsage, testBuildWithLocalFiles, + testBuildLocalExporter, }) } From 97b773a04b8e8765c90916025a68ac47cf6add21 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Nov 2017 16:56:58 -0800 Subject: [PATCH 4/4] buildctl: add integration test for containerd exporter Signed-off-by: Tonis Tiigi --- cmd/buildctl/build_test.go | 36 +++++++++++++++++++++++++ cmd/buildctl/buildctl_test.go | 1 + util/testutil/integration/containerd.go | 11 +++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/cmd/buildctl/build_test.go b/cmd/buildctl/build_test.go index aaf21ef9..1962b5a2 100644 --- a/cmd/buildctl/build_test.go +++ b/cmd/buildctl/build_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -9,7 +10,9 @@ import ( "path/filepath" "testing" + "github.com/containerd/containerd" "github.com/containerd/containerd/fs/fstest" + "github.com/containerd/containerd/namespaces" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/util/testutil/integration" "github.com/stretchr/testify/require" @@ -64,6 +67,39 @@ func testBuildLocalExporter(t *testing.T, sb integration.Sandbox) { require.Equal(t, string(dt), "bar") } +func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) { + t.Parallel() + + var cdAddress string + if cd, ok := sb.(interface { + ContainerdAddress() string + }); !ok { + t.Skip("only for containerd worker") + } else { + cdAddress = cd.ContainerdAddress() + } + + st := llb.Image("busybox"). + Run(llb.Shlex("sh -c 'echo -n bar > /foo'")) + + rdr, err := marshal(st.Root()) + require.NoError(t, err) + + cmd := sb.Cmd("build --no-progress --exporter=image --exporter-opt name=example.com/moby/imageexporter:test") + cmd.Stdin = rdr + err = cmd.Run() + require.NoError(t, err) + + client, err := containerd.New(cdAddress) + require.NoError(t, err) + defer client.Close() + + ctx := namespaces.WithNamespace(context.Background(), "buildkit") + + _, err = client.ImageService().Get(ctx, "example.com/moby/imageexporter:test") + require.NoError(t, err) +} + func marshal(st llb.State) (io.Reader, error) { def, err := st.Marshal() if err != nil { diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go index e781384f..c166ec88 100644 --- a/cmd/buildctl/buildctl_test.go +++ b/cmd/buildctl/buildctl_test.go @@ -11,5 +11,6 @@ func TestCLIIntegration(t *testing.T) { testDiskUsage, testBuildWithLocalFiles, testBuildLocalExporter, + testBuildContainerdExporter, }) } diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go index 6b1c3172..95d8ed36 100644 --- a/util/testutil/integration/containerd.go +++ b/util/testutil/integration/containerd.go @@ -70,5 +70,14 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) { } deferF.append(stop) - return &sandbox{address: builddSock, logs: logs}, cl, nil + return &cdsandbox{address: address, sandbox: sandbox{address: builddSock, logs: logs}}, cl, nil +} + +type cdsandbox struct { + sandbox + address string +} + +func (s *cdsandbox) ContainerdAddress() string { + return s.address }