From 0d17ac323e26c1183e8ae1e975497ee8cf6613e3 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 8 Mar 2019 16:15:42 -0800 Subject: [PATCH] fileop: updates with new fsutil copy pkg Signed-off-by: Tonis Tiigi --- cache/contenthash/checksum.go | 3 +- client/client_test.go | 16 +- client/llb/fileop.go | 21 ++- cmd/buildkitd/main.go | 2 + frontend/dockerfile/dockerfile2llb/convert.go | 11 +- frontend/dockerfile/dockerfile_test.go | 76 +++++++- solver/llbsolver/file/backend.go | 164 ++++++++++-------- solver/llbsolver/file/chown.go | 62 ------- solver/llbsolver/file/chown_unix.go | 7 - solver/llbsolver/file/chown_windows.go | 14 -- solver/llbsolver/file/unpack.go | 61 +++++++ solver/llbsolver/file/user.go | 21 +-- solver/llbsolver/ops/file.go | 12 +- 13 files changed, 282 insertions(+), 188 deletions(-) delete mode 100644 solver/llbsolver/file/chown.go delete mode 100644 solver/llbsolver/file/chown_unix.go delete mode 100644 solver/llbsolver/file/chown_windows.go create mode 100644 solver/llbsolver/file/unpack.go diff --git a/cache/contenthash/checksum.go b/cache/contenthash/checksum.go index 6cf5bb9f..0b878215 100644 --- a/cache/contenthash/checksum.go +++ b/cache/contenthash/checksum.go @@ -558,12 +558,13 @@ func (cc *cacheContext) lazyChecksum(ctx context.Context, m *mount, p string) (* } func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *iradix.Txn, m *mount, k []byte, follow bool) (*CacheRecord, bool, error) { + origk := k k, cr, err := getFollowLinks(root, k, follow) if err != nil { return nil, false, err } if cr == nil { - return nil, false, errors.Wrapf(errNotFound, "%s not found", convertKeyToPath(k)) + return nil, false, errors.Wrapf(errNotFound, "%q not found", convertKeyToPath(origk)) } if cr.Digest != "" { return cr, false, nil diff --git a/client/client_test.go b/client/client_test.go index 14c03680..cf41ac20 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -672,8 +672,12 @@ func testFileOpMkdirMkfile(t *testing.T, sb integration.Sandbox) { defer os.RemoveAll(destDir) _, err = c.Solve(context.TODO(), def, SolveOpt{ - Exporter: ExporterLocal, - ExporterOutputDir: destDir, + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, }, nil) require.NoError(t, err) @@ -722,8 +726,12 @@ func testFileOpCopyRm(t *testing.T, sb integration.Sandbox) { defer os.RemoveAll(destDir) _, err = c.Solve(context.TODO(), def, SolveOpt{ - Exporter: ExporterLocal, - ExporterOutputDir: destDir, + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, LocalDirs: map[string]string{ "mylocal": dir, "mylocal2": dir2, diff --git a/client/llb/fileop.go b/client/llb/fileop.go index 731aa619..21dd9061 100644 --- a/client/llb/fileop.go +++ b/client/llb/fileop.go @@ -149,7 +149,7 @@ type fileActionMkdir struct { func (a *fileActionMkdir) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction { return &pb.FileAction_Mkdir{ Mkdir: &pb.FileActionMkDir{ - Path: normalizePath(parent, a.file), + Path: normalizePath(parent, a.file, false), Mode: int32(a.mode & 0777), MakeParents: a.info.MakeParents, Owner: a.info.ChownOpt.marshal(base), @@ -318,7 +318,7 @@ type fileActionMkfile struct { func (a *fileActionMkfile) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction { return &pb.FileAction_Mkfile{ Mkfile: &pb.FileActionMkFile{ - Path: normalizePath(parent, a.file), + Path: normalizePath(parent, a.file, false), Mode: int32(a.mode & 0777), Data: a.dt, Owner: a.info.ChownOpt.marshal(base), @@ -382,7 +382,7 @@ type fileActionRm struct { func (a *fileActionRm) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction { return &pb.FileAction_Rm{ Rm: &pb.FileActionRm{ - Path: normalizePath(parent, a.file), + Path: normalizePath(parent, a.file, false), AllowNotFound: a.info.AllowNotFound, AllowWildcard: a.info.AllowWildcard, }, @@ -451,7 +451,7 @@ type fileActionCopy struct { func (a *fileActionCopy) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction { c := &pb.FileActionCopy{ Src: a.sourcePath(), - Dest: normalizePath(parent, a.dest), + Dest: normalizePath(parent, a.dest, true), Owner: a.info.ChownOpt.marshal(base), AllowWildcard: a.info.AllowWildcard, AllowEmptyWildcard: a.info.AllowEmptyWildcard, @@ -683,11 +683,22 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, return f.Load() } -func normalizePath(parent, p string) string { +func normalizePath(parent, p string, keepSlash bool) string { + origPath := p p = path.Clean(p) if !path.IsAbs(p) { p = path.Join("/", parent, p) } + if keepSlash { + if strings.HasSuffix(origPath, "/") && !strings.HasSuffix(p, "/") { + p += "/" + } else if strings.HasSuffix(origPath, "/.") { + if p != "/" { + p += "/" + } + p += "." + } + } return p } diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index c9e42565..507f7091 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -19,6 +19,7 @@ import ( "github.com/containerd/containerd/pkg/seed" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/sys" + "github.com/docker/docker/pkg/reexec" "github.com/docker/go-connections/sockets" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/moby/buildkit/cache/remotecache" @@ -55,6 +56,7 @@ import ( func init() { apicaps.ExportedProduct = "buildkit" seed.WithTimeAndRand() + reexec.Init() } type workerInitializerOpt struct { diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index cb9b1e7a..419796b4 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -474,7 +474,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { case *instructions.WorkdirCommand: err = dispatchWorkdir(d, c, true, &opt) case *instructions.AddCommand: - err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, "", opt) + err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, c.Chown, opt) if err == nil { for _, src := range c.Sources() { if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { @@ -660,7 +660,8 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo } d.image.Config.WorkingDir = wd if commit { - if opt != nil && useFileOp(opt.buildArgValues, opt.llbCaps) { + withLayer := false + if wd != "/" && opt != nil && useFileOp(opt.buildArgValues, opt.llbCaps) { mkdirOpt := []llb.MkdirOption{llb.WithParents(true)} if user := d.image.Config.User; user != "" { mkdirOpt = append(mkdirOpt, llb.WithUser(user)) @@ -670,9 +671,9 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo platform = *d.platform } d.state = d.state.File(llb.Mkdir(wd, 0755, mkdirOpt...), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, c.String(), d.state.Env())), d.prefixPlatform, &platform))) + withLayer = true } - - return commitToHistory(&d.image, "WORKDIR "+wd, false, nil) + return commitToHistory(&d.image, "WORKDIR "+wd, withLayer, nil) } return nil } @@ -1285,7 +1286,7 @@ func prefixCommand(ds *dispatchState, str string, prefixPlatform bool, platform func useFileOp(args map[string]string, caps *apicaps.CapSet) bool { enabled := fileOpEnabled if v, ok := args["BUILDKIT_USE_FILEOP"]; ok { - if b, err := strconv.ParseBool(v); err != nil { + if b, err := strconv.ParseBool(v); err == nil { enabled = b } } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 34c38929..c0f44e8d 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -81,6 +81,7 @@ var allTests = []integration.Test{ testCopyChownCreateDest, testEmptyDestDir, testSymlinkedDockerfile, + testDockerfileAddArchiveWildcard, } var opts []integration.TestOpt @@ -1309,6 +1310,79 @@ ADD %s /newname.tar.gz require.Equal(t, buf2.Bytes(), dt) } +func testDockerfileAddArchiveWildcard(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + + buf := bytes.NewBuffer(nil) + tw := tar.NewWriter(buf) + expectedContent := []byte("content0") + err := tw.WriteHeader(&tar.Header{ + Name: "foo", + Typeflag: tar.TypeReg, + Size: int64(len(expectedContent)), + Mode: 0644, + }) + require.NoError(t, err) + _, err = tw.Write(expectedContent) + require.NoError(t, err) + err = tw.Close() + require.NoError(t, err) + + buf2 := bytes.NewBuffer(nil) + tw = tar.NewWriter(buf2) + expectedContent = []byte("content1") + err = tw.WriteHeader(&tar.Header{ + Name: "bar", + Typeflag: tar.TypeReg, + Size: int64(len(expectedContent)), + Mode: 0644, + }) + require.NoError(t, err) + _, err = tw.Write(expectedContent) + require.NoError(t, err) + err = tw.Close() + require.NoError(t, err) + + dockerfile := []byte(` +FROM scratch +ADD *.tar /dest +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("t.tar", buf.Bytes(), 0600), + fstest.CreateFile("b.tar", buf2.Bytes(), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + c, err := client.New(context.TODO(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + _, err = f.Solve(context.TODO(), c, client.SolveOpt{ + Exporter: client.ExporterLocal, + ExporterOutputDir: destDir, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest/foo")) + require.NoError(t, err) + require.Equal(t, "content0", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "dest/bar")) + require.NoError(t, err) + require.Equal(t, "content1", string(dt)) +} + func testSymlinkDestination(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) f.RequiresBuildctl(t) @@ -2061,7 +2135,7 @@ COPY sub/dir1 subdest6 require.NoError(t, err) require.Equal(t, "foo-contents", string(dt)) - dt, err = ioutil.ReadFile(filepath.Join(destDir, "subdest/dir1/dir2/foo")) + dt, err = ioutil.ReadFile(filepath.Join(destDir, "subdest/dir2/foo")) require.NoError(t, err) require.Equal(t, "foo-contents", string(dt)) diff --git a/solver/llbsolver/file/backend.go b/solver/llbsolver/file/backend.go index 554d869b..05e201c4 100644 --- a/solver/llbsolver/file/backend.go +++ b/solver/llbsolver/file/backend.go @@ -6,25 +6,33 @@ import ( "log" "os" "path/filepath" + "strings" + "time" "github.com/containerd/continuity/fs" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes" "github.com/moby/buildkit/solver/pb" "github.com/pkg/errors" - "github.com/sirupsen/logrus" copy "github.com/tonistiigi/fsutil/copy" - "golang.org/x/sys/unix" ) -func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *uidgid) error { +func timestampToTime(ts int64) *time.Time { + if ts == -1 { + return nil + } + tm := time.Unix(ts/1e9, ts%1e9) + return &tm +} + +func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *copy.ChownOpt) error { p, err := fs.RootPath(d, filepath.Join(filepath.Join("/", action.Path))) if err != nil { return err } if action.MakeParents { - if err := mkdirAll(p, os.FileMode(action.Mode)&0777, user); err != nil { + if err := copy.MkdirAll(p, os.FileMode(action.Mode)&0777, user, timestampToTime(action.Timestamp)); err != nil { return err } } else { @@ -34,26 +42,18 @@ func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *uidgi } return err } - if user != nil { - if err := os.Chown(p, user.uid, user.gid); err != nil { - return err - } + if err := copy.Chown(p, user); err != nil { + return err } - - } - - if action.Timestamp != -1 { - st := unix.Timespec{Sec: action.Timestamp / 1e9, Nsec: action.Timestamp % 1e9} - timespec := []unix.Timespec{st, st} - if err := unix.UtimesNanoAt(unix.AT_FDCWD, p, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil { - return errors.Wrapf(err, "failed to utime %s", p) + if err := copy.Utimes(p, timestampToTime(action.Timestamp)); err != nil { + return err } } return nil } -func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *uidgid) error { +func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *copy.ChownOpt) error { p, err := fs.RootPath(d, filepath.Join(filepath.Join("/", action.Path))) if err != nil { return err @@ -63,18 +63,12 @@ func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *uid return err } - if user != nil { - if err := os.Chown(p, user.uid, user.gid); err != nil { - return err - } + if err := copy.Chown(p, user); err != nil { + return err } - if action.Timestamp != -1 { - st := unix.Timespec{Sec: action.Timestamp / 1e9, Nsec: action.Timestamp % 1e9} - timespec := []unix.Timespec{st, st} - if err := unix.UtimesNanoAt(unix.AT_FDCWD, p, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil { - return errors.Wrapf(err, "failed to utime %s", p) - } + if err := copy.Utimes(p, timestampToTime(action.Timestamp)); err != nil { + return err } return nil @@ -96,50 +90,18 @@ func rm(ctx context.Context, d string, action pb.FileActionRm) error { return nil } -func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *uidgid) error { - // // src is the source path - // Src string `protobuf:"bytes,1,opt,name=src,proto3" json:"src,omitempty"` - // // dest path - // Dest string `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"` - // // optional owner override - // Owner *ChownOpt `protobuf:"bytes,4,opt,name=owner" json:"owner,omitempty"` - // // optional permission bits override - // Mode int32 `protobuf:"varint,5,opt,name=mode,proto3" json:"mode,omitempty"` - // // followSymlink resolves symlinks in src - // FollowSymlink bool `protobuf:"varint,6,opt,name=followSymlink,proto3" json:"followSymlink,omitempty"` - // // dirCopyContents only copies contents if src is a directory - // DirCopyContents bool `protobuf:"varint,7,opt,name=dirCopyContents,proto3" json:"dirCopyContents,omitempty"` - // // attemptUnpackDockerCompatibility detects if src is an archive to unpack it instead - // AttemptUnpackDockerCompatibility bool `protobuf:"varint,8,opt,name=attemptUnpackDockerCompatibility,proto3" json:"attemptUnpackDockerCompatibility,omitempty"` - // // createDestPath creates dest path directories if needed - // CreateDestPath bool `protobuf:"varint,9,opt,name=createDestPath,proto3" json:"createDestPath,omitempty"` - // // allowWildcard allows filepath.Match wildcards in src path - // AllowWildcard bool `protobuf:"varint,10,opt,name=allowWildcard,proto3" json:"allowWildcard,omitempty"` - // // allowEmptyWildcard doesn't fail the whole copy if wildcard doesn't resolve to files - // AllowEmptyWildcard bool `protobuf:"varint,11,opt,name=allowEmptyWildcard,proto3" json:"allowEmptyWildcard,omitempty"` - // // optional created time override - // Timestamp int64 `protobuf:"varint,12,opt,name=timestamp,proto3" json:"timestamp,omitempty"` +func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *copy.ChownOpt) error { + srcPath := cleanPath(action.Src) + destPath := cleanPath(action.Dest) - srcp, err := fs.RootPath(src, filepath.Join(filepath.Join("/", action.Src))) - if err != nil { - return err - } - - destp, err := fs.RootPath(dest, filepath.Join(filepath.Join("/", action.Dest))) - if err != nil { - return err - } - - var opt []copy.Opt - - if action.AllowWildcard { - opt = append(opt, copy.AllowWildcards) - } - - if u != nil { - opt = append(opt, func(ci *copy.CopyInfo) { - ci.Chown = ©.ChownOpt{Uid: u.uid, Gid: u.gid} - }) + if !action.CreateDestPath { + p, err := fs.RootPath(dest, filepath.Join(filepath.Join("/", action.Dest))) + if err != nil { + return err + } + if _, err := os.Lstat(filepath.Dir(p)); err != nil { + return errors.Wrapf(err, "failed to stat %s", action.Dest) + } } xattrErrorHandler := func(dst, src, key string, err error) error { @@ -147,15 +109,71 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u * return nil } - opt = append(opt, copy.WithXAttrErrorHandler(xattrErrorHandler)) + opt := []copy.Opt{ + func(ci *copy.CopyInfo) { + ci.Chown = u + ci.Utime = timestampToTime(action.Timestamp) + if m := int(action.Mode); m != -1 { + ci.Mode = &m + } + ci.CopyDirContents = action.DirCopyContents + ci.FollowLinks = action.FollowSymlink + }, + copy.WithXAttrErrorHandler(xattrErrorHandler), + } - if err := copy.Copy(ctx, srcp, destp, opt...); err != nil { + if !action.AllowWildcard { + if action.AttemptUnpackDockerCompatibility { + if ok, err := unpack(ctx, src, srcPath, dest, destPath, u, timestampToTime(action.Timestamp)); err != nil { + return err + } else if ok { + return nil + } + } + return copy.Copy(ctx, src, srcPath, dest, destPath, opt...) + } + + m, err := copy.ResolveWildcards(src, srcPath, action.FollowSymlink) + if err != nil { return err } + if len(m) == 0 { + if action.AllowEmptyWildcard { + return nil + } + return errors.Errorf("%s not found", srcPath) + } + + for _, s := range m { + if action.AttemptUnpackDockerCompatibility { + if ok, err := unpack(ctx, src, s, dest, destPath, u, timestampToTime(action.Timestamp)); err != nil { + return err + } else if ok { + continue + } + } + if err := copy.Copy(ctx, src, s, dest, destPath, opt...); err != nil { + return err + } + } + return nil } +func cleanPath(s string) string { + s2 := filepath.Join("/", s) + if strings.HasSuffix(s, "/.") { + if s2 != "/" { + s2 += "/" + } + s2 += "." + } else if strings.HasSuffix(s, "/") && s2 != "/" { + s2 += "/" + } + return s2 +} + type Backend struct { } @@ -244,7 +262,5 @@ func (fb *Backend) Copy(ctx context.Context, m1, m2, user, group fileoptypes.Mou return err } - logrus.Debugf("copy %+v %+v %+v", action.Owner, user, group) - return docopy(ctx, src, dest, action, u) } diff --git a/solver/llbsolver/file/chown.go b/solver/llbsolver/file/chown.go deleted file mode 100644 index 4c305cf8..00000000 --- a/solver/llbsolver/file/chown.go +++ /dev/null @@ -1,62 +0,0 @@ -package file - -import ( - "os" - "syscall" -) - -// mkdirAll is forked os.MkdirAll -func mkdirAll(path string, perm os.FileMode, user *uidgid) error { - // Fast path: if we can tell whether path is a directory or file, stop with success or error. - dir, err := os.Stat(path) - if err == nil { - if dir.IsDir() { - return nil - } - return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} - } - - // Slow path: make sure parent exists and then call Mkdir for path. - i := len(path) - for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. - i-- - } - - j := i - for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. - j-- - } - - if j > 1 { - // Create parent. - err = mkdirAll(fixRootDirectory(path[:j-1]), perm, user) - if err != nil { - return err - } - } - - dir, err1 := os.Lstat(path) - if err1 == nil && dir.IsDir() { - return nil - } - - // Parent now exists; invoke Mkdir and use its result. - err = os.Mkdir(path, perm) - if err != nil { - // Handle arguments like "foo/." by - // double-checking that directory doesn't exist. - dir, err1 := os.Lstat(path) - if err1 == nil && dir.IsDir() { - return nil - } - return err - } - - if user != nil { - if err := os.Chown(path, user.uid, user.gid); err != nil { - return err - } - } - - return nil -} diff --git a/solver/llbsolver/file/chown_unix.go b/solver/llbsolver/file/chown_unix.go deleted file mode 100644 index 4796302b..00000000 --- a/solver/llbsolver/file/chown_unix.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !windows - -package file - -func fixRootDirectory(p string) string { - return p -} diff --git a/solver/llbsolver/file/chown_windows.go b/solver/llbsolver/file/chown_windows.go deleted file mode 100644 index b0874673..00000000 --- a/solver/llbsolver/file/chown_windows.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build windows - -package file - -import "os" - -func fixRootDirectory(p string) string { - if len(p) == len(`\\?\c:`) { - if os.IsPathSeparator(p[0]) && os.IsPathSeparator(p[1]) && p[2] == '?' && os.IsPathSeparator(p[3]) && p[5] == ':' { - return p + `\` - } - } - return p -} diff --git a/solver/llbsolver/file/unpack.go b/solver/llbsolver/file/unpack.go new file mode 100644 index 00000000..395b1379 --- /dev/null +++ b/solver/llbsolver/file/unpack.go @@ -0,0 +1,61 @@ +package file + +import ( + "archive/tar" + "context" + "os" + "time" + + "github.com/containerd/continuity/fs" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" + copy "github.com/tonistiigi/fsutil/copy" +) + +func unpack(ctx context.Context, srcRoot string, src string, destRoot string, dest string, user *copy.ChownOpt, tm *time.Time) (bool, error) { + src, err := fs.RootPath(srcRoot, src) + if err != nil { + return false, err + } + if !isArchivePath(src) { + return false, nil + } + + dest, err = fs.RootPath(destRoot, dest) + if err != nil { + return false, err + } + if err := copy.MkdirAll(dest, 0755, user, tm); err != nil { + return false, err + } + + file, err := os.Open(src) + if err != nil { + return false, err + } + defer file.Close() + + return true, chrootarchive.Untar(file, dest, nil) +} + +func isArchivePath(path string) bool { + fi, err := os.Lstat(path) + if err != nil { + return false + } + if fi.Mode()&os.ModeType != 0 { + return false + } + file, err := os.Open(path) + if err != nil { + return false + } + defer file.Close() + rdr, err := archive.DecompressStream(file) + if err != nil { + return false + } + r := tar.NewReader(rdr) + _, err = r.Next() + return err == nil +} diff --git a/solver/llbsolver/file/user.go b/solver/llbsolver/file/user.go index 665319b7..3cc74852 100644 --- a/solver/llbsolver/file/user.go +++ b/solver/llbsolver/file/user.go @@ -9,17 +9,14 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/runc/libcontainer/user" "github.com/pkg/errors" + copy "github.com/tonistiigi/fsutil/copy" ) -type uidgid struct { - uid, gid int -} - -func readUser(chopt *pb.ChownOpt, mu, mg fileoptypes.Mount) (*uidgid, error) { +func readUser(chopt *pb.ChownOpt, mu, mg fileoptypes.Mount) (*copy.ChownOpt, error) { if chopt == nil { return nil, nil } - var us uidgid + var us copy.ChownOpt if chopt.User != nil { switch u := chopt.User.User.(type) { case *pb.UserOpt_ByName: @@ -61,12 +58,12 @@ func readUser(chopt *pb.ChownOpt, mu, mg fileoptypes.Mount) (*uidgid, error) { } if len(users) > 0 { - us.uid = users[0].Uid - us.gid = users[0].Gid + us.Uid = users[0].Uid + us.Gid = users[0].Gid } case *pb.UserOpt_ByID: - us.uid = int(u.ByID) - us.gid = int(u.ByID) + us.Uid = int(u.ByID) + us.Gid = int(u.ByID) } } @@ -111,10 +108,10 @@ func readUser(chopt *pb.ChownOpt, mu, mg fileoptypes.Mount) (*uidgid, error) { } if len(groups) > 0 { - us.gid = groups[0].Gid + us.Gid = groups[0].Gid } case *pb.UserOpt_ByID: - us.gid = int(u.ByID) + us.Gid = int(u.ByID) } } diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go index 379d1df9..23bcad4d 100644 --- a/solver/llbsolver/ops/file.go +++ b/solver/llbsolver/ops/file.go @@ -88,8 +88,8 @@ func (f *fileOp) CacheMap(ctx context.Context, index int) (*solver.CacheMap, boo markInvalid(action.Input) processOwner(p.Owner, selectors) if action.SecondaryInput != -1 && int(action.SecondaryInput) < f.numInputs { - p.Src = path.Base(p.Src) addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink) + p.Src = path.Base(p.Src) } dt, err = json.Marshal(p) if err != nil { @@ -375,7 +375,7 @@ func (s *FileOpSolver) validate(idx int, inputs []fileoptypes.Ref, actions []*pb } func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptypes.Ref, actions []*pb.FileAction) (input, error) { - inp, err := s.g.Do(ctx, fmt.Sprintf("inp-%d", idx), func(ctx context.Context) (interface{}, error) { + inp, err := s.g.Do(ctx, fmt.Sprintf("inp-%d", idx), func(ctx context.Context) (_ interface{}, err error) { s.mu.Lock() inp := s.ins[idx] s.mu.Unlock() @@ -391,14 +391,18 @@ func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptyp return inp, nil } + var inpMount, inpMountSecondary fileoptypes.Mount var toRelease []fileoptypes.Mount + var inpMountPrepared bool defer func() { for _, m := range toRelease { m.Release(context.TODO()) } + if err != nil && inpMount != nil && inpMountPrepared { + inpMount.Release(context.TODO()) + } }() - var inpMount, inpMountSecondary fileoptypes.Mount action := actions[idx-len(inputs)] loadInput := func(ctx context.Context) func() error { @@ -413,6 +417,7 @@ func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptyp return err } inpMount = m + inpMountPrepared = true return nil } inpMount = inp.mount @@ -511,6 +516,7 @@ func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptyp return nil, err } inpMount = m + inpMountPrepared = true } switch a := action.Action.(type) {