JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #431 from tonistiigi/dockerfile-symlinks 

dockerfile: detect source symlinks with targets
docker-18.09
Akihiro Suda 2018-06-08 21:57:53 +09:00 committed by GitHub
parent 6ee0fece94 055dcb6c09
commit cf4cc2d6d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 356 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
client/client_test.go
View File

@ -21,6 +21,7 @@ import (
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/continuity/fs/fstest"
"github.com/docker/distribution/manifest/schema2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/identity"
@ -50,9 +51,64 @@ func TestClientIntegration(t *testing.T) {
testBasicCacheImportExport,
testCachedMounts,
testProxyEnv,
testLocalSymlinkEscape,
})
}
func testLocalSymlinkEscape(t *testing.T, sb integration.Sandbox) {
t.Parallel()
requiresLinux(t)
c, err := New(sb.Address())
require.NoError(t, err)
defer c.Close()
test := []byte(`set -x
[[ -L /mount/foo ]]
[[ -L /mount/sub/bar ]]
[[ -L /mount/bax ]]
[[ -f /mount/bay ]]
[[ ! -f /mount/baz ]]
[[ ! -f /mount/etc/passwd ]]
[[ ! -f /mount/etc/group ]]
[[ $(readlink /mount/foo) == "/etc/passwd" ]]
[[ $(readlink /mount/sub/bar) == "../../../etc/group" ]]
`)
dir, err := tmpdir(
// point to absolute path that is not part of dir
fstest.Symlink("/etc/passwd", "foo"),
fstest.CreateDir("sub", 0700),
// point outside of the dir
fstest.Symlink("../../../etc/group", "sub/bar"),
// regular valid symlink
fstest.Symlink("bay", "bax"),
// target for symlink (not requested)
fstest.CreateFile("bay", []byte{}, 0600),
// unused file that shouldn't be included
fstest.CreateFile("baz", []byte{}, 0600),
fstest.CreateFile("test.sh", test, 0700),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
local := llb.Local("mylocal", llb.FollowPaths([]string{
"test.sh", "foo", "sub/bar", "bax",
}))
st := llb.Image("busybox:latest").
Run(llb.Shlex(`sh /mount/test.sh`), llb.AddMount("/mount", local, llb.Readonly))
def, err := st.Marshal()
require.NoError(t, err)
_, err = c.Solve(context.TODO(), def, SolveOpt{
LocalDirs: map[string]string{
"mylocal": dir,
},
}, nil)
require.NoError(t, err)
}
func testRelativeWorkDir(t *testing.T, sb integration.Sandbox) {
t.Parallel()
requiresLinux(t)
@ -1182,3 +1238,14 @@ func testInvalidExporter(t *testing.T, sb integration.Sandbox) {
checkAllReleasable(t, c, sb, true)
}
func tmpdir(appliers ...fstest.Applier) (string, error) {
tmpdir, err := ioutil.TempDir("", "buildkit-client")
if err != nil {
return "", err
}
if err := fstest.Apply(appliers...).Apply(tmpdir); err != nil {
return "", err
}
return tmpdir, nil
}

15
client/llb/source.go
View File

@ -210,6 +210,9 @@ func Local(name string, opts ...LocalOption) State {
if gi.IncludePatterns != "" {
attrs[pb.AttrIncludePatterns] = gi.IncludePatterns
}
if gi.FollowPaths != "" {
attrs[pb.AttrFollowPaths] = gi.FollowPaths
}
if gi.ExcludePatterns != "" {
attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns
}
@ -248,6 +251,17 @@ func IncludePatterns(p []string) LocalOption {
})
}
func FollowPaths(p []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
if len(p) == 0 {
li.FollowPaths = ""
return
}
dt, _ := json.Marshal(p) // empty on error
li.FollowPaths = string(dt)
})
}
func ExcludePatterns(p []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
if len(p) == 0 {
@ -270,6 +284,7 @@ type LocalInfo struct {
SessionID string
IncludePatterns string
ExcludePatterns string
FollowPaths string
SharedKeyHint string
}

2
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -267,7 +267,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
llb.SharedKeyHint(localNameContext),
}
if includePatterns := normalizeContextPaths(ctxPaths); includePatterns != nil {
opts = append(opts, llb.IncludePatterns(includePatterns))
opts = append(opts, llb.FollowPaths(includePatterns))
}
bc := llb.Local(localNameContext, opts...)
if opt.BuildContext != nil {

39
frontend/dockerfile/dockerfile_test.go
View File

@ -65,9 +65,48 @@ func TestIntegration(t *testing.T) {
testPullScratch,
testSymlinkDestination,
testHTTPDockerfile,
testCopySymlinks,
})
}
func testCopySymlinks(t *testing.T, sb integration.Sandbox) {
t.Parallel()
dockerfile := []byte(`
FROM scratch
COPY foo /
COPY sub/l* alllinks/
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateFile("bar", []byte(`bar-contents`), 0600),
fstest.Symlink("bar", "foo"),
fstest.CreateDir("sub", 0700),
fstest.CreateFile("sub/lfile", []byte(`baz-contents`), 0600),
fstest.Symlink("subfile", "sub/l0"),
fstest.CreateFile("sub/subfile", []byte(`subfile-contents`), 0600),
fstest.Symlink("second", "sub/l1"),
fstest.Symlink("baz", "sub/second"),
fstest.CreateFile("sub/baz", []byte(`baz-contents`), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(sb.Address())
require.NoError(t, err)
defer c.Close()
_, err = c.Solve(context.TODO(), nil, client.SolveOpt{
Frontend: "dockerfile.v0",
LocalDirs: map[string]string{
builder.LocalNameDockerfile: dir,
builder.LocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
}
func testHTTPDockerfile(t *testing.T, sb integration.Sandbox) {
t.Parallel()

3
session/filesync/diffcopy.go
View File

@ -12,10 +12,11 @@ import (
"google.golang.org/grpc"
)
func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error {
func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error {
return fsutil.Send(stream.Context(), stream, dir, &fsutil.WalkOpt{
ExcludePatterns: excludes,
IncludePatterns: includes,
FollowPaths: followPaths,
Map: _map,
}, progress)
}

14
session/filesync/filesync.go
View File

@ -18,6 +18,7 @@ const (
keyOverrideExcludes = "override-excludes"
keyIncludePatterns = "include-patterns"
keyExcludePatterns = "exclude-patterns"
keyFollowPaths = "followpaths"
keyDirName = "dir-name"
)
@ -87,6 +88,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
}
includes := opts[keyIncludePatterns]
followPaths := opts[keyFollowPaths]
var progress progressCb
if sp.p != nil {
progress = sp.p
@ -98,7 +101,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
doneCh = sp.doneCh
sp.doneCh = nil
}
err := pr.sendFn(stream, dir.Dir, includes, excludes, progress, dir.Map)
err := pr.sendFn(stream, dir.Dir, includes, excludes, followPaths, progress, dir.Map)
if doneCh != nil {
if err != nil {
doneCh <- err
@ -117,7 +120,7 @@ type progressCb func(int, bool)
type protocol struct {
name string
sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error
sendFn func(stream grpc.Stream, srcDir string, includes, excludes, followPaths []string, progress progressCb, _map func(*fsutil.Stat) bool) error
recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb) error
}
@ -142,6 +145,7 @@ type FSSendRequestOpt struct {
Name string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory
DestDir string
CacheUpdater CacheUpdater
@ -181,6 +185,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
opts[keyExcludePatterns] = opt.ExcludePatterns
}
if opt.FollowPaths != nil {
opts[keyFollowPaths] = opt.FollowPaths
}
opts[keyDirName] = []string{opt.Name}
ctx, cancel := context.WithCancel(ctx)
@ -261,7 +269,7 @@ func CopyToCaller(ctx context.Context, srcPath string, c session.Caller, progres
return err
}
return sendDiffCopy(cc, srcPath, nil, nil, progress, nil)
return sendDiffCopy(cc, srcPath, nil, nil, nil, progress, nil)
}
func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) {

1
solver/pb/attr.go
View File

@ -4,6 +4,7 @@ const AttrKeepGitDir = "git.keepgitdir"
const AttrFullRemoteURL = "git.fullurl"
const AttrLocalSessionID = "local.session"
const AttrIncludePatterns = "local.includepattern"
const AttrFollowPaths = "local.followpaths"
const AttrExcludePatterns = "local.excludepatterns"
const AttrSharedKeyHint = "local.sharedkeyhint"
const AttrLLBDefinitionFilename = "llbbuild.filename"

10
source/identifier.go
View File

@ -2,6 +2,7 @@ package source
import (
"encoding/json"
"fmt"
"strconv"
"strings"
@ -69,6 +70,7 @@ func FromLLB(op *pb.Op_Source) (Identifier, error) {
}
if id, ok := id.(*LocalIdentifier); ok {
for k, v := range op.Source.Attrs {
fmt.Printf("kv %q %q\n", k, v)
switch k {
case pb.AttrLocalSessionID:
id.SessionID = v
@ -88,6 +90,13 @@ func FromLLB(op *pb.Op_Source) (Identifier, error) {
return nil, err
}
id.ExcludePatterns = patterns
case pb.AttrFollowPaths:
var paths []string
if err := json.Unmarshal([]byte(v), &paths); err != nil {
return nil, err
}
id.FollowPaths = paths
fmt.Printf("FollowPaths %#v\n", paths)
case pb.AttrSharedKeyHint:
id.SharedKeyHint = v
}
@ -153,6 +162,7 @@ type LocalIdentifier struct {
SessionID string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
SharedKeyHint string
}

1
source/local/local.go
View File

@ -159,6 +159,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
Name: ls.src.Name,
IncludePatterns: ls.src.IncludePatterns,
ExcludePatterns: ls.src.ExcludePatterns,
FollowPaths: ls.src.FollowPaths,
OverrideExcludes: false,
DestDir: dest,
CacheUpdater: &cacheUpdater{cc},

2
vendor.conf
View File

@ -40,7 +40,7 @@ github.com/BurntSushi/locker a6e239ea1c69bff1cfdb20c4b73dadf52f784b6a
github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281
github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f
github.com/tonistiigi/fsutil 30b4fcc516fc682ec95ad17c13e6634667560c0a
github.com/tonistiigi/fsutil 8839685ae8c3c8bd67d0ce28e9b3157b23c1c7a5
github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b

150
vendor/github.com/tonistiigi/fsutil/followlinks.go generated vendored Normal file
View File

@ -0,0 +1,150 @@
package fsutil
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sort"
strings "strings"
"github.com/pkg/errors"
)
func FollowLinks(root string, paths []string) ([]string, error) {
r := &symlinkResolver{root: root, resolved: map[string]struct{}{}}
for _, p := range paths {
if err := r.append(p); err != nil {
return nil, err
}
}
res := make([]string, 0, len(r.resolved))
for r := range r.resolved {
res = append(res, r)
}
sort.Strings(res)
return dedupePaths(res), nil
}
type symlinkResolver struct {
root string
resolved map[string]struct{}
}
func (r *symlinkResolver) append(p string) error {
p = filepath.Join(".", p)
current := "."
for {
parts := strings.SplitN(p, string(filepath.Separator), 2)
current = filepath.Join(current, parts[0])
targets, err := r.readSymlink(current, true)
if err != nil {
return err
}
p = ""
if len(parts) == 2 {
p = parts[1]
}
if p == "" || targets != nil {
if _, ok := r.resolved[current]; ok {
return nil
}
}
if targets != nil {
r.resolved[current] = struct{}{}
for _, target := range targets {
if err := r.append(filepath.Join(target, p)); err != nil {
return err
}
}
return nil
}
if p == "" {
r.resolved[current] = struct{}{}
return nil
}
}
}
func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) {
realPath := filepath.Join(r.root, p)
base := filepath.Base(p)
if allowWildcard && containsWildcards(base) {
fis, err := ioutil.ReadDir(filepath.Dir(realPath))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, errors.Wrapf(err, "failed to read dir %s", filepath.Dir(realPath))
}
var out []string
for _, f := range fis {
if ok, _ := filepath.Match(base, f.Name()); ok {
res, err := r.readSymlink(filepath.Join(filepath.Dir(p), f.Name()), false)
if err != nil {
return nil, err
}
out = append(out, res...)
}
}
return out, nil
}
fi, err := os.Lstat(realPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, errors.Wrapf(err, "failed to lstat %s", realPath)
}
if fi.Mode()&os.ModeSymlink == 0 {
return nil, nil
}
link, err := os.Readlink(realPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to readlink %s", realPath)
}
link = filepath.Clean(link)
if filepath.IsAbs(link) {
return []string{link}, nil
}
return []string{
filepath.Join(string(filepath.Separator), filepath.Join(filepath.Dir(p), link)),
}, nil
}
func containsWildcards(name string) bool {
isWindows := runtime.GOOS == "windows"
for i := 0; i < len(name); i++ {
ch := name[i]
if ch == '\\' && !isWindows {
i++
} else if ch == '*' || ch == '?' || ch == '[' {
return true
}
}
return false
}
// dedupePaths expects input as a sorted list
func dedupePaths(in []string) []string {
out := make([]string, 0, len(in))
var last string
for _, s := range in {
// if one of the paths is root there is no filter
if s == "." {
return nil
}
if strings.HasPrefix(s, last+string(filepath.Separator)) {
continue
}
out = append(out, s)
last = s
}
return out
}

100
vendor/github.com/tonistiigi/fsutil/walker.go generated vendored
View File

@ -15,7 +15,10 @@ import (
type WalkOpt struct {
IncludePatterns []string
ExcludePatterns []string
Map func(*Stat) bool
// FollowPaths contains symlinks that are resolved into include patterns
// before performing the fs walk
FollowPaths []string
Map func(*Stat) bool
}
func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error {
@ -39,8 +42,25 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
}
var includePatterns []string
if opt != nil && opt.IncludePatterns != nil {
includePatterns = make([]string, len(opt.IncludePatterns))
for k := range opt.IncludePatterns {
includePatterns[k] = filepath.Clean(opt.IncludePatterns[k])
}
}
if opt != nil && opt.FollowPaths != nil {
targets, err := FollowLinks(p, opt.FollowPaths)
if err != nil {
return err
}
if targets != nil {
includePatterns = append(includePatterns, targets...)
includePatterns = dedupePaths(includePatterns)
}
}
var lastIncludedDir string
var includePatternPrefixes []string
seenFiles := make(map[uint64]string)
return filepath.Walk(root, func(path string, fi os.FileInfo, err error) (retErr error) {
@ -66,34 +86,31 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
if opt != nil {
if opt.IncludePatterns != nil {
if includePatternPrefixes == nil {
includePatternPrefixes = patternPrefixes(opt.IncludePatterns)
}
matched := false
if includePatterns != nil {
skip := false
if lastIncludedDir != "" {
if strings.HasPrefix(path, lastIncludedDir+string(filepath.Separator)) {
matched = true
skip = true
}
}
if !matched {
for _, p := range opt.IncludePatterns {
if m, _ := filepath.Match(p, path); m {
if !skip {
matched := false
partial := true
for _, p := range includePatterns {
if ok, p := matchPrefix(p, path); ok {
matched = true
break
if !p {
partial = false
break
}
}
}
if matched && fi.IsDir() {
lastIncludedDir = path
}
}
if !matched {
if !fi.IsDir() {
if !matched {
return nil
} else {
if noPossiblePrefixMatch(path, includePatternPrefixes) {
return filepath.SkipDir
}
}
if !partial && fi.IsDir() {
lastIncludedDir = path
}
}
}
@ -199,29 +216,28 @@ func (s *StatInfo) Sys() interface{} {
return s.Stat
}
func patternPrefixes(patterns []string) []string {
pfxs := make([]string, 0, len(patterns))
for _, ptrn := range patterns {
idx := strings.IndexFunc(ptrn, func(ch rune) bool {
return ch == '*' || ch == '?' || ch == '[' || ch == '\\'
})
if idx == -1 {
idx = len(ptrn)
}
pfxs = append(pfxs, ptrn[:idx])
func matchPrefix(pattern, name string) (bool, bool) {
count := strings.Count(name, string(filepath.Separator))
partial := false
if strings.Count(pattern, string(filepath.Separator)) > count {
pattern = trimUntilIndex(pattern, string(filepath.Separator), count)
partial = true
}
return pfxs
m, _ := filepath.Match(pattern, name)
return m, partial
}
func noPossiblePrefixMatch(p string, pfxs []string) bool {
for _, pfx := range pfxs {
chk := p
if len(pfx) < len(p) {
chk = p[:len(pfx)]
}
if strings.HasPrefix(pfx, chk) {
return false
func trimUntilIndex(str, sep string, count int) string {
s := str
i := 0
c := 0
for {
idx := strings.Index(s, sep)
s = s[idx+len(sep):]
i += idx + len(sep)
c++
if c >= count {
return str[:i-len(sep)]
}
}
return true
}