Merge pull request #218 from tonistiigi/dockerignore

dockerfile: dockerignore support

client/llb/source.go

@@ -193,6 +193,12 @@ func Local(name string, opts ...LocalOption) State {
 	if gi.IncludePatterns != "" {
 		attrs[pb.AttrIncludePatterns] = gi.IncludePatterns
 	}
+	if gi.ExcludePatterns != "" {
+		attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns
+	}
+	if gi.SharedKeyHint != "" {
+		attrs[pb.AttrSharedKeyHint] = gi.SharedKeyHint
+	}
 
 	source := NewSource("local://"+name, attrs, gi.Metadata())
 	return NewState(source.Output())
@@ -216,15 +222,38 @@ func SessionID(id string) LocalOption {
 
 func IncludePatterns(p []string) LocalOption {
 	return localOptionFunc(func(li *LocalInfo) {
+		if len(p) == 0 {
+			li.IncludePatterns = ""
+			return
+		}
 		dt, _ := json.Marshal(p) // empty on error
 		li.IncludePatterns = string(dt)
 	})
 }
 
+func ExcludePatterns(p []string) LocalOption {
+	return localOptionFunc(func(li *LocalInfo) {
+		if len(p) == 0 {
+			li.ExcludePatterns = ""
+			return
+		}
+		dt, _ := json.Marshal(p) // empty on error
+		li.ExcludePatterns = string(dt)
+	})
+}
+
+func SharedKeyHint(h string) LocalOption {
+	return localOptionFunc(func(li *LocalInfo) {
+		li.SharedKeyHint = h
+	})
+}
+
 type LocalInfo struct {
 	opMetaWrapper
 	SessionID       string
 	IncludePatterns string
+	ExcludePatterns string
+	SharedKeyHint   string
 }
 
 func HTTP(url string, opts ...HTTPOption) State {

client/local.go

@@ -1,5 +0,0 @@
-package client
-
-func getSharedKey(dir string) (string, error) {
-	return dir, nil // not implemented
-}

frontend/dockerfile/builder/build.go

@@ -1,15 +1,18 @@
 package builder
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"path"
 	"strings"
 
+	"github.com/docker/docker/builder/dockerignore"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
 	"github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -19,6 +22,7 @@ const (
 	keyFilename           = "filename"
 	exporterImageConfig   = "containerimage.config"
 	defaultDockerfileName = "Dockerfile"
+	dockerignoreFilename  = ".dockerignore"
 	buildArgPrefix        = "build-arg:"
 	gitPrefix             = "git://"
 )
@@ -37,6 +41,7 @@ func Build(ctx context.Context, c client.Client) error {
 	src := llb.Local(LocalNameDockerfile,
 		llb.IncludePatterns([]string{filename}),
 		llb.SessionID(c.SessionID()),
+		llb.SharedKeyHint(defaultDockerfileName),
 	)
 	var buildContext *llb.State
 	if strings.HasPrefix(opts[LocalNameContext], gitPrefix) {
@@ -48,15 +53,52 @@ func Build(ctx context.Context, c client.Client) error {
 		return err
 	}
 
-	ref, err := c.Solve(ctx, def.ToPB(), "", nil, false)
+	eg, ctx2 := errgroup.WithContext(ctx)
+	var dtDockerfile []byte
+	eg.Go(func() error {
+		ref, err := c.Solve(ctx2, def.ToPB(), "", nil, false)
 		if err != nil {
 			return err
 		}
 
-	dtDockerfile, err := ref.ReadFile(ctx, filename)
+		dtDockerfile, err = ref.ReadFile(ctx2, filename)
 		if err != nil {
 			return err
 		}
+		return nil
+	})
+	var excludes []string
+	eg.Go(func() error {
+		dockerignoreState := buildContext
+		if dockerignoreState == nil {
+			st := llb.Local(LocalNameContext,
+				llb.SessionID(c.SessionID()),
+				llb.IncludePatterns([]string{dockerignoreFilename}),
+				llb.SharedKeyHint(dockerignoreFilename),
+			)
+			dockerignoreState = &st
+		}
+		def, err := dockerignoreState.Marshal()
+		if err != nil {
+			return err
+		}
+		ref, err := c.Solve(ctx2, def.ToPB(), "", nil, false)
+		if err != nil {
+			return err
+		}
+		dtDockerignore, err := ref.ReadFile(ctx2, dockerignoreFilename)
+		if err == nil {
+			excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dtDockerignore))
+			if err != nil {
+				return errors.Wrap(err, "failed to parse dockerignore")
+			}
+		}
+		return nil
+	})
+
+	if err := eg.Wait(); err != nil {
+		return err
+	}
 
 	st, img, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, dockerfile2llb.ConvertOpt{
 		Target:       opts[keyTarget],
@@ -64,6 +106,7 @@ func Build(ctx context.Context, c client.Client) error {
 		BuildArgs:    filterBuildArgs(opts),
 		SessionID:    c.SessionID(),
 		BuildContext: buildContext,
+		Excludes:     excludes,
 	})
 
 	if err != nil {

frontend/dockerfile/dockerfile2llb/convert.go

@@ -38,6 +38,7 @@ type ConvertOpt struct {
 	BuildArgs    map[string]string
 	SessionID    string
 	BuildContext *llb.State
+	Excludes     []string
 }
 
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) {
@@ -166,8 +167,11 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	if err := eg.Wait(); err != nil {
 		return nil, nil, err
 	}
-
+	buildContext := llb.Local(localNameContext,
-	buildContext := llb.Local(localNameContext, llb.SessionID(opt.SessionID))
+		llb.SessionID(opt.SessionID),
+		llb.ExcludePatterns(opt.Excludes),
+		llb.SharedKeyHint(localNameContext),
+	)
 	if opt.BuildContext != nil {
 		buildContext = *opt.BuildContext
 	}

frontend/dockerfile/dockerfile_test.go

@@ -40,6 +40,7 @@ func TestIntegration(t *testing.T) {
 		testExportedHistory,
 		testExposeExpansion,
 		testUser,
+		testDockerignore,
 	})
 }
 
@@ -503,6 +504,78 @@ EXPOSE 5000
 	require.Equal(t, "5000/tcp", ports[2])
 }
 
+func testDockerignore(t *testing.T, sb integration.Sandbox) {
+	t.Parallel()
+
+	dockerfile := []byte(`
+FROM scratch
+COPY . .
+`)
+
+	dockerignore := []byte(`
+ba*
+Dockerfile
+!bay
+.dockerignore
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		fstest.CreateFile("foo", []byte(`foo-contents`), 0600),
+		fstest.CreateFile("bar", []byte(`bar-contents`), 0600),
+		fstest.CreateFile("baz", []byte(`baz-contents`), 0600),
+		fstest.CreateFile("bay", []byte(`bay-contents`), 0600),
+		fstest.CreateFile(".dockerignore", dockerignore, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	err = c.Solve(context.TODO(), nil, client.SolveOpt{
+		Frontend: "dockerfile.v0",
+		Exporter: client.ExporterLocal,
+		ExporterAttrs: map[string]string{
+			"output": destDir,
+		},
+		LocalDirs: map[string]string{
+			builder.LocalNameDockerfile: dir,
+			builder.LocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, "foo-contents", string(dt))
+
+	_, err = os.Stat(filepath.Join(destDir, ".dockerignore"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	_, err = os.Stat(filepath.Join(destDir, "Dockerfile"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	_, err = os.Stat(filepath.Join(destDir, "bar"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	_, err = os.Stat(filepath.Join(destDir, "baz"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "bay"))
+	require.NoError(t, err)
+	require.Equal(t, "bay-contents", string(dt))
+}
+
 func testExportedHistory(t *testing.T, sb integration.Sandbox) {
 	t.Parallel()
 

session/filesync/filesync.go

@@ -17,6 +17,7 @@ import (
 const (
 	keyOverrideExcludes = "override-excludes"
 	keyIncludePatterns  = "include-patterns"
+	keyExcludePatterns  = "exclude-patterns"
 	keyDirName          = "dir-name"
 )
 
@@ -55,7 +56,7 @@ func (sp *fsSyncProvider) TarStream(stream FileSync_TarStreamServer) error {
 	return sp.handle("tarstream", stream)
 }
 
-func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error {
+func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retErr error) {
 	var pr *protocol
 	for _, p := range supportedProtocols {
 		if method == p.name && isProtoSupported(p.name) {
@@ -80,8 +81,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error
 		return errors.Errorf("no access allowed to dir %q", dirName)
 	}
 
-	var excludes []string
+	excludes := opts[keyExcludePatterns]
-	if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" {
+	if len(dir.Excludes) != 0 && (len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true") {
 		excludes = dir.Excludes
 	}
 	includes := opts[keyIncludePatterns]
@@ -140,7 +141,8 @@ var supportedProtocols = []protocol{
 type FSSendRequestOpt struct {
 	Name             string
 	IncludePatterns  []string
-	OverrideExcludes bool
+	ExcludePatterns  []string
+	OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory
 	DestDir          string
 	CacheUpdater     CacheUpdater
 	ProgressCb       func(int, bool)
@@ -175,6 +177,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
 		opts[keyIncludePatterns] = opt.IncludePatterns
 	}
 
+	if opt.ExcludePatterns != nil {
+		opts[keyExcludePatterns] = opt.ExcludePatterns
+	}
+
 	opts[keyDirName] = []string{opt.Name}
 
 	ctx, cancel := context.WithCancel(ctx)

solver/pb/attr.go

@@ -3,6 +3,8 @@ package pb
 const AttrKeepGitDir = "git.keepgitdir"
 const AttrLocalSessionID = "local.session"
 const AttrIncludePatterns = "local.includepattern"
+const AttrExcludePatterns = "local.excludepatterns"
+const AttrSharedKeyHint = "local.sharedkeyhint"
 const AttrLLBDefinitionFilename = "llbbuild.filename"
 
 const AttrHTTPChecksum = "http.checksum"

source/identifier.go

@@ -80,6 +80,14 @@ func FromLLB(op *pb.Op_Source) (Identifier, error) {
 					return nil, err
 				}
 				id.IncludePatterns = patterns
+			case pb.AttrExcludePatterns:
+				var patterns []string
+				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
+					return nil, err
+				}
+				id.ExcludePatterns = patterns
+			case pb.AttrSharedKeyHint:
+				id.SharedKeyHint = v
 			}
 		}
 	}
@@ -142,6 +150,8 @@ type LocalIdentifier struct {
 	Name            string
 	SessionID       string
 	IncludePatterns []string
+	ExcludePatterns []string
+	SharedKeyHint   string
 }
 
 func NewLocalIdentifier(str string) (*LocalIdentifier, error) {

source/local/local.go

@@ -79,7 +79,8 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context) (string, error) {
 	dt, err := json.Marshal(struct {
 		SessionID       string
 		IncludePatterns []string
-	}{SessionID: sessionID, IncludePatterns: ls.src.IncludePatterns})
+		ExcludePatterns []string
+	}{SessionID: sessionID, IncludePatterns: ls.src.IncludePatterns, ExcludePatterns: ls.src.ExcludePatterns})
 	if err != nil {
 		return "", err
 	}
@@ -101,7 +102,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
 		return nil, err
 	}
 
-	sharedKey := keySharedKey + ":" + ls.src.Name + ":" + caller.SharedKey()
+	sharedKey := keySharedKey + ":" + ls.src.Name + ":" + ls.src.SharedKeyHint + ":" + caller.SharedKey() // TODO: replace caller.SharedKey() with source based hint from client(absolute-path+nodeid)
 
 	var mutable cache.MutableRef
 	sis, err := ls.md.Search(sharedKey)
@@ -157,6 +158,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
 	opt := filesync.FSSendRequestOpt{
 		Name:             ls.src.Name,
 		IncludePatterns:  ls.src.IncludePatterns,
+		ExcludePatterns:  ls.src.ExcludePatterns,
 		OverrideExcludes: false,
 		DestDir:          dest,
 		CacheUpdater:     &cacheUpdater{cc},

vendor/github.com/docker/docker/builder/dockerignore/dockerignore.go

@@ -0,0 +1,64 @@
+package dockerignore
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+)
+
+// ReadAll reads a .dockerignore file and returns the list of file patterns
+// to ignore. Note this will trim whitespace from each line as well
+// as use GO's "clean" func to get the shortest/cleanest path for each.
+func ReadAll(reader io.Reader) ([]string, error) {
+	if reader == nil {
+		return nil, nil
+	}
+
+	scanner := bufio.NewScanner(reader)
+	var excludes []string
+	currentLine := 0
+
+	utf8bom := []byte{0xEF, 0xBB, 0xBF}
+	for scanner.Scan() {
+		scannedBytes := scanner.Bytes()
+		// We trim UTF8 BOM
+		if currentLine == 0 {
+			scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom)
+		}
+		pattern := string(scannedBytes)
+		currentLine++
+		// Lines starting with # (comments) are ignored before processing
+		if strings.HasPrefix(pattern, "#") {
+			continue
+		}
+		pattern = strings.TrimSpace(pattern)
+		if pattern == "" {
+			continue
+		}
+		// normalize absolute paths to paths relative to the context
+		// (taking care of '!' prefix)
+		invert := pattern[0] == '!'
+		if invert {
+			pattern = strings.TrimSpace(pattern[1:])
+		}
+		if len(pattern) > 0 {
+			pattern = filepath.Clean(pattern)
+			pattern = filepath.ToSlash(pattern)
+			if len(pattern) > 1 && pattern[0] == '/' {
+				pattern = pattern[1:]
+			}
+		}
+		if invert {
+			pattern = "!" + pattern
+		}
+
+		excludes = append(excludes, pattern)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("Error reading .dockerignore: %v", err)
+	}
+	return excludes, nil
+}