Merge pull request #2081 from tonistiigi/local-differ-none

Allow none differ on local source to avoid false Dockerfile matches

client/client_test.go

@@ -19,6 +19,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -129,6 +130,7 @@ func TestIntegration(t *testing.T) {
 		testStargzLazyPull,
 		testFileOpInputSwap,
 		testRelativeMountpoint,
+		testLocalSourceDiffer,
 	}, mirrors)
 
 	integration.Run(t, []integration.Test{
@@ -1272,6 +1274,86 @@ func testFileOpInputSwap(t *testing.T, sb integration.Sandbox) {
 	require.Contains(t, err.Error(), "bar: no such file")
 }
 
+func testLocalSourceDiffer(t *testing.T, sb integration.Sandbox) {
+	for _, d := range []llb.DiffType{llb.DiffNone, llb.DiffMetadata} {
+		t.Run(fmt.Sprintf("differ=%s", d), func(t *testing.T) {
+			testLocalSourceWithDiffer(t, sb, d)
+		})
+	}
+}
+
+func testLocalSourceWithDiffer(t *testing.T, sb integration.Sandbox, d llb.DiffType) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dir, err := tmpdir(
+		fstest.CreateFile("foo", []byte("foo"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	tv := syscall.NsecToTimespec(time.Now().UnixNano())
+
+	err = syscall.UtimesNano(filepath.Join(dir, "foo"), []syscall.Timespec{tv, tv})
+	require.NoError(t, err)
+
+	st := llb.Local("mylocal"+string(d), llb.Differ(d, false))
+
+	def, err := st.Marshal(context.TODO())
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			"mylocal" + string(d): dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("foo"), dt)
+
+	err = ioutil.WriteFile(filepath.Join(dir, "foo"), []byte("bar"), 0600)
+	require.NoError(t, err)
+
+	err = syscall.UtimesNano(filepath.Join(dir, "foo"), []syscall.Timespec{tv, tv})
+	require.NoError(t, err)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			"mylocal" + string(d): dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	if d == llb.DiffMetadata {
+		require.Equal(t, []byte("foo"), dt)
+	}
+	if d == llb.DiffNone {
+		require.Equal(t, []byte("bar"), dt)
+	}
+}
+
 func testFileOpRmWildcard(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(sb.Context(), sb.Address())

client/llb/source.go

@@ -369,6 +369,12 @@ func Local(name string, opts ...LocalOption) State {
 		attrs[pb.AttrSharedKeyHint] = gi.SharedKeyHint
 		addCap(&gi.Constraints, pb.CapSourceLocalSharedKeyHint)
 	}
+	if gi.Differ.Type != "" {
+		attrs[pb.AttrLocalDiffer] = string(gi.Differ.Type)
+		if gi.Differ.Required {
+			addCap(&gi.Constraints, pb.CapSourceLocalDiffer)
+		}
+	}
 
 	addCap(&gi.Constraints, pb.CapSourceLocal)
 
@@ -431,6 +437,32 @@ func SharedKeyHint(h string) LocalOption {
 	})
 }
 
+func Differ(t DiffType, required bool) LocalOption {
+	return localOptionFunc(func(li *LocalInfo) {
+		li.Differ = DifferInfo{
+			Type:     t,
+			Required: required,
+		}
+	})
+}
+
+type DiffType string
+
+const (
+	// DiffNone will do no file comparisons, all files in the Local source will
+	// be retransmitted.
+	DiffNone DiffType = pb.AttrLocalDifferNone
+	// DiffMetadata will compare file metadata (size, modified time, mode, owner,
+	// group, device and link name) to determine if the files in the Local source need
+	// to be retransmitted.  This is the default behavior.
+	DiffMetadata DiffType = pb.AttrLocalDifferMetadata
+)
+
+type DifferInfo struct {
+	Type     DiffType
+	Required bool
+}
+
 type LocalInfo struct {
 	constraintsWrapper
 	SessionID       string
@@ -438,6 +470,7 @@ type LocalInfo struct {
 	ExcludePatterns string
 	FollowPaths     string
 	SharedKeyHint   string
+	Differ          DifferInfo
 }
 
 func HTTP(url string, opts ...HTTPOption) State {

frontend/dockerfile/builder/build.go

@@ -143,6 +143,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 		llb.SessionID(c.BuildOpts().SessionID),
 		llb.SharedKeyHint(localNameDockerfile),
 		dockerfile2llb.WithInternalName(name),
+		llb.Differ(llb.DiffNone, false),
 	)
 
 	fileop := useFileOp(opts, &caps)
@@ -302,6 +303,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 					llb.FollowPaths([]string{dockerignoreFilename}),
 					llb.SharedKeyHint(localNameContext+"-"+dockerignoreFilename),
 					dockerfile2llb.WithInternalName("load "+dockerignoreFilename),
+					llb.Differ(llb.DiffNone, false),
 				)
 				dockerignoreState = &st
 			}

session/filesync/diffcopy.go

@@ -70,7 +70,7 @@ func (wc *streamWriterCloser) Close() error {
 	return nil
 }
 
-func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, filter func(string, *fstypes.Stat) bool) (err error) {
+func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter func(string, *fstypes.Stat) bool) (err error) {
 	st := time.Now()
 	defer func() {
 		logrus.Debugf("diffcopy took: %v", time.Since(st))
@@ -93,6 +93,7 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
 		ContentHasher: ch,
 		ProgressCb:    progress,
 		Filter:        fsutil.FilterFunc(filter),
+		Differ:        differ,
 	}))
 }
 

session/filesync/filesync.go

@@ -130,7 +130,7 @@ type progressCb func(int, bool)
 type protocol struct {
 	name   string
 	sendFn func(stream Stream, fs fsutil.FS, progress progressCb) error
-	recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, mapFunc func(string, *fstypes.Stat) bool) error
+	recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc func(string, *fstypes.Stat) bool) error
 }
 
 func isProtoSupported(p string) bool {
@@ -160,6 +160,7 @@ type FSSendRequestOpt struct {
 	CacheUpdater     CacheUpdater
 	ProgressCb       func(int, bool)
 	Filter           func(string, *fstypes.Stat) bool
+	Differ           fsutil.DiffType
 }
 
 // CacheUpdater is an object capable of sending notifications for the cache hash changes
@@ -227,7 +228,7 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
 		panic(fmt.Sprintf("invalid protocol: %q", pr.name))
 	}
 
-	return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Filter)
+	return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter)
 }
 
 // NewFSSyncTargetDir allows writing into a directory

solver/pb/attr.go

@@ -12,6 +12,7 @@ const AttrIncludePatterns = "local.includepattern"
 const AttrFollowPaths = "local.followpaths"
 const AttrExcludePatterns = "local.excludepatterns"
 const AttrSharedKeyHint = "local.sharedkeyhint"
+
 const AttrLLBDefinitionFilename = "llbbuild.filename"
 
 const AttrHTTPChecksum = "http.checksum"
@@ -26,4 +27,8 @@ const AttrImageResolveModeForcePull = "pull"
 const AttrImageResolveModePreferLocal = "local"
 const AttrImageRecordType = "image.recordtype"
 
+const AttrLocalDiffer = "local.differ"
+const AttrLocalDifferNone = "none"
+const AttrLocalDifferMetadata = "metadata"
+
 type IsFileAction = isFileAction_Action

solver/pb/caps.go

@@ -18,6 +18,7 @@ const (
 	CapSourceLocalFollowPaths     apicaps.CapID = "source.local.followpaths"
 	CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
 	CapSourceLocalSharedKeyHint   apicaps.CapID = "source.local.sharedkeyhint"
+	CapSourceLocalDiffer          apicaps.CapID = "source.local.differ"
 
 	CapSourceGit              apicaps.CapID = "source.git"
 	CapSourceGitKeepDir       apicaps.CapID = "source.git.keepgitdir"
@@ -118,6 +119,13 @@ func init() {
 		Enabled: true,
 		Status:  apicaps.CapStatusExperimental,
 	})
+
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceLocalDiffer,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceGit,
 		Enabled: true,

source/identifier.go

@@ -11,6 +11,7 @@ import (
 	digest "github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/tonistiigi/fsutil"
 )
 
 var (
@@ -146,6 +147,13 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
 				id.FollowPaths = paths
 			case pb.AttrSharedKeyHint:
 				id.SharedKeyHint = v
+			case pb.AttrLocalDiffer:
+				switch v {
+				case pb.AttrLocalDifferMetadata, "":
+					id.Differ = fsutil.DiffMetadata
+				case pb.AttrLocalDifferNone:
+					id.Differ = fsutil.DiffNone
+				}
 			}
 		}
 	}
@@ -214,6 +222,7 @@ type LocalIdentifier struct {
 	ExcludePatterns []string
 	FollowPaths     []string
 	SharedKeyHint   string
+	Differ          fsutil.DiffType
 }
 
 func NewLocalIdentifier(str string) (*LocalIdentifier, error) {

source/local/local.go

@@ -178,6 +178,7 @@ func (ls *localSourceHandler) snapshot(ctx context.Context, s session.Group, cal
 		DestDir:          dest,
 		CacheUpdater:     &cacheUpdater{cc, mount.IdentityMapping()},
 		ProgressCb:       newProgressHandler(ctx, "transferring "+ls.src.Name+":"),
+		Differ:           ls.src.Differ,
 	}
 
 	if idmap := mount.IdentityMapping(); idmap != nil {