add differ support for local source

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

client/client_test.go

@@ -19,6 +19,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -126,6 +127,7 @@ func TestIntegration(t *testing.T) {
 		testStargzLazyPull,
 		testFileOpInputSwap,
 		testRelativeMountpoint,
+		testLocalSourceDiffer,
 	}, mirrors)
 
 	integration.Run(t, []integration.Test{
@@ -1269,6 +1271,86 @@ func testFileOpInputSwap(t *testing.T, sb integration.Sandbox) {
 	require.Contains(t, err.Error(), "bar: no such file")
 }
 
+func testLocalSourceDiffer(t *testing.T, sb integration.Sandbox) {
+	for _, d := range []llb.DiffType{llb.DiffNone, llb.DiffMetadata} {
+		t.Run(fmt.Sprintf("differ=%s", d), func(t *testing.T) {
+			testLocalSourceWithDiffer(t, sb, d)
+		})
+	}
+}
+
+func testLocalSourceWithDiffer(t *testing.T, sb integration.Sandbox, d llb.DiffType) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dir, err := tmpdir(
+		fstest.CreateFile("foo", []byte("foo"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	tv := syscall.NsecToTimespec(time.Now().UnixNano())
+
+	err = syscall.UtimesNano(filepath.Join(dir, "foo"), []syscall.Timespec{tv, tv})
+	require.NoError(t, err)
+
+	st := llb.Local("mylocal"+string(d), llb.Differ(d, false))
+
+	def, err := st.Marshal(context.TODO())
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			"mylocal" + string(d): dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("foo"), dt)
+
+	err = ioutil.WriteFile(filepath.Join(dir, "foo"), []byte("bar"), 0600)
+	require.NoError(t, err)
+
+	err = syscall.UtimesNano(filepath.Join(dir, "foo"), []syscall.Timespec{tv, tv})
+	require.NoError(t, err)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			"mylocal" + string(d): dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	if d == llb.DiffMetadata {
+		require.Equal(t, []byte("foo"), dt)
+	}
+	if d == llb.DiffNone {
+		require.Equal(t, []byte("bar"), dt)
+	}
+}
+
 func testFileOpRmWildcard(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(context.TODO(), sb.Address())

client/llb/source.go

@@ -361,6 +361,12 @@ func Local(name string, opts ...LocalOption) State {
 		attrs[pb.AttrSharedKeyHint] = gi.SharedKeyHint
 		addCap(&gi.Constraints, pb.CapSourceLocalSharedKeyHint)
 	}
+	if gi.Differ.Type != "" {
+		attrs[pb.AttrLocalDiffer] = string(gi.Differ.Type)
+		if gi.Differ.Required {
+			addCap(&gi.Constraints, pb.CapSourceLocalDiffer)
+		}
+	}
 
 	addCap(&gi.Constraints, pb.CapSourceLocal)
 
@@ -423,6 +429,26 @@ func SharedKeyHint(h string) LocalOption {
 	})
 }
 
+func Differ(t DiffType, required bool) LocalOption {
+	return localOptionFunc(func(li *LocalInfo) {
+		li.Differ = DifferInfo{
+			Type:     t,
+			Required: required,
+		}
+	})
+}
+
+type DiffType string
+
+const DiffNone DiffType = pb.AttrLocalDifferNone
+const DiffMetadata DiffType = pb.AttrLocalDifferMetadata
+const DiffContent DiffType = pb.AttrLocalDifferContent
+
+type DifferInfo struct {
+	Type     DiffType
+	Required bool
+}
+
 type LocalInfo struct {
 	constraintsWrapper
 	SessionID       string
@@ -430,6 +456,7 @@ type LocalInfo struct {
 	ExcludePatterns string
 	FollowPaths     string
 	SharedKeyHint   string
+	Differ          DifferInfo
 }
 
 func HTTP(url string, opts ...HTTPOption) State {

session/filesync/diffcopy.go

@@ -70,7 +70,7 @@ func (wc *streamWriterCloser) Close() error {
 	return nil
 }
 
-func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, filter func(string, *fstypes.Stat) bool) (err error) {
+func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter func(string, *fstypes.Stat) bool) (err error) {
 	st := time.Now()
 	defer func() {
 		logrus.Debugf("diffcopy took: %v", time.Since(st))
@@ -93,6 +93,7 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
 		ContentHasher: ch,
 		ProgressCb:    progress,
 		Filter:        fsutil.FilterFunc(filter),
+		Differ:        differ,
 	}))
 }
 

session/filesync/filesync.go

@@ -24,6 +24,7 @@ const (
 	keyFollowPaths        = "followpaths"
 	keyDirName            = "dir-name"
 	keyExporterMetaPrefix = "exporter-md-"
+	keyDiffer             = "differ"
 )
 
 type fsSyncProvider struct {
@@ -130,7 +131,7 @@ type progressCb func(int, bool)
 type protocol struct {
 	name   string
 	sendFn func(stream Stream, fs fsutil.FS, progress progressCb) error
-	recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, mapFunc func(string, *fstypes.Stat) bool) error
+	recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc func(string, *fstypes.Stat) bool) error
 }
 
 func isProtoSupported(p string) bool {
@@ -160,6 +161,7 @@ type FSSendRequestOpt struct {
 	CacheUpdater     CacheUpdater
 	ProgressCb       func(int, bool)
 	Filter           func(string, *fstypes.Stat) bool
+	Differ           fsutil.DiffType
 }
 
 // CacheUpdater is an object capable of sending notifications for the cache hash changes
@@ -227,7 +229,7 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
 		panic(fmt.Sprintf("invalid protocol: %q", pr.name))
 	}
 
-	return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Filter)
+	return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter)
 }
 
 // NewFSSyncTargetDir allows writing into a directory

solver/pb/attr.go

@@ -12,6 +12,7 @@ const AttrIncludePatterns = "local.includepattern"
 const AttrFollowPaths = "local.followpaths"
 const AttrExcludePatterns = "local.excludepatterns"
 const AttrSharedKeyHint = "local.sharedkeyhint"
+
 const AttrLLBDefinitionFilename = "llbbuild.filename"
 
 const AttrHTTPChecksum = "http.checksum"
@@ -26,4 +27,9 @@ const AttrImageResolveModeForcePull = "pull"
 const AttrImageResolveModePreferLocal = "local"
 const AttrImageRecordType = "image.recordtype"
 
+const AttrLocalDiffer = "local.differ"
+const AttrLocalDifferNone = "none"
+const AttrLocalDifferMetadata = "metadata"
+const AttrLocalDifferContent = "content"
+
 type IsFileAction = isFileAction_Action

solver/pb/caps.go

@@ -18,6 +18,7 @@ const (
 	CapSourceLocalFollowPaths     apicaps.CapID = "source.local.followpaths"
 	CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
 	CapSourceLocalSharedKeyHint   apicaps.CapID = "source.local.sharedkeyhint"
+	CapSourceLocalDiffer          apicaps.CapID = "source.local.differ"
 
 	CapSourceGit              apicaps.CapID = "source.git"
 	CapSourceGitKeepDir       apicaps.CapID = "source.git.keepgitdir"
@@ -118,6 +119,13 @@ func init() {
 		Enabled: true,
 		Status:  apicaps.CapStatusExperimental,
 	})
+
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceLocalDiffer,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceGit,
 		Enabled: true,

source/identifier.go

@@ -11,6 +11,7 @@ import (
 	digest "github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/tonistiigi/fsutil"
 )
 
 var (
@@ -146,6 +147,15 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
 				id.FollowPaths = paths
 			case pb.AttrSharedKeyHint:
 				id.SharedKeyHint = v
+			case pb.AttrLocalDiffer:
+				switch v {
+				case pb.AttrLocalDifferMetadata, "":
+					id.Differ = fsutil.DiffMetadata
+				case pb.AttrLocalDifferNone:
+					id.Differ = fsutil.DiffNone
+				case pb.AttrLocalDifferContent:
+					id.Differ = fsutil.DiffContent
+				}
 			}
 		}
 	}
@@ -214,6 +224,7 @@ type LocalIdentifier struct {
 	ExcludePatterns []string
 	FollowPaths     []string
 	SharedKeyHint   string
+	Differ          fsutil.DiffType
 }
 
 func NewLocalIdentifier(str string) (*LocalIdentifier, error) {

source/local/local.go

@@ -178,6 +178,7 @@ func (ls *localSourceHandler) snapshot(ctx context.Context, s session.Group, cal
 		DestDir:          dest,
 		CacheUpdater:     &cacheUpdater{cc, mount.IdentityMapping()},
 		ProgressCb:       newProgressHandler(ctx, "transferring "+ls.src.Name+":"),
+		Differ:           ls.src.Differ,
 	}
 
 	if idmap := mount.IdentityMapping(); idmap != nil {