JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2081 from tonistiigi/local-differ-none 

Allow none differ on local source to avoid false Dockerfile matches
v0.9
Akihiro Suda 2021-07-01 09:42:24 +09:00 committed by GitHub
parent 921b0de92e ee39f5dd68
commit be8ab28a0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 145 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
client/client_test.go
View File

@ -19,6 +19,7 @@ import (
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
@ -129,6 +130,7 @@ func TestIntegration(t *testing.T) {
testStargzLazyPull,
testFileOpInputSwap,
testRelativeMountpoint,
testLocalSourceDiffer,
}, mirrors)
integration.Run(t, []integration.Test{
@ -1272,6 +1274,86 @@ func testFileOpInputSwap(t *testing.T, sb integration.Sandbox) {
require.Contains(t, err.Error(), "bar: no such file")
}
func testLocalSourceDiffer(t *testing.T, sb integration.Sandbox) {
for _, d := range []llb.DiffType{llb.DiffNone, llb.DiffMetadata} {
t.Run(fmt.Sprintf("differ=%s", d), func(t *testing.T) {
testLocalSourceWithDiffer(t, sb, d)
})
}
}
func testLocalSourceWithDiffer(t *testing.T, sb integration.Sandbox, d llb.DiffType) {
requiresLinux(t)
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
dir, err := tmpdir(
fstest.CreateFile("foo", []byte("foo"), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
tv := syscall.NsecToTimespec(time.Now().UnixNano())
err = syscall.UtimesNano(filepath.Join(dir, "foo"), []syscall.Timespec{tv, tv})
require.NoError(t, err)
st := llb.Local("mylocal"+string(d), llb.Differ(d, false))
def, err := st.Marshal(context.TODO())
require.NoError(t, err)
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = c.Solve(context.TODO(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
"mylocal" + string(d): dir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo"))
require.NoError(t, err)
require.Equal(t, []byte("foo"), dt)
err = ioutil.WriteFile(filepath.Join(dir, "foo"), []byte("bar"), 0600)
require.NoError(t, err)
err = syscall.UtimesNano(filepath.Join(dir, "foo"), []syscall.Timespec{tv, tv})
require.NoError(t, err)
_, err = c.Solve(context.TODO(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
"mylocal" + string(d): dir,
},
}, nil)
require.NoError(t, err)
dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo"))
require.NoError(t, err)
if d == llb.DiffMetadata {
require.Equal(t, []byte("foo"), dt)
}
if d == llb.DiffNone {
require.Equal(t, []byte("bar"), dt)
}
}
func testFileOpRmWildcard(t *testing.T, sb integration.Sandbox) {
requiresLinux(t)
c, err := New(sb.Context(), sb.Address())

33
client/llb/source.go
View File

@ -369,6 +369,12 @@ func Local(name string, opts ...LocalOption) State {
attrs[pb.AttrSharedKeyHint] = gi.SharedKeyHint
addCap(&gi.Constraints, pb.CapSourceLocalSharedKeyHint)
}
if gi.Differ.Type != "" {
attrs[pb.AttrLocalDiffer] = string(gi.Differ.Type)
if gi.Differ.Required {
addCap(&gi.Constraints, pb.CapSourceLocalDiffer)
}
}
addCap(&gi.Constraints, pb.CapSourceLocal)
@ -431,6 +437,32 @@ func SharedKeyHint(h string) LocalOption {
})
}
func Differ(t DiffType, required bool) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
li.Differ = DifferInfo{
Type: t,
Required: required,
}
})
}
type DiffType string
const (
// DiffNone will do no file comparisons, all files in the Local source will
// be retransmitted.
DiffNone DiffType = pb.AttrLocalDifferNone
// DiffMetadata will compare file metadata (size, modified time, mode, owner,
// group, device and link name) to determine if the files in the Local source need
// to be retransmitted. This is the default behavior.
DiffMetadata DiffType = pb.AttrLocalDifferMetadata
)
type DifferInfo struct {
Type DiffType
Required bool
}
type LocalInfo struct {
constraintsWrapper
SessionID string
@ -438,6 +470,7 @@ type LocalInfo struct {
ExcludePatterns string
FollowPaths string
SharedKeyHint string
Differ DifferInfo
}
func HTTP(url string, opts ...HTTPOption) State {

2
frontend/dockerfile/builder/build.go
View File

@ -143,6 +143,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
llb.SessionID(c.BuildOpts().SessionID),
llb.SharedKeyHint(localNameDockerfile),
dockerfile2llb.WithInternalName(name),
llb.Differ(llb.DiffNone, false),
)
fileop := useFileOp(opts, &caps)
@ -302,6 +303,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
llb.FollowPaths([]string{dockerignoreFilename}),
llb.SharedKeyHint(localNameContext+"-"+dockerignoreFilename),
dockerfile2llb.WithInternalName("load "+dockerignoreFilename),
llb.Differ(llb.DiffNone, false),
)
dockerignoreState = &st
}

3
session/filesync/diffcopy.go
View File

@ -70,7 +70,7 @@ func (wc *streamWriterCloser) Close() error {
return nil
}
func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, filter func(string, *fstypes.Stat) bool) (err error) {
func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter func(string, *fstypes.Stat) bool) (err error) {
st := time.Now()
defer func() {
logrus.Debugf("diffcopy took: %v", time.Since(st))
@ -93,6 +93,7 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
ContentHasher: ch,
ProgressCb: progress,
Filter: fsutil.FilterFunc(filter),
Differ: differ,
}))
}

5
session/filesync/filesync.go
View File

@ -130,7 +130,7 @@ type progressCb func(int, bool)
type protocol struct {
name string
sendFn func(stream Stream, fs fsutil.FS, progress progressCb) error
recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, mapFunc func(string, *fstypes.Stat) bool) error
recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc func(string, *fstypes.Stat) bool) error
}
func isProtoSupported(p string) bool {
@ -160,6 +160,7 @@ type FSSendRequestOpt struct {
CacheUpdater CacheUpdater
ProgressCb func(int, bool)
Filter func(string, *fstypes.Stat) bool
Differ fsutil.DiffType
}
// CacheUpdater is an object capable of sending notifications for the cache hash changes
@ -227,7 +228,7 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
panic(fmt.Sprintf("invalid protocol: %q", pr.name))
}
return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Filter)
return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter)
}
// NewFSSyncTargetDir allows writing into a directory

5
solver/pb/attr.go
View File

@ -12,6 +12,7 @@ const AttrIncludePatterns = "local.includepattern"
const AttrFollowPaths = "local.followpaths"
const AttrExcludePatterns = "local.excludepatterns"
const AttrSharedKeyHint = "local.sharedkeyhint"
const AttrLLBDefinitionFilename = "llbbuild.filename"
const AttrHTTPChecksum = "http.checksum"
@ -26,4 +27,8 @@ const AttrImageResolveModeForcePull = "pull"
const AttrImageResolveModePreferLocal = "local"
const AttrImageRecordType = "image.recordtype"
const AttrLocalDiffer = "local.differ"
const AttrLocalDifferNone = "none"
const AttrLocalDifferMetadata = "metadata"
type IsFileAction = isFileAction_Action

8
solver/pb/caps.go
View File

@ -18,6 +18,7 @@ const (
CapSourceLocalFollowPaths apicaps.CapID = "source.local.followpaths"
CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint"
CapSourceLocalDiffer apicaps.CapID = "source.local.differ"
CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
@ -118,6 +119,13 @@ func init() {
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceLocalDiffer,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceGit,
Enabled: true,

9
source/identifier.go
View File

@ -11,6 +11,7 @@ import (
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
)
var (
@ -146,6 +147,13 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
id.FollowPaths = paths
case pb.AttrSharedKeyHint:
id.SharedKeyHint = v
case pb.AttrLocalDiffer:
switch v {
case pb.AttrLocalDifferMetadata, "":
id.Differ = fsutil.DiffMetadata
case pb.AttrLocalDifferNone:
id.Differ = fsutil.DiffNone
}
}
}
}
@ -214,6 +222,7 @@ type LocalIdentifier struct {
ExcludePatterns []string
FollowPaths []string
SharedKeyHint string
Differ fsutil.DiffType
}
func NewLocalIdentifier(str string) (*LocalIdentifier, error) {

1
source/local/local.go
View File

@ -178,6 +178,7 @@ func (ls *localSourceHandler) snapshot(ctx context.Context, s session.Group, cal
DestDir: dest,
CacheUpdater: &cacheUpdater{cc, mount.IdentityMapping()},
ProgressCb: newProgressHandler(ctx, "transferring "+ls.src.Name+":"),
Differ: ls.src.Differ,
}
if idmap := mount.IdentityMapping(); idmap != nil {