JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #218 from tonistiigi/dockerignore 

dockerfile: dockerignore support
docker-18.09
Akihiro Suda 2017-12-18 11:40:55 +09:00 committed by GitHub
parent 51fc6661bb 2d3f36d359
commit 8c9d66f7da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 247 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
client/llb/source.go
View File

@ -193,6 +193,12 @@ func Local(name string, opts ...LocalOption) State {
if gi.IncludePatterns != "" {
attrs[pb.AttrIncludePatterns] = gi.IncludePatterns
}
if gi.ExcludePatterns != "" {
attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns
}
if gi.SharedKeyHint != "" {
attrs[pb.AttrSharedKeyHint] = gi.SharedKeyHint
}
source := NewSource("local://"+name, attrs, gi.Metadata())
return NewState(source.Output())
@ -216,15 +222,38 @@ func SessionID(id string) LocalOption {
func IncludePatterns(p []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
if len(p) == 0 {
li.IncludePatterns = ""
return
}
dt, _ := json.Marshal(p) // empty on error
li.IncludePatterns = string(dt)
})
}
func ExcludePatterns(p []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
if len(p) == 0 {
li.ExcludePatterns = ""
return
}
dt, _ := json.Marshal(p) // empty on error
li.ExcludePatterns = string(dt)
})
}
func SharedKeyHint(h string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
li.SharedKeyHint = h
})
}
type LocalInfo struct {
opMetaWrapper
SessionID string
IncludePatterns string
ExcludePatterns string
SharedKeyHint string
}
func HTTP(url string, opts ...HTTPOption) State {

5
client/local.go
View File

@ -1,5 +0,0 @@
package client
func getSharedKey(dir string) (string, error) {
return dir, nil // not implemented
}

55
frontend/dockerfile/builder/build.go
View File

@ -1,15 +1,18 @@
package builder
import (
"bytes"
"context"
"encoding/json"
"path"
"strings"
"github.com/docker/docker/builder/dockerignore"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
"github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
const (
@ -19,6 +22,7 @@ const (
keyFilename = "filename"
exporterImageConfig = "containerimage.config"
defaultDockerfileName = "Dockerfile"
dockerignoreFilename = ".dockerignore"
buildArgPrefix = "build-arg:"
gitPrefix = "git://"
)
@ -37,6 +41,7 @@ func Build(ctx context.Context, c client.Client) error {
src := llb.Local(LocalNameDockerfile,
llb.IncludePatterns([]string{filename}),
llb.SessionID(c.SessionID()),
llb.SharedKeyHint(defaultDockerfileName),
)
var buildContext *llb.State
if strings.HasPrefix(opts[LocalNameContext], gitPrefix) {
@ -48,13 +53,50 @@ func Build(ctx context.Context, c client.Client) error {
return err
}
ref, err := c.Solve(ctx, def.ToPB(), "", nil, false)
if err != nil {
return err
}
eg, ctx2 := errgroup.WithContext(ctx)
var dtDockerfile []byte
eg.Go(func() error {
ref, err := c.Solve(ctx2, def.ToPB(), "", nil, false)
if err != nil {
return err
}
dtDockerfile, err := ref.ReadFile(ctx, filename)
if err != nil {
dtDockerfile, err = ref.ReadFile(ctx2, filename)
if err != nil {
return err
}
return nil
})
var excludes []string
eg.Go(func() error {
dockerignoreState := buildContext
if dockerignoreState == nil {
st := llb.Local(LocalNameContext,
llb.SessionID(c.SessionID()),
llb.IncludePatterns([]string{dockerignoreFilename}),
llb.SharedKeyHint(dockerignoreFilename),
)
dockerignoreState = &st
}
def, err := dockerignoreState.Marshal()
if err != nil {
return err
}
ref, err := c.Solve(ctx2, def.ToPB(), "", nil, false)
if err != nil {
return err
}
dtDockerignore, err := ref.ReadFile(ctx2, dockerignoreFilename)
if err == nil {
excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dtDockerignore))
if err != nil {
return errors.Wrap(err, "failed to parse dockerignore")
}
}
return nil
})
if err := eg.Wait(); err != nil {
return err
}
@ -64,6 +106,7 @@ func Build(ctx context.Context, c client.Client) error {
BuildArgs: filterBuildArgs(opts),
SessionID: c.SessionID(),
BuildContext: buildContext,
Excludes: excludes,
})
if err != nil {

8
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -38,6 +38,7 @@ type ConvertOpt struct {
BuildArgs map[string]string
SessionID string
BuildContext *llb.State
Excludes []string
}
func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) {
@ -166,8 +167,11 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if err := eg.Wait(); err != nil {
return nil, nil, err
}
buildContext := llb.Local(localNameContext, llb.SessionID(opt.SessionID))
buildContext := llb.Local(localNameContext,
llb.SessionID(opt.SessionID),
llb.ExcludePatterns(opt.Excludes),
llb.SharedKeyHint(localNameContext),
)
if opt.BuildContext != nil {
buildContext = *opt.BuildContext
}

73
frontend/dockerfile/dockerfile_test.go
View File

@ -40,6 +40,7 @@ func TestIntegration(t *testing.T) {
testExportedHistory,
testExposeExpansion,
testUser,
testDockerignore,
})
}
@ -503,6 +504,78 @@ EXPOSE 5000
require.Equal(t, "5000/tcp", ports[2])
}
func testDockerignore(t *testing.T, sb integration.Sandbox) {
t.Parallel()
dockerfile := []byte(`
FROM scratch
COPY . .
`)
dockerignore := []byte(`
ba*
Dockerfile
!bay
.dockerignore
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateFile("foo", []byte(`foo-contents`), 0600),
fstest.CreateFile("bar", []byte(`bar-contents`), 0600),
fstest.CreateFile("baz", []byte(`baz-contents`), 0600),
fstest.CreateFile("bay", []byte(`bay-contents`), 0600),
fstest.CreateFile(".dockerignore", dockerignore, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
err = c.Solve(context.TODO(), nil, client.SolveOpt{
Frontend: "dockerfile.v0",
Exporter: client.ExporterLocal,
ExporterAttrs: map[string]string{
"output": destDir,
},
LocalDirs: map[string]string{
builder.LocalNameDockerfile: dir,
builder.LocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo"))
require.NoError(t, err)
require.Equal(t, "foo-contents", string(dt))
_, err = os.Stat(filepath.Join(destDir, ".dockerignore"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
_, err = os.Stat(filepath.Join(destDir, "Dockerfile"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
_, err = os.Stat(filepath.Join(destDir, "bar"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
_, err = os.Stat(filepath.Join(destDir, "baz"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "bay"))
require.NoError(t, err)
require.Equal(t, "bay-contents", string(dt))
}
func testExportedHistory(t *testing.T, sb integration.Sandbox) {
t.Parallel()

14
session/filesync/filesync.go
View File

@ -17,6 +17,7 @@ import (
const (
keyOverrideExcludes = "override-excludes"
keyIncludePatterns = "include-patterns"
keyExcludePatterns = "exclude-patterns"
keyDirName = "dir-name"
)
@ -55,7 +56,7 @@ func (sp *fsSyncProvider) TarStream(stream FileSync_TarStreamServer) error {
return sp.handle("tarstream", stream)
}
func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error {
func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retErr error) {
var pr *protocol
for _, p := range supportedProtocols {
if method == p.name && isProtoSupported(p.name) {
@ -80,8 +81,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error
return errors.Errorf("no access allowed to dir %q", dirName)
}
var excludes []string
if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" {
excludes := opts[keyExcludePatterns]
if len(dir.Excludes) != 0 && (len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true") {
excludes = dir.Excludes
}
includes := opts[keyIncludePatterns]
@ -140,7 +141,8 @@ var supportedProtocols = []protocol{
type FSSendRequestOpt struct {
Name string
IncludePatterns []string
OverrideExcludes bool
ExcludePatterns []string
OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory
DestDir string
CacheUpdater CacheUpdater
ProgressCb func(int, bool)
@ -175,6 +177,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
opts[keyIncludePatterns] = opt.IncludePatterns
}
if opt.ExcludePatterns != nil {
opts[keyExcludePatterns] = opt.ExcludePatterns
}
opts[keyDirName] = []string{opt.Name}
ctx, cancel := context.WithCancel(ctx)

2
solver/pb/attr.go
View File

@ -3,6 +3,8 @@ package pb
const AttrKeepGitDir = "git.keepgitdir"
const AttrLocalSessionID = "local.session"
const AttrIncludePatterns = "local.includepattern"
const AttrExcludePatterns = "local.excludepatterns"
const AttrSharedKeyHint = "local.sharedkeyhint"
const AttrLLBDefinitionFilename = "llbbuild.filename"
const AttrHTTPChecksum = "http.checksum"

10
source/identifier.go
View File

@ -80,6 +80,14 @@ func FromLLB(op *pb.Op_Source) (Identifier, error) {
return nil, err
}
id.IncludePatterns = patterns
case pb.AttrExcludePatterns:
var patterns []string
if err := json.Unmarshal([]byte(v), &patterns); err != nil {
return nil, err
}
id.ExcludePatterns = patterns
case pb.AttrSharedKeyHint:
id.SharedKeyHint = v
}
}
}
@ -142,6 +150,8 @@ type LocalIdentifier struct {
Name string
SessionID string
IncludePatterns []string
ExcludePatterns []string
SharedKeyHint string
}
func NewLocalIdentifier(str string) (*LocalIdentifier, error) {

6
source/local/local.go
View File

@ -79,7 +79,8 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context) (string, error) {
dt, err := json.Marshal(struct {
SessionID string
IncludePatterns []string
}{SessionID: sessionID, IncludePatterns: ls.src.IncludePatterns})
ExcludePatterns []string
}{SessionID: sessionID, IncludePatterns: ls.src.IncludePatterns, ExcludePatterns: ls.src.ExcludePatterns})
if err != nil {
return "", err
}
@ -101,7 +102,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
return nil, err
}
sharedKey := keySharedKey + ":" + ls.src.Name + ":" + caller.SharedKey()
sharedKey := keySharedKey + ":" + ls.src.Name + ":" + ls.src.SharedKeyHint + ":" + caller.SharedKey() // TODO: replace caller.SharedKey() with source based hint from client(absolute-path+nodeid)
var mutable cache.MutableRef
sis, err := ls.md.Search(sharedKey)
@ -157,6 +158,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
opt := filesync.FSSendRequestOpt{
Name: ls.src.Name,
IncludePatterns: ls.src.IncludePatterns,
ExcludePatterns: ls.src.ExcludePatterns,
OverrideExcludes: false,
DestDir: dest,
CacheUpdater: &cacheUpdater{cc},

64
vendor/github.com/docker/docker/builder/dockerignore/dockerignore.go generated vendored Normal file
View File

@ -0,0 +1,64 @@
package dockerignore
import (
"bufio"
"bytes"
"fmt"
"io"
"path/filepath"
"strings"
)
// ReadAll reads a .dockerignore file and returns the list of file patterns
// to ignore. Note this will trim whitespace from each line as well
// as use GO's "clean" func to get the shortest/cleanest path for each.
func ReadAll(reader io.Reader) ([]string, error) {
if reader == nil {
return nil, nil
}
scanner := bufio.NewScanner(reader)
var excludes []string
currentLine := 0
utf8bom := []byte{0xEF, 0xBB, 0xBF}
for scanner.Scan() {
scannedBytes := scanner.Bytes()
// We trim UTF8 BOM
if currentLine == 0 {
scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom)
}
pattern := string(scannedBytes)
currentLine++
// Lines starting with # (comments) are ignored before processing
if strings.HasPrefix(pattern, "#") {
continue
}
pattern = strings.TrimSpace(pattern)
if pattern == "" {
continue
}
// normalize absolute paths to paths relative to the context
// (taking care of '!' prefix)
invert := pattern[0] == '!'
if invert {
pattern = strings.TrimSpace(pattern[1:])
}
if len(pattern) > 0 {
pattern = filepath.Clean(pattern)
pattern = filepath.ToSlash(pattern)
if len(pattern) > 1 && pattern[0] == '/' {
pattern = pattern[1:]
}
}
if invert {
pattern = "!" + pattern
}
excludes = append(excludes, pattern)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("Error reading .dockerignore: %v", err)
}
return excludes, nil
}