Merge pull request #78 from tonistiigi/multiple-local-sources

source: add multiple dir support for local source

README.md

@@ -59,6 +59,7 @@ Different versions of the example scripts show different ways of describing the
 - `./examples/buildkit0` - uses only exec operations, defines a full stage per component.
 - `./examples/buildkit1` - cloning git repositories has been separated for extra concurrency.
 - `./examples/buildkit2` - uses git sources directly instead of running `git clone`, allowing better performance and much safer caching.
+- `./examples/buildkit3` - allows using local source files for separate components eg. `./buildkit3 --runc=local | buildctl build --local runc-src=some/local/path`  
 
 #### Supported runc version
 

client/client_test.go

@@ -106,6 +106,6 @@ func testBuildMultiMount(t *testing.T, address string) {
 	err = llb.WriteTo(dt, buf)
 	assert.Nil(t, err)
 
-	err = c.Solve(context.TODO(), buf, nil, "", nil, "")
+	err = c.Solve(context.TODO(), buf, SolveOpt{}, nil)
 	assert.Nil(t, err)
 }

client/solve.go

@@ -21,7 +21,15 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveStatus, exporter string, exporterAttrs map[string]string, localDir string) error {
+type SolveOpt struct {
+	Exporter      string
+	ExporterAttrs map[string]string
+	LocalDirs     map[string]string
+	SharedKey     string
+	// Session string
+}
+
+func (c *Client) Solve(ctx context.Context, r io.Reader, opt SolveOpt, statusChan chan *SolveStatus) error {
 	defer func() {
 		if statusChan != nil {
 			close(statusChan)
@@ -37,7 +45,8 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS
 		return errors.New("invalid empty definition")
 	}
 
-	if err := validateLocals(def, localDir); err != nil {
+	syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
+	if err != nil {
 		return err
 	}
 
@@ -47,19 +56,13 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS
 	statusContext, cancelStatus := context.WithCancel(context.Background())
 	defer cancelStatus()
 
-	sharedKey, err := getSharedKey(localDir)
+	s, err := session.NewSession(defaultSessionName(), opt.SharedKey)
-	if err != nil {
-		return errors.Wrap(err, "failed to get build shared key")
-	}
-	s, err := session.NewSession(filepath.Base(localDir), sharedKey)
 	if err != nil {
 		return errors.Wrap(err, "failed to create session")
 	}
 
-	if localDir != "" {
+	if len(syncedDirs) > 0 {
-		_, dir, _ := parseLocalDir(localDir)
+		s.Allow(filesync.NewFSSyncProvider(syncedDirs))
-		workdirProvider := filesync.NewFSSyncProvider(dir, nil)
-		s.Allow(workdirProvider)
 	}
 
 	eg.Go(func() error {
@@ -78,8 +81,8 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS
 		_, err = c.controlClient().Solve(ctx, &controlapi.SolveRequest{
 			Ref:           ref,
 			Definition:    def,
-			Exporter:      exporter,
+			Exporter:      opt.Exporter,
-			ExporterAttrs: exporterAttrs,
+			ExporterAttrs: opt.ExporterAttrs,
 			Session:       s.ID(),
 		})
 		if err != nil {
@@ -152,43 +155,40 @@ func generateID() string {
 	return hex.EncodeToString(b)
 }
 
-func validateLocals(defs [][]byte, localDir string) error {
+func prepareSyncedDirs(defs [][]byte, localDirs map[string]string) ([]filesync.SyncedDir, error) {
-	k, _, err := parseLocalDir(localDir)
+	for _, d := range localDirs {
-	if err != nil {
+		fi, err := os.Stat(d)
-		return err
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not find %s", d)
+		}
+		if !fi.IsDir() {
+			return nil, errors.Errorf("%s not a directory", d)
+		}
 	}
+	dirs := make([]filesync.SyncedDir, 0, len(localDirs))
 	for _, dt := range defs {
 		var op pb.Op
 		if err := (&op).Unmarshal(dt); err != nil {
-			return errors.Wrap(err, "failed to parse llb proto op")
+			return nil, errors.Wrap(err, "failed to parse llb proto op")
 		}
 		if src := op.GetSource(); src != nil {
 			if strings.HasPrefix(src.Identifier, "local://") { // TODO: just make a type property
 				name := strings.TrimPrefix(src.Identifier, "local://")
-				if name != k {
+				d, ok := localDirs[name]
-					return errors.Errorf("local directory %s not enabled", name)
+				if !ok {
+					return nil, errors.Errorf("local directory %s not enabled", name)
 				}
+				dirs = append(dirs, filesync.SyncedDir{Name: name, Dir: d}) // TODO: excludes
 			}
 		}
 	}
-	return nil
+	return dirs, nil
 }
 
-func parseLocalDir(str string) (string, string, error) {
+func defaultSessionName() string {
-	if str == "" {
+	wd, err := os.Getwd()
-		return "", "", nil
-	}
-	parts := strings.SplitN(str, "=", 2)
-	if len(parts) != 2 {
-		return "", "", errors.Errorf("invalid local indentifier %q, need name=dir", str)
-	}
-	fi, err := os.Stat(parts[1])
 	if err != nil {
-		return "", "", errors.Wrapf(err, "could not find %s", parts[1])
+		return "unknown"
 	}
-	if !fi.IsDir() {
+	return filepath.Base(wd)
-		return "", "", errors.Errorf("%s not a directory", parts[1])
-	}
-	return parts[0], parts[1], nil
-
 }

cmd/buildctl/build.go

@@ -33,7 +33,7 @@ var buildCommand = cli.Command{
 			Name:  "no-progress",
 			Usage: "Don't show interactive progress",
 		},
-		cli.StringFlag{
+		cli.StringSliceFlag{
 			Name:  "local",
 			Usage: "Allow build access to the local directory",
 		},
@@ -64,8 +64,17 @@ func build(clicontext *cli.Context) error {
 		return errors.Wrap(err, "invalid exporter-opt")
 	}
 
+	localDirs, err := attrMap(clicontext.StringSlice("local"))
+	if err != nil {
+		return errors.Wrap(err, "invalid local")
+	}
+
 	eg.Go(func() error {
-		return c.Solve(ctx, os.Stdin, ch, clicontext.String("exporter"), exporterAttrs, clicontext.String("local"))
+		return c.Solve(ctx, os.Stdin, client.SolveOpt{
+			Exporter:      clicontext.String("exporter"),
+			ExporterAttrs: exporterAttrs,
+			LocalDirs:     localDirs,
+		}, ch)
 	})
 
 	eg.Go(func() error {

examples/buildkit3/buildkit.go

@@ -12,7 +12,7 @@ type buildOpt struct {
 	target     string
 	containerd string
 	runc       string
-	local      bool
+	buildkit   string
 }
 
 func main() {
@@ -20,7 +20,7 @@ func main() {
 	flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
 	flag.StringVar(&opt.containerd, "containerd", "master", "containerd version")
 	flag.StringVar(&opt.runc, "runc", "v1.0.0-rc3", "runc version")
-	flag.BoolVar(&opt.local, "local", false, "use local buildkit source")
+	flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version")
 	flag.Parse()
 
 	bk := buildkit(opt)
@@ -52,17 +52,25 @@ func goRepo(s *llb.State, repo string, src *llb.State) func(ro ...llb.RunOption)
 
 func runc(version string) *llb.State {
 	repo := "github.com/opencontainers/runc"
-	return goRepo(goBuildBase(), repo, llb.Git(repo, version))(
+	src := llb.Git(repo, version)
+	if version == "local" {
+		src = llb.Local("runc-src")
+	}
+	return goRepo(goBuildBase(), repo, src)(
 		llb.Shlex("go build -o /out/runc ./"),
 	)
 }
 
 func containerd(version string) *llb.State {
 	repo := "github.com/containerd/containerd"
+	src := llb.Git(repo, version, llb.KeepGitDir())
+	if version == "local" {
+		src = llb.Local("containerd-src")
+	}
 	return goRepo(
 		goBuildBase().
 			Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).Root(),
-		repo, llb.Git(repo, version, llb.KeepGitDir()))(
+		repo, src)(
 		llb.Shlex("go build -o /out/containerd ./cmd/containerd"),
 	)
 }
@@ -70,7 +78,7 @@ func containerd(version string) *llb.State {
 func buildkit(opt buildOpt) *llb.State {
 	repo := "github.com/moby/buildkit"
 	src := llb.Git(repo, "master")
-	if opt.local {
+	if opt.buildkit == "local" {
 		src = llb.Local("buildkit-src")
 	}
 	run := goRepo(goBuildBase(), repo, src)
@@ -109,6 +117,6 @@ func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption {
 func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State {
 	cpImage := llb.Image("docker.io/library/alpine:latest")
 	cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath))
-	cp.AddMount("/src", src)
+	cp.AddMount("/src", src, llb.Readonly)
 	return cp.AddMount("/dest", dest)
 }

session/filesync/filesync.go

@@ -16,20 +16,28 @@ import (
 const (
 	keyOverrideExcludes = "override-excludes"
 	keyIncludePatterns  = "include-patterns"
+	keyDirName          = "dir-name"
 )
 
 type fsSyncProvider struct {
-	root     string
+	dirs   map[string]SyncedDir
-	excludes []string
+	p      progressCb
-	p        progressCb
+	doneCh chan error
-	doneCh   chan error
+}
+
+type SyncedDir struct {
+	Name     string
+	Dir      string
+	Excludes []string
 }
 
 // NewFSSyncProvider creates a new provider for sending files from client
-func NewFSSyncProvider(root string, excludes []string) session.Attachable {
+func NewFSSyncProvider(dirs []SyncedDir) session.Attachable {
 	p := &fsSyncProvider{
-		root:     root,
+		dirs: map[string]SyncedDir{},
-		excludes: excludes,
+	}
+	for _, d := range dirs {
+		p.dirs[d.Name] = d
 	}
 	return p
 }
@@ -59,9 +67,19 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error
 
 	opts, _ := metadata.FromContext(stream.Context()) // if no metadata continue with empty object
 
+	name, ok := opts[keyDirName]
+	if !ok || len(name) != 1 {
+		return errors.New("no dir name in request")
+	}
+
+	dir, ok := sp.dirs[name[0]]
+	if !ok {
+		return errors.Errorf("no access allowed to dir %q", name[0])
+	}
+
 	var excludes []string
 	if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" {
-		excludes = sp.excludes
+		excludes = dir.Excludes
 	}
 	includes := opts[keyIncludePatterns]
 
@@ -76,7 +94,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error
 		doneCh = sp.doneCh
 		sp.doneCh = nil
 	}
-	err := pr.sendFn(stream, sp.root, includes, excludes, progress)
+	err := pr.sendFn(stream, dir.Dir, includes, excludes, progress)
 	if doneCh != nil {
 		if err != nil {
 			doneCh <- err
@@ -122,6 +140,7 @@ var supportedProtocols = []protocol{
 
 // FSSendRequestOpt defines options for FSSend request
 type FSSendRequestOpt struct {
+	Name             string
 	IncludePatterns  []string
 	OverrideExcludes bool
 	DestDir          string
@@ -156,6 +175,8 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
 		opts[keyIncludePatterns] = opt.IncludePatterns
 	}
 
+	opts[keyDirName] = []string{opt.Name}
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 

session/filesync/filesync_test.go

@@ -32,7 +32,7 @@ func TestFileSyncIncludePatterns(t *testing.T) {
 	m, err := session.NewManager()
 	require.NoError(t, err)
 
-	fs := NewFSSyncProvider(tmpDir, nil)
+	fs := NewFSSyncProvider([]SyncedDir{{Name: "test0", Dir: tmpDir}})
 	s.Allow(fs)
 
 	dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
@@ -49,6 +49,7 @@ func TestFileSyncIncludePatterns(t *testing.T) {
 			return err
 		}
 		if err := FSSync(ctx, c, FSSendRequestOpt{
+			Name:            "test0",
 			DestDir:         destDir,
 			IncludePatterns: []string{"ba*"},
 		}); err != nil {

source/local/local.go

@@ -137,6 +137,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
 	}()
 
 	opt := filesync.FSSendRequestOpt{
+		Name:             ls.src.Name,
 		IncludePatterns:  nil,
 		OverrideExcludes: false,
 		DestDir:          dest,