diff --git a/README.md b/README.md index 7c90bdd6..65237846 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Different versions of the example scripts show different ways of describing the - `./examples/buildkit0` - uses only exec operations, defines a full stage per component. - `./examples/buildkit1` - cloning git repositories has been separated for extra concurrency. - `./examples/buildkit2` - uses git sources directly instead of running `git clone`, allowing better performance and much safer caching. +- `./examples/buildkit3` - allows using local source files for separate components eg. `./buildkit3 --runc=local | buildctl build --local runc-src=some/local/path` #### Supported runc version diff --git a/client/client_test.go b/client/client_test.go index 3b07ef67..c122d3a9 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -106,6 +106,6 @@ func testBuildMultiMount(t *testing.T, address string) { err = llb.WriteTo(dt, buf) assert.Nil(t, err) - err = c.Solve(context.TODO(), buf, nil, "", nil, "") + err = c.Solve(context.TODO(), buf, SolveOpt{}, nil) assert.Nil(t, err) } diff --git a/client/solve.go b/client/solve.go index cfe5d7d6..a2500e4d 100644 --- a/client/solve.go +++ b/client/solve.go @@ -21,7 +21,15 @@ import ( "golang.org/x/sync/errgroup" ) -func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveStatus, exporter string, exporterAttrs map[string]string, localDir string) error { +type SolveOpt struct { + Exporter string + ExporterAttrs map[string]string + LocalDirs map[string]string + SharedKey string + // Session string +} + +func (c *Client) Solve(ctx context.Context, r io.Reader, opt SolveOpt, statusChan chan *SolveStatus) error { defer func() { if statusChan != nil { close(statusChan) @@ -37,7 +45,8 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS return errors.New("invalid empty definition") } - if err := validateLocals(def, localDir); err != nil { + syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs) + if err != nil { return err } @@ -47,19 +56,13 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS statusContext, cancelStatus := context.WithCancel(context.Background()) defer cancelStatus() - sharedKey, err := getSharedKey(localDir) - if err != nil { - return errors.Wrap(err, "failed to get build shared key") - } - s, err := session.NewSession(filepath.Base(localDir), sharedKey) + s, err := session.NewSession(defaultSessionName(), opt.SharedKey) if err != nil { return errors.Wrap(err, "failed to create session") } - if localDir != "" { - _, dir, _ := parseLocalDir(localDir) - workdirProvider := filesync.NewFSSyncProvider(dir, nil) - s.Allow(workdirProvider) + if len(syncedDirs) > 0 { + s.Allow(filesync.NewFSSyncProvider(syncedDirs)) } eg.Go(func() error { @@ -78,8 +81,8 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS _, err = c.controlClient().Solve(ctx, &controlapi.SolveRequest{ Ref: ref, Definition: def, - Exporter: exporter, - ExporterAttrs: exporterAttrs, + Exporter: opt.Exporter, + ExporterAttrs: opt.ExporterAttrs, Session: s.ID(), }) if err != nil { @@ -152,43 +155,40 @@ func generateID() string { return hex.EncodeToString(b) } -func validateLocals(defs [][]byte, localDir string) error { - k, _, err := parseLocalDir(localDir) - if err != nil { - return err +func prepareSyncedDirs(defs [][]byte, localDirs map[string]string) ([]filesync.SyncedDir, error) { + for _, d := range localDirs { + fi, err := os.Stat(d) + if err != nil { + return nil, errors.Wrapf(err, "could not find %s", d) + } + if !fi.IsDir() { + return nil, errors.Errorf("%s not a directory", d) + } } + dirs := make([]filesync.SyncedDir, 0, len(localDirs)) for _, dt := range defs { var op pb.Op if err := (&op).Unmarshal(dt); err != nil { - return errors.Wrap(err, "failed to parse llb proto op") + return nil, errors.Wrap(err, "failed to parse llb proto op") } if src := op.GetSource(); src != nil { if strings.HasPrefix(src.Identifier, "local://") { // TODO: just make a type property name := strings.TrimPrefix(src.Identifier, "local://") - if name != k { - return errors.Errorf("local directory %s not enabled", name) + d, ok := localDirs[name] + if !ok { + return nil, errors.Errorf("local directory %s not enabled", name) } + dirs = append(dirs, filesync.SyncedDir{Name: name, Dir: d}) // TODO: excludes } } } - return nil + return dirs, nil } -func parseLocalDir(str string) (string, string, error) { - if str == "" { - return "", "", nil - } - parts := strings.SplitN(str, "=", 2) - if len(parts) != 2 { - return "", "", errors.Errorf("invalid local indentifier %q, need name=dir", str) - } - fi, err := os.Stat(parts[1]) +func defaultSessionName() string { + wd, err := os.Getwd() if err != nil { - return "", "", errors.Wrapf(err, "could not find %s", parts[1]) + return "unknown" } - if !fi.IsDir() { - return "", "", errors.Errorf("%s not a directory", parts[1]) - } - return parts[0], parts[1], nil - + return filepath.Base(wd) } diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index 6e597ea3..9680dab9 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -33,7 +33,7 @@ var buildCommand = cli.Command{ Name: "no-progress", Usage: "Don't show interactive progress", }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "local", Usage: "Allow build access to the local directory", }, @@ -64,8 +64,17 @@ func build(clicontext *cli.Context) error { return errors.Wrap(err, "invalid exporter-opt") } + localDirs, err := attrMap(clicontext.StringSlice("local")) + if err != nil { + return errors.Wrap(err, "invalid local") + } + eg.Go(func() error { - return c.Solve(ctx, os.Stdin, ch, clicontext.String("exporter"), exporterAttrs, clicontext.String("local")) + return c.Solve(ctx, os.Stdin, client.SolveOpt{ + Exporter: clicontext.String("exporter"), + ExporterAttrs: exporterAttrs, + LocalDirs: localDirs, + }, ch) }) eg.Go(func() error { diff --git a/examples/buildkit3/buildkit.go b/examples/buildkit3/buildkit.go index c59c19f6..f23c4b00 100644 --- a/examples/buildkit3/buildkit.go +++ b/examples/buildkit3/buildkit.go @@ -12,7 +12,7 @@ type buildOpt struct { target string containerd string runc string - local bool + buildkit string } func main() { @@ -20,7 +20,7 @@ func main() { flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") flag.StringVar(&opt.containerd, "containerd", "master", "containerd version") flag.StringVar(&opt.runc, "runc", "v1.0.0-rc3", "runc version") - flag.BoolVar(&opt.local, "local", false, "use local buildkit source") + flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version") flag.Parse() bk := buildkit(opt) @@ -52,17 +52,25 @@ func goRepo(s *llb.State, repo string, src *llb.State) func(ro ...llb.RunOption) func runc(version string) *llb.State { repo := "github.com/opencontainers/runc" - return goRepo(goBuildBase(), repo, llb.Git(repo, version))( + src := llb.Git(repo, version) + if version == "local" { + src = llb.Local("runc-src") + } + return goRepo(goBuildBase(), repo, src)( llb.Shlex("go build -o /out/runc ./"), ) } func containerd(version string) *llb.State { repo := "github.com/containerd/containerd" + src := llb.Git(repo, version, llb.KeepGitDir()) + if version == "local" { + src = llb.Local("containerd-src") + } return goRepo( goBuildBase(). Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).Root(), - repo, llb.Git(repo, version, llb.KeepGitDir()))( + repo, src)( llb.Shlex("go build -o /out/containerd ./cmd/containerd"), ) } @@ -70,7 +78,7 @@ func containerd(version string) *llb.State { func buildkit(opt buildOpt) *llb.State { repo := "github.com/moby/buildkit" src := llb.Git(repo, "master") - if opt.local { + if opt.buildkit == "local" { src = llb.Local("buildkit-src") } run := goRepo(goBuildBase(), repo, src) @@ -109,6 +117,6 @@ func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption { func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State { cpImage := llb.Image("docker.io/library/alpine:latest") cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath)) - cp.AddMount("/src", src) + cp.AddMount("/src", src, llb.Readonly) return cp.AddMount("/dest", dest) } diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index 2a1ea365..0bd94e00 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -16,20 +16,28 @@ import ( const ( keyOverrideExcludes = "override-excludes" keyIncludePatterns = "include-patterns" + keyDirName = "dir-name" ) type fsSyncProvider struct { - root string - excludes []string - p progressCb - doneCh chan error + dirs map[string]SyncedDir + p progressCb + doneCh chan error +} + +type SyncedDir struct { + Name string + Dir string + Excludes []string } // NewFSSyncProvider creates a new provider for sending files from client -func NewFSSyncProvider(root string, excludes []string) session.Attachable { +func NewFSSyncProvider(dirs []SyncedDir) session.Attachable { p := &fsSyncProvider{ - root: root, - excludes: excludes, + dirs: map[string]SyncedDir{}, + } + for _, d := range dirs { + p.dirs[d.Name] = d } return p } @@ -59,9 +67,19 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error opts, _ := metadata.FromContext(stream.Context()) // if no metadata continue with empty object + name, ok := opts[keyDirName] + if !ok || len(name) != 1 { + return errors.New("no dir name in request") + } + + dir, ok := sp.dirs[name[0]] + if !ok { + return errors.Errorf("no access allowed to dir %q", name[0]) + } + var excludes []string if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" { - excludes = sp.excludes + excludes = dir.Excludes } includes := opts[keyIncludePatterns] @@ -76,7 +94,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error doneCh = sp.doneCh sp.doneCh = nil } - err := pr.sendFn(stream, sp.root, includes, excludes, progress) + err := pr.sendFn(stream, dir.Dir, includes, excludes, progress) if doneCh != nil { if err != nil { doneCh <- err @@ -122,6 +140,7 @@ var supportedProtocols = []protocol{ // FSSendRequestOpt defines options for FSSend request type FSSendRequestOpt struct { + Name string IncludePatterns []string OverrideExcludes bool DestDir string @@ -156,6 +175,8 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { opts[keyIncludePatterns] = opt.IncludePatterns } + opts[keyDirName] = []string{opt.Name} + ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/session/filesync/filesync_test.go b/session/filesync/filesync_test.go index fc4c58c3..b054504a 100644 --- a/session/filesync/filesync_test.go +++ b/session/filesync/filesync_test.go @@ -32,7 +32,7 @@ func TestFileSyncIncludePatterns(t *testing.T) { m, err := session.NewManager() require.NoError(t, err) - fs := NewFSSyncProvider(tmpDir, nil) + fs := NewFSSyncProvider([]SyncedDir{{Name: "test0", Dir: tmpDir}}) s.Allow(fs) dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn))) @@ -49,6 +49,7 @@ func TestFileSyncIncludePatterns(t *testing.T) { return err } if err := FSSync(ctx, c, FSSendRequestOpt{ + Name: "test0", DestDir: destDir, IncludePatterns: []string{"ba*"}, }); err != nil { diff --git a/source/local/local.go b/source/local/local.go index 157be1fd..34cf53e4 100644 --- a/source/local/local.go +++ b/source/local/local.go @@ -137,6 +137,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable }() opt := filesync.FSSendRequestOpt{ + Name: ls.src.Name, IncludePatterns: nil, OverrideExcludes: false, DestDir: dest,