Merge pull request #40 from tonistiigi/llb-update

llb: add more features to client package

client/llb/llb.go

@@ -2,194 +2,30 @@ package llb
 
 import (
 	_ "crypto/sha256"
-	"fmt"
 	"sort"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/google/shlex"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/util/system"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 )
 
-type RunOption func(m Meta) Meta
+var errNotFound = errors.Errorf("not found")
 
-type SourceOp struct {
+type RunOption func(m Meta) (Meta, error)
-	id string
-}
-
-type ExecOp struct {
-	meta   Meta
-	mounts []*Mount
-	root   *Mount
-}
-
-type Meta struct {
-	Args []string
-	Env  []string
-	Cwd  string
-}
-
-type Mount struct {
-	op         *ExecOp
-	dest       string
-	mount      *Mount
-	src        *SourceOp
-	output     bool
-	inputIndex int64
-}
-
-func NewMeta(args ...string) Meta {
-	m := Meta{}
-	m = m.addEnv("PATH", system.DefaultPathEnv)
-	m = m.setArgs(args...)
-	m.Cwd = "/"
-	return m
-}
-
-func (m *Meta) ensurePrivate() {
-	m.Env = append([]string{}, m.Env...)
-	m.Args = append([]string{}, m.Args...)
-}
-
-func (m Meta) addEnv(k, v string) Meta {
-	(&m).ensurePrivate()
-	// TODO: flatten
-	m.Env = append(m.Env, k+"="+v)
-	return m
-}
-
-func (m Meta) setArgs(args ...string) Meta {
-	m.Args = append([]string{}, args...)
-	return m
-}
-
-func Shlex(str string, v ...string) RunOption {
-	return func(m Meta) Meta {
-		vi := make([]interface{}, 0, len(v))
-		for _, v := range v {
-			vi = append(vi, v)
-		}
-		sp, err := shlex.Split(fmt.Sprintf(str, vi...))
-		if err != nil {
-			panic(err) // TODO
-		}
-		(&m).ensurePrivate()
-		return m.setArgs(sp...)
-	}
-}
-
-type State struct {
-	src      *SourceOp
-	exec     *ExecOp
-	meta     Meta
-	mount    *Mount
-	metaNext Meta
-}
-
-type ExecState struct {
-	State
-}
-
-func (s *State) Validate() error {
-	if s.src != nil {
-		if err := s.src.Validate(); err != nil {
-			return err
-		}
-	}
-	if s.exec != nil {
-		if err := s.exec.Validate(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *State) Run(opts ...RunOption) *ExecState {
-	var es ExecState
-	meta := s.metaNext
-	for _, o := range opts {
-		meta = o(meta)
-	}
-	exec := newExec(meta, s.src, s.mount)
-	es.exec = exec
-	es.mount = exec.root
-	es.metaNext = meta
-	es.meta = meta
-	return &es
-}
-
-func (s *State) AddEnv(k, v string) *State {
-	s.metaNext = s.metaNext.addEnv(k, v)
-	return s
-}
-func (s *State) Dir(wd string) *State {
-	s.metaNext.Cwd = wd
-	return s
-}
-
-func (s *State) Marshal() ([][]byte, error) {
-	if err := s.Validate(); err != nil {
-		return nil, err
-	}
-	cache := make(map[digest.Digest]struct{})
-	var list [][]byte
-	var err error
-	if s.src != nil { // TODO: fix repetition
-		_, list, err = s.src.recursiveMarshal(nil, cache)
-	} else if s.exec != nil {
-		_, list, err = s.exec.root.recursiveMarshal(nil, cache)
-	} else {
-		_, list, err = s.mount.recursiveMarshal(nil, cache)
-	}
-	return list, err
-}
-
-func (s *ExecState) AddMount(dest string, mount *State) *State {
-	m := &Mount{
-		dest:   dest,
-		src:    mount.src,
-		mount:  mount.mount,
-		op:     s.exec,
-		output: true, // TODO: should be set only if something inherits
-	}
-	var newState State
-	newState.meta = s.meta
-	newState.metaNext = s.meta
-	newState.mount = m
-	s.exec.mounts = append(s.exec.mounts, m)
-	return &newState
-}
-
-func (s *ExecState) Root() *State {
-	return &s.State
-}
-
-func newExec(meta Meta, src *SourceOp, m *Mount) *ExecOp {
-	exec := &ExecOp{
-		meta:   meta,
-		mounts: []*Mount{},
-		root: &Mount{
-			dest:   "/",
-			src:    src,
-			mount:  m,
-			output: true,
-		},
-	}
-	exec.root.op = exec
-	exec.mounts = append(exec.mounts, exec.root)
-	return exec
-}
 
 func Source(id string) *State {
 	return &State{
 		metaNext: NewMeta(),
-		src:      &SourceOp{id: id},
+		source:   &source{id: id},
 	}
 }
 
-func (so *SourceOp) Validate() error {
+type source struct {
+	id string
+}
+
+func (so *source) Validate() error {
 	// TODO: basic identifier validation
 	if so.id == "" {
 		return errors.Errorf("source identifier can't be empty")
@@ -197,7 +33,7 @@ func (so *SourceOp) Validate() error {
 	return nil
 }
 
-func (so *SourceOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
+func (so *source) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
 	if err := so.Validate(); err != nil {
 		return "", nil, err
 	}
@@ -213,15 +49,21 @@ func Image(ref string) *State {
 	return Source("docker-image://" + ref) // controversial
 }
 
-func (eo *ExecOp) Validate() error {
+type exec struct {
+	meta   Meta
+	mounts []*mount
+	root   *mount
+}
+
+func (eo *exec) Validate() error {
 	for _, m := range eo.mounts {
-		if m.src != nil {
+		if m.source != nil {
-			if err := m.src.Validate(); err != nil {
+			if err := m.source.Validate(); err != nil {
 				return err
 			}
 		}
-		if m.mount != nil {
+		if m.parent != nil {
-			if err := m.mount.op.Validate(); err != nil {
+			if err := m.parent.execState.exec.Validate(); err != nil {
 				return err
 			}
 		}
@@ -230,21 +72,12 @@ func (eo *ExecOp) Validate() error {
 	return nil
 }
 
-func (eo *ExecOp) Marshal() ([][]byte, error) {
+func (eo *exec) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
-	if err := eo.Validate(); err != nil {
-		return nil, err
-	}
-	cache := make(map[digest.Digest]struct{})
-	_, list, err := eo.recursiveMarshal(nil, cache)
-	return list, err
-}
-
-func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
 	peo := &pb.ExecOp{
 		Meta: &pb.Meta{
-			Args: eo.meta.Args,
+			Args: eo.meta.args,
-			Env:  eo.meta.Env,
+			Env:  eo.meta.env.ToArray(),
-			Cwd:  eo.meta.Cwd,
+			Cwd:  eo.meta.cwd,
 		},
 	}
 
@@ -263,15 +96,13 @@ func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct
 	for _, m := range eo.mounts {
 		var dgst digest.Digest
 		var err error
-		var op interface{}
+		if m.source != nil {
-		if m.src != nil {
+			dgst, list, err = m.source.marshalTo(list, cache)
-			op = m.src
 		} else {
-			op = m.mount.op
+			dgst, list, err = m.parent.execState.exec.marshalTo(list, cache)
 		}
-		dgst, list, err = recursiveMarshalAny(op, list, cache)
 		if err != nil {
-			return "", nil, err
+			return "", list, err
 		}
 		inputIndex := len(pop.Inputs)
 		for i := range pop.Inputs {
@@ -282,8 +113,8 @@ func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct
 		}
 		if inputIndex == len(pop.Inputs) {
 			var mountIndex int64
-			if m.mount != nil {
+			if m.parent != nil {
-				mountIndex = m.mount.inputIndex
+				mountIndex = m.parent.outputIndex
 			}
 			pop.Inputs = append(pop.Inputs, &pb.Input{
 				Digest: dgst,
@@ -295,34 +126,46 @@ func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct
 			Input: int64(inputIndex),
 			Dest:  m.dest,
 		}
-		if m.output {
+		if m.hasOutput {
 			pm.Output = outputIndex
 			outputIndex++
 		} else {
 			pm.Output = -1
 		}
-		m.inputIndex = outputIndex - 1
+		m.outputIndex = outputIndex - 1
 		peo.Mounts = append(peo.Mounts, pm)
 	}
 
 	return appendResult(pop, list, cache)
 }
 
-func (m *Mount) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
+type mount struct {
-	if m.op == nil {
+	execState *ExecState
+	dest      string
+	// ro bool
+	// either parent or source has to be set
+	parent      *mount
+	source      *source
+	hasOutput   bool  // TODO: remove
+	outputIndex int64 // filled in after marshal
+	state       *ExecState
+}
+
+func (m *mount) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
+	if m.execState == nil {
 		return "", nil, errors.Errorf("invalid mount")
 	}
 	var dgst digest.Digest
-	dgst, list, err := m.op.recursiveMarshal(list, cache)
+	dgst, list, err := m.execState.exec.marshalTo(list, cache)
 	if err != nil {
 		return "", list, err
 	}
-	for _, m2 := range m.op.mounts {
+	for _, m2 := range m.execState.exec.mounts {
 		if m2 == m {
 			po := &pb.Op{}
 			po.Inputs = append(po.Inputs, &pb.Input{
 				Digest: dgst,
-				Index:  int64(m.inputIndex),
+				Index:  int64(m.outputIndex),
 			})
 			return appendResult(po, list, cache)
 		}
@@ -343,14 +186,3 @@ func appendResult(p proto.Marshaler, list [][]byte, cache map[digest.Digest]stru
 	cache[dgst] = struct{}{}
 	return dgst, list, nil
 }
-
-func recursiveMarshalAny(op interface{}, list [][]byte, cache map[digest.Digest]struct{}) (dgst digest.Digest, out [][]byte, err error) {
-	switch op := op.(type) {
-	case *ExecOp:
-		return op.recursiveMarshal(list, cache)
-	case *SourceOp:
-		return op.recursiveMarshal(list, cache)
-	default:
-		return "", nil, errors.Errorf("invalid operation")
-	}
-}

client/llb/meta.go

@@ -0,0 +1,143 @@
+package llb
+
+import (
+	"fmt"
+
+	"github.com/google/shlex"
+	"github.com/moby/buildkit/util/system"
+)
+
+func NewMeta(args ...string) Meta {
+	m := Meta{}
+	m, _ = AddEnv("PATH", system.DefaultPathEnv)(m)
+	m, _ = Args(args...)(m)
+	m, _ = Dir("/")(m)
+	return m
+}
+
+type Meta struct {
+	args []string
+	env  envList
+	cwd  string
+}
+
+func AddEnv(key, value string) RunOption {
+	return AddEnvf(key, value)
+}
+func AddEnvf(key, value string, v ...interface{}) RunOption {
+	return func(m Meta) (Meta, error) {
+		m.env = m.env.AddOrReplace(key, fmt.Sprintf(value, v...))
+		return m, nil
+	}
+}
+
+func ClearEnv() RunOption {
+	return func(m Meta) (Meta, error) {
+		m.env = NewMeta().env
+		return m, nil
+	}
+}
+
+func DelEnv(key string) RunOption {
+	return func(m Meta) (Meta, error) {
+		m.env = m.env.Delete(key)
+		return m, nil
+	}
+}
+
+func Args(args ...string) RunOption {
+	return func(m Meta) (Meta, error) {
+		m.args = args
+		return m, nil
+	}
+}
+
+func Dir(str string) RunOption {
+	return Dirf(str)
+}
+func Dirf(str string, v ...interface{}) RunOption {
+	return func(m Meta) (Meta, error) {
+		m.cwd = fmt.Sprintf(str, v...)
+		return m, nil
+	}
+}
+
+func Reset(s *State) RunOption {
+	return func(m Meta) (Meta, error) {
+		if s == nil {
+			return NewMeta(), nil
+		}
+		return s.metaNext, nil
+	}
+}
+
+func (m Meta) Env(key string) (string, bool) {
+	return m.env.Get(key)
+}
+
+func (m Meta) Dir() string {
+	return m.cwd
+}
+
+func (m Meta) Args() []string {
+	return append([]string{}, m.args...)
+}
+
+func Shlex(str string) RunOption {
+	return Shlexf(str)
+}
+
+func Shlexf(str string, v ...interface{}) RunOption {
+	return func(m Meta) (Meta, error) {
+		args, err := shlex.Split(fmt.Sprintf(str, v...))
+		if err != nil {
+			return m, err
+		}
+		return Args(args...)(m)
+	}
+}
+
+type envList []keyValue
+
+type keyValue struct {
+	key   string
+	value string
+}
+
+func (e envList) AddOrReplace(k, v string) envList {
+	e = e.Delete(k)
+	e = append(e, keyValue{key: k, value: v})
+	return e
+}
+
+func (e envList) Delete(k string) envList {
+	e = append([]keyValue(nil), e...)
+	if i, ok := e.index(k); ok {
+		return append(e[:i], e[i+1:]...)
+	}
+	return e
+}
+
+func (e envList) Get(k string) (string, bool) {
+	if index, ok := e.index(k); ok {
+		return e[index].value, true
+	}
+	return "", false
+}
+
+func (e envList) index(k string) (int, bool) {
+	for i, kv := range e {
+		if kv.key == k {
+			return i, true
+		}
+	}
+	return -1, false
+}
+
+func (e envList) ToArray() []string {
+	out := make([]string, 0, len(e))
+	for _, kv := range e {
+		out = append(out, kv.key+"="+kv.value)
+	}
+	return out
+}

client/llb/meta_test.go

@@ -0,0 +1,61 @@
+package llb
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultMeta(t *testing.T) {
+	m := NewMeta()
+	path, ok := m.Env("PATH")
+	assert.True(t, ok)
+	assert.NotEmpty(t, path)
+
+	cwd := m.Dir()
+	assert.NotEmpty(t, cwd)
+}
+
+func TestReset(t *testing.T) {
+	m := NewMeta()
+	wd := m.Dir()
+	path, _ := m.Env("PATH")
+	m, _ = Dir("/foo")(m)
+	m, _ = AddEnv("FOO", "bar")(m)
+
+	m, _ = Reset(nil)(m)
+	assert.Equal(t, wd, m.Dir())
+	path2, _ := m.Env("PATH")
+	assert.Equal(t, path, path2)
+}
+
+func TestEnv(t *testing.T) {
+	m := NewMeta()
+	m, _ = AddEnv("FOO", "bar")(m)
+	m2, _ := AddEnv("FOO", "baz")(m)
+	m2, _ = AddEnv("BAR", "abc")(m2)
+
+	v, ok := m.Env("FOO")
+	assert.True(t, ok)
+	assert.Equal(t, "bar", v)
+
+	_, ok = m.Env("BAR")
+	assert.False(t, ok)
+
+	v, ok = m2.Env("FOO")
+	assert.True(t, ok)
+	assert.Equal(t, "baz", v)
+
+	v, ok = m2.Env("BAR")
+	assert.True(t, ok)
+	assert.Equal(t, "abc", v)
+}
+
+func TestShlex(t *testing.T) {
+	m, err := Shlex("echo foo")(Meta{})
+	assert.Nil(t, err)
+	assert.Equal(t, []string{"echo", "foo"}, m.Args())
+
+	m, err = Shlex("echo \"foo")(Meta{})
+	assert.Error(t, err)
+}

client/llb/state.go

@@ -0,0 +1,170 @@
+package llb
+
+import (
+	_ "crypto/sha256"
+
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+type StateOption func(s *State) *State
+
+// State represents modifiable llb state
+type State struct {
+	source   *source
+	exec     *exec
+	meta     Meta
+	mount    *mount
+	metaNext Meta // this meta will be used for the next Run()
+	err      error
+}
+
+// TODO: add state.Reset() state.Save() state.Restore()
+
+// Validate checks that every node has been set up properly
+func (s *State) Validate() error {
+	if s.source != nil {
+		if err := s.source.Validate(); err != nil {
+			return err
+		}
+	}
+	if s.exec != nil {
+		if err := s.exec.Validate(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *State) Run(opts ...RunOption) *ExecState {
+	var es ExecState
+	meta := s.metaNext
+	var err error
+	for _, o := range opts {
+		meta, err = o(meta)
+	}
+
+	exec := &exec{
+		meta:   meta,
+		mounts: []*mount{},
+		root: &mount{
+			dest:      "/",
+			source:    s.source,
+			parent:    s.mount,
+			hasOutput: true,
+		},
+	}
+	exec.root.execState = &es
+	exec.mounts = append(exec.mounts, exec.root)
+
+	es.exec = exec
+	es.mount = exec.root
+	es.metaNext = meta
+	es.meta = meta
+	es.err = err
+	return &es
+}
+
+func (s *State) AddEnv(key, value string) *State {
+	return s.AddEnvf(key, value)
+}
+func (s *State) AddEnvf(key, value string, v ...interface{}) *State {
+	s.metaNext, _ = AddEnvf(key, value, v...)(s.metaNext)
+	return s
+}
+
+func (s *State) DelEnv(key string) *State {
+	s.metaNext, _ = DelEnv(key)(s.metaNext)
+	return s
+}
+func (s *State) ClearEnv() *State {
+	s.metaNext, _ = ClearEnv()(s.metaNext)
+	return s
+}
+func (s *State) GetEnv(key string) (string, bool) {
+	return s.metaNext.Env(key)
+}
+
+func (s *State) Dir(str string) *State {
+	return s.Dirf(str)
+}
+func (s *State) Dirf(str string, v ...interface{}) *State {
+	s.metaNext, _ = Dirf(str, v...)(s.metaNext)
+	return s
+}
+
+func (s *State) GetDir() string {
+	return s.metaNext.Dir()
+}
+
+func (s *State) Args(args ...string) *State {
+	s.metaNext, _ = Args(args...)(s.metaNext)
+	return s
+}
+
+func (s *State) Reset(src *State) *State {
+	copy := *s
+	copy.metaNext, _ = Reset(src)(s.metaNext)
+	return &copy
+}
+
+func (s *State) With(so ...StateOption) *State {
+	for _, o := range so {
+		s = o(s)
+	}
+	return s
+}
+
+func (s *State) Marshal() (list [][]byte, err error) {
+	if err := s.Validate(); err != nil {
+		return nil, err
+	}
+	cache := make(map[digest.Digest]struct{})
+	if s.source != nil {
+		_, list, err = s.source.marshalTo(nil, cache)
+	} else if s.exec != nil {
+		_, list, err = s.exec.root.marshalTo(nil, cache)
+	} else {
+		_, list, err = s.mount.marshalTo(nil, cache)
+	}
+	return
+}
+
+// ExecState is a state with a active leaf pointing to a run exec command.
+// Mounts can be added only to this state.
+type ExecState struct {
+	State
+}
+
+func (s *ExecState) AddMount(dest string, mountState *State) *State {
+	m := &mount{
+		dest:      dest,
+		source:    mountState.source,
+		parent:    mountState.mount,
+		execState: s,
+		hasOutput: true, // TODO: should be set only if something inherits
+	}
+	var newState State
+	newState.meta = s.meta
+	newState.metaNext = s.metaNext
+	newState.mount = m
+	s.exec.mounts = append(s.exec.mounts, m)
+	return &newState
+}
+
+func (s *ExecState) Root() *State {
+	return &s.State
+}
+
+func (s *ExecState) GetMount(target string) (*State, error) {
+	for _, m := range s.exec.mounts {
+		if m.dest == target {
+			var newState State
+			newState.meta = m.execState.meta
+			newState.metaNext = m.execState.metaNext
+			newState.mount = m
+			return &newState, nil
+		}
+	}
+	return nil, errors.WithStack(errNotFound)
+}

client/llb/state_test.go

@@ -0,0 +1,28 @@
+package llb
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStateMeta(t *testing.T) {
+	s := Source("foo")
+	s = s.AddEnv("BAR", "abc").Dir("/foo/bar")
+
+	v, ok := s.GetEnv("BAR")
+	assert.True(t, ok)
+	assert.Equal(t, "abc", v)
+
+	assert.Equal(t, "/foo/bar", s.GetDir())
+
+	s2 := Source("foo2")
+	s2 = s2.AddEnv("BAZ", "def").Reset(s)
+
+	_, ok = s2.GetEnv("BAZ")
+	assert.False(t, ok)
+
+	v, ok = s2.GetEnv("BAR")
+	assert.True(t, ok)
+	assert.Equal(t, "abc", v)
+}

examples/buildkit/buildkit.go

@@ -33,50 +33,67 @@ func goBuildBase() *llb.State {
 
 func runc(version string) *llb.State {
 	return goBuildBase().
-		Run(llb.Shlex("git clone https://github.com/opencontainers/runc.git /go/src/github.com/opencontainers/runc")).
+		With(goFromGit("github.com/opencontainers/runc", version)).
-		Dir("/go/src/github.com/opencontainers/runc").
+		Run(llb.Shlex("go build -o /usr/bin/runc ./")).
-		Run(llb.Shlex("git checkout -q %s", version)).
+		Root()
-		Run(llb.Shlex("go build -o /usr/bin/runc ./")).Root()
 }
 
 func containerd(version string) *llb.State {
 	return goBuildBase().
 		Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).
-		Run(llb.Shlex("git clone https://github.com/containerd/containerd.git /go/src/github.com/containerd/containerd")).
+		With(goFromGit("github.com/containerd/containerd", version)).
-		Dir("/go/src/github.com/containerd/containerd").
-		Run(llb.Shlex("git checkout -q %s", version)).
 		Run(llb.Shlex("make bin/containerd")).Root()
 }
 
 func buildkit(withContainerd bool) *llb.State {
-	src := goBuildBase().
+	src := goBuildBase().With(goFromGit("github.com/moby/buildkit", "master"))
-		Run(llb.Shlex("git clone https://github.com/moby/buildkit.git /go/src/github.com/moby/buildkit")).
-		Dir("/go/src/github.com/moby/buildkit")
 
 	builddStandalone := src.
-		Run(llb.Shlex("go build -o /bin/buildd-standalone -tags standalone ./cmd/buildd"))
+		Run(llb.Shlex("go build -o /bin/buildd-standalone -tags standalone ./cmd/buildd")).Root()
 
 	builddContainerd := src.
-		Run(llb.Shlex("go build -o /bin/buildd-containerd -tags containerd ./cmd/buildd"))
+		Run(llb.Shlex("go build -o /bin/buildd-containerd -tags containerd ./cmd/buildd")).Root()
 
 	buildctl := src.
-		Run(llb.Shlex("go build -o /bin/buildctl ./cmd/buildctl"))
+		Run(llb.Shlex("go build -o /bin/buildctl ./cmd/buildctl")).Root()
+
+	r := llb.Image("docker.io/library/alpine:latest").With(
+		copyFrom(buildctl, "/bin/buildctl", "/bin/"),
+		copyFrom(runc("v1.0.0-rc3"), "/usr/bin/runc", "/bin/"),
+	)
 
-	r := llb.Image("docker.io/library/alpine:latest")
-	r = copy(buildctl.Root(), "/bin/buildctl", r, "/bin/")
-	r = copy(runc("v1.0.0-rc3"), "/usr/bin/runc", r, "/bin/")
 	if withContainerd {
-		r = copy(containerd("master"), "/go/src/github.com/containerd/containerd/bin/containerd", r, "/bin/")
+		return r.With(
-		r = copy(builddContainerd.Root(), "/bin/buildd-containerd", r, "/bin/")
+			copyFrom(containerd("master"), "/go/src/github.com/containerd/containerd/bin/containerd", "/bin/"),
-	} else {
+			copyFrom(builddContainerd, "/bin/buildd-containerd", "/bin/"))
-		r = copy(builddStandalone.Root(), "/bin/buildd-standalone", r, "/bin/")
 	}
-	return r
+	return r.With(copyFrom(builddStandalone, "/bin/buildd-standalone", "/bin/"))
 }
 
+// goFromGit is a helper for cloning a git repo, checking out a tag and copying
+// source directory into
+func goFromGit(repo, tag string) llb.StateOption {
+	src := llb.Image("docker.io/library/alpine:latest").
+		Run(llb.Shlex("apk add --no-cache git")).
+		Run(llb.Shlexf("git clone https://%[1]s.git /go/src/%[1]s", repo)).
+		Dirf("/go/src/%s", repo).
+		Run(llb.Shlexf("git checkout -q %s", tag)).Root()
+	return func(s *llb.State) *llb.State {
+		return s.With(copyFrom(src, "/go", "/")).Reset(s).Dir(src.GetDir())
+	}
+}
+
+// copyFrom has similar semantics as `COPY --from`
+func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption {
+	return func(s *llb.State) *llb.State {
+		return copy(src, srcPath, s, destPath)
+	}
+}
+
+// copy copies files between 2 states using cp until there is no copyOp
 func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State {
 	cpImage := llb.Image("docker.io/library/alpine:latest")
-	cp := cpImage.Run(llb.Shlex("cp -a /src%s /dest%s", srcPath, destPath))
+	cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath))
 	cp.AddMount("/src", src)
 	return cp.AddMount("/dest", dest)
 }