JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #84 from tonistiigi/extendable-llb 

Extendable llb
docker-18.09
Akihiro Suda 2017-07-24 11:59:30 +09:00 committed by GitHub
parent f0424c367f ce6a428195
commit 93b5a797a3
13 changed files with 610 additions and 618 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

227
client/llb/exec.go Normal file
View File

@ -0,0 +1,227 @@
package llb
import (
_ "crypto/sha256"
"sort"
"github.com/moby/buildkit/solver/pb"
"github.com/pkg/errors"
)
type Meta struct {
Args []string
Env EnvList
Cwd string
}
func NewExecOp(root Output, meta Meta) *ExecOp {
e := &ExecOp{meta: meta}
rootMount := &mount{
target: pb.RootMount,
source: root,
}
e.mounts = append(e.mounts, rootMount)
e.root = &output{vertex: e, getIndex: e.getMountIndexFn(rootMount)}
rootMount.output = e.root
return e
}
type mount struct {
target string
readonly bool
source Output
output Output
// hasOutput bool
}
type ExecOp struct {
root Output
mounts []*mount
meta Meta
}
func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Output {
m := &mount{
target: target,
source: source,
}
for _, o := range opt {
o(m)
}
e.mounts = append(e.mounts, m)
m.output = &output{vertex: e, getIndex: e.getMountIndexFn(m)}
return m.output
}
func (e *ExecOp) GetMount(target string) Output {
for _, m := range e.mounts {
if m.target == target {
return m.output
}
}
return nil
}
func (e *ExecOp) Validate() error {
if len(e.meta.Args) == 0 {
return errors.Errorf("arguments are required")
}
if e.meta.Cwd == "" {
return errors.Errorf("working directory is required")
}
for _, m := range e.mounts {
if m.source != nil {
if err := m.source.Vertex().Validate(); err != nil {
return nil
}
}
}
return nil
}
func (e *ExecOp) Marshal() ([]byte, error) {
if err := e.Validate(); err != nil {
return nil, err
}
// make sure mounts are sorted
sort.Slice(e.mounts, func(i, j int) bool {
return e.mounts[i].target < e.mounts[j].target
})
peo := &pb.ExecOp{
Meta: &pb.Meta{
Args: e.meta.Args,
Env: e.meta.Env.ToArray(),
Cwd: e.meta.Cwd,
},
}
pop := &pb.Op{
Op: &pb.Op_Exec{
Exec: peo,
},
}
for i, m := range e.mounts {
inputIndex := pb.InputIndex(len(pop.Inputs))
if m.source != nil {
inp, err := m.source.ToInput()
if err != nil {
return nil, err
}
newInput := true
for i, inp2 := range pop.Inputs {
if *inp == *inp2 {
inputIndex = pb.InputIndex(i)
newInput = false
break
}
}
if newInput {
pop.Inputs = append(pop.Inputs, inp)
}
} else {
inputIndex = 0
}
pm := &pb.Mount{
Input: inputIndex,
Dest: m.target,
Readonly: m.readonly,
Output: pb.OutputIndex(i),
}
peo.Mounts = append(peo.Mounts, pm)
}
return pop.Marshal()
}
func (e *ExecOp) Output() Output {
return e.root
}
func (e *ExecOp) Inputs() (inputs []Output) {
mm := map[Output]struct{}{}
for _, m := range e.mounts {
if m.source != nil {
mm[m.source] = struct{}{}
}
}
for o := range mm {
inputs = append(inputs, o)
}
return
}
func (e *ExecOp) getMountIndexFn(m *mount) func() (pb.OutputIndex, error) {
return func() (pb.OutputIndex, error) {
// make sure mounts are sorted
sort.Slice(e.mounts, func(i, j int) bool {
return e.mounts[i].target < e.mounts[j].target
})
for i, m2 := range e.mounts {
if m == m2 {
return pb.OutputIndex(i), nil
}
}
return pb.OutputIndex(0), errors.Errorf("invalid mount")
}
}
type ExecState struct {
State
exec *ExecOp
}
func (e ExecState) AddMount(target string, source State, opt ...MountOption) State {
return NewState(e.exec.AddMount(target, source.Output(), opt...))
}
func (e ExecState) GetMount(target string) State {
return NewState(e.exec.GetMount(target))
}
func (e ExecState) Root() State {
return e.State
}
type MountOption func(*mount)
func Readonly(m *mount) {
m.readonly = true
}
type RunOption func(es ExecInfo) ExecInfo
func Shlex(str string) RunOption {
return Shlexf(str)
}
func Shlexf(str string, v ...interface{}) RunOption {
return func(ei ExecInfo) ExecInfo {
ei.State = shlexf(str, v...)(ei.State)
return ei
}
}
func AddMount(dest string, mountState *State, opts ...MountOption) RunOption {
return func(ei ExecInfo) ExecInfo {
ei.Mounts = append(ei.Mounts, MountInfo{dest, mountState.Output(), opts})
return ei
}
}
type ExecInfo struct {
State State
Mounts []MountInfo
}
type MountInfo struct {
Target string
Source Output
Opts []MountOption
}

233
client/llb/llb.go
View File

@ -1,233 +0,0 @@
package llb
import (
_ "crypto/sha256"
"sort"
"github.com/gogo/protobuf/proto"
"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
var errNotFound = errors.Errorf("not found")
func Source(id string) *State {
return &State{
metaNext: NewMeta(),
source: &source{id: id, attrs: map[string]string{}},
}
}
type source struct {
id string
attrs map[string]string
scratch bool
}
func (so *source) Validate() error {
// TODO: basic identifier validation
if so.id == "" {
return errors.Errorf("source identifier can't be empty")
}
return nil
}
func (so *source) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
if so.scratch {
return "", list, nil
}
if err := so.Validate(); err != nil {
return "", nil, err
}
po := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{Identifier: so.id, Attrs: so.attrs},
},
}
return appendResult(po, list, cache)
}
func Image(ref string) *State {
return Source("docker-image://" + ref) // controversial
}
func Git(remote, ref string, opts ...GitOption) *State {
id := remote
if ref != "" {
id += "#" + ref
}
state := Source("git://" + id)
for _, opt := range opts {
opt(state.source)
}
return state
}
type GitOption func(*source)
func KeepGitDir() GitOption {
return func(s *source) {
s.attrs[pb.AttrKeepGitDir] = "true"
}
}
func Scratch() *State {
s := Source("scratch")
s.source.scratch = true
return s
}
func Local(name string) *State {
return Source("local://" + name)
}
type LocalOption func(*source)
func SessionID(id string) LocalOption {
return func(s *source) {
s.attrs[pb.AttrLocalSessionID] = id
}
}
type exec struct {
meta Meta
mounts []*mount
root *mount
}
func (eo *exec) Validate() error {
for _, m := range eo.mounts {
if m.source != nil {
if err := m.source.Validate(); err != nil {
return err
}
}
if m.parent != nil {
if err := m.parent.execState.exec.Validate(); err != nil {
return err
}
}
}
// TODO: validate meta
return nil
}
func (eo *exec) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
peo := &pb.ExecOp{
Meta: &pb.Meta{
Args: eo.meta.args,
Env: eo.meta.env.ToArray(),
Cwd: eo.meta.cwd,
},
}
pop := &pb.Op{
Op: &pb.Op_Exec{
Exec: peo,
},
}
sort.Slice(eo.mounts, func(i, j int) bool {
return eo.mounts[i].dest < eo.mounts[j].dest
})
var outputIndex int64 = 0
for _, m := range eo.mounts {
var dgst digest.Digest
var err error
if m.source != nil {
dgst, list, err = m.source.marshalTo(list, cache)
} else {
dgst, list, err = m.parent.execState.exec.marshalTo(list, cache)
}
if err != nil {
return "", list, err
}
inputIndex := len(pop.Inputs)
for i := range pop.Inputs {
if pop.Inputs[i].Digest == dgst {
inputIndex = i
break
}
}
if dgst == "" {
inputIndex = pb.Empty
}
if inputIndex == len(pop.Inputs) {
var mountIndex int64
if m.parent != nil {
mountIndex = m.parent.outputIndex
}
pop.Inputs = append(pop.Inputs, &pb.Input{
Digest: dgst,
Index: mountIndex,
})
}
pm := &pb.Mount{
Input: int64(inputIndex),
Dest: m.dest,
Readonly: m.readonly,
}
if m.hasOutput {
pm.Output = outputIndex
outputIndex++
} else {
pm.Output = pb.SkipOutput
}
m.outputIndex = outputIndex - 1
peo.Mounts = append(peo.Mounts, pm)
}
return appendResult(pop, list, cache)
}
type mount struct {
execState *ExecState
dest string
readonly bool
// either parent or source has to be set
parent *mount
source *source
hasOutput bool // TODO: remove
outputIndex int64 // filled in after marshal
state *ExecState
}
func (m *mount) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
if m.execState == nil {
return "", nil, errors.Errorf("invalid mount")
}
var dgst digest.Digest
dgst, list, err := m.execState.exec.marshalTo(list, cache)
if err != nil {
return "", list, err
}
for _, m2 := range m.execState.exec.mounts {
if m2 == m {
po := &pb.Op{}
po.Inputs = append(po.Inputs, &pb.Input{
Digest: dgst,
Index: int64(m.outputIndex),
})
return appendResult(po, list, cache)
}
}
return "", nil, errors.Errorf("invalid mount")
}
func appendResult(p proto.Marshaler, list [][]byte, cache map[digest.Digest]struct{}) (dgst digest.Digest, out [][]byte, err error) {
dt, err := p.Marshal()
if err != nil {
return "", nil, err
}
dgst = digest.FromBytes(dt)
if _, ok := cache[dgst]; ok {
return dgst, list, nil
}
list = append(list, dt)
cache[dgst] = struct{}{}
return dgst, list, nil
}

141
client/llb/meta.go
View File

@ -4,126 +4,121 @@ import (
"fmt"
"github.com/google/shlex"
"github.com/moby/buildkit/util/system"
)
func NewMeta(arg ...string) Meta {
m := Meta{}
m, _ = addEnv("PATH", system.DefaultPathEnv)(m)
m, _ = args(arg...)(m)
m, _ = dir("/")(m)
return m
}
type contextKeyT string
type Meta struct {
args []string
env envList
cwd string
}
var (
keyArgs = contextKeyT("llb.exec.args")
keyDir = contextKeyT("llb.exec.dir")
keyEnv = contextKeyT("llb.exec.env")
)
type metaOption func(Meta) (Meta, error)
func addEnv(key, value string) metaOption {
func addEnv(key, value string) StateOption {
return addEnvf(key, value)
}
func addEnvf(key, value string, v ...interface{}) metaOption {
return func(m Meta) (Meta, error) {
m.env = m.env.AddOrReplace(key, fmt.Sprintf(value, v...))
return m, nil
func addEnvf(key, value string, v ...interface{}) StateOption {
return func(s State) State {
return s.WithValue(keyEnv, getEnv(s).AddOrReplace(key, fmt.Sprintf(value, v...)))
}
}
func clearEnv() StateOption {
return func(s State) State {
return s.WithValue(keyEnv, EnvList{})
}
}
func delEnv(key string) StateOption {
return func(s State) State {
return s.WithValue(keyEnv, getEnv(s).Delete(key))
}
}
func clearEnv() metaOption {
return func(m Meta) (Meta, error) {
m.env = NewMeta().env
return m, nil
}
}
func delEnv(key string) metaOption {
return func(m Meta) (Meta, error) {
m.env = m.env.Delete(key)
return m, nil
}
}
func args(args ...string) metaOption {
return func(m Meta) (Meta, error) {
m.args = args
return m, nil
}
}
func dir(str string) metaOption {
func dir(str string) StateOption {
return dirf(str)
}
func dirf(str string, v ...interface{}) metaOption {
return func(m Meta) (Meta, error) {
m.cwd = fmt.Sprintf(str, v...)
return m, nil
func dirf(str string, v ...interface{}) StateOption {
return func(s State) State {
return s.WithValue(keyDir, fmt.Sprintf(str, v...))
}
}
func reset(s *State) metaOption {
return func(m Meta) (Meta, error) {
if s == nil {
return NewMeta(), nil
}
return s.metaNext, nil
func reset(s_ State) StateOption {
return func(s State) State {
s = NewState(s.Output())
s.ctx = s_.ctx
return s
}
}
func (m Meta) Env(key string) (string, bool) {
return m.env.Get(key)
func getEnv(s State) EnvList {
v := s.Value(keyEnv)
if v != nil {
return v.(EnvList)
}
return EnvList{}
}
func (m Meta) Dir() string {
return m.cwd
func getDir(s State) string {
v := s.Value(keyDir)
if v != nil {
return v.(string)
}
return ""
}
func (m Meta) Args() []string {
return append([]string{}, m.args...)
func getArgs(s State) []string {
v := s.Value(keyArgs)
if v != nil {
return v.([]string)
}
return nil
}
func shlexf(str string, v ...interface{}) metaOption {
return func(m Meta) (Meta, error) {
func args(args ...string) StateOption {
return func(s State) State {
return s.WithValue(keyArgs, args)
}
}
func shlexf(str string, v ...interface{}) StateOption {
return func(s State) State {
arg, err := shlex.Split(fmt.Sprintf(str, v...))
if err != nil {
return m, err
// TODO: handle error
}
return args(arg...)(m)
return args(arg...)(s)
}
}
type envList []keyValue
type EnvList []KeyValue
type keyValue struct {
type KeyValue struct {
key string
value string
}
func (e envList) AddOrReplace(k, v string) envList {
func (e EnvList) AddOrReplace(k, v string) EnvList {
e = e.Delete(k)
e = append(e, keyValue{key: k, value: v})
e = append(e, KeyValue{key: k, value: v})
return e
}
func (e envList) Delete(k string) envList {
e = append([]keyValue(nil), e...)
if i, ok := e.index(k); ok {
func (e EnvList) Delete(k string) EnvList {
e = append([]KeyValue(nil), e...)
if i, ok := e.Index(k); ok {
return append(e[:i], e[i+1:]...)
}
return e
}
func (e envList) Get(k string) (string, bool) {
if index, ok := e.index(k); ok {
func (e EnvList) Get(k string) (string, bool) {
if index, ok := e.Index(k); ok {
return e[index].value, true
}
return "", false
}
func (e envList) index(k string) (int, bool) {
func (e EnvList) Index(k string) (int, bool) {
for i, kv := range e {
if kv.key == k {
return i, true
@ -132,7 +127,7 @@ func (e envList) index(k string) (int, bool) {
return -1, false
}
func (e envList) ToArray() []string {
func (e EnvList) ToArray() []string {
out := make([]string, 0, len(e))
for _, kv := range e {
out = append(out, kv.key+"="+kv.value)

61
client/llb/meta_test.go
View File

@ -1,61 +0,0 @@
package llb
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDefaultMeta(t *testing.T) {
m := NewMeta()
path, ok := m.Env("PATH")
assert.True(t, ok)
assert.NotEmpty(t, path)
cwd := m.Dir()
assert.NotEmpty(t, cwd)
}
func TestReset(t *testing.T) {
m := NewMeta()
wd := m.Dir()
path, _ := m.Env("PATH")
m, _ = dir("/foo")(m)
m, _ = addEnv("FOO", "bar")(m)
m, _ = reset(nil)(m)
assert.Equal(t, wd, m.Dir())
path2, _ := m.Env("PATH")
assert.Equal(t, path, path2)
}
func TestEnv(t *testing.T) {
m := NewMeta()
m, _ = addEnv("FOO", "bar")(m)
m2, _ := addEnv("FOO", "baz")(m)
m2, _ = addEnv("BAR", "abc")(m2)
v, ok := m.Env("FOO")
assert.True(t, ok)
assert.Equal(t, "bar", v)
_, ok = m.Env("BAR")
assert.False(t, ok)
v, ok = m2.Env("FOO")
assert.True(t, ok)
assert.Equal(t, "baz", v)
v, ok = m2.Env("BAR")
assert.True(t, ok)
assert.Equal(t, "abc", v)
}
func TestShlex(t *testing.T) {
m, err := shlexf("echo foo")(Meta{})
assert.Nil(t, err)
assert.Equal(t, []string{"echo", "foo"}, m.Args())
_, err = shlexf("echo \"foo")(Meta{})
assert.Error(t, err)
}

121
client/llb/source.go Normal file
View File

@ -0,0 +1,121 @@
package llb
import (
_ "crypto/sha256"
"github.com/moby/buildkit/solver/pb"
"github.com/pkg/errors"
)
type SourceOp struct {
id string
attrs map[string]string
output Output
}
func NewSource(id string, attrs map[string]string) *SourceOp {
s := &SourceOp{
id: id,
attrs: attrs,
}
s.output = &output{vertex: s}
return s
}
func (s *SourceOp) Validate() error {
if s.id == "" {
return errors.Errorf("source identifier can't be empty")
}
return nil
}
func (s *SourceOp) Marshal() ([]byte, error) {
if err := s.Validate(); err != nil {
return nil, err
}
proto := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{Identifier: s.id, Attrs: s.attrs},
},
}
return proto.Marshal()
}
func (s *SourceOp) Output() Output {
return s.output
}
func (s *SourceOp) Inputs() []Output {
return nil
}
func Source(id string) State {
return NewState(NewSource(id, nil).Output())
}
func Image(ref string) State {
return Source("docker-image://" + ref) // controversial
}
func Git(remote, ref string, opts ...GitOption) State {
id := remote
if ref != "" {
id += "#" + ref
}
gi := &GitInfo{}
for _, o := range opts {
o(gi)
}
attrs := map[string]string{}
if gi.KeepGitDir {
attrs[pb.AttrKeepGitDir] = "true"
}
source := NewSource("git://"+id, attrs)
return NewState(source.Output())
}
type GitOption func(*GitInfo)
type GitInfo struct {
KeepGitDir bool
}
func KeepGitDir() GitOption {
return func(gi *GitInfo) {
gi.KeepGitDir = true
}
}
func Scratch() State {
return NewState(nil)
}
func Local(name string, opts ...LocalOption) State {
gi := &LocalInfo{}
for _, o := range opts {
o(gi)
}
attrs := map[string]string{}
if gi.SessionID != "" {
attrs[pb.AttrLocalSessionID] = gi.SessionID
}
source := NewSource("local://"+name, attrs)
return NewState(source.Output())
}
type LocalOption func(*LocalInfo)
func SessionID(id string) LocalOption {
return func(li *LocalInfo) {
li.SessionID = id
}
}
type LocalInfo struct {
SessionID string
}

340
client/llb/state.go
View File

@ -1,249 +1,189 @@
package llb
import (
_ "crypto/sha256"
"context"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/system"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type StateOption func(s *State) *State
type StateOption func(State) State
type Output interface {
ToInput() (*pb.Input, error)
Vertex() Vertex
}
type Vertex interface {
Validate() error
Marshal() ([]byte, error)
Output() Output
Inputs() []Output
}
func NewState(o Output) State {
s := State{
out: o,
ctx: context.Background(),
}
s = dir("/")(s)
s = addEnv("PATH", system.DefaultPathEnv)(s)
return s
}
// State represents modifiable llb state
type State struct {
source *source
exec *exec
meta Meta
mount *mount
metaNext Meta // this meta will be used for the next Run()
err error
out Output
ctx context.Context
}
// TODO: add state.Reset() state.Save() state.Restore()
func (s State) WithValue(k, v interface{}) State {
return State{
out: s.out,
ctx: context.WithValue(s.ctx, k, v),
}
}
// Validate checks that every node has been set up properly
func (s *State) Validate() error {
if s.source != nil {
if err := s.source.Validate(); err != nil {
return err
func (s State) Value(k interface{}) interface{} {
return s.ctx.Value(k)
}
func (s State) Marshal() ([][]byte, error) {
list, err := marshal(s.Output().Vertex(), nil, map[digest.Digest]struct{}{})
if err != nil {
return nil, err
}
inp, err := s.Output().ToInput()
if err != nil {
return nil, err
}
proto := &pb.Op{Inputs: []*pb.Input{inp}}
dt, err := proto.Marshal()
if err != nil {
return nil, err
}
list = append(list, dt)
return list, nil
}
func marshal(v Vertex, list [][]byte, cache map[digest.Digest]struct{}) (out [][]byte, err error) {
for _, inp := range v.Inputs() {
var err error
list, err = marshal(inp.Vertex(), list, cache)
if err != nil {
return nil, err
}
}
if s.exec != nil {
if err := s.exec.Validate(); err != nil {
return err
}
dt, err := v.Marshal()
if err != nil {
return nil, err
}
return nil
dgst := digest.FromBytes(dt)
if _, ok := cache[dgst]; ok {
return list, nil
}
list = append(list, dt)
cache[dgst] = struct{}{}
return list, nil
}
func (s *State) Run(opts ...RunOption) *ExecState {
var es ExecState
meta := s.metaNext
exec := &exec{
meta: meta,
mounts: []*mount{},
root: &mount{
dest: pb.RootMount,
source: s.source,
parent: s.mount,
hasOutput: true,
},
}
exec.root.execState = &es
exec.mounts = append(exec.mounts, exec.root)
es.exec = exec
es.mount = exec.root
es.meta = meta
es.metaNext = es.meta
var err error
for _, o := range opts {
es = *o(&es)
}
es.exec.meta = es.meta
es.err = err
return &es
func (s State) Validate() error {
return s.Output().Vertex().Validate()
}
func (s *State) AddEnv(key, value string) *State {
func (s State) Output() Output {
return s.out
}
func (s State) WithOutput(o Output) State {
return State{
out: o,
ctx: s.ctx,
}
}
func (s State) Run(ro ...RunOption) ExecState {
ei := ExecInfo{State: s}
for _, o := range ro {
ei = o(ei)
}
meta := Meta{
Args: getArgs(ei.State),
Cwd: getDir(ei.State),
Env: getEnv(ei.State),
}
exec := NewExecOp(s.Output(), meta)
for _, m := range ei.Mounts {
exec.AddMount(m.Target, m.Source, m.Opts...)
}
return ExecState{
State: s.WithOutput(exec.Output()),
exec: exec,
}
}
func (s State) AddEnv(key, value string) State {
return s.AddEnvf(key, value)
}
func (s *State) AddEnvf(key, value string, v ...interface{}) *State {
s.metaNext, _ = addEnvf(key, value, v...)(s.metaNext)
return s
func (s State) AddEnvf(key, value string, v ...interface{}) State {
return addEnvf(key, value, v...)(s)
}
func (s *State) DelEnv(key string) *State {
s.metaNext, _ = delEnv(key)(s.metaNext)
return s
}
func (s *State) ClearEnv() *State {
s.metaNext, _ = clearEnv()(s.metaNext)
return s
}
func (s *State) GetEnv(key string) (string, bool) {
return s.metaNext.Env(key)
}
func (s *State) Dir(str string) *State {
func (s State) Dir(str string) State {
return s.Dirf(str)
}
func (s *State) Dirf(str string, v ...interface{}) *State {
s.metaNext, _ = dirf(str, v...)(s.metaNext)
return s
func (s State) Dirf(str string, v ...interface{}) State {
return dirf(str, v...)(s)
}
func (s *State) GetDir() string {
return s.metaNext.Dir()
func (s State) GetEnv(key string) (string, bool) {
return getEnv(s).Get(key)
}
func (s *State) Args(arg ...string) *State {
s.metaNext, _ = args(arg...)(s.metaNext)
return s
func (s State) GetDir() string {
return getDir(s)
}
func (s *State) Reset(src *State) *State {
copy := *s
copy.metaNext, _ = reset(src)(s.metaNext)
return &copy
func (s State) GetArgs() []string {
return getArgs(s)
}
func (s *State) With(so ...StateOption) *State {
func (s State) Reset(s2 State) State {
return reset(s2)(s)
}
func (s State) With(so ...StateOption) State {
for _, o := range so {
s = o(s)
}
return s
}
func (s *State) Marshal() (list [][]byte, err error) {
if err := s.Validate(); err != nil {
return nil, err
}
cache := make(map[digest.Digest]struct{})
if s.source != nil {
_, list, err = s.source.marshalTo(nil, cache)
} else if s.exec != nil {
_, list, err = s.exec.root.marshalTo(nil, cache)
} else {
_, list, err = s.mount.marshalTo(nil, cache)
}
return
type output struct {
vertex Vertex
getIndex func() (pb.OutputIndex, error)
}
// ExecState is a state with a active leaf pointing to a run exec command.
// Mounts can be added only to this state.
type ExecState struct {
State
}
func (s *ExecState) AddMount(dest string, mountState *State, opts ...MountOption) *State {
m := &mount{
dest: dest,
source: mountState.source,
parent: mountState.mount,
execState: s,
hasOutput: true, // TODO: should be set only if something inherits
}
for _, opt := range opts {
opt(m)
}
var newState State
newState.meta = s.meta
newState.metaNext = s.metaNext
newState.mount = m
s.exec.mounts = append(s.exec.mounts, m)
return &newState
}
func (s *ExecState) Root() *State {
return &s.State
}
func (s *ExecState) GetMount(target string) (*State, error) {
for _, m := range s.exec.mounts {
if m.dest == target {
var newState State
newState.meta = m.execState.meta
newState.metaNext = m.execState.metaNext
newState.mount = m
return &newState, nil
func (o *output) ToInput() (*pb.Input, error) {
var index pb.OutputIndex
if o.getIndex != nil {
var err error
index, err = o.getIndex()
if err != nil {
return nil, err
}
}
return nil, errors.WithStack(errNotFound)
}
func (s *ExecState) updateMeta(fn metaOption) *ExecState {
meta, err := fn(s.meta)
s.meta = meta
dt, err := o.vertex.Marshal()
if err != nil {
s.err = err
return nil, err
}
return s
dgst := digest.FromBytes(dt)
return &pb.Input{Digest: dgst, Index: index}, nil
}
type RunOption func(es *ExecState) *ExecState
type MountOption func(*mount)
func Readonly(m *mount) {
m.readonly = true
}
func AddMount(dest string, mountState *State, opts ...MountOption) RunOption {
return func(es *ExecState) *ExecState {
es.AddMount(dest, mountState, opts...)
return nil
}
}
func Shlex(str string) RunOption {
return Shlexf(str)
}
func Shlexf(str string, v ...interface{}) RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(shlexf(str, v...))
}
}
func AddEnv(key, value string) RunOption {
return AddEnvf(key, value)
}
func AddEnvf(key, value string, v ...interface{}) RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(addEnvf(key, value, v...))
}
}
func DelEnv(key string) RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(delEnv(key))
}
}
func ClearEnv() RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(clearEnv())
}
}
func Dir(str string) RunOption {
return Dirf(str)
}
func Dirf(str string, v ...interface{}) RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(dirf(str, v...))
}
}
func Args(arg ...string) RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(args(arg...))
}
}
func Reset(src *State) RunOption {
return func(es *ExecState) *ExecState {
return es.updateMeta(reset(src))
}
func (o *output) Vertex() Vertex {
return o.vertex
}

10
examples/buildkit0/buildkit.go
View File

@ -31,7 +31,7 @@ func main() {
llb.WriteTo(dt, os.Stdout)
}
func goBuildBase() *llb.State {
func goBuildBase() llb.State {
goAlpine := llb.Image("docker.io/library/golang:1.8-alpine")
return goAlpine.
AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv).
@ -40,7 +40,7 @@ func goBuildBase() *llb.State {
Run(llb.Shlex("apk add --no-cache git make")).Root()
}
func runc(version string) *llb.State {
func runc(version string) llb.State {
return goBuildBase().
Run(llb.Shlex("git clone https://github.com/opencontainers/runc.git /go/src/github.com/opencontainers/runc")).
Dir("/go/src/github.com/opencontainers/runc").
@ -48,7 +48,7 @@ func runc(version string) *llb.State {
Run(llb.Shlex("go build -o /usr/bin/runc ./")).Root()
}
func containerd(version string) *llb.State {
func containerd(version string) llb.State {
return goBuildBase().
Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).
Run(llb.Shlex("git clone https://github.com/containerd/containerd.git /go/src/github.com/containerd/containerd")).
@ -57,7 +57,7 @@ func containerd(version string) *llb.State {
Run(llb.Shlex("make bin/containerd")).Root()
}
func buildkit(opt buildOpt) *llb.State {
func buildkit(opt buildOpt) llb.State {
src := goBuildBase().
Run(llb.Shlex("git clone https://github.com/moby/buildkit.git /go/src/github.com/moby/buildkit")).
Dir("/go/src/github.com/moby/buildkit")
@ -83,7 +83,7 @@ func buildkit(opt buildOpt) *llb.State {
return r
}
func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State {
func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State {
cpImage := llb.Image("docker.io/library/alpine:latest")
cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath))
cp.AddMount("/src", src)

16
examples/buildkit1/buildkit.go
View File

@ -31,7 +31,7 @@ func main() {
llb.WriteTo(dt, os.Stdout)
}
func goBuildBase() *llb.State {
func goBuildBase() llb.State {
goAlpine := llb.Image("docker.io/library/golang:1.8-alpine")
return goAlpine.
AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv).
@ -40,21 +40,21 @@ func goBuildBase() *llb.State {
Run(llb.Shlex("apk add --no-cache git make")).Root()
}
func runc(version string) *llb.State {
func runc(version string) llb.State {
return goBuildBase().
With(goFromGit("github.com/opencontainers/runc", version)).
Run(llb.Shlex("go build -o /usr/bin/runc ./")).
Root()
}
func containerd(version string) *llb.State {
func containerd(version string) llb.State {
return goBuildBase().
Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).
With(goFromGit("github.com/containerd/containerd", version)).
Run(llb.Shlex("make bin/containerd")).Root()
}
func buildkit(opt buildOpt) *llb.State {
func buildkit(opt buildOpt) llb.State {
src := goBuildBase().With(goFromGit("github.com/moby/buildkit", "master"))
builddStandalone := src.
@ -87,20 +87,20 @@ func goFromGit(repo, tag string) llb.StateOption {
Run(llb.Shlexf("git clone https://%[1]s.git /go/src/%[1]s", repo)).
Dirf("/go/src/%s", repo).
Run(llb.Shlexf("git checkout -q %s", tag)).Root()
return func(s *llb.State) *llb.State {
return func(s llb.State) llb.State {
return s.With(copyFrom(src, "/go", "/")).Reset(s).Dir(src.GetDir())
}
}
// copyFrom has similar semantics as `COPY --from`
func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption {
return func(s *llb.State) *llb.State {
func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption {
return func(s llb.State) llb.State {
return copy(src, srcPath, s, destPath)
}
}
// copy copies files between 2 states using cp until there is no copyOp
func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State {
func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State {
cpImage := llb.Image("docker.io/library/alpine:latest")
cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath))
cp.AddMount("/src", src)

20
examples/buildkit2/buildkit.go
View File

@ -31,7 +31,7 @@ func main() {
llb.WriteTo(dt, os.Stdout)
}
func goBuildBase() *llb.State {
func goBuildBase() llb.State {
goAlpine := llb.Image("docker.io/library/golang:1.8-alpine")
return goAlpine.
AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv).
@ -39,22 +39,22 @@ func goBuildBase() *llb.State {
Run(llb.Shlex("apk add --no-cache g++ linux-headers make")).Root()
}
func goRepo(s *llb.State, repo, ref string, g ...llb.GitOption) func(ro ...llb.RunOption) *llb.State {
func goRepo(s llb.State, repo, ref string, g ...llb.GitOption) func(ro ...llb.RunOption) llb.State {
dir := "/go/src/" + repo
return func(ro ...llb.RunOption) *llb.State {
return func(ro ...llb.RunOption) llb.State {
es := s.Dir(dir).Run(ro...)
es.AddMount(dir, llb.Git(repo, ref, g...))
return es.AddMount(dir+"/bin", llb.Scratch())
}
}
func runc(version string) *llb.State {
func runc(version string) llb.State {
return goRepo(goBuildBase(), "github.com/opencontainers/runc", version)(
llb.Shlex("go build -o ./bin/runc ./"),
)
}
func containerd(version string) *llb.State {
func containerd(version string) llb.State {
return goRepo(
goBuildBase().
Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).Root(),
@ -63,7 +63,7 @@ func containerd(version string) *llb.State {
)
}
func buildkit(opt buildOpt) *llb.State {
func buildkit(opt buildOpt) llb.State {
run := goRepo(goBuildBase(), "github.com/moby/buildkit", "master")
builddStandalone := run(llb.Shlex("go build -o ./bin/buildd-standalone -tags standalone ./cmd/buildd"))
@ -85,19 +85,19 @@ func buildkit(opt buildOpt) *llb.State {
return r.With(copyAll(builddStandalone, "/bin"))
}
func copyAll(src *llb.State, destPath string) llb.StateOption {
func copyAll(src llb.State, destPath string) llb.StateOption {
return copyFrom(src, "/.", destPath)
}
// copyFrom has similar semantics as `COPY --from`
func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption {
return func(s *llb.State) *llb.State {
func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption {
return func(s llb.State) llb.State {
return copy(src, srcPath, s, destPath)
}
}
// copy copies files between 2 states using cp until there is no copyOp
func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State {
func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State {
cpImage := llb.Image("docker.io/library/alpine:latest")
cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath))
cp.AddMount("/src", src)

20
examples/buildkit3/buildkit.go
View File

@ -33,7 +33,7 @@ func main() {
llb.WriteTo(dt, os.Stdout)
}
func goBuildBase() *llb.State {
func goBuildBase() llb.State {
goAlpine := llb.Image("docker.io/library/golang:1.8-alpine")
return goAlpine.
AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv).
@ -41,16 +41,16 @@ func goBuildBase() *llb.State {
Run(llb.Shlex("apk add --no-cache g++ linux-headers make")).Root()
}
func goRepo(s *llb.State, repo string, src *llb.State) func(ro ...llb.RunOption) *llb.State {
func goRepo(s llb.State, repo string, src llb.State) func(ro ...llb.RunOption) llb.State {
dir := "/go/src/" + repo
return func(ro ...llb.RunOption) *llb.State {
return func(ro ...llb.RunOption) llb.State {
es := s.Dir(dir).Run(ro...)
es.AddMount(dir, src, llb.Readonly)
return es.AddMount("/out", llb.Scratch())
}
}
func runc(version string) *llb.State {
func runc(version string) llb.State {
repo := "github.com/opencontainers/runc"
src := llb.Git(repo, version)
if version == "local" {
@ -61,7 +61,7 @@ func runc(version string) *llb.State {
)
}
func containerd(version string) *llb.State {
func containerd(version string) llb.State {
repo := "github.com/containerd/containerd"
src := llb.Git(repo, version, llb.KeepGitDir())
if version == "local" {
@ -75,7 +75,7 @@ func containerd(version string) *llb.State {
)
}
func buildkit(opt buildOpt) *llb.State {
func buildkit(opt buildOpt) llb.State {
repo := "github.com/moby/buildkit"
src := llb.Git(repo, "master")
if opt.buildkit == "local" {
@ -102,19 +102,19 @@ func buildkit(opt buildOpt) *llb.State {
return r.With(copyAll(builddStandalone, "/bin"))
}
func copyAll(src *llb.State, destPath string) llb.StateOption {
func copyAll(src llb.State, destPath string) llb.StateOption {
return copyFrom(src, "/.", destPath)
}
// copyFrom has similar semantics as `COPY --from`
func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption {
return func(s *llb.State) *llb.State {
func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption {
return func(s llb.State) llb.State {
return copy(src, srcPath, s, destPath)
}
}
// copy copies files between 2 states using cp until there is no copyOp
func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State {
func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State {
cpImage := llb.Image("docker.io/library/alpine:latest")
cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath))
cp.AddMount("/src", src, llb.Readonly)

7
solver/pb/const.go
View File

@ -1,5 +1,8 @@
package pb
type InputIndex int64
type OutputIndex int64
const RootMount = "/"
const SkipOutput = -1 // TODO: custom type
const Empty = -1 // TODO: custom type
const SkipOutput OutputIndex = -1
const Empty InputIndex = -1

24
solver/pb/ops.pb.go
View File

@ -171,7 +171,7 @@ func _Op_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bo
type Input struct {
Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"`
Index int64 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"`
Index OutputIndex `protobuf:"varint,2,opt,name=index,proto3,customtype=OutputIndex" json:"index"`
}
func (m *Input) Reset() { *m = Input{} }
@ -212,11 +212,11 @@ func (m *Meta) String() string { return proto.CompactTextString(m) }
func (*Meta) ProtoMessage() {}
type Mount struct {
Input int64 `protobuf:"varint,1,opt,name=input,proto3" json:"input,omitempty"`
Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"`
Dest string `protobuf:"bytes,3,opt,name=dest,proto3" json:"dest,omitempty"`
Output int64 `protobuf:"varint,4,opt,name=output,proto3" json:"output,omitempty"`
Readonly bool `protobuf:"varint,5,opt,name=readonly,proto3" json:"readonly,omitempty"`
Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"`
Dest string `protobuf:"bytes,3,opt,name=dest,proto3" json:"dest,omitempty"`
Output OutputIndex `protobuf:"varint,4,opt,name=output,proto3,customtype=OutputIndex" json:"output"`
Readonly bool `protobuf:"varint,5,opt,name=readonly,proto3" json:"readonly,omitempty"`
}
func (m *Mount) Reset() { *m = Mount{} }
@ -240,8 +240,8 @@ func (m *CopyOp) GetSrc() []*CopySource {
}
type CopySource struct {
Input int64 `protobuf:"varint,1,opt,name=input,proto3" json:"input,omitempty"`
Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"`
Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"`
}
func (m *CopySource) Reset() { *m = CopySource{} }
@ -1085,7 +1085,7 @@ func (m *Input) Unmarshal(data []byte) error {
}
b := data[iNdEx]
iNdEx++
m.Index |= (int64(b) & 0x7F) << shift
m.Index |= (OutputIndex(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -1405,7 +1405,7 @@ func (m *Mount) Unmarshal(data []byte) error {
}
b := data[iNdEx]
iNdEx++
m.Input |= (int64(b) & 0x7F) << shift
m.Input |= (InputIndex(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -1482,7 +1482,7 @@ func (m *Mount) Unmarshal(data []byte) error {
}
b := data[iNdEx]
iNdEx++
m.Output |= (int64(b) & 0x7F) << shift
m.Output |= (OutputIndex(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -1681,7 +1681,7 @@ func (m *CopySource) Unmarshal(data []byte) error {
}
b := data[iNdEx]
iNdEx++
m.Input |= (int64(b) & 0x7F) << shift
m.Input |= (InputIndex(b) & 0x7F) << shift
if b < 0x80 {
break
}

8
solver/pb/ops.proto
View File

@ -15,7 +15,7 @@ message Op {
message Input {
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
int64 index = 2;
int64 index = 2 [(gogoproto.customtype) = "OutputIndex", (gogoproto.nullable) = false];
}
message ExecOp {
@ -30,10 +30,10 @@ message Meta {
}
message Mount {
int64 input = 1;
int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false];
string selector = 2;
string dest = 3;
int64 output = 4;
int64 output = 4 [(gogoproto.customtype) = "OutputIndex", (gogoproto.nullable) = false];
bool readonly = 5;
}
@ -43,7 +43,7 @@ message CopyOp {
}
message CopySource {
int64 input = 1;
int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false];
string selector = 2;
}