JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

llb: add more helpers and tests to llb client 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2017-06-25 21:10:02 -07:00
parent afd6ad7031
commit 991ab8d9a3
5 changed files with 433 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

266
client/llb/llb.go
View File

@ -2,194 +2,30 @@ package llb
import (
_ "crypto/sha256"
"fmt"
"sort"
"github.com/gogo/protobuf/proto"
"github.com/google/shlex"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/system"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type RunOption func(m Meta) Meta
var errNotFound = errors.Errorf("not found")
type SourceOp struct {
id string
}
type ExecOp struct {
meta Meta
mounts []*Mount
root *Mount
}
type Meta struct {
Args []string
Env []string
Cwd string
}
type Mount struct {
op *ExecOp
dest string
mount *Mount
src *SourceOp
output bool
inputIndex int64
}
func NewMeta(args ...string) Meta {
m := Meta{}
m = m.addEnv("PATH", system.DefaultPathEnv)
m = m.setArgs(args...)
m.Cwd = "/"
return m
}
func (m *Meta) ensurePrivate() {
m.Env = append([]string{}, m.Env...)
m.Args = append([]string{}, m.Args...)
}
func (m Meta) addEnv(k, v string) Meta {
(&m).ensurePrivate()
// TODO: flatten
m.Env = append(m.Env, k+"="+v)
return m
}
func (m Meta) setArgs(args ...string) Meta {
m.Args = append([]string{}, args...)
return m
}
func Shlex(str string, v ...string) RunOption {
return func(m Meta) Meta {
vi := make([]interface{}, 0, len(v))
for _, v := range v {
vi = append(vi, v)
}
sp, err := shlex.Split(fmt.Sprintf(str, vi...))
if err != nil {
panic(err) // TODO
}
(&m).ensurePrivate()
return m.setArgs(sp...)
}
}
type State struct {
src *SourceOp
exec *ExecOp
meta Meta
mount *Mount
metaNext Meta
}
type ExecState struct {
State
}
func (s *State) Validate() error {
if s.src != nil {
if err := s.src.Validate(); err != nil {
return err
}
}
if s.exec != nil {
if err := s.exec.Validate(); err != nil {
return err
}
}
return nil
}
func (s *State) Run(opts ...RunOption) *ExecState {
var es ExecState
meta := s.metaNext
for _, o := range opts {
meta = o(meta)
}
exec := newExec(meta, s.src, s.mount)
es.exec = exec
es.mount = exec.root
es.metaNext = meta
es.meta = meta
return &es
}
func (s *State) AddEnv(k, v string) *State {
s.metaNext = s.metaNext.addEnv(k, v)
return s
}
func (s *State) Dir(wd string) *State {
s.metaNext.Cwd = wd
return s
}
func (s *State) Marshal() ([][]byte, error) {
if err := s.Validate(); err != nil {
return nil, err
}
cache := make(map[digest.Digest]struct{})
var list [][]byte
var err error
if s.src != nil { // TODO: fix repetition
_, list, err = s.src.recursiveMarshal(nil, cache)
} else if s.exec != nil {
_, list, err = s.exec.root.recursiveMarshal(nil, cache)
} else {
_, list, err = s.mount.recursiveMarshal(nil, cache)
}
return list, err
}
func (s *ExecState) AddMount(dest string, mount *State) *State {
m := &Mount{
dest: dest,
src: mount.src,
mount: mount.mount,
op: s.exec,
output: true, // TODO: should be set only if something inherits
}
var newState State
newState.meta = s.meta
newState.metaNext = s.meta
newState.mount = m
s.exec.mounts = append(s.exec.mounts, m)
return &newState
}
func (s *ExecState) Root() *State {
return &s.State
}
func newExec(meta Meta, src *SourceOp, m *Mount) *ExecOp {
exec := &ExecOp{
meta: meta,
mounts: []*Mount{},
root: &Mount{
dest: "/",
src: src,
mount: m,
output: true,
},
}
exec.root.op = exec
exec.mounts = append(exec.mounts, exec.root)
return exec
}
type RunOption func(m Meta) (Meta, error)
func Source(id string) *State {
return &State{
metaNext: NewMeta(),
src: &SourceOp{id: id},
source: &source{id: id},
}
}
func (so *SourceOp) Validate() error {
type source struct {
id string
}
func (so *source) Validate() error {
// TODO: basic identifier validation
if so.id == "" {
return errors.Errorf("source identifier can't be empty")
@ -197,7 +33,7 @@ func (so *SourceOp) Validate() error {
return nil
}
func (so *SourceOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
func (so *source) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
if err := so.Validate(); err != nil {
return "", nil, err
}
@ -213,15 +49,21 @@ func Image(ref string) *State {
return Source("docker-image://" + ref) // controversial
}
func (eo *ExecOp) Validate() error {
type exec struct {
meta Meta
mounts []*mount
root *mount
}
func (eo *exec) Validate() error {
for _, m := range eo.mounts {
if m.src != nil {
if err := m.src.Validate(); err != nil {
if m.source != nil {
if err := m.source.Validate(); err != nil {
return err
}
}
if m.mount != nil {
if err := m.mount.op.Validate(); err != nil {
if m.parent != nil {
if err := m.parent.execState.exec.Validate(); err != nil {
return err
}
}
@ -230,21 +72,12 @@ func (eo *ExecOp) Validate() error {
return nil
}
func (eo *ExecOp) Marshal() ([][]byte, error) {
if err := eo.Validate(); err != nil {
return nil, err
}
cache := make(map[digest.Digest]struct{})
_, list, err := eo.recursiveMarshal(nil, cache)
return list, err
}
func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
func (eo *exec) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
peo := &pb.ExecOp{
Meta: &pb.Meta{
Args: eo.meta.Args,
Env: eo.meta.Env,
Cwd: eo.meta.Cwd,
Args: eo.meta.args,
Env: eo.meta.env.ToArray(),
Cwd: eo.meta.cwd,
},
}
@ -263,15 +96,13 @@ func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct
for _, m := range eo.mounts {
var dgst digest.Digest
var err error
var op interface{}
if m.src != nil {
op = m.src
if m.source != nil {
dgst, list, err = m.source.marshalTo(list, cache)
} else {
op = m.mount.op
dgst, list, err = m.parent.execState.exec.marshalTo(list, cache)
}
dgst, list, err = recursiveMarshalAny(op, list, cache)
if err != nil {
return "", nil, err
return "", list, err
}
inputIndex := len(pop.Inputs)
for i := range pop.Inputs {
@ -282,8 +113,8 @@ func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct
}
if inputIndex == len(pop.Inputs) {
var mountIndex int64
if m.mount != nil {
mountIndex = m.mount.inputIndex
if m.parent != nil {
mountIndex = m.parent.outputIndex
}
pop.Inputs = append(pop.Inputs, &pb.Input{
Digest: dgst,
@ -295,34 +126,46 @@ func (eo *ExecOp) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct
Input: int64(inputIndex),
Dest: m.dest,
}
if m.output {
if m.hasOutput {
pm.Output = outputIndex
outputIndex++
} else {
pm.Output = -1
}
m.inputIndex = outputIndex - 1
m.outputIndex = outputIndex - 1
peo.Mounts = append(peo.Mounts, pm)
}
return appendResult(pop, list, cache)
}
func (m *Mount) recursiveMarshal(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
if m.op == nil {
type mount struct {
execState *ExecState
dest string
// ro bool
// either parent or source has to be set
parent *mount
source *source
hasOutput bool // TODO: remove
outputIndex int64 // filled in after marshal
state *ExecState
}
func (m *mount) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) {
if m.execState == nil {
return "", nil, errors.Errorf("invalid mount")
}
var dgst digest.Digest
dgst, list, err := m.op.recursiveMarshal(list, cache)
dgst, list, err := m.execState.exec.marshalTo(list, cache)
if err != nil {
return "", list, err
}
for _, m2 := range m.op.mounts {
for _, m2 := range m.execState.exec.mounts {
if m2 == m {
po := &pb.Op{}
po.Inputs = append(po.Inputs, &pb.Input{
Digest: dgst,
Index: int64(m.inputIndex),
Index: int64(m.outputIndex),
})
return appendResult(po, list, cache)
}
@ -343,14 +186,3 @@ func appendResult(p proto.Marshaler, list [][]byte, cache map[digest.Digest]stru
cache[dgst] = struct{}{}
return dgst, list, nil
}
func recursiveMarshalAny(op interface{}, list [][]byte, cache map[digest.Digest]struct{}) (dgst digest.Digest, out [][]byte, err error) {
switch op := op.(type) {
case *ExecOp:
return op.recursiveMarshal(list, cache)
case *SourceOp:
return op.recursiveMarshal(list, cache)
default:
return "", nil, errors.Errorf("invalid operation")
}
}

133
client/llb/meta.go Normal file
View File

@ -0,0 +1,133 @@
package llb
import (
"fmt"
"github.com/google/shlex"
"github.com/moby/buildkit/util/system"
)
func NewMeta(args ...string) Meta {
m := Meta{}
m, _ = AddEnv("PATH", system.DefaultPathEnv)(m)
m, _ = Args(args...)(m)
m, _ = Dir("/")(m)
return m
}
type Meta struct {
args []string
env envList
cwd string
}
func AddEnv(key, value string, v ...interface{}) RunOption {
return func(m Meta) (Meta, error) {
m.env = m.env.AddOrReplace(key, fmt.Sprintf(value, v...))
return m, nil
}
}
func ClearEnv() RunOption {
return func(m Meta) (Meta, error) {
m.env = NewMeta().env
return m, nil
}
}
func DelEnv(key string) RunOption {
return func(m Meta) (Meta, error) {
m.env = m.env.Delete(key)
return m, nil
}
}
func Args(args ...string) RunOption {
return func(m Meta) (Meta, error) {
m.args = args
return m, nil
}
}
func Dir(str string, v ...interface{}) RunOption {
return func(m Meta) (Meta, error) {
m.cwd = fmt.Sprintf(str, v...)
return m, nil
}
}
func Reset(s *State) RunOption {
return func(m Meta) (Meta, error) {
if s == nil {
return NewMeta(), nil
}
return s.metaNext, nil
}
}
func (m Meta) Env(key string) (string, bool) {
return m.env.Get(key)
}
func (m Meta) Dir() string {
return m.cwd
}
func (m Meta) Args() []string {
return append([]string{}, m.args...)
}
func Shlex(str string, v ...interface{}) RunOption {
return func(m Meta) (Meta, error) {
args, err := shlex.Split(fmt.Sprintf(str, v...))
if err != nil {
return m, err
}
return Args(args...)(m)
}
}
type envList []keyValue
type keyValue struct {
key string
value string
}
func (e envList) AddOrReplace(k, v string) envList {
e = e.Delete(k)
e = append(e, keyValue{key: k, value: v})
return e
}
func (e envList) Delete(k string) envList {
e = append([]keyValue(nil), e...)
if i, ok := e.index(k); ok {
return append(e[:i], e[i+1:]...)
}
return e
}
func (e envList) Get(k string) (string, bool) {
if index, ok := e.index(k); ok {
return e[index].value, true
}
return "", false
}
func (e envList) index(k string) (int, bool) {
for i, kv := range e {
if kv.key == k {
return i, true
}
}
return -1, false
}
func (e envList) ToArray() []string {
out := make([]string, 0, len(e))
for _, kv := range e {
out = append(out, kv.key+"="+kv.value)
}
return out
}

61
client/llb/meta_test.go Normal file
View File

@ -0,0 +1,61 @@
package llb
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDefaultMeta(t *testing.T) {
m := NewMeta()
path, ok := m.Env("PATH")
assert.True(t, ok)
assert.NotEmpty(t, path)
cwd := m.Dir()
assert.NotEmpty(t, cwd)
}
func TestReset(t *testing.T) {
m := NewMeta()
wd := m.Dir()
path, _ := m.Env("PATH")
m, _ = Dir("/foo")(m)
m, _ = AddEnv("FOO", "bar")(m)
m, _ = Reset(nil)(m)
assert.Equal(t, wd, m.Dir())
path2, _ := m.Env("PATH")
assert.Equal(t, path, path2)
}
func TestEnv(t *testing.T) {
m := NewMeta()
m, _ = AddEnv("FOO", "bar")(m)
m2, _ := AddEnv("FOO", "baz")(m)
m2, _ = AddEnv("BAR", "abc")(m2)
v, ok := m.Env("FOO")
assert.True(t, ok)
assert.Equal(t, "bar", v)
_, ok = m.Env("BAR")
assert.False(t, ok)
v, ok = m2.Env("FOO")
assert.True(t, ok)
assert.Equal(t, "baz", v)
v, ok = m2.Env("BAR")
assert.True(t, ok)
assert.Equal(t, "abc", v)
}
func TestShlex(t *testing.T) {
m, err := Shlex("echo foo")(Meta{})
assert.Nil(t, err)
assert.Equal(t, []string{"echo", "foo"}, m.Args())
m, err = Shlex("echo \"foo")(Meta{})
assert.Error(t, err)
}

162
client/llb/state.go Normal file
View File

@ -0,0 +1,162 @@
package llb
import (
_ "crypto/sha256"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type StateOption func(s *State) *State
// State represents modifiable llb state
type State struct {
source *source
exec *exec
meta Meta
mount *mount
metaNext Meta // this meta will be used for the next Run()
err error
}
// TODO: add state.Reset() state.Save() state.Restore()
// Validate checks that every node has been set up properly
func (s *State) Validate() error {
if s.source != nil {
if err := s.source.Validate(); err != nil {
return err
}
}
if s.exec != nil {
if err := s.exec.Validate(); err != nil {
return err
}
}
return nil
}
func (s *State) Run(opts ...RunOption) *ExecState {
var es ExecState
meta := s.metaNext
var err error
for _, o := range opts {
meta, err = o(meta)
}
exec := &exec{
meta: meta,
mounts: []*mount{},
root: &mount{
dest: "/",
source: s.source,
parent: s.mount,
hasOutput: true,
},
}
exec.root.execState = &es
exec.mounts = append(exec.mounts, exec.root)
es.exec = exec
es.mount = exec.root
es.metaNext = meta
es.meta = meta
es.err = err
return &es
}
func (s *State) AddEnv(key, value string, v ...interface{}) *State {
s.metaNext, _ = AddEnv(key, value, v...)(s.metaNext)
return s
}
func (s *State) DelEnv(key string) *State {
s.metaNext, _ = DelEnv(key)(s.metaNext)
return s
}
func (s *State) ClearEnv() *State {
s.metaNext, _ = ClearEnv()(s.metaNext)
return s
}
func (s *State) GetEnv(key string) (string, bool) {
return s.metaNext.Env(key)
}
func (s *State) Dir(str string, v ...interface{}) *State {
s.metaNext, _ = Dir(str, v...)(s.metaNext)
return s
}
func (s *State) GetDir() string {
return s.metaNext.Dir()
}
func (s *State) Args(args ...string) *State {
s.metaNext, _ = Args(args...)(s.metaNext)
return s
}
func (s *State) Reset(src *State) *State {
copy := *s
copy.metaNext, _ = Reset(src)(s.metaNext)
return &copy
}
func (s *State) With(so ...StateOption) *State {
for _, o := range so {
s = o(s)
}
return s
}
func (s *State) Marshal() (list [][]byte, err error) {
if err := s.Validate(); err != nil {
return nil, err
}
cache := make(map[digest.Digest]struct{})
if s.source != nil {
_, list, err = s.source.marshalTo(nil, cache)
} else if s.exec != nil {
_, list, err = s.exec.root.marshalTo(nil, cache)
} else {
_, list, err = s.mount.marshalTo(nil, cache)
}
return
}
// ExecState is a state with a active leaf pointing to a run exec command.
// Mounts can be added only to this state.
type ExecState struct {
State
}
func (s *ExecState) AddMount(dest string, mountState *State) *State {
m := &mount{
dest: dest,
source: mountState.source,
parent: mountState.mount,
execState: s,
hasOutput: true, // TODO: should be set only if something inherits
}
var newState State
newState.meta = s.meta
newState.metaNext = s.metaNext
newState.mount = m
s.exec.mounts = append(s.exec.mounts, m)
return &newState
}
func (s *ExecState) Root() *State {
return &s.State
}
func (s *ExecState) GetMount(target string) (*State, error) {
for _, m := range s.exec.mounts {
if m.dest == target {
var newState State
newState.meta = m.execState.meta
newState.metaNext = m.execState.metaNext
newState.mount = m
return &newState, nil
}
}
return nil, errors.WithStack(errNotFound)
}

28
client/llb/state_test.go Normal file
View File

@ -0,0 +1,28 @@
package llb
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestStateMeta(t *testing.T) {
s := Source("foo")
s = s.AddEnv("BAR", "abc").Dir("/foo/bar")
v, ok := s.GetEnv("BAR")
assert.True(t, ok)
assert.Equal(t, "abc", v)
assert.Equal(t, "/foo/bar", s.GetDir())
s2 := Source("foo2")
s2 = s2.AddEnv("BAZ", "def").Reset(s)
_, ok = s2.GetEnv("BAZ")
assert.False(t, ok)
v, ok = s2.GetEnv("BAR")
assert.True(t, ok)
assert.Equal(t, "abc", v)
}