JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

llb: allow defining metadata on nodes 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2017-12-06 11:36:47 -08:00
parent d66501e254
commit 303d4ccab9
9 changed files with 199 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
client/llb/exec.go
View File

@ -14,8 +14,8 @@ type Meta struct {
Cwd string
}
func NewExecOp(root Output, meta Meta, readOnly bool) *ExecOp {
e := &ExecOp{meta: meta}
func NewExecOp(root Output, meta Meta, readOnly bool, md OpMetadata) *ExecOp {
e := &ExecOp{meta: meta, cachedOpMetadata: md}
rootMount := &mount{
target: pb.RootMount,
source: root,
@ -46,7 +46,7 @@ type ExecOp struct {
mounts []*mount
meta Meta
cachedPB []byte
cachedOpMetadata *pb.OpMetadata
cachedOpMetadata OpMetadata
}
func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Output {
@ -93,9 +93,9 @@ func (e *ExecOp) Validate() error {
return nil
}
func (e *ExecOp) Marshal() ([]byte, *pb.OpMetadata, error) {
func (e *ExecOp) Marshal() ([]byte, *OpMetadata, error) {
if e.cachedPB != nil {
return e.cachedPB, e.cachedOpMetadata, nil
return e.cachedPB, &e.cachedOpMetadata, nil
}
if err := e.Validate(); err != nil {
return nil, nil, err
@ -166,8 +166,7 @@ func (e *ExecOp) Marshal() ([]byte, *pb.OpMetadata, error) {
return nil, nil, err
}
e.cachedPB = dt
e.cachedOpMetadata = &pb.OpMetadata{}
return dt, e.cachedOpMetadata, nil
return dt, &e.cachedOpMetadata, nil
}
func (e *ExecOp) Output() Output {
@ -237,23 +236,29 @@ func SourcePath(src string) MountOption {
}
}
type RunOption func(es ExecInfo) ExecInfo
type RunOption interface {
SetRunOption(es *ExecInfo)
}
type runOptionFunc func(*ExecInfo)
func (fn runOptionFunc) SetRunOption(ei *ExecInfo) {
fn(ei)
}
func Shlex(str string) RunOption {
return Shlexf(str)
}
func Shlexf(str string, v ...interface{}) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = shlexf(str, v...)(ei.State)
return ei
}
})
}
func Args(a []string) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = args(a...)(ei.State)
return ei
}
})
}
func AddEnv(key, value string) RunOption {
@ -261,41 +266,36 @@ func AddEnv(key, value string) RunOption {
}
func AddEnvf(key, value string, v ...interface{}) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.AddEnvf(key, value, v...)
return ei
}
})
}
func Dir(str string) RunOption {
return Dirf(str)
}
func Dirf(str string, v ...interface{}) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.Dirf(str, v...)
return ei
}
})
}
func Reset(s State) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.Reset(s)
return ei
}
})
}
func With(so ...StateOption) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.With(so...)
return ei
}
})
}
func AddMount(dest string, mountState State, opts ...MountOption) RunOption {
return func(ei ExecInfo) ExecInfo {
return runOptionFunc(func(ei *ExecInfo) {
ei.Mounts = append(ei.Mounts, MountInfo{dest, mountState.Output(), opts})
return ei
}
})
}
func ReadonlyRootFS(ei ExecInfo) ExecInfo {
@ -304,6 +304,7 @@ func ReadonlyRootFS(ei ExecInfo) ExecInfo {
}
type ExecInfo struct {
opMetaWrapper
State State
Mounts []MountInfo
ReadonlyRootFS bool

6
client/llb/imagemetaresolver/resolver.go
View File

@ -22,9 +22,9 @@ import (
var defaultImageMetaResolver llb.ImageMetaResolver
var defaultImageMetaResolverOnce sync.Once
func WithDefault(ii *llb.ImageInfo) {
llb.WithMetaResolver(Default())(ii)
}
var WithDefault = llb.ImageOptionFunc(func(ii *llb.ImageInfo) {
llb.WithMetaResolver(Default()).SetImageOption(ii)
})
func New() llb.ImageMetaResolver {
return &imageMetaResolver{

18
client/llb/llbbuild/llbbuild.go
View File

@ -17,14 +17,14 @@ func NewBuildOp(source llb.Output, opt ...BuildOption) llb.Vertex {
for _, o := range opt {
o(info)
}
return &build{source: source, info: info}
return &build{source: source, info: info, cachedOpMetadata: info.OpMetadata}
}
type build struct {
source llb.Output
info *BuildInfo
cachedPB []byte
cachedOpMetadata *pb.OpMetadata
cachedOpMetadata llb.OpMetadata
}
func (b *build) ToInput() (*pb.Input, error) {
@ -45,9 +45,9 @@ func (b *build) Validate() error {
return nil
}
func (b *build) Marshal() ([]byte, *pb.OpMetadata, error) {
func (b *build) Marshal() ([]byte, *llb.OpMetadata, error) {
if b.cachedPB != nil {
return b.cachedPB, b.cachedOpMetadata, nil
return b.cachedPB, &b.cachedOpMetadata, nil
}
pbo := &pb.BuildOp{
Builder: pb.LLBBuilder,
@ -79,8 +79,7 @@ func (b *build) Marshal() ([]byte, *pb.OpMetadata, error) {
return nil, nil, err
}
b.cachedPB = dt
b.cachedOpMetadata = &pb.OpMetadata{}
return dt, b.cachedOpMetadata, nil
return dt, &b.cachedOpMetadata, nil
}
func (b *build) Output() llb.Output {
@ -92,6 +91,7 @@ func (b *build) Inputs() []llb.Output {
}
type BuildInfo struct {
llb.OpMetadata
DefinitionFilename string
}
@ -102,3 +102,9 @@ func WithFilename(fn string) BuildOption {
b.DefinitionFilename = fn
}
}
func WithMetadata(md llb.MetadataOpt) BuildOption {
return func(b *BuildInfo) {
md.SetMetadataOption(&b.OpMetadata)
}
}

10
client/llb/marshal.go
View File

@ -16,9 +16,9 @@ type Definition struct {
}
func (def *Definition) ToPB() *pb.Definition {
md := make(map[digest.Digest]pb.OpMetadata)
md := make(map[digest.Digest]OpMetadata)
for k, v := range def.Metadata {
md[k] = v.OpMetadata
md[k] = v
}
return &pb.Definition{
Def: def.Def,
@ -30,13 +30,11 @@ func (def *Definition) FromPB(x *pb.Definition) {
def.Def = x.Def
def.Metadata = make(map[digest.Digest]OpMetadata)
for k, v := range x.Metadata {
def.Metadata[k] = OpMetadata{v}
def.Metadata[k] = v
}
}
type OpMetadata struct {
pb.OpMetadata
}
type OpMetadata = pb.OpMetadata
func WriteTo(def *Definition, w io.Writer) error {
b, err := def.ToPB().Marshal()

4
client/llb/resolver.go
View File

@ -6,9 +6,9 @@ import (
)
func WithMetaResolver(mr ImageMetaResolver) ImageOption {
return func(ii *ImageInfo) {
return ImageOptionFunc(func(ii *ImageInfo) {
ii.metaResolver = mr
}
})
}
type ImageMetaResolver interface {

108
client/llb/source.go
View File

@ -19,14 +19,15 @@ type SourceOp struct {
attrs map[string]string
output Output
cachedPB []byte
cachedOpMetadata *pb.OpMetadata
cachedOpMetadata OpMetadata
err error
}
func NewSource(id string, attrs map[string]string) *SourceOp {
func NewSource(id string, attrs map[string]string, md OpMetadata) *SourceOp {
s := &SourceOp{
id: id,
attrs: attrs,
cachedOpMetadata: md,
}
s.output = &output{vertex: s}
return s
@ -42,9 +43,9 @@ func (s *SourceOp) Validate() error {
return nil
}
func (s *SourceOp) Marshal() ([]byte, *pb.OpMetadata, error) {
func (s *SourceOp) Marshal() ([]byte, *OpMetadata, error) {
if s.cachedPB != nil {
return s.cachedPB, s.cachedOpMetadata, nil
return s.cachedPB, &s.cachedOpMetadata, nil
}
if err := s.Validate(); err != nil {
return nil, nil, err
@ -60,8 +61,7 @@ func (s *SourceOp) Marshal() ([]byte, *pb.OpMetadata, error) {
return nil, nil, err
}
s.cachedPB = dt
s.cachedOpMetadata = &pb.OpMetadata{}
return dt, s.cachedOpMetadata, nil
return dt, &s.cachedOpMetadata, nil
}
func (s *SourceOp) Output() Output {
@ -73,7 +73,7 @@ func (s *SourceOp) Inputs() []Output {
}
func Source(id string) State {
return NewState(NewSource(id, nil).Output())
return NewState(NewSource(id, nil, OpMetadata{}).Output())
}
func Image(ref string, opts ...ImageOption) State {
@ -81,13 +81,13 @@ func Image(ref string, opts ...ImageOption) State {
if err == nil {
ref = reference.TagNameOnly(r).String()
}
src := NewSource("docker-image://"+ref, nil) // controversial
if err != nil {
src.err = err
}
var info ImageInfo
for _, opt := range opts {
opt(&info)
opt.SetImageOption(&info)
}
src := NewSource("docker-image://"+ref, nil, info.Metadata()) // controversial
if err != nil {
src.err = err
}
if info.metaResolver != nil {
_, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref)
@ -123,9 +123,18 @@ func Image(ref string, opts ...ImageOption) State {
return NewState(src.Output())
}
type ImageOption func(*ImageInfo)
type ImageOption interface {
SetImageOption(*ImageInfo)
}
type ImageOptionFunc func(*ImageInfo)
func (fn ImageOptionFunc) SetImageOption(ii *ImageInfo) {
fn(ii)
}
type ImageInfo struct {
opMetaWrapper
metaResolver ImageMetaResolver
}
@ -137,27 +146,34 @@ func Git(remote, ref string, opts ...GitOption) State {
gi := &GitInfo{}
for _, o := range opts {
o(gi)
o.SetGitOption(gi)
}
attrs := map[string]string{}
if gi.KeepGitDir {
attrs[pb.AttrKeepGitDir] = "true"
}
source := NewSource("git://"+id, attrs)
source := NewSource("git://"+id, attrs, gi.Metadata())
return NewState(source.Output())
}
type GitOption func(*GitInfo)
type GitOption interface {
SetGitOption(*GitInfo)
}
type gitOptionFunc func(*GitInfo)
func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {
fn(gi)
}
type GitInfo struct {
opMetaWrapper
KeepGitDir bool
}
func KeepGitDir() GitOption {
return func(gi *GitInfo) {
return gitOptionFunc(func(gi *GitInfo) {
gi.KeepGitDir = true
}
})
}
func Scratch() State {
@ -168,7 +184,7 @@ func Local(name string, opts ...LocalOption) State {
gi := &LocalInfo{}
for _, o := range opts {
o(gi)
o.SetLocalOption(gi)
}
attrs := map[string]string{}
if gi.SessionID != "" {
@ -178,26 +194,35 @@ func Local(name string, opts ...LocalOption) State {
attrs[pb.AttrIncludePatterns] = gi.IncludePatterns
}
source := NewSource("local://"+name, attrs)
source := NewSource("local://"+name, attrs, gi.Metadata())
return NewState(source.Output())
}
type LocalOption func(*LocalInfo)
type LocalOption interface {
SetLocalOption(*LocalInfo)
}
type localOptionFunc func(*LocalInfo)
func (fn localOptionFunc) SetLocalOption(li *LocalInfo) {
fn(li)
}
func SessionID(id string) LocalOption {
return func(li *LocalInfo) {
return localOptionFunc(func(li *LocalInfo) {
li.SessionID = id
}
})
}
func IncludePatterns(p []string) LocalOption {
return func(li *LocalInfo) {
return localOptionFunc(func(li *LocalInfo) {
dt, _ := json.Marshal(p) // empty on error
li.IncludePatterns = string(dt)
}
})
}
type LocalInfo struct {
opMetaWrapper
SessionID string
IncludePatterns string
}
@ -205,7 +230,7 @@ type LocalInfo struct {
func HTTP(url string, opts ...HTTPOption) State {
hi := &HTTPInfo{}
for _, o := range opts {
o(hi)
o.SetHTTPOption(hi)
}
attrs := map[string]string{}
if hi.Checksum != "" {
@ -224,11 +249,12 @@ func HTTP(url string, opts ...HTTPOption) State {
attrs[pb.AttrHTTPGID] = strconv.Itoa(hi.GID)
}
source := NewSource(url, attrs)
source := NewSource(url, attrs, hi.Metadata())
return NewState(source.Output())
}
type HTTPInfo struct {
opMetaWrapper
Checksum digest.Digest
Filename string
Perm int
@ -236,29 +262,37 @@ type HTTPInfo struct {
GID int
}
type HTTPOption func(*HTTPInfo)
type HTTPOption interface {
SetHTTPOption(*HTTPInfo)
}
type httpOptionFunc func(*HTTPInfo)
func (fn httpOptionFunc) SetHTTPOption(hi *HTTPInfo) {
fn(hi)
}
func Checksum(dgst digest.Digest) HTTPOption {
return func(hi *HTTPInfo) {
return httpOptionFunc(func(hi *HTTPInfo) {
hi.Checksum = dgst
}
})
}
func Chmod(perm os.FileMode) HTTPOption {
return func(hi *HTTPInfo) {
return httpOptionFunc(func(hi *HTTPInfo) {
hi.Perm = int(perm) & 0777
}
})
}
func Filename(name string) HTTPOption {
return func(hi *HTTPInfo) {
return httpOptionFunc(func(hi *HTTPInfo) {
hi.Filename = name
}
})
}
func Chown(uid, gid int) HTTPOption {
return func(hi *HTTPInfo) {
return httpOptionFunc(func(hi *HTTPInfo) {
hi.UID = uid
hi.GID = gid
}
})
}

84
client/llb/state.go
View File

@ -17,7 +17,7 @@ type Output interface {
type Vertex interface {
Validate() error
Marshal() ([]byte, *pb.OpMetadata, error)
Marshal() ([]byte, *OpMetadata, error)
Output() Output
Inputs() []Output
}
@ -48,14 +48,14 @@ func (s State) Value(k interface{}) interface{} {
return s.ctx.Value(k)
}
func (s State) Marshal() (*Definition, error) {
func (s State) Marshal(md ...MetadataOpt) (*Definition, error) {
def := &Definition{
Metadata: make(map[digest.Digest]OpMetadata, 0),
}
if s.Output() == nil {
return def, nil
}
def, err := marshal(s.Output().Vertex(), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{})
def, err := marshal(s.Output().Vertex(), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, md)
if err != nil {
return def, err
}
@ -72,10 +72,10 @@ func (s State) Marshal() (*Definition, error) {
return def, nil
}
func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}) (*Definition, error) {
func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, md []MetadataOpt) (*Definition, error) {
for _, inp := range v.Inputs() {
var err error
def, err = marshal(inp.Vertex(), def, cache, vertexCache)
def, err = marshal(inp.Vertex(), def, cache, vertexCache, md)
if err != nil {
return def, err
}
@ -90,13 +90,17 @@ func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertex
}
vertexCache[v] = struct{}{}
dgst := digest.FromBytes(dt)
if opMeta != nil {
m := mergeMetadata(def.Metadata[dgst], *opMeta)
for _, f := range md {
f.SetMetadataOption(&m)
}
def.Metadata[dgst] = m
}
if _, ok := cache[dgst]; ok {
return def, nil
}
def.Def = append(def.Def, dt)
if opMeta != nil {
def.Metadata[dgst] = OpMetadata{*opMeta}
}
cache[dgst] = struct{}{}
return def, nil
}
@ -117,9 +121,9 @@ func (s State) WithOutput(o Output) State {
}
func (s State) Run(ro ...RunOption) ExecState {
ei := ExecInfo{State: s}
ei := &ExecInfo{State: s}
for _, o := range ro {
ei = o(ei)
o.SetRunOption(ei)
}
meta := Meta{
Args: getArgs(ei.State),
@ -127,7 +131,7 @@ func (s State) Run(ro ...RunOption) ExecState {
Env: getEnv(ei.State),
}
exec := NewExecOp(s.Output(), meta, ei.ReadonlyRootFS)
exec := NewExecOp(s.Output(), meta, ei.ReadonlyRootFS, ei.Metadata())
for _, m := range ei.Mounts {
exec.AddMount(m.Target, m.Source, m.Opts...)
}
@ -202,3 +206,61 @@ func (o *output) ToInput() (*pb.Input, error) {
func (o *output) Vertex() Vertex {
return o.vertex
}
type MetadataOpt interface {
SetMetadataOption(*OpMetadata)
RunOption
LocalOption
HTTPOption
ImageOption
GitOption
}
type metadataOptFunc func(m *OpMetadata)
func (fn metadataOptFunc) SetMetadataOption(m *OpMetadata) {
fn(m)
}
func (fn metadataOptFunc) SetRunOption(ei *ExecInfo) {
ei.ApplyMetadata(fn)
}
func (fn metadataOptFunc) SetLocalOption(li *LocalInfo) {
li.ApplyMetadata(fn)
}
func (fn metadataOptFunc) SetHTTPOption(hi *HTTPInfo) {
hi.ApplyMetadata(fn)
}
func (fn metadataOptFunc) SetImageOption(ii *ImageInfo) {
ii.ApplyMetadata(fn)
}
func (fn metadataOptFunc) SetGitOption(gi *GitInfo) {
gi.ApplyMetadata(fn)
}
func mergeMetadata(m1, m2 OpMetadata) OpMetadata {
if m2.IgnoreCache {
m1.IgnoreCache = true
}
return m1
}
var IgnoreCache = metadataOptFunc(func(md *OpMetadata) {
md.IgnoreCache = true
})
type opMetaWrapper struct {
OpMetadata
}
func (mw *opMetaWrapper) ApplyMetadata(f func(m *OpMetadata)) {
f(&mw.OpMetadata)
}
func (mw *opMetaWrapper) Metadata() OpMetadata {
return mw.OpMetadata
}

2
cmd/buildctl/build.go
View File

@ -85,7 +85,7 @@ func read(r io.Reader, clicontext *cli.Context) (*llb.Definition, error) {
if !ok {
opMetadata = llb.OpMetadata{}
}
opMetadata.IgnoreCache = true
llb.IgnoreCache(&opMetadata)
def.Metadata[dgst] = opMetadata
}
}

2
cmd/buildctl/debug/dumpllb.go
View File

@ -74,7 +74,7 @@ func loadLLB(r io.Reader) ([]llbOp, error) {
return nil, errors.Wrap(err, "failed to parse op")
}
dgst := digest.FromBytes(dt)
ent := llbOp{Op: op, Digest: dgst, OpMetadata: def.Metadata[dgst].OpMetadata}
ent := llbOp{Op: op, Digest: dgst, OpMetadata: def.Metadata[dgst]}
ops = append(ops, ent)
}
return ops, nil