JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

llb: force platform in llb and allow constraints 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2018-06-21 14:09:38 -07:00
parent 25d975ce6e
commit dc9de85069
9 changed files with 220 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
client/llb/exec.go
View File

@ -17,8 +17,8 @@ type Meta struct {
ProxyEnv *ProxyEnv
}
func NewExecOp(root Output, meta Meta, readOnly bool, md OpMetadata) *ExecOp {
e := &ExecOp{meta: meta, cachedOpMetadata: md}
func NewExecOp(root Output, meta Meta, readOnly bool, c Constraints) *ExecOp {
e := &ExecOp{meta: meta, constraints: c}
rootMount := &mount{
target: pb.RootMount,
source: root,
@ -48,13 +48,12 @@ type mount struct {
}
type ExecOp struct {
root Output
mounts []*mount
meta Meta
cachedPBDigest digest.Digest
cachedPB []byte
cachedOpMetadata OpMetadata
isValidated bool
MarshalCache
root Output
mounts []*mount
meta Meta
constraints Constraints
isValidated bool
}
func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Output {
@ -73,7 +72,7 @@ func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Outp
} else {
m.output = &output{vertex: e, getIndex: e.getMountIndexFn(m)}
}
e.cachedPB = nil
e.Store(nil, nil, nil)
e.isValidated = false
return m.output
}
@ -108,9 +107,9 @@ func (e *ExecOp) Validate() error {
return nil
}
func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if e.cachedPB != nil {
return e.cachedPBDigest, e.cachedPB, &e.cachedOpMetadata, nil
func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
if e.Cached(c) {
return e.Load()
}
if err := e.Validate(); err != nil {
return "", nil, nil, err
@ -138,10 +137,9 @@ func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
}
}
pop := &pb.Op{
Op: &pb.Op_Exec{
Exec: peo,
},
pop, md := MarshalConstraints(c, &e.constraints)
pop.Op = &pb.Op_Exec{
Exec: peo,
}
outIndex := 0
@ -151,7 +149,7 @@ func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if m.tmpfs {
return "", nil, nil, errors.Errorf("tmpfs mounts must use scratch")
}
inp, err := m.source.ToInput()
inp, err := m.source.ToInput(c)
if err != nil {
return "", nil, nil, err
}
@ -210,9 +208,8 @@ func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if err != nil {
return "", nil, nil, err
}
e.cachedPBDigest = digest.FromBytes(dt)
e.cachedPB = dt
return e.cachedPBDigest, dt, &e.cachedOpMetadata, nil
e.Store(dt, md, c)
return e.Load()
}
func (e *ExecOp) Output() Output {
@ -376,7 +373,7 @@ func WithProxy(ps ProxyEnv) RunOption {
}
type ExecInfo struct {
opMetaWrapper
constraintsWrapper
State State
Mounts []MountInfo
ReadonlyRootFS bool

43
client/llb/llbbuild/llbbuild.go
View File

@ -17,19 +17,20 @@ func NewBuildOp(source llb.Output, opt ...BuildOption) llb.Vertex {
for _, o := range opt {
o(info)
}
return &build{source: source, info: info, cachedOpMetadata: info.OpMetadata}
return &build{source: source, info: info, constraints: info.Constraints}
}
type build struct {
source llb.Output
info *BuildInfo
cachedPBDigest digest.Digest
cachedPB []byte
cachedOpMetadata llb.OpMetadata
llb.MarshalCache
source llb.Output
info *BuildInfo
cachedPBDigest digest.Digest
cachedPB []byte
constraints llb.Constraints
}
func (b *build) ToInput() (*pb.Input, error) {
dgst, _, _, err := b.Marshal()
func (b *build) ToInput(c *llb.Constraints) (*pb.Input, error) {
dgst, _, _, err := b.Marshal(c)
if err != nil {
return nil, err
}
@ -44,9 +45,9 @@ func (b *build) Validate() error {
return nil
}
func (b *build) Marshal() (digest.Digest, []byte, *llb.OpMetadata, error) {
if b.cachedPB != nil {
return b.cachedPBDigest, b.cachedPB, &b.cachedOpMetadata, nil
func (b *build) Marshal(c *llb.Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
if b.Cached(c) {
return b.Load()
}
pbo := &pb.BuildOp{
Builder: pb.LLBBuilder,
@ -60,13 +61,12 @@ func (b *build) Marshal() (digest.Digest, []byte, *llb.OpMetadata, error) {
pbo.Attrs[pb.AttrLLBDefinitionFilename] = b.info.DefinitionFilename
}
pop := &pb.Op{
Op: &pb.Op_Build{
Build: pbo,
},
pop, md := llb.MarshalConstraints(c, &b.constraints)
pop.Op = &pb.Op_Build{
Build: pbo,
}
inp, err := b.source.ToInput()
inp, err := b.source.ToInput(c)
if err != nil {
return "", nil, nil, err
}
@ -77,9 +77,8 @@ func (b *build) Marshal() (digest.Digest, []byte, *llb.OpMetadata, error) {
if err != nil {
return "", nil, nil, err
}
b.cachedPB = dt
b.cachedPBDigest = digest.FromBytes(dt)
return b.cachedPBDigest, dt, &b.cachedOpMetadata, nil
b.Store(dt, md, c)
return b.Load()
}
func (b *build) Output() llb.Output {
@ -91,7 +90,7 @@ func (b *build) Inputs() []llb.Output {
}
type BuildInfo struct {
llb.OpMetadata
llb.Constraints
DefinitionFilename string
}
@ -103,8 +102,8 @@ func WithFilename(fn string) BuildOption {
}
}
func WithMetadata(md llb.MetadataOpt) BuildOption {
func WithConstraints(co llb.ConstraintsOpt) BuildOption {
return func(b *BuildInfo) {
md.SetMetadataOption(&b.OpMetadata)
co.SetConstraintsOption(&b.Constraints)
}
}

4
client/llb/llbbuild/llbbuild_test.go
View File

@ -12,7 +12,7 @@ import (
func TestMarshal(t *testing.T) {
t.Parallel()
b := NewBuildOp(newDummyOutput("foobar"), WithFilename("myfilename"))
dgst, dt, opMeta, err := b.Marshal()
dgst, dt, opMeta, err := b.Marshal(&llb.Constraints{})
_ = opMeta
require.NoError(t, err)
@ -42,7 +42,7 @@ type dummyOutput struct {
dgst digest.Digest
}
func (d *dummyOutput) ToInput() (*pb.Input, error) {
func (d *dummyOutput) ToInput(*llb.Constraints) (*pb.Input, error) {
return &pb.Input{
Digest: d.dgst,
Index: pb.OutputIndex(7), // random constant

62
client/llb/marshal.go
View File

@ -4,6 +4,7 @@ import (
"io"
"io/ioutil"
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
)
@ -12,11 +13,11 @@ import (
// Corresponds to the Definition structure defined in solver/pb.Definition.
type Definition struct {
Def [][]byte
Metadata map[digest.Digest]OpMetadata
Metadata map[digest.Digest]pb.OpMetadata
}
func (def *Definition) ToPB() *pb.Definition {
md := make(map[digest.Digest]OpMetadata)
md := make(map[digest.Digest]pb.OpMetadata)
for k, v := range def.Metadata {
md[k] = v
}
@ -28,14 +29,12 @@ func (def *Definition) ToPB() *pb.Definition {
func (def *Definition) FromPB(x *pb.Definition) {
def.Def = x.Def
def.Metadata = make(map[digest.Digest]OpMetadata)
def.Metadata = make(map[digest.Digest]pb.OpMetadata)
for k, v := range x.Metadata {
def.Metadata[k] = v
}
}
type OpMetadata = pb.OpMetadata
func WriteTo(def *Definition, w io.Writer) error {
b, err := def.ToPB().Marshal()
if err != nil {
@ -58,3 +57,56 @@ func ReadFrom(r io.Reader) (*Definition, error) {
def.FromPB(&pbDef)
return &def, nil
}
func MarshalConstraints(base, override *Constraints) (*pb.Op, *pb.OpMetadata) {
c := base
c.WorkerConstraints = append([]string{}, c.WorkerConstraints...)
if p := override.Platform; p != nil {
c.Platform = p
}
for _, wc := range override.WorkerConstraints {
c.WorkerConstraints = append(c.WorkerConstraints, wc)
}
c.Metadata = mergeMetadata(c.Metadata, override.Metadata)
if c.Platform == nil {
defaultPlatform := platforms.Normalize(platforms.DefaultSpec())
c.Platform = &defaultPlatform
}
return &pb.Op{
Platform: &pb.Platform{
OS: c.Platform.OS,
Architecture: c.Platform.Architecture,
Variant: c.Platform.Variant,
OSVersion: c.Platform.OSVersion,
OSFeatures: c.Platform.OSFeatures,
},
Constraints: &pb.WorkerConstraints{
Filter: c.WorkerConstraints,
},
}, &c.Metadata
}
type MarshalCache struct {
digest digest.Digest
dt []byte
md *pb.OpMetadata
constraints *Constraints
}
func (mc *MarshalCache) Cached(c *Constraints) bool {
return mc.dt != nil && mc.constraints == c
}
func (mc *MarshalCache) Load() (digest.Digest, []byte, *pb.OpMetadata, error) {
return mc.digest, mc.dt, mc.md, nil
}
func (mc *MarshalCache) Store(dt []byte, md *pb.OpMetadata, c *Constraints) {
mc.digest = digest.FromBytes(dt)
mc.dt = dt
mc.md = md
mc.constraints = c
}

53
client/llb/source.go
View File

@ -15,20 +15,19 @@ import (
)
type SourceOp struct {
id string
attrs map[string]string
output Output
cachedPBDigest digest.Digest
cachedPB []byte
cachedOpMetadata OpMetadata
err error
MarshalCache
id string
attrs map[string]string
output Output
constraints Constraints
err error
}
func NewSource(id string, attrs map[string]string, md OpMetadata) *SourceOp {
func NewSource(id string, attrs map[string]string, c Constraints) *SourceOp {
s := &SourceOp{
id: id,
attrs: attrs,
cachedOpMetadata: md,
id: id,
attrs: attrs,
constraints: c,
}
s.output = &output{vertex: s}
return s
@ -44,26 +43,26 @@ func (s *SourceOp) Validate() error {
return nil
}
func (s *SourceOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if s.cachedPB != nil {
return s.cachedPBDigest, s.cachedPB, &s.cachedOpMetadata, nil
func (s *SourceOp) Marshal(constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
if s.Cached(constraints) {
return s.Load()
}
if err := s.Validate(); err != nil {
return "", nil, nil, err
}
proto := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{Identifier: s.id, Attrs: s.attrs},
},
proto, md := MarshalConstraints(constraints, &s.constraints)
proto.Op = &pb.Op_Source{
Source: &pb.SourceOp{Identifier: s.id, Attrs: s.attrs},
}
dt, err := proto.Marshal()
if err != nil {
return "", nil, nil, err
}
s.cachedPB = dt
s.cachedPBDigest = digest.FromBytes(dt)
return s.cachedPBDigest, dt, &s.cachedOpMetadata, nil
s.Store(dt, md, constraints)
return s.Load()
}
func (s *SourceOp) Output() Output {
@ -74,10 +73,6 @@ func (s *SourceOp) Inputs() []Output {
return nil
}
func Source(id string) State {
return NewState(NewSource(id, nil, OpMetadata{}).Output())
}
func Image(ref string, opts ...ImageOption) State {
r, err := reference.ParseNormalizedNamed(ref)
if err == nil {
@ -136,7 +131,7 @@ func (fn ImageOptionFunc) SetImageOption(ii *ImageInfo) {
}
type ImageInfo struct {
opMetaWrapper
constraintsWrapper
metaResolver ImageMetaResolver
}
@ -183,7 +178,7 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {
}
type GitInfo struct {
opMetaWrapper
constraintsWrapper
KeepGitDir bool
}
@ -280,7 +275,7 @@ func SharedKeyHint(h string) LocalOption {
}
type LocalInfo struct {
opMetaWrapper
constraintsWrapper
SessionID string
IncludePatterns string
ExcludePatterns string
@ -315,7 +310,7 @@ func HTTP(url string, opts ...HTTPOption) State {
}
type HTTPInfo struct {
opMetaWrapper
constraintsWrapper
Checksum digest.Digest
Filename string
Perm int

141
client/llb/state.go
View File

@ -3,21 +3,23 @@ package llb
import (
"context"
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/system"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
type StateOption func(State) State
type Output interface {
ToInput() (*pb.Input, error)
ToInput(*Constraints) (*pb.Input, error)
Vertex() Vertex
}
type Vertex interface {
Validate() error
Marshal() (digest.Digest, []byte, *OpMetadata, error)
Marshal(*Constraints) (digest.Digest, []byte, *pb.OpMetadata, error)
Output() Output
Inputs() []Output
}
@ -48,18 +50,27 @@ func (s State) Value(k interface{}) interface{} {
return s.ctx.Value(k)
}
func (s State) Marshal(md ...MetadataOpt) (*Definition, error) {
func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
def := &Definition{
Metadata: make(map[digest.Digest]OpMetadata, 0),
Metadata: make(map[digest.Digest]pb.OpMetadata, 0),
}
if s.Output() == nil {
return def, nil
}
def, err := marshal(s.Output().Vertex(), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, md)
defaultPlatform := platforms.Normalize(platforms.DefaultSpec())
c := &Constraints{
Platform: &defaultPlatform,
}
for _, o := range co {
o.SetConstraintsOption(c)
}
def, err := marshal(s.Output().Vertex(), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, c)
if err != nil {
return def, err
}
inp, err := s.Output().ToInput()
inp, err := s.Output().ToInput(c)
if err != nil {
return def, err
}
@ -72,29 +83,25 @@ func (s State) Marshal(md ...MetadataOpt) (*Definition, error) {
return def, nil
}
func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, md []MetadataOpt) (*Definition, error) {
func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, c *Constraints) (*Definition, error) {
if _, ok := vertexCache[v]; ok {
return def, nil
}
for _, inp := range v.Inputs() {
var err error
def, err = marshal(inp.Vertex(), def, cache, vertexCache, md)
def, err = marshal(inp.Vertex(), def, cache, vertexCache, c)
if err != nil {
return def, err
}
}
dgst, dt, opMeta, err := v.Marshal()
dgst, dt, opMeta, err := v.Marshal(c)
if err != nil {
return def, err
}
vertexCache[v] = struct{}{}
if opMeta != nil {
m := mergeMetadata(def.Metadata[dgst], *opMeta)
for _, f := range md {
f.SetMetadataOption(&m)
}
def.Metadata[dgst] = m
def.Metadata[dgst] = mergeMetadata(def.Metadata[dgst], *opMeta)
}
if _, ok := cache[dgst]; ok {
return def, nil
@ -191,7 +198,7 @@ type output struct {
err error
}
func (o *output) ToInput() (*pb.Input, error) {
func (o *output) ToInput(c *Constraints) (*pb.Input, error) {
if o.err != nil {
return nil, o.err
}
@ -203,7 +210,7 @@ func (o *output) ToInput() (*pb.Input, error) {
return nil, err
}
}
dgst, _, _, err := o.vertex.Marshal()
dgst, _, _, err := o.vertex.Marshal(c)
if err != nil {
return nil, err
}
@ -214,8 +221,8 @@ func (o *output) Vertex() Vertex {
return o.vertex
}
type MetadataOpt interface {
SetMetadataOption(*OpMetadata)
type ConstraintsOpt interface {
SetConstraintsOption(*Constraints)
RunOption
LocalOption
HTTPOption
@ -223,33 +230,33 @@ type MetadataOpt interface {
GitOption
}
type metadataOptFunc func(m *OpMetadata)
type constraintsOptFunc func(m *Constraints)
func (fn metadataOptFunc) SetMetadataOption(m *OpMetadata) {
func (fn constraintsOptFunc) SetConstraintsOption(m *Constraints) {
fn(m)
}
func (fn metadataOptFunc) SetRunOption(ei *ExecInfo) {
ei.ApplyMetadata(fn)
func (fn constraintsOptFunc) SetRunOption(ei *ExecInfo) {
ei.applyConstraints(fn)
}
func (fn metadataOptFunc) SetLocalOption(li *LocalInfo) {
li.ApplyMetadata(fn)
func (fn constraintsOptFunc) SetLocalOption(li *LocalInfo) {
li.applyConstraints(fn)
}
func (fn metadataOptFunc) SetHTTPOption(hi *HTTPInfo) {
hi.ApplyMetadata(fn)
func (fn constraintsOptFunc) SetHTTPOption(hi *HTTPInfo) {
hi.applyConstraints(fn)
}
func (fn metadataOptFunc) SetImageOption(ii *ImageInfo) {
ii.ApplyMetadata(fn)
func (fn constraintsOptFunc) SetImageOption(ii *ImageInfo) {
ii.applyConstraints(fn)
}
func (fn metadataOptFunc) SetGitOption(gi *GitInfo) {
gi.ApplyMetadata(fn)
func (fn constraintsOptFunc) SetGitOption(gi *GitInfo) {
gi.applyConstraints(fn)
}
func mergeMetadata(m1, m2 OpMetadata) OpMetadata {
func mergeMetadata(m1, m2 pb.OpMetadata) pb.OpMetadata {
if m2.IgnoreCache {
m1.IgnoreCache = true
}
@ -268,49 +275,81 @@ func mergeMetadata(m1, m2 OpMetadata) OpMetadata {
return m1
}
var IgnoreCache = metadataOptFunc(func(md *OpMetadata) {
md.IgnoreCache = true
var IgnoreCache = constraintsOptFunc(func(c *Constraints) {
c.Metadata.IgnoreCache = true
})
func WithDescription(m map[string]string) MetadataOpt {
return metadataOptFunc(func(md *OpMetadata) {
md.Description = m
func WithDescription(m map[string]string) ConstraintsOpt {
return constraintsOptFunc(func(c *Constraints) {
c.Metadata.Description = m
})
}
// WithExportCache forces results for this vertex to be exported with the cache
func WithExportCache() MetadataOpt {
return metadataOptFunc(func(md *OpMetadata) {
md.ExportCache = &pb.ExportCache{Value: true}
func WithExportCache() ConstraintsOpt {
return constraintsOptFunc(func(c *Constraints) {
c.Metadata.ExportCache = &pb.ExportCache{Value: true}
})
}
// WithoutExportCache sets results for this vertex to be not exported with
// the cache
func WithoutExportCache() MetadataOpt {
return metadataOptFunc(func(md *OpMetadata) {
func WithoutExportCache() ConstraintsOpt {
return constraintsOptFunc(func(c *Constraints) {
// ExportCache with value false means to disable exporting
md.ExportCache = &pb.ExportCache{Value: false}
c.Metadata.ExportCache = &pb.ExportCache{Value: false}
})
}
// WithoutDefaultExportCache resets the cache export for the vertex to use
// the default defined by the build configuration.
func WithoutDefaultExportCache() MetadataOpt {
return metadataOptFunc(func(md *OpMetadata) {
func WithoutDefaultExportCache() ConstraintsOpt {
return constraintsOptFunc(func(c *Constraints) {
// nil means no vertex based config has been set
md.ExportCache = nil
c.Metadata.ExportCache = nil
})
}
type opMetaWrapper struct {
OpMetadata
type constraintsWrapper struct {
Constraints
}
func (mw *opMetaWrapper) ApplyMetadata(f func(m *OpMetadata)) {
f(&mw.OpMetadata)
func (cw *constraintsWrapper) applyConstraints(f func(c *Constraints)) {
f(&cw.Constraints)
}
func (mw *opMetaWrapper) Metadata() OpMetadata {
return mw.OpMetadata
func (cw *constraintsWrapper) Metadata() Constraints {
return cw.Constraints
}
type Constraints struct {
Platform *specs.Platform
WorkerConstraints []string
Metadata pb.OpMetadata
}
func Platform(p specs.Platform) ConstraintsOpt {
return constraintsOptFunc(func(c *Constraints) {
c.Platform = &p
})
}
var (
LinuxAmd64 = Platform(specs.Platform{OS: "linux", Architecture: "amd64"})
LinuxArmhf = Platform(specs.Platform{OS: "linux", Architecture: "arm", Variant: "v7"})
LinuxArm = LinuxArmhf
LinuxArmel = Platform(specs.Platform{OS: "linux", Architecture: "arm", Variant: "v6"})
LinuxArm64 = Platform(specs.Platform{OS: "linux", Architecture: "arm64"})
LinuxS390x = Platform(specs.Platform{OS: "linux", Architecture: "s390x"})
LinuxPpc64le = Platform(specs.Platform{OS: "linux", Architecture: "ppc64le"})
Darwin = Platform(specs.Platform{OS: "darwin", Architecture: "amd64"})
Windows = Platform(specs.Platform{OS: "windows", Architecture: "amd64"})
)
func Require(filters ...string) ConstraintsOpt {
return constraintsOptFunc(func(c *Constraints) {
for _, f := range filters {
c.WorkerConstraints = append(c.WorkerConstraints, f)
}
})
}

4
client/llb/state_test.go
View File

@ -9,7 +9,7 @@ import (
func TestStateMeta(t *testing.T) {
t.Parallel()
s := Source("foo")
s := Image("foo")
s = s.AddEnv("BAR", "abc").Dir("/foo/bar")
v, ok := s.GetEnv("BAR")
@ -18,7 +18,7 @@ func TestStateMeta(t *testing.T) {
assert.Equal(t, "/foo/bar", s.GetDir())
s2 := Source("foo2")
s2 := Image("foo2")
s2 = s2.AddEnv("BAZ", "def").Reset(s)
_, ok = s2.GetEnv("BAZ")

7
cmd/buildctl/build.go
View File

@ -87,10 +87,11 @@ func read(r io.Reader, clicontext *cli.Context) (*llb.Definition, error) {
dgst := digest.FromBytes(dt)
opMetadata, ok := def.Metadata[dgst]
if !ok {
opMetadata = llb.OpMetadata{}
opMetadata = pb.OpMetadata{}
}
llb.IgnoreCache(&opMetadata)
def.Metadata[dgst] = opMetadata
c := llb.Constraints{Metadata: opMetadata}
llb.IgnoreCache(&c)
def.Metadata[dgst] = c.Metadata
}
}
return def, nil

2
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -771,7 +771,7 @@ func getArgValue(arg instructions.ArgCommand) string {
return v
}
func dfCmd(cmd interface{}) llb.MetadataOpt {
func dfCmd(cmd interface{}) llb.ConstraintsOpt {
// TODO: add fmt.Stringer to instructions.Command to remove interface{}
var cmdStr string
if cmd, ok := cmd.(fmt.Stringer); ok {