From ecf2e8dbc9122e1d7c53895d4e73911f7dab2e64 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 27 Feb 2018 16:06:01 +0900 Subject: [PATCH] llb: update docs Signed-off-by: Akihiro Suda --- README.md | 14 +++++++++++++- solver/pb/ops.pb.go | 33 +++++++++++++++++++++++++++------ solver/pb/ops.proto | 26 +++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 20b7a8d3..392c10d2 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,19 @@ We are open to adding more backends. #### Exploring LLB -BuildKit builds are based on a binary intermediate format called LLB that is used for defining the dependency graph for processes running part of your build. +BuildKit builds are based on a binary intermediate format called LLB that is used for defining the dependency graph for processes running part of your build. tl;dr: LLB is to Dockerfile what LLVM IR is to C. + +- Marshaled as Protobuf messages +- Concurrently executable +- Efficiently cacheable +- Vendor-neutral (i.e. non-Dockerfile languages can be easily implemented) + +See [`solver/pb/ops.proto`](./solver/pb/ops.proto) for the format definition. + +Currently, following high-level languages has been implemented for LLB: + +- Dockerfile (See [Exploring Dockerfiles](#exploring-dockerfiles)) +- (open a PR to add your own language) For understanding the basics of LLB, `examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit itself and its dependencies using the `client` package. Running one of these scripts generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build. diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go index cffaf6be..4c24bfea 100644 --- a/solver/pb/ops.pb.go +++ b/solver/pb/ops.pb.go @@ -4,6 +4,9 @@ /* Package pb is a generated protocol buffer package. + Package pb provides the protobuf definition of LLB: low-level builder instruction. + LLB is DAG-structured; Op represents a vertex, and Definition represents a graph. + It is generated from these files: ops.proto @@ -44,7 +47,9 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +// Op represents a vertex of the LLB DAG. type Op struct { + // inputs is a set of input edges. Inputs []*Input `protobuf:"bytes,1,rep,name=inputs" json:"inputs,omitempty"` // Types that are valid to be assigned to Op: // *Op_Exec @@ -237,9 +242,12 @@ func _Op_OneofSizer(msg proto.Message) (n int) { return n } +// Input represents an input edge for an Op. type Input struct { + // digest of the marshaled input Op Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` - Index OutputIndex `protobuf:"varint,2,opt,name=index,proto3,customtype=OutputIndex" json:"index"` + // output index of the input Op + Index OutputIndex `protobuf:"varint,2,opt,name=index,proto3,customtype=OutputIndex" json:"index"` } func (m *Input) Reset() { *m = Input{} } @@ -247,6 +255,7 @@ func (m *Input) String() string { return proto.CompactTextString(m) } func (*Input) ProtoMessage() {} func (*Input) Descriptor() ([]byte, []int) { return fileDescriptorOps, []int{1} } +// ExecOp executes a command in a container. type ExecOp struct { Meta *Meta `protobuf:"bytes,1,opt,name=meta" json:"meta,omitempty"` Mounts []*Mount `protobuf:"bytes,2,rep,name=mounts" json:"mounts,omitempty"` @@ -271,6 +280,7 @@ func (m *ExecOp) GetMounts() []*Mount { return nil } +// Meta is a set of arguments for ExecOp. // Meta is unrelated to LLB metadata. // FIXME: rename (ExecContext? ExecArgs?) type Meta struct { @@ -313,6 +323,7 @@ func (m *Meta) GetUser() string { return "" } +// Mount specifies how to mount an input Op as a filesystem. type Mount struct { Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"` Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"` @@ -347,6 +358,7 @@ func (m *Mount) GetReadonly() bool { return false } +// CopyOp copies files across Ops. type CopyOp struct { Src []*CopySource `protobuf:"bytes,1,rep,name=src" json:"src,omitempty"` Dest string `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"` @@ -371,6 +383,7 @@ func (m *CopyOp) GetDest() string { return "" } +// CopySource specifies a source for CopyOp. type CopySource struct { Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"` Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"` @@ -388,10 +401,13 @@ func (m *CopySource) GetSelector() string { return "" } +// SourceOp specifies a source such as build contexts and images. type SourceOp struct { - // source type? - Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"` - Attrs map[string]string `protobuf:"bytes,2,rep,name=attrs" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // TODO: use source type or any type instead of URL protocol. + // identifier e.g. local://, docker-image://, git://, https://... + Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"` + // attrs are defined in attr.go + Attrs map[string]string `protobuf:"bytes,2,rep,name=attrs" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *SourceOp) Reset() { *m = SourceOp{} } @@ -413,6 +429,7 @@ func (m *SourceOp) GetAttrs() map[string]string { return nil } +// BuildOp is used for nested build invocation. type BuildOp struct { Builder InputIndex `protobuf:"varint,1,opt,name=builder,proto3,customtype=InputIndex" json:"builder"` Inputs map[string]*BuildInput `protobuf:"bytes,2,rep,name=inputs" json:"inputs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` @@ -446,6 +463,7 @@ func (m *BuildOp) GetAttrs() map[string]string { return nil } +// BuildInput is used for BuildOp. type BuildInput struct { Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"` } @@ -455,8 +473,9 @@ func (m *BuildInput) String() string { return proto.CompactTextString func (*BuildInput) ProtoMessage() {} func (*BuildInput) Descriptor() ([]byte, []int) { return fileDescriptorOps, []int{9} } -// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex by both "script" and build client (e.g. buildctl). +// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex and overridable on the run time. type OpMetadata struct { + // ignore_cache specifies to ignore the cache for this Op. IgnoreCache bool `protobuf:"varint,1,opt,name=ignore_cache,json=ignoreCache,proto3" json:"ignore_cache,omitempty"` // Description can be used for keeping any text fields that builder doesn't parse Description map[string]string `protobuf:"bytes,2,rep,name=description" json:"description,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -508,8 +527,10 @@ func (m *WorkerConstraint) GetFilter() []string { // Definition is the LLB definition structure with per-vertex metadata entries type Definition struct { + // def is a list of marshaled Op messages Def [][]byte `protobuf:"bytes,1,rep,name=def" json:"def,omitempty"` - // key = LLB op digest string. Currently, empty string is not expected but may change in the future. + // metadata contains metadata for the each of the Op messages. + // A key must be an LLB op digest string. Currently, empty string is not expected as a key, but it may change in the future. Metadata map[github_com_opencontainers_go_digest.Digest]OpMetadata `protobuf:"bytes,2,rep,name=metadata,castkey=github.com/opencontainers/go-digest.Digest" json:"metadata" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` } diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto index 912f82c8..a2d85817 100644 --- a/solver/pb/ops.proto +++ b/solver/pb/ops.proto @@ -1,10 +1,14 @@ syntax = "proto3"; +// Package pb provides the protobuf definition of LLB: low-level builder instruction. +// LLB is DAG-structured; Op represents a vertex, and Definition represents a graph. package pb; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +// Op represents a vertex of the LLB DAG. message Op { + // inputs is a set of input edges. repeated Input inputs = 1; oneof op { ExecOp exec = 2; @@ -14,16 +18,21 @@ message Op { } } +// Input represents an input edge for an Op. message Input { + // digest of the marshaled input Op string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; + // output index of the input Op int64 index = 2 [(gogoproto.customtype) = "OutputIndex", (gogoproto.nullable) = false]; } +// ExecOp executes a command in a container. message ExecOp { Meta meta = 1; repeated Mount mounts = 2; } +// Meta is a set of arguments for ExecOp. // Meta is unrelated to LLB metadata. // FIXME: rename (ExecContext? ExecArgs?) message Meta { @@ -33,6 +42,7 @@ message Meta { string user = 4; } +// Mount specifies how to mount an input Op as a filesystem. message Mount { int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; string selector = 2; @@ -41,22 +51,28 @@ message Mount { bool readonly = 5; } +// CopyOp copies files across Ops. message CopyOp { repeated CopySource src = 1; string dest = 2; } +// CopySource specifies a source for CopyOp. message CopySource { int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; string selector = 2; } +// SourceOp specifies a source such as build contexts and images. message SourceOp { - // source type? + // TODO: use source type or any type instead of URL protocol. + // identifier e.g. local://, docker-image://, git://, https://... string identifier = 1; + // attrs are defined in attr.go map attrs = 2; } +// BuildOp is used for nested build invocation. message BuildOp { int64 builder = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; map inputs = 2; @@ -65,12 +81,14 @@ message BuildOp { // outputs } +// BuildInput is used for BuildOp. message BuildInput { int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; } -// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex by both "script" and build client (e.g. buildctl). +// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex and overridable on the run time. message OpMetadata { + // ignore_cache specifies to ignore the cache for this Op. bool ignore_cache = 1; // Description can be used for keeping any text fields that builder doesn't parse map description = 2; @@ -84,7 +102,9 @@ message WorkerConstraint { // Definition is the LLB definition structure with per-vertex metadata entries message Definition { + // def is a list of marshaled Op messages repeated bytes def = 1; - // key = LLB op digest string. Currently, empty string is not expected but may change in the future. + // metadata contains metadata for the each of the Op messages. + // A key must be an LLB op digest string. Currently, empty string is not expected as a key, but it may change in the future. map metadata = 2 [(gogoproto.castkey) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; }