llb: update docs

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>

README.md

@@ -58,7 +58,19 @@ We are open to adding more backends.
 
 #### Exploring LLB
 
-BuildKit builds are based on a binary intermediate format called LLB that is used for defining the dependency graph for processes running part of your build.
+BuildKit builds are based on a binary intermediate format called LLB that is used for defining the dependency graph for processes running part of your build. tl;dr: LLB is to Dockerfile what LLVM IR is to C.
+
+- Marshaled as Protobuf messages
+- Concurrently executable
+- Efficiently cacheable
+- Vendor-neutral (i.e. non-Dockerfile languages can be easily implemented)
+
+See [`solver/pb/ops.proto`](./solver/pb/ops.proto) for the format definition.
+
+Currently, following high-level languages has been implemented for LLB:
+
+- Dockerfile (See [Exploring Dockerfiles](#exploring-dockerfiles))
+- (open a PR to add your own language)
 
 For understanding the basics of LLB, `examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit itself and its dependencies using the `client` package. Running one of these scripts generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build.
 

solver/pb/ops.pb.go

@@ -4,6 +4,9 @@
 /*
 	Package pb is a generated protocol buffer package.
 
+	Package pb provides the protobuf definition of LLB: low-level builder instruction.
+	LLB is DAG-structured; Op represents a vertex, and Definition represents a graph.
+
 	It is generated from these files:
 		ops.proto
 
@@ -44,7 +47,9 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
 
+// Op represents a vertex of the LLB DAG.
 type Op struct {
+	// inputs is a set of input edges.
 	Inputs []*Input `protobuf:"bytes,1,rep,name=inputs" json:"inputs,omitempty"`
 	// Types that are valid to be assigned to Op:
 	//	*Op_Exec
@@ -237,9 +242,12 @@ func _Op_OneofSizer(msg proto.Message) (n int) {
 	return n
 }
 
+// Input represents an input edge for an Op.
 type Input struct {
+	// digest of the marshaled input Op
 	Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"`
-	Index  OutputIndex                                `protobuf:"varint,2,opt,name=index,proto3,customtype=OutputIndex" json:"index"`
+	// output index of the input Op
+	Index OutputIndex `protobuf:"varint,2,opt,name=index,proto3,customtype=OutputIndex" json:"index"`
 }
 
 func (m *Input) Reset()                    { *m = Input{} }
@@ -247,6 +255,7 @@ func (m *Input) String() string            { return proto.CompactTextString(m) }
 func (*Input) ProtoMessage()               {}
 func (*Input) Descriptor() ([]byte, []int) { return fileDescriptorOps, []int{1} }
 
+// ExecOp executes a command in a container.
 type ExecOp struct {
 	Meta   *Meta    `protobuf:"bytes,1,opt,name=meta" json:"meta,omitempty"`
 	Mounts []*Mount `protobuf:"bytes,2,rep,name=mounts" json:"mounts,omitempty"`
@@ -271,6 +280,7 @@ func (m *ExecOp) GetMounts() []*Mount {
 	return nil
 }
 
+// Meta is a set of arguments for ExecOp.
 // Meta is unrelated to LLB metadata.
 // FIXME: rename (ExecContext? ExecArgs?)
 type Meta struct {
@@ -313,6 +323,7 @@ func (m *Meta) GetUser() string {
 	return ""
 }
 
+// Mount specifies how to mount an input Op as a filesystem.
 type Mount struct {
 	Input    InputIndex  `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
 	Selector string      `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"`
@@ -347,6 +358,7 @@ func (m *Mount) GetReadonly() bool {
 	return false
 }
 
+// CopyOp copies files across Ops.
 type CopyOp struct {
 	Src  []*CopySource `protobuf:"bytes,1,rep,name=src" json:"src,omitempty"`
 	Dest string        `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"`
@@ -371,6 +383,7 @@ func (m *CopyOp) GetDest() string {
 	return ""
 }
 
+// CopySource specifies a source for CopyOp.
 type CopySource struct {
 	Input    InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
 	Selector string     `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"`
@@ -388,10 +401,13 @@ func (m *CopySource) GetSelector() string {
 	return ""
 }
 
+// SourceOp specifies a source such as build contexts and images.
 type SourceOp struct {
-	// source type?
+	// TODO: use source type or any type instead of URL protocol.
-	Identifier string            `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"`
+	// identifier e.g. local://, docker-image://, git://, https://...
-	Attrs      map[string]string `protobuf:"bytes,2,rep,name=attrs" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"`
+	// attrs are defined in attr.go
+	Attrs map[string]string `protobuf:"bytes,2,rep,name=attrs" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (m *SourceOp) Reset()                    { *m = SourceOp{} }
@@ -413,6 +429,7 @@ func (m *SourceOp) GetAttrs() map[string]string {
 	return nil
 }
 
+// BuildOp is used for nested build invocation.
 type BuildOp struct {
 	Builder InputIndex             `protobuf:"varint,1,opt,name=builder,proto3,customtype=InputIndex" json:"builder"`
 	Inputs  map[string]*BuildInput `protobuf:"bytes,2,rep,name=inputs" json:"inputs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"`
@@ -446,6 +463,7 @@ func (m *BuildOp) GetAttrs() map[string]string {
 	return nil
 }
 
+// BuildInput is used for BuildOp.
 type BuildInput struct {
 	Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
 }
@@ -455,8 +473,9 @@ func (m *BuildInput) String() string            { return proto.CompactTextString
 func (*BuildInput) ProtoMessage()               {}
 func (*BuildInput) Descriptor() ([]byte, []int) { return fileDescriptorOps, []int{9} }
 
-// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex by both "script" and build client (e.g. buildctl).
+// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex and overridable on the run time.
 type OpMetadata struct {
+	// ignore_cache specifies to ignore the cache for this Op.
 	IgnoreCache bool `protobuf:"varint,1,opt,name=ignore_cache,json=ignoreCache,proto3" json:"ignore_cache,omitempty"`
 	// Description can be used for keeping any text fields that builder doesn't parse
 	Description      map[string]string `protobuf:"bytes,2,rep,name=description" json:"description,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
@@ -508,8 +527,10 @@ func (m *WorkerConstraint) GetFilter() []string {
 
 // Definition is the LLB definition structure with per-vertex metadata entries
 type Definition struct {
+	// def is a list of marshaled Op messages
 	Def [][]byte `protobuf:"bytes,1,rep,name=def" json:"def,omitempty"`
-	// key = LLB op digest string. Currently, empty string is not expected but may change in the future.
+	// metadata contains metadata for the each of the Op messages.
+	// A key must be an LLB op digest string. Currently, empty string is not expected as a key, but it may change in the future.
 	Metadata map[github_com_opencontainers_go_digest.Digest]OpMetadata `protobuf:"bytes,2,rep,name=metadata,castkey=github.com/opencontainers/go-digest.Digest" json:"metadata" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"`
 }
 

solver/pb/ops.proto

@@ -1,10 +1,14 @@
 syntax = "proto3";
 
+// Package pb provides the protobuf definition of LLB: low-level builder instruction.
+// LLB is DAG-structured; Op represents a vertex, and Definition represents a graph.
 package pb;
 
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
 
+// Op represents a vertex of the LLB DAG.
 message Op {
+	// inputs is a set of input edges.
 	repeated Input inputs = 1;
 	oneof op {
 		ExecOp exec = 2;
@@ -14,16 +18,21 @@ message Op {
 	 }
 }
 
+// Input represents an input edge for an Op.
 message Input {
+	// digest of the marshaled input Op
 	string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
+	// output index of the input Op
 	int64 index = 2 [(gogoproto.customtype) = "OutputIndex", (gogoproto.nullable) = false];
 }
 
+// ExecOp executes a command in a container.
 message ExecOp {
 	Meta meta = 1;
 	repeated Mount mounts = 2;
 }
 
+// Meta is a set of arguments for ExecOp.
 // Meta is unrelated to LLB metadata.
 // FIXME: rename (ExecContext? ExecArgs?)
 message Meta {
@@ -33,6 +42,7 @@ message Meta {
 	string user = 4;
 }
 
+// Mount specifies how to mount an input Op as a filesystem.
 message Mount {
 	int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false];
 	string selector = 2;
@@ -41,22 +51,28 @@ message Mount {
 	bool readonly = 5;
 }
 
+// CopyOp copies files across Ops.
 message CopyOp {
 	repeated CopySource src = 1;
 	string dest = 2;
 }
 
+// CopySource specifies a source for CopyOp.
 message CopySource {
 	int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false];
 	string selector = 2;
 }
 
+// SourceOp specifies a source such as build contexts and images.
 message SourceOp {
-	// source type?
+	// TODO: use source type or any type instead of URL protocol.
+	// identifier e.g. local://, docker-image://, git://, https://...
 	string identifier = 1;
+	// attrs are defined in attr.go
 	map<string, string> attrs = 2;
 }
 
+// BuildOp is used for nested build invocation.
 message BuildOp {
 	int64 builder = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false];
 	map<string, BuildInput> inputs = 2;
@@ -65,12 +81,14 @@ message BuildOp {
 	// outputs
 }
 
+// BuildInput is used for BuildOp.
 message BuildInput {
 	int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false];
 }
 
-// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex by both "script" and build client (e.g. buildctl).
+// OpMetadata is a per-vertex metadata entry, which can be defined for arbitrary Op vertex and overridable on the run time.
 message OpMetadata {
+	// ignore_cache specifies to ignore the cache for this Op.
 	bool ignore_cache = 1;
 	// Description can be used for keeping any text fields that builder doesn't parse
 	map<string, string> description = 2;  
@@ -84,7 +102,9 @@ message WorkerConstraint {
 
 // Definition is the LLB definition structure with per-vertex metadata entries
 message Definition {
+	// def is a list of marshaled Op messages
 	repeated bytes def = 1;
-	// key = LLB op digest string. Currently, empty string is not expected but may change in the future.
+	// metadata contains metadata for the each of the Op messages.
+	// A key must be an LLB op digest string. Currently, empty string is not expected as a key, but it may change in the future.
 	map<string, OpMetadata> metadata = 2 [(gogoproto.castkey) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
 }