JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

frontend: support for subrequests 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
v0.8
Tonis Tiigi 2020-10-06 21:03:41 -07:00
parent 0e916c2697
commit b4fad847ac
12 changed files with 330 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
frontend/dockerfile/builder/build.go
View File

@ -343,6 +343,10 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
return nil, capsError
}
if res, ok, err := checkSubRequest(ctx, opts); ok {
return res, err
}
exportMap := len(targetPlatforms) > 1
if v := opts[keyMultiPlatform]; v != "" {

3
frontend/dockerfile/builder/caps.go
View File

@ -10,7 +10,8 @@ import (
)
var enabledCaps = map[string]struct{}{
"moby.buildkit.frontend.inputs": {},
"moby.buildkit.frontend.inputs": {},
"moby.buildkit.frontend.subrequests": {},
}
func validateCaps(req string) (forward bool, err error) {

39
frontend/dockerfile/builder/subrequests.go Normal file
View File

@ -0,0 +1,39 @@
package builder
import (
"context"
"encoding/json"
"github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/frontend/subrequests"
"github.com/moby/buildkit/solver/errdefs"
)
func checkSubRequest(ctx context.Context, opts map[string]string) (*client.Result, bool, error) {
req, ok := opts["requestid"]
if !ok {
return nil, false, nil
}
switch req {
case subrequests.RequestSubrequestsDescribe:
res, err := describe()
return res, true, err
default:
return nil, true, errdefs.NewUnsupportedSubrequestError(req)
}
}
func describe() (*client.Result, error) {
all := []subrequests.Request{
subrequests.SubrequestsDescribeDefinition,
}
dt, err := json.MarshalIndent(all, " ", "")
if err != nil {
return nil, err
}
res := client.NewResult()
res.Metadata = map[string][]byte{
"result.json": dt,
}
return res, nil
}

2
frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
View File

@ -28,7 +28,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \
FROM scratch AS release
LABEL moby.buildkit.frontend.network.none="true"
LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs"
LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests"
COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend
ENTRYPOINT ["/bin/dockerfile-frontend"]

82
frontend/dockerfile/dockerfile_test.go
View File

@ -32,9 +32,11 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/builder"
"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/frontend/subrequests"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/testutil"
"github.com/moby/buildkit/util/testutil/httpserver"
@ -102,6 +104,7 @@ var allTests = []integration.Test{
testFrontendInputs,
testErrorsSourceMap,
testMultiArgs,
testFrontendSubrequests,
}
var fileOpTests = []integration.Test{
@ -4685,6 +4688,85 @@ COPY foo foo2
require.Equal(t, expected, actual)
}
func testFrontendSubrequests(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
dockerfile := []byte(`
FROM scratch
COPY Dockerfile Dockerfile
`)
if gf, ok := f.(*gatewayFrontend); ok {
dockerfile = []byte(fmt.Sprintf("#syntax=%s\n\n%s", gf.gw, dockerfile))
}
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
called := false
frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
reqs, err := subrequests.Describe(ctx, c)
require.NoError(t, err)
require.True(t, len(reqs) > 0)
hasDescribe := false
for _, req := range reqs {
if req.Name == "frontend.subrequests.describe" {
hasDescribe = true
require.Equal(t, subrequests.RequestType("rpc"), req.Type)
require.NotEqual(t, req.Version, "")
require.True(t, len(req.Metadata) > 0)
}
}
require.True(t, hasDescribe)
_, err = c.Solve(ctx, gateway.SolveRequest{
FrontendOpt: map[string]string{
"requestid": "frontend.subrequests.notexist",
"frontend.caps": "moby.buildkit.frontend.subrequests",
},
Frontend: "dockerfile.v0",
})
require.Error(t, err)
var reqErr *errdefs.UnsupportedSubrequestError
require.True(t, errors.As(err, &reqErr))
require.Equal(t, "frontend.subrequests.notexist", reqErr.GetName())
_, err = c.Solve(ctx, gateway.SolveRequest{
FrontendOpt: map[string]string{
"frontend.caps": "moby.buildkit.frontend.notexistcap",
},
Frontend: "dockerfile.v0",
})
require.Error(t, err)
var capErr *errdefs.UnsupportedFrontendCapError
require.True(t, errors.As(err, &capErr))
require.Equal(t, "moby.buildkit.frontend.notexistcap", capErr.GetName())
called = true
return nil, nil
}
_, err = c.Build(context.TODO(), client.SolveOpt{
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
},
}, "", frontend, nil)
require.NoError(t, err)
require.True(t, called)
}
func tmpdir(appliers ...fstest.Applier) (string, error) {
tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile")
if err != nil {

63
frontend/subrequests/describe.go Normal file
View File

@ -0,0 +1,63 @@
package subrequests
import (
"context"
"encoding/json"
"github.com/moby/buildkit/frontend/gateway/client"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/errdefs"
"github.com/pkg/errors"
)
const RequestSubrequestsDescribe = "frontend.subrequests.describe"
var SubrequestsDescribeDefinition = Request{
Name: RequestSubrequestsDescribe,
Version: "1.0.0",
Type: TypeRPC,
Description: "List available subrequest types",
Metadata: []Named{
{
Name: "result.json",
},
},
}
func Describe(ctx context.Context, c client.Client) ([]Request, error) {
gwcaps := c.BuildOpts().Caps
if err := (&gwcaps).Supports(gwpb.CapFrontendCaps); err != nil {
return nil, errdefs.NewUnsupportedSubrequestError(RequestSubrequestsDescribe)
}
res, err := c.Solve(ctx, client.SolveRequest{
FrontendOpt: map[string]string{
"requestid": RequestSubrequestsDescribe,
"frontend.caps": "moby.buildkit.frontend.subrequests",
},
Frontend: "dockerfile.v0",
})
if err != nil {
var reqErr *errdefs.UnsupportedSubrequestError
if errors.As(err, &reqErr) {
return nil, err
}
var capErr *errdefs.UnsupportedFrontendCapError
if errors.As(err, &capErr) {
return nil, errdefs.NewUnsupportedSubrequestError(RequestSubrequestsDescribe)
}
return nil, err
}
dt, ok := res.Metadata["result.json"]
if !ok {
return nil, errors.Errorf("no result.json metadata in response")
}
var reqs []Request
if err := json.Unmarshal(dt, &reqs); err != nil {
return nil, errors.Wrap(err, "failed to parse describe result")
}
return reqs, nil
}

21
frontend/subrequests/types.go Normal file
View File

@ -0,0 +1,21 @@
package subrequests
type Request struct {
Name string `json:"name"`
Version string `json:"version"`
Type RequestType `json:"type"`
Description string `json:"description"`
Opts []Named `json:"opts"`
Inputs []Named `json:"inputs"`
Metadata []Named `json:"metadata"`
Refs []Named `json:"refs"`
}
type Named struct {
Name string `json:"name"`
Description string `json:"description"`
}
type RequestType string
const TypeRPC RequestType = "rpc"

68
solver/errdefs/errdefs.pb.go
View File

@ -143,27 +143,67 @@ func (m *FrontendCap) GetName() string {
return ""
}
type Subrequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Subrequest) Reset() { *m = Subrequest{} }
func (m *Subrequest) String() string { return proto.CompactTextString(m) }
func (*Subrequest) ProtoMessage() {}
func (*Subrequest) Descriptor() ([]byte, []int) {
return fileDescriptor_689dc58a5060aff5, []int{3}
}
func (m *Subrequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Subrequest.Unmarshal(m, b)
}
func (m *Subrequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Subrequest.Marshal(b, m, deterministic)
}
func (m *Subrequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_Subrequest.Merge(m, src)
}
func (m *Subrequest) XXX_Size() int {
return xxx_messageInfo_Subrequest.Size(m)
}
func (m *Subrequest) XXX_DiscardUnknown() {
xxx_messageInfo_Subrequest.DiscardUnknown(m)
}
var xxx_messageInfo_Subrequest proto.InternalMessageInfo
func (m *Subrequest) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func init() {
proto.RegisterType((*Vertex)(nil), "errdefs.Vertex")
proto.RegisterType((*Source)(nil), "errdefs.Source")
proto.RegisterType((*FrontendCap)(nil), "errdefs.FrontendCap")
proto.RegisterType((*Subrequest)(nil), "errdefs.Subrequest")
}
func init() { proto.RegisterFile("errdefs.proto", fileDescriptor_689dc58a5060aff5) }
var fileDescriptor_689dc58a5060aff5 = []byte{
// 200 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x2c, 0xce, 0xc1, 0x4a, 0x04, 0x31,
0x0c, 0x06, 0x60, 0x56, 0x97, 0xca, 0x66, 0xd0, 0x43, 0x0f, 0xb2, 0x78, 0x9a, 0xed, 0x69, 0x0f,
0xd2, 0xc2, 0xfa, 0x08, 0x82, 0xe0, 0x49, 0xa8, 0xe0, 0x7d, 0xba, 0xcd, 0x8c, 0xc5, 0x9d, 0xa6,
0xa4, 0x1d, 0xd1, 0xb7, 0x97, 0xa9, 0xbd, 0xe5, 0xcf, 0x97, 0x90, 0xc0, 0x2d, 0x32, 0x7b, 0x1c,
0xb3, 0x4e, 0x4c, 0x85, 0xe4, 0x4d, 0x8b, 0x0f, 0x8f, 0x53, 0x28, 0x9f, 0x8b, 0xd3, 0x67, 0x9a,
0xcd, 0x4c, 0xee, 0xd7, 0xb8, 0x25, 0x5c, 0xfc, 0x57, 0x28, 0x26, 0xd3, 0xe5, 0x1b, 0xd9, 0x24,
0x67, 0x28, 0xb5, 0x35, 0xd5, 0x83, 0xf8, 0x40, 0x2e, 0xf8, 0x23, 0xef, 0x41, 0xf8, 0x30, 0x61,
0x2e, 0xfb, 0x4d, 0xbf, 0x39, 0xee, 0x6c, 0x4b, 0xea, 0x0d, 0xc4, 0x3b, 0x2d, 0x7c, 0x46, 0xa9,
0x60, 0x1b, 0xe2, 0x48, 0xd5, 0xbb, 0xd3, 0x9d, 0x4e, 0x4e, 0xff, 0xcb, 0x6b, 0x1c, 0xc9, 0x56,
0x93, 0x07, 0x10, 0x3c, 0xc4, 0x09, 0xf3, 0xfe, 0xaa, 0xbf, 0x3e, 0x76, 0xa7, 0xdd, 0x3a, 0x65,
0xd7, 0x8e, 0x6d, 0xa0, 0x0e, 0xd0, 0xbd, 0x30, 0xc5, 0x82, 0xd1, 0x3f, 0x0f, 0x49, 0x4a, 0xd8,
0xc6, 0x61, 0xc6, 0x76, 0xb5, 0xd6, 0x4e, 0xd4, 0xe7, 0x9e, 0xfe, 0x02, 0x00, 0x00, 0xff, 0xff,
0x80, 0xf5, 0x33, 0xf9, 0xe4, 0x00, 0x00, 0x00,
// 213 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x8e, 0xc1, 0x4a, 0x03, 0x31,
0x10, 0x86, 0xa9, 0x96, 0x48, 0x67, 0xd1, 0x43, 0x0e, 0x52, 0x3c, 0x6d, 0x73, 0xea, 0x41, 0x36,
0x50, 0x1f, 0x41, 0x10, 0x3c, 0x09, 0x5b, 0xf0, 0xbe, 0x69, 0x66, 0xd7, 0x60, 0x37, 0x13, 0x27,
0x89, 0xe8, 0xdb, 0xcb, 0xc6, 0x1c, 0x7b, 0x9b, 0x7f, 0xbe, 0x6f, 0x98, 0x1f, 0x6e, 0x91, 0xd9,
0xe2, 0x18, 0xbb, 0xc0, 0x94, 0x48, 0xde, 0xd4, 0xf8, 0xf0, 0x38, 0xb9, 0xf4, 0x91, 0x4d, 0x77,
0xa2, 0x59, 0xcf, 0x64, 0x7e, 0xb5, 0xc9, 0xee, 0x6c, 0x3f, 0x5d, 0xd2, 0x91, 0xce, 0xdf, 0xc8,
0x3a, 0x18, 0x4d, 0xa1, 0x9e, 0xa9, 0x16, 0xc4, 0x3b, 0x72, 0xc2, 0x1f, 0x79, 0x0f, 0xc2, 0xba,
0x09, 0x63, 0xda, 0xae, 0xda, 0xd5, 0x7e, 0xd3, 0xd7, 0xa4, 0xde, 0x40, 0x1c, 0x29, 0xf3, 0x09,
0xa5, 0x82, 0xb5, 0xf3, 0x23, 0x15, 0xde, 0x1c, 0xee, 0xba, 0x60, 0xba, 0x7f, 0xf2, 0xea, 0x47,
0xea, 0x0b, 0x93, 0x3b, 0x10, 0x3c, 0xf8, 0x09, 0xe3, 0xf6, 0xaa, 0xbd, 0xde, 0x37, 0x87, 0xcd,
0x62, 0xf5, 0xcb, 0xa6, 0xaf, 0x40, 0xed, 0xa0, 0x79, 0x61, 0xf2, 0x09, 0xbd, 0x7d, 0x1e, 0x82,
0x94, 0xb0, 0xf6, 0xc3, 0x8c, 0xf5, 0x6b, 0x99, 0x55, 0x0b, 0x70, 0xcc, 0x86, 0xf1, 0x2b, 0x63,
0x4c, 0x97, 0x0c, 0x23, 0x4a, 0xfd, 0xa7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x75, 0x4b, 0xfe,
0xad, 0x06, 0x01, 0x00, 0x00,
}

4
solver/errdefs/errdefs.proto
View File

@ -16,3 +16,7 @@ message Source {
message FrontendCap {
string name = 1;
}
message Subrequest {
string name = 1;
}

41
solver/errdefs/subrequest.go Normal file
View File

@ -0,0 +1,41 @@
package errdefs
import (
fmt "fmt"
"github.com/containerd/typeurl"
"github.com/moby/buildkit/util/grpcerrors"
)
func init() {
typeurl.Register((*Subrequest)(nil), "github.com/moby/buildkit", "errdefs.Subrequest+json")
}
type UnsupportedSubrequestError struct {
Subrequest
error
}
func (e *UnsupportedSubrequestError) Error() string {
msg := fmt.Sprintf("unsupported request %s", e.Subrequest.Name)
if e.error != nil {
msg += ": " + e.error.Error()
}
return msg
}
func (e *UnsupportedSubrequestError) Unwrap() error {
return e.error
}
func (e *UnsupportedSubrequestError) ToProto() grpcerrors.TypedErrorProto {
return &e.Subrequest
}
func NewUnsupportedSubrequestError(name string) error {
return &UnsupportedSubrequestError{Subrequest: Subrequest{Name: name}}
}
func (v *Subrequest) WrapError(err error) error {
return &UnsupportedSubrequestError{error: err, Subrequest: *v}
}

9
util/grpcerrors/grpcerrors.go
View File

@ -100,9 +100,10 @@ func Code(err error) codes.Code {
Unwrap() error
})
if ok {
return Code(wrapped.Unwrap())
if err := wrapped.Unwrap(); err != nil {
return Code(err)
}
}
return status.FromContextError(err).Code()
}
@ -124,7 +125,9 @@ func AsGRPCStatus(err error) (*status.Status, bool) {
Unwrap() error
})
if ok {
return AsGRPCStatus(wrapped.Unwrap())
if err := wrapped.Unwrap(); err != nil {
return AsGRPCStatus(err)
}
}
return nil, false

13
util/grpcerrors/intercept.go
View File

@ -2,15 +2,28 @@ package grpcerrors
import (
"context"
"log"
"os"
"github.com/pkg/errors"
"google.golang.org/grpc"
)
func UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
resp, err = handler(ctx, req)
oldErr := err
if err != nil {
err = ToGRPC(err)
}
if oldErr != nil && err == nil {
logErr := errors.Wrap(err, "invalid grpc error conversion")
if os.Getenv("BUILDKIT_DEBUG_PANIC_ON_ERROR") == "1" {
panic(logErr)
}
log.Printf("%v", logErr)
err = oldErr
}
return resp, err
}