frontend: support for subrequests

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

frontend/dockerfile/builder/build.go

@@ -343,6 +343,10 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 		return nil, capsError
 	}
 
+	if res, ok, err := checkSubRequest(ctx, opts); ok {
+		return res, err
+	}
+
 	exportMap := len(targetPlatforms) > 1
 
 	if v := opts[keyMultiPlatform]; v != "" {

frontend/dockerfile/builder/caps.go

@@ -11,6 +11,7 @@ import (
 
 var enabledCaps = map[string]struct{}{
 	"moby.buildkit.frontend.inputs":      {},
+	"moby.buildkit.frontend.subrequests": {},
 }
 
 func validateCaps(req string) (forward bool, err error) {

frontend/dockerfile/builder/subrequests.go

@@ -0,0 +1,39 @@
+package builder
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/moby/buildkit/frontend/subrequests"
+	"github.com/moby/buildkit/solver/errdefs"
+)
+
+func checkSubRequest(ctx context.Context, opts map[string]string) (*client.Result, bool, error) {
+	req, ok := opts["requestid"]
+	if !ok {
+		return nil, false, nil
+	}
+	switch req {
+	case subrequests.RequestSubrequestsDescribe:
+		res, err := describe()
+		return res, true, err
+	default:
+		return nil, true, errdefs.NewUnsupportedSubrequestError(req)
+	}
+}
+
+func describe() (*client.Result, error) {
+	all := []subrequests.Request{
+		subrequests.SubrequestsDescribeDefinition,
+	}
+	dt, err := json.MarshalIndent(all, "  ", "")
+	if err != nil {
+		return nil, err
+	}
+	res := client.NewResult()
+	res.Metadata = map[string][]byte{
+		"result.json": dt,
+	}
+	return res, nil
+}

frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile

@@ -28,7 +28,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \
 
 FROM scratch AS release
 LABEL moby.buildkit.frontend.network.none="true"
-LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs"
+LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests"
 COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend
 ENTRYPOINT ["/bin/dockerfile-frontend"]
 

frontend/dockerfile/dockerfile_test.go

@@ -32,9 +32,11 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/builder"
 	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
 	gateway "github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/moby/buildkit/frontend/subrequests"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
+	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/testutil"
 	"github.com/moby/buildkit/util/testutil/httpserver"
@@ -102,6 +104,7 @@ var allTests = []integration.Test{
 	testFrontendInputs,
 	testErrorsSourceMap,
 	testMultiArgs,
+	testFrontendSubrequests,
 }
 
 var fileOpTests = []integration.Test{
@@ -4685,6 +4688,85 @@ COPY foo foo2
 	require.Equal(t, expected, actual)
 }
 
+func testFrontendSubrequests(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dockerfile := []byte(`
+FROM scratch
+COPY Dockerfile Dockerfile
+`)
+
+	if gf, ok := f.(*gatewayFrontend); ok {
+		dockerfile = []byte(fmt.Sprintf("#syntax=%s\n\n%s", gf.gw, dockerfile))
+	}
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	called := false
+
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		reqs, err := subrequests.Describe(ctx, c)
+		require.NoError(t, err)
+
+		require.True(t, len(reqs) > 0)
+
+		hasDescribe := false
+
+		for _, req := range reqs {
+			if req.Name == "frontend.subrequests.describe" {
+				hasDescribe = true
+				require.Equal(t, subrequests.RequestType("rpc"), req.Type)
+				require.NotEqual(t, req.Version, "")
+				require.True(t, len(req.Metadata) > 0)
+			}
+		}
+		require.True(t, hasDescribe)
+
+		_, err = c.Solve(ctx, gateway.SolveRequest{
+			FrontendOpt: map[string]string{
+				"requestid":     "frontend.subrequests.notexist",
+				"frontend.caps": "moby.buildkit.frontend.subrequests",
+			},
+			Frontend: "dockerfile.v0",
+		})
+		require.Error(t, err)
+		var reqErr *errdefs.UnsupportedSubrequestError
+		require.True(t, errors.As(err, &reqErr))
+		require.Equal(t, "frontend.subrequests.notexist", reqErr.GetName())
+
+		_, err = c.Solve(ctx, gateway.SolveRequest{
+			FrontendOpt: map[string]string{
+				"frontend.caps": "moby.buildkit.frontend.notexistcap",
+			},
+			Frontend: "dockerfile.v0",
+		})
+		require.Error(t, err)
+		var capErr *errdefs.UnsupportedFrontendCapError
+		require.True(t, errors.As(err, &capErr))
+		require.Equal(t, "moby.buildkit.frontend.notexistcap", capErr.GetName())
+
+		called = true
+		return nil, nil
+	}
+
+	_, err = c.Build(context.TODO(), client.SolveOpt{
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+		},
+	}, "", frontend, nil)
+	require.NoError(t, err)
+
+	require.True(t, called)
+}
+
 func tmpdir(appliers ...fstest.Applier) (string, error) {
 	tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile")
 	if err != nil {

frontend/subrequests/describe.go

@@ -0,0 +1,63 @@
+package subrequests
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/moby/buildkit/frontend/gateway/client"
+	gwpb "github.com/moby/buildkit/frontend/gateway/pb"
+	"github.com/moby/buildkit/solver/errdefs"
+	"github.com/pkg/errors"
+)
+
+const RequestSubrequestsDescribe = "frontend.subrequests.describe"
+
+var SubrequestsDescribeDefinition = Request{
+	Name:        RequestSubrequestsDescribe,
+	Version:     "1.0.0",
+	Type:        TypeRPC,
+	Description: "List available subrequest types",
+	Metadata: []Named{
+		{
+			Name: "result.json",
+		},
+	},
+}
+
+func Describe(ctx context.Context, c client.Client) ([]Request, error) {
+	gwcaps := c.BuildOpts().Caps
+
+	if err := (&gwcaps).Supports(gwpb.CapFrontendCaps); err != nil {
+		return nil, errdefs.NewUnsupportedSubrequestError(RequestSubrequestsDescribe)
+	}
+
+	res, err := c.Solve(ctx, client.SolveRequest{
+		FrontendOpt: map[string]string{
+			"requestid":     RequestSubrequestsDescribe,
+			"frontend.caps": "moby.buildkit.frontend.subrequests",
+		},
+		Frontend: "dockerfile.v0",
+	})
+	if err != nil {
+		var reqErr *errdefs.UnsupportedSubrequestError
+		if errors.As(err, &reqErr) {
+			return nil, err
+		}
+		var capErr *errdefs.UnsupportedFrontendCapError
+		if errors.As(err, &capErr) {
+			return nil, errdefs.NewUnsupportedSubrequestError(RequestSubrequestsDescribe)
+		}
+		return nil, err
+	}
+
+	dt, ok := res.Metadata["result.json"]
+	if !ok {
+		return nil, errors.Errorf("no result.json metadata in response")
+	}
+
+	var reqs []Request
+	if err := json.Unmarshal(dt, &reqs); err != nil {
+		return nil, errors.Wrap(err, "failed to parse describe result")
+	}
+	return reqs, nil
+}

frontend/subrequests/types.go

@@ -0,0 +1,21 @@
+package subrequests
+
+type Request struct {
+	Name        string      `json:"name"`
+	Version     string      `json:"version"`
+	Type        RequestType `json:"type"`
+	Description string      `json:"description"`
+	Opts        []Named     `json:"opts"`
+	Inputs      []Named     `json:"inputs"`
+	Metadata    []Named     `json:"metadata"`
+	Refs        []Named     `json:"refs"`
+}
+
+type Named struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+type RequestType string
+
+const TypeRPC RequestType = "rpc"

solver/errdefs/errdefs.pb.go

@@ -143,27 +143,67 @@ func (m *FrontendCap) GetName() string {
 	return ""
 }
 
+type Subrequest struct {
+	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Subrequest) Reset()         { *m = Subrequest{} }
+func (m *Subrequest) String() string { return proto.CompactTextString(m) }
+func (*Subrequest) ProtoMessage()    {}
+func (*Subrequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_689dc58a5060aff5, []int{3}
+}
+func (m *Subrequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Subrequest.Unmarshal(m, b)
+}
+func (m *Subrequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Subrequest.Marshal(b, m, deterministic)
+}
+func (m *Subrequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Subrequest.Merge(m, src)
+}
+func (m *Subrequest) XXX_Size() int {
+	return xxx_messageInfo_Subrequest.Size(m)
+}
+func (m *Subrequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_Subrequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Subrequest proto.InternalMessageInfo
+
+func (m *Subrequest) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
 func init() {
 	proto.RegisterType((*Vertex)(nil), "errdefs.Vertex")
 	proto.RegisterType((*Source)(nil), "errdefs.Source")
 	proto.RegisterType((*FrontendCap)(nil), "errdefs.FrontendCap")
+	proto.RegisterType((*Subrequest)(nil), "errdefs.Subrequest")
 }
 
 func init() { proto.RegisterFile("errdefs.proto", fileDescriptor_689dc58a5060aff5) }
 
 var fileDescriptor_689dc58a5060aff5 = []byte{
-	// 200 bytes of a gzipped FileDescriptorProto
+	// 213 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x2c, 0xce, 0xc1, 0x4a, 0x04, 0x31,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x8e, 0xc1, 0x4a, 0x03, 0x31,
-	0x0c, 0x06, 0x60, 0x56, 0x97, 0xca, 0x66, 0xd0, 0x43, 0x0f, 0xb2, 0x78, 0x9a, 0xed, 0x69, 0x0f,
+	0x10, 0x86, 0xa9, 0x96, 0x48, 0x67, 0xd1, 0x43, 0x0e, 0x52, 0x3c, 0x6d, 0x73, 0xea, 0x41, 0x36,
-	0xd2, 0xc2, 0xfa, 0x08, 0x82, 0xe0, 0x49, 0xa8, 0xe0, 0x7d, 0xba, 0xcd, 0x8c, 0xc5, 0x9d, 0xa6,
+	0x50, 0x1f, 0x41, 0x10, 0x3c, 0x09, 0x5b, 0xf0, 0xbe, 0x69, 0x66, 0xd7, 0x60, 0x37, 0x13, 0x27,
-	0xa4, 0x1d, 0xd1, 0xb7, 0x97, 0xa9, 0xbd, 0xe5, 0xcf, 0x97, 0x90, 0xc0, 0x2d, 0x32, 0x7b, 0x1c,
+	0x89, 0xe8, 0xdb, 0xcb, 0xc6, 0x1c, 0x7b, 0x9b, 0x7f, 0xbe, 0x6f, 0x98, 0x1f, 0x6e, 0x91, 0xd9,
-	0xb3, 0x4e, 0x4c, 0x85, 0xe4, 0x4d, 0x8b, 0x0f, 0x8f, 0x53, 0x28, 0x9f, 0x8b, 0xd3, 0x67, 0x9a,
+	0xe2, 0x18, 0xbb, 0xc0, 0x94, 0x48, 0xde, 0xd4, 0xf8, 0xf0, 0x38, 0xb9, 0xf4, 0x91, 0x4d, 0x77,
-	0xcd, 0x4c, 0xee, 0xd7, 0xb8, 0x25, 0x5c, 0xfc, 0x57, 0x28, 0x26, 0xd3, 0xe5, 0x1b, 0xd9, 0x24,
+	0xa2, 0x59, 0xcf, 0x64, 0x7e, 0xb5, 0xc9, 0xee, 0x6c, 0x3f, 0x5d, 0xd2, 0x91, 0xce, 0xdf, 0xc8,
-	0x67, 0x28, 0xb5, 0x35, 0xd5, 0x83, 0xf8, 0x40, 0x2e, 0xf8, 0x23, 0xef, 0x41, 0xf8, 0x30, 0x61,
+	0x3a, 0x18, 0x4d, 0xa1, 0x9e, 0xa9, 0x16, 0xc4, 0x3b, 0x72, 0xc2, 0x1f, 0x79, 0x0f, 0xc2, 0xba,
-	0x2e, 0xfb, 0x4d, 0xbf, 0x39, 0xee, 0x6c, 0x4b, 0xea, 0x0d, 0xc4, 0x3b, 0x2d, 0x7c, 0x46, 0xa9,
+	0x09, 0x63, 0xda, 0xae, 0xda, 0xd5, 0x7e, 0xd3, 0xd7, 0xa4, 0xde, 0x40, 0x1c, 0x29, 0xf3, 0x09,
-	0x60, 0x1b, 0xe2, 0x48, 0xd5, 0xbb, 0xd3, 0x9d, 0x4e, 0x4e, 0xff, 0xcb, 0x6b, 0x1c, 0xc9, 0x56,
+	0xa5, 0x82, 0xb5, 0xf3, 0x23, 0x15, 0xde, 0x1c, 0xee, 0xba, 0x60, 0xba, 0x7f, 0xf2, 0xea, 0x47,
-	0x93, 0x07, 0x10, 0x3c, 0xc4, 0x09, 0xf3, 0xfe, 0xaa, 0xbf, 0x3e, 0x76, 0xa7, 0xdd, 0x3a, 0x65,
+	0xea, 0x0b, 0x93, 0x3b, 0x10, 0x3c, 0xf8, 0x09, 0xe3, 0xf6, 0xaa, 0xbd, 0xde, 0x37, 0x87, 0xcd,
-	0xd7, 0x8e, 0x6d, 0xa0, 0x0e, 0xd0, 0xbd, 0x30, 0xc5, 0x82, 0xd1, 0x3f, 0x0f, 0x49, 0x4a, 0xd8,
+	0x62, 0xf5, 0xcb, 0xa6, 0xaf, 0x40, 0xed, 0xa0, 0x79, 0x61, 0xf2, 0x09, 0xbd, 0x7d, 0x1e, 0x82,
-	0xc6, 0x61, 0xc6, 0x76, 0xb5, 0xd6, 0x4e, 0xd4, 0xe7, 0x9e, 0xfe, 0x02, 0x00, 0x00, 0xff, 0xff,
+	0x94, 0xb0, 0xf6, 0xc3, 0x8c, 0xf5, 0x6b, 0x99, 0x55, 0x0b, 0x70, 0xcc, 0x86, 0xf1, 0x2b, 0x63,
-	0x80, 0xf5, 0x33, 0xf9, 0xe4, 0x00, 0x00, 0x00,
+	0x4c, 0x97, 0x0c, 0x23, 0x4a, 0xfd, 0xa7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x75, 0x4b, 0xfe,
+	0xad, 0x06, 0x01, 0x00, 0x00,
 }

solver/errdefs/errdefs.proto

@@ -16,3 +16,7 @@ message Source {
 message FrontendCap {
 	string name = 1;
 }
+
+message Subrequest {
+	string name = 1;
+}

solver/errdefs/subrequest.go

@@ -0,0 +1,41 @@
+package errdefs
+
+import (
+	fmt "fmt"
+
+	"github.com/containerd/typeurl"
+	"github.com/moby/buildkit/util/grpcerrors"
+)
+
+func init() {
+	typeurl.Register((*Subrequest)(nil), "github.com/moby/buildkit", "errdefs.Subrequest+json")
+}
+
+type UnsupportedSubrequestError struct {
+	Subrequest
+	error
+}
+
+func (e *UnsupportedSubrequestError) Error() string {
+	msg := fmt.Sprintf("unsupported request %s", e.Subrequest.Name)
+	if e.error != nil {
+		msg += ": " + e.error.Error()
+	}
+	return msg
+}
+
+func (e *UnsupportedSubrequestError) Unwrap() error {
+	return e.error
+}
+
+func (e *UnsupportedSubrequestError) ToProto() grpcerrors.TypedErrorProto {
+	return &e.Subrequest
+}
+
+func NewUnsupportedSubrequestError(name string) error {
+	return &UnsupportedSubrequestError{Subrequest: Subrequest{Name: name}}
+}
+
+func (v *Subrequest) WrapError(err error) error {
+	return &UnsupportedSubrequestError{error: err, Subrequest: *v}
+}

util/grpcerrors/grpcerrors.go

@@ -100,9 +100,10 @@ func Code(err error) codes.Code {
 		Unwrap() error
 	})
 	if ok {
-		return Code(wrapped.Unwrap())
+		if err := wrapped.Unwrap(); err != nil {
+			return Code(err)
+		}
 	}
-
 	return status.FromContextError(err).Code()
 }
 
@@ -124,7 +125,9 @@ func AsGRPCStatus(err error) (*status.Status, bool) {
 		Unwrap() error
 	})
 	if ok {
-		return AsGRPCStatus(wrapped.Unwrap())
+		if err := wrapped.Unwrap(); err != nil {
+			return AsGRPCStatus(err)
+		}
 	}
 
 	return nil, false

util/grpcerrors/intercept.go

@@ -2,15 +2,28 @@ package grpcerrors
 
 import (
 	"context"
+	"log"
+	"os"
 
+	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 )
 
 func UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
 	resp, err = handler(ctx, req)
+	oldErr := err
 	if err != nil {
 		err = ToGRPC(err)
 	}
+	if oldErr != nil && err == nil {
+		logErr := errors.Wrap(err, "invalid grpc error conversion")
+		if os.Getenv("BUILDKIT_DEBUG_PANIC_ON_ERROR") == "1" {
+			panic(logErr)
+		}
+		log.Printf("%v", logErr)
+		err = oldErr
+	}
+
 	return resp, err
 }