buildkit/solver/llbsolver/solver.go

411 lines
11 KiB
Go
Raw Normal View History

package llbsolver
import (
"context"
"fmt"
"strings"
"time"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/client"
controlgateway "github.com/moby/buildkit/control/gateway"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/gateway"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
const keyEntitlements = "llb.entitlements"
type ExporterRequest struct {
Exporter exporter.ExporterInstance
CacheExporter remotecache.Exporter
CacheExportMode solver.CacheExportMode
}
// ResolveWorkerFunc returns default worker for the temporary default non-distributed use cases
type ResolveWorkerFunc func() (worker.Worker, error)
type Solver struct {
workerController *worker.Controller
solver *solver.Solver
resolveWorker ResolveWorkerFunc
eachWorker func(func(worker.Worker) error) error
frontends map[string]frontend.Frontend
resolveCacheImporterFuncs map[string]remotecache.ResolveCacheImporterFunc
gatewayForwarder *controlgateway.GatewayForwarder
sm *session.Manager
entitlements []string
}
func New(wc *worker.Controller, f map[string]frontend.Frontend, cache solver.CacheManager, resolveCI map[string]remotecache.ResolveCacheImporterFunc, gatewayForwarder *controlgateway.GatewayForwarder, sm *session.Manager, ents []string) (*Solver, error) {
s := &Solver{
workerController: wc,
resolveWorker: defaultResolver(wc),
eachWorker: allWorkers(wc),
frontends: f,
resolveCacheImporterFuncs: resolveCI,
gatewayForwarder: gatewayForwarder,
sm: sm,
entitlements: ents,
}
s.solver = solver.NewSolver(solver.SolverOpt{
ResolveOpFunc: s.resolver(),
DefaultCache: cache,
})
return s, nil
}
func (s *Solver) resolver() solver.ResolveOpFunc {
return func(v solver.Vertex, b solver.Builder) (solver.Op, error) {
w, err := s.resolveWorker()
if err != nil {
return nil, err
}
return w.ResolveOp(v, s.Bridge(b), s.sm)
}
}
func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
return &llbBridge{
builder: b,
frontends: s.frontends,
resolveWorker: s.resolveWorker,
eachWorker: s.eachWorker,
resolveCacheImporterFuncs: s.resolveCacheImporterFuncs,
cms: map[string]solver.CacheManager{},
sm: s.sm,
}
}
func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement) (*client.SolveResponse, error) {
j, err := s.solver.NewJob(id)
if err != nil {
return nil, err
}
defer j.Discard()
set, err := entitlements.WhiteList(ent, supportedEntitlements(s.entitlements))
if err != nil {
return nil, err
}
j.SetValue(keyEntitlements, set)
j.SessionID = sessionID
var res *frontend.Result
if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" {
fwd := gateway.NewBridgeForwarder(ctx, s.Bridge(j), s.workerController, req.FrontendInputs, sessionID)
defer fwd.Discard()
if err := s.gatewayForwarder.RegisterBuild(ctx, id, fwd); err != nil {
return nil, err
}
defer s.gatewayForwarder.UnregisterBuild(ctx, id)
var err error
select {
case <-fwd.Done():
res, err = fwd.Result()
case <-ctx.Done():
err = ctx.Err()
}
if err != nil {
return nil, err
}
} else {
res, err = s.Bridge(j).Solve(ctx, req, sessionID)
if err != nil {
return nil, err
}
}
defer func() {
res.EachRef(func(ref solver.ResultProxy) error {
go ref.Release(context.TODO())
return nil
})
}()
eg, ctx2 := errgroup.WithContext(ctx)
res.EachRef(func(ref solver.ResultProxy) error {
eg.Go(func() error {
_, err := ref.Result(ctx2)
return err
})
return nil
})
if err := eg.Wait(); err != nil {
return nil, err
}
var exporterResponse map[string]string
if e := exp.Exporter; e != nil {
inp := exporter.Source{
Metadata: res.Metadata,
}
solver: avoid dereferencing nil res.Metadata I saw this: panic: assignment to entry in nil map goroutine 3173 [running]: github.com/moby/buildkit/exporter/oci.(*imageExporterInstance).Export(0xc42094ac40, 0xecac60, 0xc4211ca690, 0xed12c0, 0xc42000fca8, 0x0, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/moby/buildkit/exporter/oci/export.go:113 +0x156 github.com/moby/buildkit/solver/llbsolver.(*Solver).Solve.func2(0xecac60, 0xc4211ca690, 0xc4202207c0, 0x0) /go/src/github.com/moby/buildkit/solver/llbsolver/solver.go:132 +0x7d github.com/moby/buildkit/solver/llbsolver.inVertexContext(0xecac60, 0xc4211ca690, 0xe2ec38, 0x1d, 0xc420d43400, 0x0, 0x0) /go/src/github.com/moby/buildkit/solver/llbsolver/solver.go:203 +0x1f6 github.com/moby/buildkit/solver/llbsolver.(*Solver).Solve(0xc4203f7dc0, 0xecac60, 0xc420deb830, 0xc4203ed200, 0x19, 0xc4202ff840, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/moby/buildkit/solver/llbsolver/solver.go:131 +0x761 github.com/moby/buildkit/control.(*Controller).Solve(0xc42017e870, 0xecac60, 0xc420deb830, 0xc4201edd40, 0xc42017e870, 0x1, 0x1) /go/src/github.com/moby/buildkit/control/control.go:207 +0x4b8 github.com/moby/buildkit/api/services/control._Control_Solve_Handler.func1(0xecac60, 0xc420deb800, 0xde2180, 0xc4201edd40, 0xecac60, 0xc420deb800, 0xed67a0, 0x1595288) /go/src/github.com/moby/buildkit/api/services/control/control.pb.go:810 +0x86 github.com/moby/buildkit/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0xecac60, 0xc420deb800, 0xde2180, 0xc4201edd40, 0xc4202ff9e0, 0xc4202ffa00, 0x0, 0x0, 0xebbea0, 0xc420188310) /go/src/github.com/moby/buildkit/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:57 +0x2ba main.unaryInterceptor.func1(0xecaba0, 0xc4206afe40, 0xde2180, 0xc4201edd40, 0xc4202ff9e0, 0xc4202ffa00, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/moby/buildkit/cmd/buildkitd/main.go:330 +0x15f github.com/moby/buildkit/api/services/control._Control_Solve_Handler(0xd76a00, 0xc42017e870, 0xecac60, 0xc420deb530, 0xc42028d030, 0xc420450760, 0x0, 0x0, 0x34, 0x3) /go/src/github.com/moby/buildkit/api/services/control/control.pb.go:812 +0x167 github.com/moby/buildkit/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc420244700, 0xed39c0, 0xc420694000, 0xc420671680, 0xc420497650, 0x152cef8, 0x0, 0x0, 0x0) /go/src/github.com/moby/buildkit/vendor/google.golang.org/grpc/server.go:1011 +0x4fc github.com/moby/buildkit/vendor/google.golang.org/grpc.(*Server).handleStream(0xc420244700, 0xed39c0, 0xc420694000, 0xc420671680, 0x0) /go/src/github.com/moby/buildkit/vendor/google.golang.org/grpc/server.go:1249 +0x1318 github.com/moby/buildkit/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42063e0f0, 0xc420244700, 0xed39c0, 0xc420694000, 0xc420671680) /go/src/github.com/moby/buildkit/vendor/google.golang.org/grpc/server.go:680 +0x9f created by github.com/moby/buildkit/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /go/src/github.com/moby/buildkit/vendor/google.golang.org/grpc/server.go:678 +0xa1 Which was due to `res, err := s.Bridge(j).Solve(ctx, req)` having `res.Metadata == nil`. There are several paths in `llbBridge.Solve()` where this can be the case, plus a case where this comes from a frontend which should not be allowed to crash the daemon. Likely introduced by d70d816dee6d or 6be1257f5d3e. Signed-off-by: Ian Campbell <ijc@docker.com>
2018-08-03 15:02:26 +00:00
if inp.Metadata == nil {
inp.Metadata = make(map[string][]byte)
}
if res := res.Ref; res != nil {
r, err := res.Result(ctx)
if err != nil {
return nil, err
}
workerRef, ok := r.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid reference: %T", r.Sys())
}
inp.Ref = workerRef.ImmutableRef
dt, err := inlineCache(ctx, exp.CacheExporter, r)
if err != nil {
return nil, err
}
if dt != nil {
inp.Metadata[exptypes.ExporterInlineCache] = dt
}
}
if res.Refs != nil {
m := make(map[string]cache.ImmutableRef, len(res.Refs))
for k, res := range res.Refs {
if res == nil {
m[k] = nil
} else {
r, err := res.Result(ctx)
if err != nil {
return nil, err
}
workerRef, ok := r.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid reference: %T", r.Sys())
}
m[k] = workerRef.ImmutableRef
dt, err := inlineCache(ctx, exp.CacheExporter, r)
if err != nil {
return nil, err
}
if dt != nil {
inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, k)] = dt
}
}
}
inp.Refs = m
}
if err := inBuilderContext(ctx, j, e.Name(), "", func(ctx context.Context, _ session.Group) error {
exporterResponse, err = e.Export(ctx, inp, j.SessionID)
return err
}); err != nil {
return nil, err
}
}
var cacheExporterResponse map[string]string
if e := exp.CacheExporter; e != nil {
if err := inBuilderContext(ctx, j, "exporting cache", "", func(ctx context.Context, _ session.Group) error {
prepareDone := oneOffProgress(ctx, "preparing build cache for export")
if err := res.EachRef(func(res solver.ResultProxy) error {
r, err := res.Result(ctx)
if err != nil {
return err
}
// all keys have same export chain so exporting others is not needed
_, err = r.CacheKeys()[0].Exporter.ExportTo(ctx, e, solver.CacheExportOpt{
Convert: workerRefConverter,
Mode: exp.CacheExportMode,
})
return err
}); err != nil {
return prepareDone(err)
}
prepareDone(nil)
cacheExporterResponse, err = e.Finalize(ctx)
return err
}); err != nil {
return nil, err
}
}
if exporterResponse == nil {
exporterResponse = make(map[string]string)
}
for k, v := range res.Metadata {
if strings.HasPrefix(k, "frontend.") {
exporterResponse[k] = string(v)
}
}
for k, v := range cacheExporterResponse {
if strings.HasPrefix(k, "cache.") {
exporterResponse[k] = v
}
}
return &client.SolveResponse{
ExporterResponse: exporterResponse,
}, nil
}
func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedResult) ([]byte, error) {
if efl, ok := e.(interface {
ExportForLayers([]digest.Digest) ([]byte, error)
}); ok {
workerRef, ok := res.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid reference: %T", res.Sys())
}
remote, err := workerRef.Worker.GetRemote(ctx, workerRef.ImmutableRef, true)
if err != nil || remote == nil {
return nil, nil
}
digests := make([]digest.Digest, 0, len(remote.Descriptors))
for _, desc := range remote.Descriptors {
digests = append(digests, desc.Digest)
}
if _, err := res.CacheKeys()[0].Exporter.ExportTo(ctx, e, solver.CacheExportOpt{
Convert: workerRefConverter,
Mode: solver.CacheExportModeMin,
}); err != nil {
return nil, err
}
return efl.ExportForLayers(digests)
}
return nil, nil
}
func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client.SolveStatus) error {
j, err := s.solver.Get(id)
if err != nil {
close(statusChan)
return err
}
return j.Status(ctx, statusChan)
}
func defaultResolver(wc *worker.Controller) ResolveWorkerFunc {
return func() (worker.Worker, error) {
return wc.GetDefault()
}
}
func allWorkers(wc *worker.Controller) func(func(w worker.Worker) error) error {
return func(f func(worker.Worker) error) error {
all, err := wc.List()
if err != nil {
return err
}
for _, w := range all {
if err := f(w); err != nil {
return err
}
}
return nil
}
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {
pw, _, _ := progress.FromContext(ctx)
now := time.Now()
st := progress.Status{
Started: &now,
}
pw.Write(id, st)
return func(err error) error {
// TODO: set error on status
now := time.Now()
st.Completed = &now
pw.Write(id, st)
pw.Close()
return err
}
}
func inBuilderContext(ctx context.Context, b solver.Builder, name, id string, f func(ctx context.Context, g session.Group) error) error {
if id == "" {
id = name
}
v := client.Vertex{
Digest: digest.FromBytes([]byte(id)),
Name: name,
}
return b.InContext(ctx, func(ctx context.Context, g session.Group) error {
pw, _, ctx := progress.FromContext(ctx, progress.WithMetadata("vertex", v.Digest))
notifyStarted(ctx, &v, false)
defer pw.Close()
err := f(ctx, g)
notifyCompleted(ctx, &v, err, false)
return err
})
}
func notifyStarted(ctx context.Context, v *client.Vertex, cached bool) {
pw, _, _ := progress.FromContext(ctx)
defer pw.Close()
now := time.Now()
v.Started = &now
v.Completed = nil
v.Cached = cached
pw.Write(v.Digest.String(), *v)
}
func notifyCompleted(ctx context.Context, v *client.Vertex, err error, cached bool) {
pw, _, _ := progress.FromContext(ctx)
defer pw.Close()
now := time.Now()
if v.Started == nil {
v.Started = &now
}
v.Completed = &now
v.Cached = cached
if err != nil {
v.Error = err.Error()
}
pw.Write(v.Digest.String(), *v)
}
func supportedEntitlements(ents []string) []entitlements.Entitlement {
out := []entitlements.Entitlement{} // nil means no filter
for _, e := range ents {
if e == string(entitlements.EntitlementNetworkHost) {
out = append(out, entitlements.EntitlementNetworkHost)
}
if e == string(entitlements.EntitlementSecurityInsecure) {
out = append(out, entitlements.EntitlementSecurityInsecure)
}
}
return out
}
func loadEntitlements(b solver.Builder) (entitlements.Set, error) {
var ent entitlements.Set = map[entitlements.Entitlement]struct{}{}
err := b.EachValue(context.TODO(), keyEntitlements, func(v interface{}) error {
set, ok := v.(entitlements.Set)
if !ok {
return errors.Errorf("invalid entitlements %T", v)
}
for k := range set {
ent[k] = struct{}{}
}
return nil
})
if err != nil {
return nil, err
}
return ent, nil
}