JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #136 from tonistiigi/solver-update 

Update solver public api
docker-18.09
Akihiro Suda 2017-10-06 11:56:04 +09:00 committed by GitHub
parent 7d0fea5b60 59910481ca
commit be1a9ed281
10 changed files with 150 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
client/solve.go
View File

@ -46,7 +46,7 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s
return errors.Errorf("invalid definition for frontend %s", opt.Frontend)
}
syncedDirs, err := prepareSyncedDirs(def.Def, opt.LocalDirs)
syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
if err != nil {
return err
}
@ -87,9 +87,13 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s
logrus.Debugf("stopping session")
s.Close()
}()
var pbd *pb.Definition
if def != nil {
pbd = def.ToPB()
}
_, err = c.controlClient().Solve(ctx, &controlapi.SolveRequest{
Ref: ref,
Definition: def.ToPB(),
Definition: pbd,
Exporter: opt.Exporter,
ExporterAttrs: opt.ExporterAttrs,
Session: s.ID(),
@ -158,7 +162,7 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s
return eg.Wait()
}
func prepareSyncedDirs(defs [][]byte, localDirs map[string]string) ([]filesync.SyncedDir, error) {
func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) ([]filesync.SyncedDir, error) {
for _, d := range localDirs {
fi, err := os.Stat(d)
if err != nil {
@ -169,24 +173,25 @@ func prepareSyncedDirs(defs [][]byte, localDirs map[string]string) ([]filesync.S
}
}
dirs := make([]filesync.SyncedDir, 0, len(localDirs))
if len(defs) == 0 {
if def == nil {
for name, d := range localDirs {
dirs = append(dirs, filesync.SyncedDir{Name: name, Dir: d})
}
}
for _, dt := range defs {
var op pb.Op
if err := (&op).Unmarshal(dt); err != nil {
return nil, errors.Wrap(err, "failed to parse llb proto op")
}
if src := op.GetSource(); src != nil {
if strings.HasPrefix(src.Identifier, "local://") { // TODO: just make a type property
name := strings.TrimPrefix(src.Identifier, "local://")
d, ok := localDirs[name]
if !ok {
return nil, errors.Errorf("local directory %s not enabled", name)
} else {
for _, dt := range def.Def {
var op pb.Op
if err := (&op).Unmarshal(dt); err != nil {
return nil, errors.Wrap(err, "failed to parse llb proto op")
}
if src := op.GetSource(); src != nil {
if strings.HasPrefix(src.Identifier, "local://") { // TODO: just make a type property
name := strings.TrimPrefix(src.Identifier, "local://")
d, ok := localDirs[name]
if !ok {
return nil, errors.Errorf("local directory %s not enabled", name)
}
dirs = append(dirs, filesync.SyncedDir{Name: name, Dir: d}) // TODO: excludes
}
dirs = append(dirs, filesync.SyncedDir{Name: name, Dir: d}) // TODO: excludes
}
}
}

8
control/control.go
View File

@ -45,6 +45,7 @@ func NewController(opt Opt) (*Controller, error) {
Worker: opt.Worker,
InstructionCache: opt.InstructionCache,
ImageSource: opt.ImageSource,
Frontends: opt.Frontends,
}),
}
return c, nil
@ -105,7 +106,12 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
}
}
if err := c.solver.Solve(ctx, req.Ref, frontend, req.Definition, expi, req.FrontendAttrs, c.opt.Frontends); err != nil {
if err := c.solver.Solve(ctx, req.Ref, solver.SolveRequest{
Frontend: frontend,
Definition: req.Definition,
Exporter: expi,
FrontendOpt: req.FrontendAttrs,
}); err != nil {
return nil, err
}
return &controlapi.SolveResponse{}, nil

8
frontend/dockerfile/dockerfile.go
View File

@ -53,7 +53,9 @@ func (f *dfFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBr
return nil, nil, err
}
ref, _, err := llbBridge.Solve(ctx, def.ToPB(), "", nil)
ref, _, err := llbBridge.Solve(ctx, frontend.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, nil, err
}
@ -112,7 +114,9 @@ func (f *dfFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBr
if err != nil {
return nil, nil, err
}
retRef, _, err = llbBridge.Solve(ctx, def.ToPB(), "", nil)
retRef, _, err = llbBridge.Solve(ctx, frontend.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, nil, err
}

8
frontend/frontend.go
View File

@ -15,7 +15,13 @@ type Frontend interface {
}
type FrontendLLBBridge interface {
Solve(ctx context.Context, def *pb.Definition, frontend string, opts map[string]string) (cache.ImmutableRef, map[string][]byte, error)
Solve(ctx context.Context, req SolveRequest) (cache.ImmutableRef, map[string][]byte, error)
ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error)
Exec(ctx context.Context, meta worker.Meta, rootfs cache.ImmutableRef, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error
}
type SolveRequest struct {
Definition *pb.Definition
Frontend string
FrontendOpt map[string]string
}

15
frontend/gateway/gateway.go
View File

@ -67,7 +67,11 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
var rootFS cache.ImmutableRef
if isDevel {
ref, exp, err := llbBridge.Solve(session.NewContext(ctx, "gateway:"+sid), nil, source, filterPrefix(opts, "gateway-"))
ref, exp, err := llbBridge.Solve(session.NewContext(ctx, "gateway:"+sid),
frontend.SolveRequest{
Frontend: source,
FrontendOpt: filterPrefix(opts, "gateway-"),
})
if err != nil {
return nil, nil, err
}
@ -105,7 +109,9 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
return nil, nil, err
}
ref, _, err := llbBridge.Solve(ctx, def.ToPB(), "", nil)
ref, _, err := llbBridge.Solve(ctx, frontend.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, nil, err
}
@ -240,7 +246,10 @@ func (lbf *llbBrideForwarder) ResolveImageConfig(ctx context.Context, req *pb.Re
}
func (lbf *llbBrideForwarder) Solve(ctx context.Context, req *pb.SolveRequest) (*pb.SolveResponse, error) {
ref, expResp, err := lbf.llbBridge.Solve(ctx, req.Definition, req.Frontend, nil)
ref, expResp, err := lbf.llbBridge.Solve(ctx, frontend.SolveRequest{
Definition: req.Definition,
Frontend: req.Frontend,
})
if err != nil {
return nil, err
}

2
hack/test-integration
View File

@ -5,4 +5,4 @@ set -eu -o pipefail -x
# update this to iidfile after 17.06
docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm .
docker run --rm -v /tmp --privileged buildkit:test go test -tags 'containerd standalone' ./client
docker run --rm buildkit:test go build ./frontend/gateway/client

6
solver/build.go
View File

@ -104,13 +104,13 @@ func (b *buildOp) Run(ctx context.Context, inputs []Reference) (outputs []Refere
f.Close()
return nil, err
}
defPB := def.ToPB()
f.Close()
lm.Unmount()
lm = nil
newref, err := b.s.loadAndSolve(ctx, b.v.Digest(), defPB)
newref, err := b.s.subBuild(ctx, b.v.Digest(), SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}

24
solver/jobs.go
View File

@ -91,30 +91,6 @@ func (jl *jobList) get(id string) (*job, error) {
}
}
func (jl *jobList) loadAndSolve(ctx context.Context, dgst digest.Digest, def *pb.Definition, f ResolveOpFunc, cache InstructionCache) (Reference, error) {
jl.mu.Lock()
st, ok := jl.actives[dgst]
if !ok {
jl.mu.Unlock()
return nil, errors.Errorf("no such parent vertex: %v", dgst)
}
var inp *Input
for j := range st.jobs {
var err error
inp, err = j.loadInternal(def, f)
if err != nil {
jl.mu.Unlock()
return nil, err
}
}
st = jl.actives[inp.Vertex.Digest()]
jl.mu.Unlock()
return getRef(st.solver, ctx, inp.Vertex.(*vertex), inp.Index, cache) // TODO: combine to pass single input
}
type job struct {
l *jobList
pr *progress.MultiReader

145
solver/solver.go
View File

@ -11,7 +11,6 @@ import (
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/bgfunc"
@ -26,10 +25,11 @@ import (
type LLBOpt struct {
SourceManager *source.Manager
CacheManager cache.Manager // TODO: this shouldn't be needed before instruction cache
CacheManager cache.Manager
Worker worker.Worker
InstructionCache InstructionCache
ImageSource source.Source
Frontends map[string]frontend.Frontend // used by nested invocations
}
func NewLLBSolver(opt LLBOpt) *Solver {
@ -45,7 +45,7 @@ func NewLLBSolver(opt LLBOpt) *Solver {
default:
return nil, nil
}
}, opt.InstructionCache, opt.ImageSource, opt.Worker, opt.CacheManager)
}, opt.InstructionCache, opt.ImageSource, opt.Worker, opt.CacheManager, opt.Frontends)
return s
}
@ -79,46 +79,55 @@ type Solver struct {
imageSource source.Source
worker worker.Worker
cm cache.Manager // TODO: remove with immutableRef.New()
frontends map[string]frontend.Frontend
}
func New(resolve ResolveOpFunc, cache InstructionCache, imageSource source.Source, worker worker.Worker, cm cache.Manager) *Solver {
return &Solver{resolve: resolve, jobs: newJobList(), cache: cache, imageSource: imageSource, worker: worker, cm: cm}
func New(resolve ResolveOpFunc, cache InstructionCache, imageSource source.Source, worker worker.Worker, cm cache.Manager, f map[string]frontend.Frontend) *Solver {
return &Solver{resolve: resolve, jobs: newJobList(), cache: cache, imageSource: imageSource, worker: worker, cm: cm, frontends: f}
}
func (s *Solver) Solve(ctx context.Context, id string, f frontend.Frontend, def *pb.Definition, exp exporter.ExporterInstance, frontendOpt map[string]string, allFrontends map[string]frontend.Frontend) error {
type SolveRequest struct {
Definition *pb.Definition
Frontend frontend.Frontend
Exporter exporter.ExporterInstance
FrontendOpt map[string]string
}
func (s *Solver) solve(ctx context.Context, j *job, req SolveRequest) (Reference, map[string][]byte, error) {
if req.Definition == nil {
if req.Frontend == nil {
return nil, nil, errors.Errorf("invalid request: no definition nor frontend")
}
return req.Frontend.Solve(ctx, s.llbBridge(j), req.FrontendOpt)
}
inp, err := j.load(req.Definition, s.resolve)
if err != nil {
return nil, nil, err
}
ref, err := j.getRef(ctx, inp.Vertex.(*vertex), inp.Index)
return ref, nil, err
}
func (s *Solver) llbBridge(j *job) *llbBridge {
return &llbBridge{job: j, Solver: s, resolveImageConfig: s.imageSource.(resolveImageConfig)}
}
func (s *Solver) Solve(ctx context.Context, id string, req SolveRequest) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
pr, ctx, closeProgressWriter := progress.NewContext(ctx)
defer closeProgressWriter()
// register a build job. vertex needs to be loaded to a job to run
ctx, j, err := s.jobs.new(ctx, id, pr, s.cache)
if err != nil {
return err
}
var ref Reference
var exporterOpt map[string][]byte
if def != nil {
var inp *Input
inp, err = j.load(def, s.resolve)
if err != nil {
j.discard()
return err
}
ref, err = j.getRef(ctx, inp.Vertex.(*vertex), inp.Index)
} else {
ref, exporterOpt, err = f.Solve(ctx, &llbBridge{
worker: s.worker,
job: j,
cm: s.cm,
resolveOp: s.resolve,
resolveImageConfig: s.imageSource.(resolveImageConfig),
allFrontends: allFrontends,
}, frontendOpt)
}
j.discard()
ref, exporterOpt, err := s.solve(ctx, j, req)
defer j.discard()
if err != nil {
return err
}
@ -135,19 +144,10 @@ func (s *Solver) Solve(ctx context.Context, id string, f frontend.Frontend, def
return err
}
if exp != nil {
v := client.Vertex{
Digest: digest.FromBytes([]byte(identity.NewID())),
Name: exp.Name(),
}
notifyStarted(ctx, &v)
pw, _, ctx := progress.FromContext(ctx, progress.WithMetadata("vertex", v.Digest))
defer pw.Close()
err := exp.Export(ctx, immutable, exporterOpt)
notifyCompleted(ctx, &v, err)
if err != nil {
return err
}
if exp := req.Exporter; exp != nil {
return inVertexContext(ctx, exp.Name(), func(ctx context.Context) error {
return exp.Export(ctx, immutable, exporterOpt)
})
}
return err
}
@ -161,8 +161,28 @@ func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client.
return j.pipe(ctx, statusChan)
}
func (s *Solver) loadAndSolve(ctx context.Context, dgst digest.Digest, def *pb.Definition) (Reference, error) {
return s.jobs.loadAndSolve(ctx, dgst, def, s.resolve, s.cache)
func (s *Solver) subBuild(ctx context.Context, dgst digest.Digest, req SolveRequest) (Reference, error) {
jl := s.jobs
jl.mu.Lock()
st, ok := jl.actives[dgst]
if !ok {
jl.mu.Unlock()
return nil, errors.Errorf("no such parent vertex: %v", dgst)
}
var inp *Input
for j := range st.jobs {
var err error
inp, err = j.loadInternal(req.Definition, s.resolve)
if err != nil {
jl.mu.Unlock()
return nil, err
}
}
st = jl.actives[inp.Vertex.Digest()]
jl.mu.Unlock()
return getRef(st.solver, ctx, inp.Vertex.(*vertex), inp.Index, s.cache) // TODO: combine to pass single input
}
type VertexSolver interface {
@ -571,39 +591,29 @@ type VertexResult struct {
// llbBridge is an helper used by frontends
type llbBridge struct {
*Solver
job *job
resolveImageConfig
job *job
resolveOp ResolveOpFunc
worker worker.Worker
allFrontends map[string]frontend.Frontend
cm cache.Manager
}
type resolveImageConfig interface {
ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error)
}
func (s *llbBridge) Solve(ctx context.Context, def *pb.Definition, frontend string, opts map[string]string) (cache.ImmutableRef, map[string][]byte, error) {
if def == nil {
f, ok := s.allFrontends[frontend]
func (s *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (cache.ImmutableRef, map[string][]byte, error) {
var f frontend.Frontend
if req.Frontend != "" {
var ok bool
f, ok = s.frontends[req.Frontend]
if !ok {
return nil, nil, errors.Errorf("invalid frontend: %s", frontend)
return nil, nil, errors.Errorf("invalid frontend: %s", req.Frontend)
}
ref, exporterOpt, err := f.Solve(ctx, s, opts)
if err != nil {
return nil, nil, err
}
immutable, ok := toImmutableRef(ref)
if !ok {
return nil, nil, errors.Errorf("invalid reference for exporting: %T", ref)
}
return immutable, exporterOpt, nil
}
inp, err := s.job.load(def, s.resolveOp)
if err != nil {
return nil, nil, err
}
ref, err := s.job.getRef(ctx, inp.Vertex.(*vertex), inp.Index)
ref, exp, err := s.solve(ctx, s.job, SolveRequest{
Definition: req.Definition,
Frontend: f,
FrontendOpt: req.FrontendOpt,
})
if err != nil {
return nil, nil, err
}
@ -611,8 +621,7 @@ func (s *llbBridge) Solve(ctx context.Context, def *pb.Definition, frontend stri
if !ok {
return nil, nil, errors.Errorf("invalid reference for exporting: %T", ref)
}
return immutable, nil, nil
return immutable, exp, nil
}
func (s *llbBridge) Exec(ctx context.Context, meta worker.Meta, rootFS cache.ImmutableRef, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {

16
solver/vertex.go
View File

@ -1,13 +1,14 @@
package solver
import (
"context"
"sync"
"time"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/progress"
digest "github.com/opencontainers/go-digest"
"golang.org/x/net/context"
)
// Vertex is one node in the build graph
@ -112,3 +113,16 @@ func notifyCompleted(ctx context.Context, v *client.Vertex, err error) {
}
pw.Write(v.Digest.String(), *v)
}
func inVertexContext(ctx context.Context, name string, f func(ctx context.Context) error) error {
v := client.Vertex{
Digest: digest.FromBytes([]byte(identity.NewID())),
Name: name,
}
pw, _, ctx := progress.FromContext(ctx, progress.WithMetadata("vertex", v.Digest))
notifyStarted(ctx, &v)
defer pw.Close()
err := f(ctx)
notifyCompleted(ctx, &v, err)
return err
}