JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9 from tonistiigi/add_solver_status 

Add solver status
docker-18.09
Tõnis Tiigi 2017-06-14 10:00:55 -07:00 committed by GitHub
parent 33b5c9b59c 7c062a8578
commit bc685ff267
15 changed files with 2420 additions and 615 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1808
api/services/control/control.pb.go
View File

File diff suppressed because it is too large Load Diff

45
api/services/control/control.proto
View File

@ -1,11 +1,18 @@
syntax = "proto3";
package control;
package moby.buildkit.v1;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "google/protobuf/timestamp.proto";
option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.unmarshaler_all) = true;
service Control {
rpc DiskUsage(DiskUsageRequest) returns (DiskUsageResponse);
rpc Solve(SolveRequest) returns (SolveResponse);
// rpc Status() returns ();
rpc Status(StatusRequest) returns (stream StatusResponse);
}
message DiskUsageRequest {
@ -28,8 +35,40 @@ message SolveRequest {
}
message SolveResponse {
repeated VertexStatus vertex = 1;
repeated Vertex vtx = 1;
}
message StatusRequest {
string Ref = 1;
}
message StatusResponse {
repeated Vertex vertexes = 1;
repeated VertexStatus statuses = 2;
repeated VertexLog logs = 3;
}
message Vertex {
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
repeated string inputs = 2 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string name = 3;
bool cached = 4;
google.protobuf.Timestamp started = 5 [(gogoproto.stdtime) = true ];
google.protobuf.Timestamp completed = 6 [(gogoproto.stdtime) = true ];
}
message VertexStatus {
string ID = 1;
string vertex = 2 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string name = 3;
int64 current = 4;
int64 total = 5;
google.protobuf.Timestamp timestamp = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
}
message VertexLog {
string vertex = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
google.protobuf.Timestamp timestamp = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
int64 stream = 3;
bytes msg = 4;
}

4
api/services/control/generate.go
View File

@ -1,3 +1,3 @@
package control
package moby_buildkit_v1
//go:generate protoc --gogoslick_out=plugins=grpc:. control.proto
//go:generate protoc -I=. -I=../../../vendor/ --gogo_out=plugins=grpc:. control.proto

49
client/graph.go Normal file
View File

@ -0,0 +1,49 @@
package client
import (
"time"
digest "github.com/opencontainers/go-digest"
)
type Vertex struct {
Digest digest.Digest
Inputs []digest.Digest
Name string
Started *time.Time
Completed *time.Time
Cached bool
Error string
}
type VertexStatus struct {
ID string
Vertex digest.Digest
Name string
Total int
Current int
Timestamp time.Time
}
type VertexLog struct {
Vertex digest.Digest
Stream int
Data []byte
Timestamp time.Time
}
type SolveStatus struct {
Vertexes []*Vertex
Statuses []*VertexStatus
Logs []*VertexLog
}
//
// type VertexEvent struct {
// ID digest.Digest
// Vertex digest.Digest
// Name string
// Total int
// Current int
// Timestamp int64
// }

30
client/solve.go
View File

@ -6,9 +6,11 @@ import (
"encoding/hex"
"io"
"github.com/Sirupsen/logrus"
"github.com/pkg/errors"
controlapi "github.com/tonistiigi/buildkit_poc/api/services/control"
"github.com/tonistiigi/buildkit_poc/client/llb"
"golang.org/x/sync/errgroup"
)
func (c *Client) Solve(ctx context.Context, r io.Reader) error {
@ -21,14 +23,40 @@ func (c *Client) Solve(ctx context.Context, r io.Reader) error {
return errors.New("invalid empty definition")
}
ref := generateID()
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
_, err = c.controlClient().Solve(ctx, &controlapi.SolveRequest{
Ref: generateID(),
Ref: ref,
Definition: def,
})
if err != nil {
return errors.Wrap(err, "failed to solve")
}
return nil
})
eg.Go(func() error {
stream, err := c.controlClient().Status(ctx, &controlapi.StatusRequest{
Ref: ref,
})
if err != nil {
return errors.Wrap(err, "failed to get status")
}
for {
resp, err := stream.Recv()
if err != nil {
if err == io.EOF {
return nil
}
return errors.Wrap(err, "failed to receive status")
}
logrus.Debugf("status: %+v", resp)
}
})
return eg.Wait()
}
func generateID() string {

41
control/control.go
View File

@ -5,10 +5,12 @@ import (
"github.com/pkg/errors"
controlapi "github.com/tonistiigi/buildkit_poc/api/services/control"
"github.com/tonistiigi/buildkit_poc/cache"
"github.com/tonistiigi/buildkit_poc/client"
"github.com/tonistiigi/buildkit_poc/solver"
"github.com/tonistiigi/buildkit_poc/source"
"github.com/tonistiigi/buildkit_poc/worker"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc"
)
@ -64,8 +66,45 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
if err != nil {
return nil, errors.Wrap(err, "failed to load")
}
if err := c.solver.Solve(ctx, v); err != nil {
if err := c.solver.Solve(ctx, req.Ref, v); err != nil {
return nil, err
}
return &controlapi.SolveResponse{}, nil
}
func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Control_StatusServer) error {
ch := make(chan *client.SolveStatus, 8)
eg, ctx := errgroup.WithContext(stream.Context())
eg.Go(func() error {
return c.solver.Status(ctx, req.Ref, ch)
})
eg.Go(func() error {
for {
select {
case <-ctx.Done():
return ctx.Err()
case ss, ok := <-ch:
if !ok {
return nil
}
sr := controlapi.StatusResponse{}
for _, v := range ss.Vertexes {
sr.Vertexes = append(sr.Vertexes, &controlapi.Vertex{
Digest: v.Digest,
Inputs: v.Inputs,
Name: v.Name,
Started: v.Started,
Completed: v.Completed,
})
}
if err := stream.SendMsg(&sr); err != nil {
return err
}
}
}
})
return eg.Wait()
}

47
control/control_standalone.go
View File

@ -13,11 +13,14 @@ import (
"github.com/containerd/containerd/archive/compression"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces"
ctdsnapshot "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/overlay"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/tonistiigi/buildkit_poc/worker/runcworker"
netcontext "golang.org/x/net/context" // TODO: fix
)
func NewStandalone(root string) (*Controller, error) {
@ -61,7 +64,7 @@ func newPullDeps(root string) (*pullDeps, error) {
a := &localApplier{root: root, content: c}
return &pullDeps{
Snapshotter: s,
Snapshotter: &nsSnapshotter{s},
ContentStore: c,
Applier: a,
}, nil
@ -73,7 +76,7 @@ type localApplier struct {
content content.Store
}
func (a *localApplier) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount) (ocispec.Descriptor, error) {
func (a *localApplier) Apply(ctx netcontext.Context, desc ocispec.Descriptor, mounts []mount.Mount) (ocispec.Descriptor, error) {
dir, err := ioutil.TempDir(a.root, "extract-")
if err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to create temporary directory")
@ -136,3 +139,43 @@ type nopCloser struct {
func (n *nopCloser) Close() error {
return nil
}
// this should be supported by containerd. currently packages are unusable without wrapping
const dummyNs = "buildkit"
type nsSnapshotter struct {
ctdsnapshot.Snapshotter
}
func (s *nsSnapshotter) Stat(ctx context.Context, key string) (ctdsnapshot.Info, error) {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Stat(ctx, key)
}
func (s *nsSnapshotter) Usage(ctx context.Context, key string) (ctdsnapshot.Usage, error) {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Usage(ctx, key)
}
func (s *nsSnapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Mounts(ctx, key)
}
func (s *nsSnapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Prepare(ctx, key, parent)
}
func (s *nsSnapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.View(ctx, key, parent)
}
func (s *nsSnapshotter) Commit(ctx context.Context, name, key string) error {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Commit(ctx, name, key)
}
func (s *nsSnapshotter) Remove(ctx context.Context, key string) error {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Remove(ctx, key)
}
func (s *nsSnapshotter) Walk(ctx context.Context, fn func(context.Context, ctdsnapshot.Info) error) error {
ctx = namespaces.WithNamespace(ctx, dummyNs)
return s.Snapshotter.Walk(ctx, fn)
}

132
solver/jobs.go Normal file
View File

@ -0,0 +1,132 @@
package solver
import (
"context"
"sync"
"time"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/tonistiigi/buildkit_poc/client"
"github.com/tonistiigi/buildkit_poc/util/progress"
)
type jobList struct {
mu sync.RWMutex
refs map[string]*job
updateCond *sync.Cond
}
func newJobList() *jobList {
jl := &jobList{
refs: make(map[string]*job),
}
jl.updateCond = sync.NewCond(jl.mu.RLocker())
return jl
}
func (jl *jobList) new(ctx context.Context, id string, g *opVertex, pr progress.ProgressReader) (*job, error) {
jl.mu.Lock()
defer jl.mu.Unlock()
if _, ok := jl.refs[id]; ok {
return nil, errors.Errorf("id %s exists", id)
}
j := &job{g: g, pr: progress.NewMultiReader(pr)}
jl.refs[id] = j
jl.updateCond.Broadcast()
go func() {
<-ctx.Done()
jl.mu.Lock()
defer jl.mu.Unlock()
delete(jl.refs, id)
}()
return jl.refs[id], nil
}
func (jl *jobList) get(id string) (*job, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
go func() {
<-ctx.Done()
jl.updateCond.Broadcast()
}()
jl.mu.RLock()
defer jl.mu.RUnlock()
for {
select {
case <-ctx.Done():
return nil, errors.Errorf("no such job %s", id)
default:
}
j, ok := jl.refs[id]
if !ok {
jl.updateCond.Wait()
continue
}
return j, nil
}
}
type job struct {
mu sync.Mutex
g *opVertex
pr *progress.MultiReader
}
func (j *job) pipe(ctx context.Context, ch chan *client.SolveStatus) error {
pr := j.pr.Reader(ctx)
for v := range walk(j.g) {
ss := &client.SolveStatus{
Vertexes: []*client.Vertex{&v.vtx},
}
select {
case <-ctx.Done():
return ctx.Err()
case ch <- ss:
}
}
for {
p, err := pr.Read(ctx) // add cancelling
if err != nil {
return err
}
if p == nil {
return nil
}
switch v := p.Sys.(type) {
case client.Vertex:
ss := &client.SolveStatus{Vertexes: []*client.Vertex{&v}}
select {
case <-ctx.Done():
return ctx.Err()
case ch <- ss:
}
}
}
}
func walk(op *opVertex) chan *opVertex {
cache := make(map[digest.Digest]struct{})
ch := make(chan *opVertex, 32)
var send func(op *opVertex)
send = func(op *opVertex) {
for _, v := range op.inputs {
send(v)
}
if _, ok := cache[op.dgst]; !ok {
ch <- op
cache[op.dgst] = struct{}{}
}
}
go func() {
send(op)
close(ch)
}()
return ch
}

186
solver/load.go
View File

@ -1,29 +1,12 @@
package solver
import (
"context"
"os"
"sync"
"golang.org/x/sync/errgroup"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/tonistiigi/buildkit_poc/cache"
"github.com/tonistiigi/buildkit_poc/client"
"github.com/tonistiigi/buildkit_poc/solver/pb"
"github.com/tonistiigi/buildkit_poc/source"
"github.com/tonistiigi/buildkit_poc/worker"
)
type opVertex struct {
mu sync.Mutex
op *pb.Op
inputs []*opVertex
refs []cache.ImmutableRef
err error
dgst digest.Digest
}
func Load(ops [][]byte) (*opVertex, error) {
if len(ops) == 0 {
return nil, errors.New("invalid empty definition")
@ -63,172 +46,25 @@ func loadReqursive(dgst digest.Digest, op *pb.Op, inputs map[digest.Digest]*pb.O
return v, nil
}
vtx := &opVertex{op: op, dgst: dgst}
inputDigests := make([]digest.Digest, 0, len(op.Inputs))
for _, in := range op.Inputs {
op, ok := inputs[digest.Digest(in.Digest)]
dgst := digest.Digest(in.Digest)
inputDigests = append(inputDigests, dgst)
op, ok := inputs[dgst]
if !ok {
return nil, errors.Errorf("failed to find %s", in)
}
sub, err := loadReqursive(digest.Digest(in.Digest), op, inputs, cache)
sub, err := loadReqursive(dgst, op, inputs, cache)
if err != nil {
return nil, err
}
vtx.inputs = append(vtx.inputs, sub)
}
vtx.vtx = client.Vertex{
Inputs: inputDigests,
Name: vtx.name(),
Digest: dgst,
}
cache[dgst] = vtx
return vtx, nil
}
type Opt struct {
SourceManager *source.Manager
CacheManager cache.Manager // TODO: this shouldn't be needed before instruction cache
Worker worker.Worker
}
func (g *opVertex) inputRequiresExport(i int) bool {
return true // TODO
}
type Solver struct {
opt Opt
}
func New(opt Opt) *Solver {
return &Solver{opt: opt}
}
func (s *Solver) Solve(ctx context.Context, g *opVertex) error {
err := g.solve(ctx, s.opt) // TODO: separate exporting
g.release(ctx)
return err
}
func (g *opVertex) release(ctx context.Context) (retErr error) {
for _, i := range g.inputs {
if err := i.release(ctx); err != nil {
retErr = err
}
}
for _, ref := range g.refs {
if ref != nil {
if err := ref.Release(ctx); err != nil {
retErr = err
}
}
}
return retErr
}
func (g *opVertex) getInputRef(i int) cache.ImmutableRef {
input := g.op.Inputs[i]
for _, v := range g.inputs {
if v.dgst == digest.Digest(input.Digest) {
return v.refs[input.Index]
}
}
return nil
}
func (g *opVertex) solve(ctx context.Context, opt Opt) (retErr error) {
g.mu.Lock()
defer g.mu.Unlock()
if g.err != nil {
return g.err
}
if len(g.refs) > 0 {
return nil
}
defer func() {
if retErr != nil {
g.err = retErr
}
}()
if len(g.inputs) > 0 {
eg, ctx := errgroup.WithContext(ctx)
for _, in := range g.inputs {
eg.Go(func() error {
err := in.solve(ctx, opt)
if err != nil {
return err
}
return nil
})
}
err := eg.Wait()
if err != nil {
return err
}
}
switch op := g.op.Op.(type) {
case *pb.Op_Source:
id, err := source.FromString(op.Source.Identifier)
if err != nil {
return err
}
ref, err := opt.SourceManager.Pull(ctx, id)
if err != nil {
return err
}
g.refs = []cache.ImmutableRef{ref}
case *pb.Op_Exec:
mounts := make(map[string]cache.Mountable)
var outputs []cache.MutableRef
defer func() {
for _, o := range outputs {
if o != nil {
s, err := o.Freeze() // TODO: log error
if err == nil {
s.Release(ctx)
}
}
}
}()
for _, m := range op.Exec.Mounts {
var mountable cache.Mountable
ref := g.getInputRef(int(m.Input))
mountable = ref
if m.Output != -1 {
active, err := opt.CacheManager.New(ctx, ref) // TODO: should be method
if err != nil {
return err
}
outputs = append(outputs, active)
mountable = active
}
mounts[m.Dest] = mountable
}
meta := worker.Meta{
Args: op.Exec.Meta.Args,
Env: op.Exec.Meta.Env,
Cwd: op.Exec.Meta.Cwd,
}
if err := opt.Worker.Exec(ctx, meta, mounts, os.Stderr, os.Stderr); err != nil {
return errors.Wrapf(err, "worker failed running %v", meta.Args)
}
g.refs = []cache.ImmutableRef{}
for i, o := range outputs {
ref, err := o.ReleaseAndCommit(ctx)
if err != nil {
return errors.Wrapf(err, "error committing %s", ref.ID())
}
g.refs = append(g.refs, ref)
outputs[i] = nil
}
default:
return errors.Errorf("invalid op type")
}
return nil
}

242
solver/solver.go Normal file
View File

@ -0,0 +1,242 @@
package solver
import (
"os"
"strings"
"sync"
"time"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/tonistiigi/buildkit_poc/cache"
"github.com/tonistiigi/buildkit_poc/client"
"github.com/tonistiigi/buildkit_poc/solver/pb"
"github.com/tonistiigi/buildkit_poc/source"
"github.com/tonistiigi/buildkit_poc/util/progress"
"github.com/tonistiigi/buildkit_poc/worker"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
type Opt struct {
SourceManager *source.Manager
CacheManager cache.Manager // TODO: this shouldn't be needed before instruction cache
Worker worker.Worker
}
type Solver struct {
opt Opt
jobs *jobList
}
func New(opt Opt) *Solver {
return &Solver{opt: opt, jobs: newJobList()}
}
func (s *Solver) Solve(ctx context.Context, id string, g *opVertex) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
pr, ctx, closeProgressWriter := progress.NewContext(ctx)
_, err := s.jobs.new(ctx, id, g, pr)
if err != nil {
return err
}
err = g.solve(ctx, s.opt) // TODO: separate exporting
closeProgressWriter()
if err != nil {
return err
}
g.release(ctx)
// TODO: export final vertex state
return err
}
func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client.SolveStatus) error {
j, err := s.jobs.get(id)
if err != nil {
return err
}
defer close(statusChan)
return j.pipe(ctx, statusChan)
}
type opVertex struct {
mu sync.Mutex
op *pb.Op
inputs []*opVertex
refs []cache.ImmutableRef
err error
dgst digest.Digest
vtx client.Vertex
}
func (g *opVertex) inputRequiresExport(i int) bool {
return true // TODO
}
func (g *opVertex) release(ctx context.Context) (retErr error) {
for _, i := range g.inputs {
if err := i.release(ctx); err != nil {
retErr = err
}
}
for _, ref := range g.refs {
if ref != nil {
if err := ref.Release(ctx); err != nil {
retErr = err
}
}
}
return retErr
}
func (g *opVertex) getInputRef(i int) cache.ImmutableRef {
input := g.op.Inputs[i]
for _, v := range g.inputs {
if v.dgst == digest.Digest(input.Digest) {
return v.refs[input.Index]
}
}
return nil
}
func (g *opVertex) solve(ctx context.Context, opt Opt) (retErr error) {
g.mu.Lock()
defer g.mu.Unlock()
if g.err != nil {
return g.err
}
if len(g.refs) > 0 {
return nil
}
defer func() {
if retErr != nil {
g.err = retErr
}
}()
if len(g.inputs) > 0 {
eg, ctx := errgroup.WithContext(ctx)
for _, in := range g.inputs {
eg.Go(func() error {
if err := in.solve(ctx, opt); err != nil {
return err
}
return nil
})
}
err := eg.Wait()
if err != nil {
return err
}
}
pw, _, ctx := progress.FromContext(ctx, g.dgst.String())
defer pw.Done()
g.notifyStarted(pw)
defer g.notifyComplete(pw)
switch op := g.op.Op.(type) {
case *pb.Op_Source:
id, err := source.FromString(op.Source.Identifier)
if err != nil {
return err
}
ref, err := opt.SourceManager.Pull(ctx, id)
if err != nil {
return err
}
g.refs = []cache.ImmutableRef{ref}
case *pb.Op_Exec:
mounts := make(map[string]cache.Mountable)
var outputs []cache.MutableRef
defer func() {
for _, o := range outputs {
if o != nil {
s, err := o.Freeze() // TODO: log error
if err == nil {
s.Release(ctx)
}
}
}
}()
for _, m := range op.Exec.Mounts {
var mountable cache.Mountable
ref := g.getInputRef(int(m.Input))
mountable = ref
if m.Output != -1 {
active, err := opt.CacheManager.New(ctx, ref) // TODO: should be method
if err != nil {
return err
}
outputs = append(outputs, active)
mountable = active
}
mounts[m.Dest] = mountable
}
meta := worker.Meta{
Args: op.Exec.Meta.Args,
Env: op.Exec.Meta.Env,
Cwd: op.Exec.Meta.Cwd,
}
if err := opt.Worker.Exec(ctx, meta, mounts, os.Stderr, os.Stderr); err != nil {
return errors.Wrapf(err, "worker failed running %v", meta.Args)
}
g.refs = []cache.ImmutableRef{}
for i, o := range outputs {
ref, err := o.ReleaseAndCommit(ctx)
if err != nil {
return errors.Wrapf(err, "error committing %s", ref.ID())
}
g.refs = append(g.refs, ref)
outputs[i] = nil
}
default:
return errors.Errorf("invalid op type")
}
return nil
}
func (g *opVertex) notifyStarted(pw progress.ProgressWriter) {
now := time.Now()
g.vtx.Started = &now
pw.Write(g.vtx)
}
func (g *opVertex) notifyComplete(pw progress.ProgressWriter) {
now := time.Now()
g.vtx.Completed = &now
pw.Write(g.vtx)
}
func (g *opVertex) name() string {
switch op := g.op.Op.(type) {
case *pb.Op_Source:
return op.Source.Identifier
case *pb.Op_Exec:
name := strings.Join(op.Exec.Meta.Args, " ")
if len(name) > 22 { // TODO: const
name = name[:20] + "..."
}
return name
default:
return "unknown"
}
}

72
util/progress/multireader.go Normal file
View File

@ -0,0 +1,72 @@
package progress
import (
"context"
"sync"
)
type MultiReader struct {
mu sync.Mutex
main ProgressReader
initialized bool
done chan struct{}
writers map[*progressWriter]func()
}
func NewMultiReader(pr ProgressReader) *MultiReader {
mr := &MultiReader{
main: pr,
writers: make(map[*progressWriter]func()),
done: make(chan struct{}),
}
return mr
}
func (mr *MultiReader) Reader(ctx context.Context) ProgressReader {
mr.mu.Lock()
defer mr.mu.Unlock()
pr, ctx, closeWriter := NewContext(ctx)
pw, _, _ := FromContext(ctx, "")
w := pw.(*progressWriter)
mr.writers[w] = closeWriter
go func() {
select {
case <-ctx.Done():
case <-mr.done:
}
mr.mu.Lock()
defer mr.mu.Unlock()
delete(mr.writers, w)
}()
if !mr.initialized {
go mr.handle()
mr.initialized = true
}
return pr
}
func (mr *MultiReader) handle() error {
for {
p, err := mr.main.Read(context.TODO())
if err != nil {
return err
}
mr.mu.Lock()
for w, c := range mr.writers {
if p == nil {
c()
} else {
w.write(*p)
}
}
mr.mu.Unlock()
if p == nil {
return nil
}
}
}

44
util/progress/progress.go
View File

@ -20,7 +20,7 @@ func FromContext(ctx context.Context, name string) (ProgressWriter, bool, contex
}
pw = newWriter(pw, name)
ctx = context.WithValue(ctx, contextKey, pw)
return pw, false, ctx
return pw, true, ctx
}
func NewContext(ctx context.Context) (ProgressReader, context.Context, func()) {
@ -30,8 +30,8 @@ func NewContext(ctx context.Context) (ProgressReader, context.Context, func()) {
}
type ProgressWriter interface {
Write(Progress) error
Done() error
Write(interface{}) error
Done() error // Close
}
type ProgressReader interface {
@ -40,16 +40,16 @@ type ProgressReader interface {
type Progress struct {
ID string
Timestamp time.Time
Done bool
Sys interface{}
}
// Progress contains a Message or...
Message string
type Status struct {
// ...progress of an action
Action string
Current int
Total int
Timestamp time.Time
Done bool
}
type progressReader struct {
@ -95,7 +95,7 @@ func (pr *progressReader) Read(ctx context.Context) (*Progress, error) {
default:
}
open := false
for _, sh := range pr.handles { // could be more efficient but unlikely that this array will be very big, maybe random ordering?
for _, sh := range pr.handles { // could be more efficient but unlikely that this array will be very big, maybe random ordering? at least remove the completed handlers.
p, ok := sh.next()
if ok {
pr.mu.Unlock()
@ -148,8 +148,12 @@ func pipe() (*progressReader, *progressWriter, func()) {
func newWriter(pw *progressWriter, name string) *progressWriter {
if pw.id != "" {
if name == "" {
name = pw.id
} else {
name = pw.id + "." + name
}
}
pw = &progressWriter{
id: name,
reader: pw.reader,
@ -163,20 +167,27 @@ type progressWriter struct {
lastP atomic.Value
done bool
reader *progressReader
byKey map[string]atomic.Value
items []atomic.Value
}
func (pw *progressWriter) Write(p Progress) error {
func (pw *progressWriter) Write(s interface{}) error {
if pw.done {
return errors.Errorf("writing to closed progresswriter %s", pw.id)
}
var p Progress
p.ID = pw.id
if p.Timestamp.IsZero() {
p.Timestamp = time.Now()
p.Sys = s
return pw.write(p)
}
pw.lastP.Store(&p)
func (pw *progressWriter) write(p Progress) error {
if p.Done {
pw.done = true
}
pw.lastP.Store(&p)
pw.reader.cond.Broadcast()
return nil
}
@ -184,21 +195,24 @@ func (pw *progressWriter) Write(p Progress) error {
func (pw *progressWriter) Done() error {
var p Progress
lastP := pw.lastP.Load().(*Progress)
p.ID = pw.id
p.Timestamp = time.Now()
if lastP != nil {
p = *lastP
if p.Done {
return nil
}
} else {
p = Progress{}
p.Sys = lastP.Sys
}
p.Done = true
return pw.Write(p)
pw.done = true
return pw.write(p)
}
type noOpWriter struct{}
func (pw *noOpWriter) Write(p Progress) error {
func (pw *noOpWriter) Write(p interface{}) error {
return nil
}

14
util/progress/progress_test.go
View File

@ -30,7 +30,8 @@ func TestProgress(t *testing.T) {
err = eg.Wait()
assert.NoError(t, err)
assert.Equal(t, 6, len(trace.items))
assert.True(t, len(trace.items) > 5)
assert.True(t, len(trace.items) <= 7)
assert.Equal(t, trace.items[len(trace.items)-1].Done, true)
}
@ -51,7 +52,7 @@ func TestProgressNested(t *testing.T) {
assert.NoError(t, err)
assert.True(t, len(trace.items) > 9) // usually 14
assert.True(t, len(trace.items) <= 14)
assert.True(t, len(trace.items) <= 15)
streams := 0
for _, t := range trace.items {
if t.Done {
@ -66,7 +67,7 @@ func calc(ctx context.Context, total int, name string) (int, error) {
defer pw.Done()
sum := 0
pw.Write(Progress{Action: "starting", Total: total})
pw.Write(Status{Action: "starting", Total: total})
for i := 1; i <= total; i++ {
select {
case <-ctx.Done():
@ -74,12 +75,13 @@ func calc(ctx context.Context, total int, name string) (int, error) {
case <-time.After(10 * time.Millisecond):
}
if i == total {
pw.Write(Progress{Action: "done", Total: total, Current: total, Done: true})
pw.Write(Status{Action: "done", Total: total, Current: total})
} else {
pw.Write(Progress{Action: "calculating", Total: total, Current: i})
pw.Write(Status{Action: "calculating", Total: total, Current: i})
}
sum += i
}
pw.Done()
return sum, nil
}
@ -90,7 +92,7 @@ func reduceCalc(ctx context.Context, total int) (int, error) {
pw, _, ctx := FromContext(ctx, "reduce")
defer pw.Done()
pw.Write(Progress{Action: "starting"})
pw.Write(Status{Action: "starting"})
// sync step
sum, err := calc(ctx, total, "synccalc")

162
vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go generated vendored Normal file
View File

@ -0,0 +1,162 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: github.com/golang/protobuf/ptypes/timestamp/timestamp.proto
/*
Package timestamp is a generated protocol buffer package.
It is generated from these files:
github.com/golang/protobuf/ptypes/timestamp/timestamp.proto
It has these top-level messages:
Timestamp
*/
package timestamp
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// A Timestamp represents a point in time independent of any time zone
// or calendar, represented as seconds and fractions of seconds at
// nanosecond resolution in UTC Epoch time. It is encoded using the
// Proleptic Gregorian Calendar which extends the Gregorian calendar
// backwards to year one. It is encoded assuming all minutes are 60
// seconds long, i.e. leap seconds are "smeared" so that no leap second
// table is needed for interpretation. Range is from
// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z.
// By restricting to that range, we ensure that we can convert to
// and from RFC 3339 date strings.
// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt).
//
// # Examples
//
// Example 1: Compute Timestamp from POSIX `time()`.
//
// Timestamp timestamp;
// timestamp.set_seconds(time(NULL));
// timestamp.set_nanos(0);
//
// Example 2: Compute Timestamp from POSIX `gettimeofday()`.
//
// struct timeval tv;
// gettimeofday(&tv, NULL);
//
// Timestamp timestamp;
// timestamp.set_seconds(tv.tv_sec);
// timestamp.set_nanos(tv.tv_usec * 1000);
//
// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`.
//
// FILETIME ft;
// GetSystemTimeAsFileTime(&ft);
// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
//
// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z
// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z.
// Timestamp timestamp;
// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL));
// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100));
//
// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`.
//
// long millis = System.currentTimeMillis();
//
// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000)
// .setNanos((int) ((millis % 1000) * 1000000)).build();
//
//
// Example 5: Compute Timestamp from current time in Python.
//
// timestamp = Timestamp()
// timestamp.GetCurrentTime()
//
// # JSON Mapping
//
// In JSON format, the Timestamp type is encoded as a string in the
// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the
// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z"
// where {year} is always expressed using four digits while {month}, {day},
// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional
// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution),
// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone
// is required, though only UTC (as indicated by "Z") is presently supported.
//
// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past
// 01:30 UTC on January 15, 2017.
//
// In JavaScript, one can convert a Date object to this format using the
// standard [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString]
// method. In Python, a standard `datetime.datetime` object can be converted
// to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime)
// with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one
// can use the Joda Time's [`ISODateTimeFormat.dateTime()`](
// http://joda-time.sourceforge.net/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime())
// to obtain a formatter capable of generating timestamps in this format.
//
//
type Timestamp struct {
// Represents seconds of UTC time since Unix epoch
// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to
// 9999-12-31T23:59:59Z inclusive.
Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"`
// Non-negative fractions of a second at nanosecond resolution. Negative
// second values with fractions must still have non-negative nanos values
// that count forward in time. Must be from 0 to 999,999,999
// inclusive.
Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"`
}
func (m *Timestamp) Reset() { *m = Timestamp{} }
func (m *Timestamp) String() string { return proto.CompactTextString(m) }
func (*Timestamp) ProtoMessage() {}
func (*Timestamp) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (*Timestamp) XXX_WellKnownType() string { return "Timestamp" }
func (m *Timestamp) GetSeconds() int64 {
if m != nil {
return m.Seconds
}
return 0
}
func (m *Timestamp) GetNanos() int32 {
if m != nil {
return m.Nanos
}
return 0
}
func init() {
proto.RegisterType((*Timestamp)(nil), "google.protobuf.Timestamp")
}
func init() {
proto.RegisterFile("github.com/golang/protobuf/ptypes/timestamp/timestamp.proto", fileDescriptor0)
}
var fileDescriptor0 = []byte{
// 190 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xb2, 0x4e, 0xcf, 0x2c, 0xc9,
0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0xcf, 0x49, 0xcc, 0x4b, 0xd7, 0x2f, 0x28,
0xca, 0x2f, 0xc9, 0x4f, 0x2a, 0x4d, 0xd3, 0x2f, 0x28, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2f, 0xc9,
0xcc, 0x4d, 0x2d, 0x2e, 0x49, 0xcc, 0x2d, 0x40, 0xb0, 0xf4, 0xc0, 0x6a, 0x84, 0xf8, 0xd3, 0xf3,
0xf3, 0xd3, 0x73, 0x52, 0xf5, 0x60, 0x3a, 0x94, 0xac, 0xb9, 0x38, 0x43, 0x60, 0x6a, 0x84, 0x24,
0xb8, 0xd8, 0x8b, 0x53, 0x93, 0xf3, 0xf3, 0x52, 0x8a, 0x25, 0x18, 0x15, 0x18, 0x35, 0x98, 0x83,
0x60, 0x5c, 0x21, 0x11, 0x2e, 0xd6, 0xbc, 0xc4, 0xbc, 0xfc, 0x62, 0x09, 0x26, 0x05, 0x46, 0x0d,
0xd6, 0x20, 0x08, 0xc7, 0xa9, 0x8e, 0x4b, 0x38, 0x39, 0x3f, 0x57, 0x0f, 0xcd, 0x4c, 0x27, 0x3e,
0xb8, 0x89, 0x01, 0x20, 0xa1, 0x00, 0xc6, 0x28, 0x6d, 0x12, 0xdc, 0xfc, 0x83, 0x91, 0x71, 0x11,
0x13, 0xb3, 0x7b, 0x80, 0xd3, 0x2a, 0x26, 0x39, 0x77, 0x88, 0xc9, 0x01, 0x50, 0xb5, 0x7a, 0xe1,
0xa9, 0x39, 0x39, 0xde, 0x79, 0xf9, 0xe5, 0x79, 0x21, 0x20, 0x3d, 0x49, 0x6c, 0x60, 0x43, 0x8c,
0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x6b, 0x59, 0x0a, 0x4d, 0x13, 0x01, 0x00, 0x00,
}

133
vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.proto generated vendored Normal file
View File

@ -0,0 +1,133 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
syntax = "proto3";
package google.protobuf;
option csharp_namespace = "Google.Protobuf.WellKnownTypes";
option cc_enable_arenas = true;
option go_package = "github.com/golang/protobuf/ptypes/timestamp";
option java_package = "com.google.protobuf";
option java_outer_classname = "TimestampProto";
option java_multiple_files = true;
option objc_class_prefix = "GPB";
// A Timestamp represents a point in time independent of any time zone
// or calendar, represented as seconds and fractions of seconds at
// nanosecond resolution in UTC Epoch time. It is encoded using the
// Proleptic Gregorian Calendar which extends the Gregorian calendar
// backwards to year one. It is encoded assuming all minutes are 60
// seconds long, i.e. leap seconds are "smeared" so that no leap second
// table is needed for interpretation. Range is from
// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z.
// By restricting to that range, we ensure that we can convert to
// and from RFC 3339 date strings.
// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt).
//
// # Examples
//
// Example 1: Compute Timestamp from POSIX `time()`.
//
// Timestamp timestamp;
// timestamp.set_seconds(time(NULL));
// timestamp.set_nanos(0);
//
// Example 2: Compute Timestamp from POSIX `gettimeofday()`.
//
// struct timeval tv;
// gettimeofday(&tv, NULL);
//
// Timestamp timestamp;
// timestamp.set_seconds(tv.tv_sec);
// timestamp.set_nanos(tv.tv_usec * 1000);
//
// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`.
//
// FILETIME ft;
// GetSystemTimeAsFileTime(&ft);
// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
//
// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z
// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z.
// Timestamp timestamp;
// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL));
// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100));
//
// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`.
//
// long millis = System.currentTimeMillis();
//
// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000)
// .setNanos((int) ((millis % 1000) * 1000000)).build();
//
//
// Example 5: Compute Timestamp from current time in Python.
//
// timestamp = Timestamp()
// timestamp.GetCurrentTime()
//
// # JSON Mapping
//
// In JSON format, the Timestamp type is encoded as a string in the
// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the
// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z"
// where {year} is always expressed using four digits while {month}, {day},
// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional
// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution),
// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone
// is required, though only UTC (as indicated by "Z") is presently supported.
//
// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past
// 01:30 UTC on January 15, 2017.
//
// In JavaScript, one can convert a Date object to this format using the
// standard [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString]
// method. In Python, a standard `datetime.datetime` object can be converted
// to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime)
// with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one
// can use the Joda Time's [`ISODateTimeFormat.dateTime()`](
// http://joda-time.sourceforge.net/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime())
// to obtain a formatter capable of generating timestamps in this format.
//
//
message Timestamp {
// Represents seconds of UTC time since Unix epoch
// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to
// 9999-12-31T23:59:59Z inclusive.
int64 seconds = 1;
// Non-negative fractions of a second at nanosecond resolution. Negative
// second values with fractions must still have non-negative nanos values
// that count forward in time. Must be from 0 to 999,999,999
// inclusive.
int32 nanos = 2;
}