From 4b41158252a5672475a728cf2f68570480b32d8a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 15 Jun 2017 16:08:20 -0700 Subject: [PATCH 1/3] solver: control vertexstatus for pull Signed-off-by: Tonis Tiigi --- api/services/control/control.pb.go | 217 ++++++++++++++++++++++------- api/services/control/control.proto | 3 + client/graph.go | 6 +- control/control.go | 12 ++ solver/jobs.go | 21 ++- solver/solver.go | 14 +- source/containerimage/pull.go | 201 +++++++++++++++++++++++++- util/progress/progress.go | 8 +- 8 files changed, 415 insertions(+), 67 deletions(-) diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go index 5a0e7d45..15111f8a 100644 --- a/api/services/control/control.pb.go +++ b/api/services/control/control.pb.go @@ -247,12 +247,15 @@ func (m *Vertex) GetCompleted() *time.Time { } type VertexStatus struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - Vertex github_com_opencontainers_go_digest.Digest `protobuf:"bytes,2,opt,name=vertex,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"vertex"` - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - Current int64 `protobuf:"varint,4,opt,name=current,proto3" json:"current,omitempty"` - Total int64 `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"` - Timestamp time.Time `protobuf:"bytes,6,opt,name=timestamp,stdtime" json:"timestamp"` + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + Vertex github_com_opencontainers_go_digest.Digest `protobuf:"bytes,2,opt,name=vertex,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"vertex"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Current int64 `protobuf:"varint,4,opt,name=current,proto3" json:"current,omitempty"` + Total int64 `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"` + // TODO: add started, completed + Timestamp time.Time `protobuf:"bytes,6,opt,name=timestamp,stdtime" json:"timestamp"` + Started *time.Time `protobuf:"bytes,7,opt,name=started,stdtime" json:"started,omitempty"` + Completed *time.Time `protobuf:"bytes,8,opt,name=completed,stdtime" json:"completed,omitempty"` } func (m *VertexStatus) Reset() { *m = VertexStatus{} } @@ -295,6 +298,20 @@ func (m *VertexStatus) GetTimestamp() time.Time { return time.Time{} } +func (m *VertexStatus) GetStarted() *time.Time { + if m != nil { + return m.Started + } + return nil +} + +func (m *VertexStatus) GetCompleted() *time.Time { + if m != nil { + return m.Completed + } + return nil +} + type VertexLog struct { Vertex github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=vertex,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"vertex"` Timestamp time.Time `protobuf:"bytes,2,opt,name=timestamp,stdtime" json:"timestamp"` @@ -870,6 +887,26 @@ func (m *VertexStatus) MarshalTo(dAtA []byte) (int, error) { return 0, err } i += n3 + if m.Started != nil { + dAtA[i] = 0x3a + i++ + i = encodeVarintControl(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.Started))) + n4, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.Started, dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + if m.Completed != nil { + dAtA[i] = 0x42 + i++ + i = encodeVarintControl(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.Completed))) + n5, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.Completed, dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + } return i, nil } @@ -897,11 +934,11 @@ func (m *VertexLog) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintControl(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.Timestamp))) - n4, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Timestamp, dAtA[i:]) + n6, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Timestamp, dAtA[i:]) if err != nil { return 0, err } - i += n4 + i += n6 if m.Stream != 0 { dAtA[i] = 0x18 i++ @@ -1096,6 +1133,14 @@ func (m *VertexStatus) Size() (n int) { } l = github_com_gogo_protobuf_types.SizeOfStdTime(m.Timestamp) n += 1 + l + sovControl(uint64(l)) + if m.Started != nil { + l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.Started) + n += 1 + l + sovControl(uint64(l)) + } + if m.Completed != nil { + l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.Completed) + n += 1 + l + sovControl(uint64(l)) + } return n } @@ -2218,6 +2263,72 @@ func (m *VertexStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Started", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthControl + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Started == nil { + m.Started = new(time.Time) + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.Started, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Completed", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthControl + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Completed == nil { + m.Completed = new(time.Time) + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.Completed, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipControl(dAtA[iNdEx:]) @@ -2506,49 +2617,49 @@ var ( func init() { proto.RegisterFile("control.proto", fileDescriptorControl) } var fileDescriptorControl = []byte{ - // 689 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x54, 0xc1, 0x6e, 0xd3, 0x4c, - 0x10, 0xfe, 0x37, 0x4e, 0x9d, 0x64, 0x92, 0x56, 0xfd, 0x57, 0x08, 0x59, 0x41, 0x24, 0xc6, 0x5c, - 0xa2, 0x4a, 0x75, 0x20, 0xc0, 0xa5, 0x48, 0x08, 0x85, 0x1c, 0x68, 0x45, 0x2f, 0xdb, 0x96, 0xbb, - 0xe3, 0x6c, 0x5d, 0xab, 0xb6, 0x37, 0x78, 0xd7, 0x51, 0xe1, 0x29, 0x78, 0x17, 0x9e, 0x01, 0xa9, - 0x47, 0xce, 0x1c, 0x0a, 0xea, 0x23, 0x70, 0xe2, 0x88, 0xbc, 0xbb, 0x76, 0x4d, 0xdb, 0x40, 0x55, - 0x4e, 0xd9, 0xd9, 0x7c, 0xf3, 0xed, 0x7c, 0xe3, 0xf9, 0x06, 0x56, 0x7d, 0x96, 0x88, 0x94, 0x45, - 0xee, 0x3c, 0x65, 0x82, 0xe1, 0xf5, 0x98, 0x4d, 0xdf, 0xbb, 0xd3, 0x2c, 0x8c, 0x66, 0xc7, 0xa1, - 0x70, 0x17, 0x8f, 0xbb, 0x9b, 0x41, 0x28, 0x8e, 0xb2, 0xa9, 0xeb, 0xb3, 0x78, 0x18, 0xb0, 0x80, - 0x0d, 0x25, 0x70, 0x9a, 0x1d, 0xca, 0x48, 0x06, 0xf2, 0xa4, 0x08, 0xba, 0xfd, 0x80, 0xb1, 0x20, - 0xa2, 0x17, 0x28, 0x11, 0xc6, 0x94, 0x0b, 0x2f, 0x9e, 0x2b, 0x80, 0x83, 0x61, 0x7d, 0x12, 0xf2, - 0xe3, 0x03, 0xee, 0x05, 0x94, 0xd0, 0x77, 0x19, 0xe5, 0xc2, 0xd9, 0x81, 0xff, 0x2b, 0x77, 0x7c, - 0xce, 0x12, 0x4e, 0xf1, 0x33, 0x30, 0x53, 0xea, 0xb3, 0x74, 0x66, 0x21, 0xdb, 0x18, 0xb4, 0x47, - 0xf7, 0xdd, 0xcb, 0xb5, 0xb9, 0x3a, 0x21, 0x07, 0x11, 0x0d, 0x76, 0x3c, 0x68, 0x57, 0xae, 0xf1, - 0x1a, 0xd4, 0xb6, 0x27, 0x16, 0xb2, 0xd1, 0xa0, 0x45, 0x6a, 0xdb, 0x13, 0x6c, 0x41, 0x63, 0x37, - 0x13, 0xde, 0x34, 0xa2, 0x56, 0xcd, 0x46, 0x83, 0x26, 0x29, 0x42, 0x7c, 0x07, 0x56, 0xb6, 0x93, - 0x03, 0x4e, 0x2d, 0x43, 0xde, 0xab, 0x00, 0x63, 0xa8, 0xef, 0x85, 0x1f, 0xa8, 0x55, 0xb7, 0xd1, - 0xc0, 0x20, 0xf2, 0xec, 0xbc, 0x84, 0xce, 0x1e, 0x8b, 0x16, 0x45, 0xf9, 0x78, 0x1d, 0x0c, 0x42, - 0x0f, 0xf5, 0x23, 0xf9, 0x11, 0xf7, 0x00, 0x26, 0xf4, 0x30, 0x4c, 0x42, 0x11, 0xb2, 0xc4, 0xaa, - 0xd9, 0xc6, 0xa0, 0x43, 0x2a, 0x37, 0xce, 0x73, 0x58, 0xd5, 0x0c, 0x5a, 0xec, 0x06, 0x18, 0x0b, - 0x71, 0xa2, 0x95, 0x5a, 0x57, 0x95, 0xbe, 0xa5, 0xa9, 0xa0, 0x27, 0x24, 0x07, 0x39, 0x0f, 0x60, - 0x75, 0x4f, 0x78, 0x22, 0xe3, 0x4b, 0xdf, 0x77, 0x3e, 0x21, 0x58, 0x2b, 0x30, 0xfa, 0x85, 0xa7, - 0xd0, 0x5c, 0x48, 0x12, 0xca, 0xff, 0xfa, 0x4c, 0x89, 0xc4, 0x5b, 0xd0, 0xe4, 0x92, 0x87, 0x72, - 0x29, 0xa3, 0x3d, 0xea, 0x2d, 0xcb, 0xd2, 0xef, 0x95, 0x78, 0x3c, 0x84, 0x7a, 0xc4, 0x02, 0x6e, - 0x19, 0x32, 0xef, 0xde, 0xb2, 0xbc, 0x37, 0x2c, 0x20, 0x12, 0xe8, 0x9c, 0xd6, 0xc0, 0x54, 0x77, - 0x78, 0x07, 0xcc, 0x59, 0x18, 0x50, 0x2e, 0x94, 0xaa, 0xf1, 0xe8, 0xf4, 0xac, 0xff, 0xdf, 0xd7, - 0xb3, 0xfe, 0x46, 0x65, 0x1a, 0xd9, 0x9c, 0x26, 0xf9, 0xf4, 0x7a, 0x61, 0x42, 0x53, 0x3e, 0x0c, - 0xd8, 0xa6, 0x4a, 0x71, 0x27, 0xf2, 0x87, 0x68, 0x86, 0x9c, 0x2b, 0x4c, 0xe6, 0x99, 0x50, 0x0a, - 0x6e, 0xc9, 0xa5, 0x18, 0xf2, 0x71, 0x48, 0xbc, 0x58, 0xcd, 0x48, 0x8b, 0xc8, 0x33, 0xbe, 0x0b, - 0xa6, 0xef, 0xf9, 0x47, 0x74, 0x26, 0x87, 0xa4, 0x49, 0x74, 0x84, 0xb7, 0xa0, 0xc1, 0x85, 0x97, - 0x0a, 0x3a, 0xb3, 0x56, 0x6c, 0x34, 0x68, 0x8f, 0xba, 0xae, 0x32, 0x87, 0x5b, 0x98, 0xc3, 0xdd, - 0x2f, 0xcc, 0x31, 0xae, 0x7f, 0xfc, 0xd6, 0x47, 0xa4, 0x48, 0xc0, 0x2f, 0xa0, 0xe5, 0xb3, 0x78, - 0x1e, 0xd1, 0x3c, 0xdb, 0xbc, 0x61, 0xf6, 0x45, 0x8a, 0xf3, 0x03, 0x41, 0xa7, 0xfa, 0x59, 0xae, - 0xf8, 0x60, 0x07, 0x4c, 0xf5, 0x91, 0xa5, 0x0d, 0x6e, 0xd9, 0x14, 0xc5, 0x70, 0x6d, 0x53, 0x2c, - 0x68, 0xf8, 0x59, 0x9a, 0xd2, 0x44, 0x68, 0xeb, 0x14, 0x61, 0xee, 0x33, 0xc1, 0x84, 0x17, 0xc9, - 0xa6, 0x18, 0x44, 0x05, 0x78, 0x0c, 0xad, 0x72, 0x53, 0xdc, 0x40, 0x70, 0x33, 0x2f, 0x57, 0x89, - 0x2e, 0xd3, 0x9c, 0xcf, 0x08, 0x5a, 0xe5, 0x4c, 0x55, 0x14, 0xa2, 0x7f, 0x56, 0xf8, 0x5b, 0x75, - 0xb5, 0x5b, 0x55, 0x97, 0x8f, 0x09, 0x17, 0x29, 0xf5, 0x62, 0xd9, 0x27, 0x83, 0xe8, 0x28, 0x77, - 0x6f, 0xcc, 0x03, 0xd9, 0xa5, 0x0e, 0xc9, 0x8f, 0xa3, 0x9f, 0x08, 0x1a, 0xaf, 0xd4, 0x5a, 0xc6, - 0xfb, 0xd0, 0x2a, 0x57, 0x23, 0x76, 0xae, 0x7a, 0xe8, 0xf2, 0x2e, 0xed, 0x3e, 0xfc, 0x23, 0x46, - 0x2f, 0x83, 0xd7, 0xb0, 0x22, 0xf7, 0x0f, 0xbe, 0xc6, 0xcd, 0xd5, 0xd5, 0xd6, 0xed, 0x2f, 0xfd, - 0x5f, 0x33, 0xed, 0x82, 0xa9, 0x27, 0xec, 0x3a, 0x68, 0x75, 0x4d, 0x75, 0xed, 0xe5, 0x00, 0x45, - 0xf6, 0x08, 0x8d, 0x3b, 0xa7, 0xe7, 0x3d, 0xf4, 0xe5, 0xbc, 0x87, 0xbe, 0x9f, 0xf7, 0xd0, 0xd4, - 0x94, 0xbd, 0x7d, 0xf2, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x08, 0xd7, 0xbb, 0xa9, 0xa5, 0x06, 0x00, - 0x00, + // 699 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x55, 0x41, 0x6f, 0xd3, 0x30, + 0x14, 0xc6, 0x4d, 0x97, 0xb6, 0xaf, 0xdd, 0x34, 0x2c, 0x34, 0x45, 0x45, 0xb4, 0x21, 0x5c, 0xaa, + 0x49, 0x4b, 0xa1, 0xc0, 0x65, 0x48, 0x08, 0x95, 0x1e, 0xd8, 0xc4, 0x2e, 0xde, 0xc6, 0x3d, 0x4d, + 0xbd, 0x2c, 0x5a, 0x12, 0x97, 0xd8, 0xa9, 0x06, 0xbf, 0x82, 0xff, 0xc2, 0x6f, 0x40, 0xda, 0x91, + 0x33, 0x87, 0x81, 0xf6, 0x03, 0x38, 0x73, 0x44, 0xb1, 0x9d, 0x2e, 0x6c, 0x2b, 0x8c, 0xed, 0x54, + 0x3f, 0xf7, 0xfb, 0x3e, 0xfb, 0x7d, 0x7e, 0xef, 0x05, 0x96, 0x7d, 0x96, 0x88, 0x94, 0x45, 0xee, + 0x34, 0x65, 0x82, 0xe1, 0xd5, 0x98, 0x8d, 0x3f, 0xb8, 0xe3, 0x2c, 0x8c, 0x26, 0x47, 0xa1, 0x70, + 0x67, 0x4f, 0xda, 0x1b, 0x41, 0x28, 0x0e, 0xb3, 0xb1, 0xeb, 0xb3, 0xb8, 0x1f, 0xb0, 0x80, 0xf5, + 0x25, 0x70, 0x9c, 0x1d, 0xc8, 0x48, 0x06, 0x72, 0xa5, 0x04, 0xda, 0xdd, 0x80, 0xb1, 0x20, 0xa2, + 0xe7, 0x28, 0x11, 0xc6, 0x94, 0x0b, 0x2f, 0x9e, 0x2a, 0x80, 0x83, 0x61, 0x75, 0x14, 0xf2, 0xa3, + 0x7d, 0xee, 0x05, 0x94, 0xd0, 0xf7, 0x19, 0xe5, 0xc2, 0xd9, 0x86, 0xbb, 0xa5, 0x3d, 0x3e, 0x65, + 0x09, 0xa7, 0xf8, 0x39, 0x98, 0x29, 0xf5, 0x59, 0x3a, 0xb1, 0x90, 0x6d, 0xf4, 0x9a, 0x83, 0x07, + 0xee, 0xc5, 0xbb, 0xb9, 0x9a, 0x90, 0x83, 0x88, 0x06, 0x3b, 0x1e, 0x34, 0x4b, 0xdb, 0x78, 0x05, + 0x2a, 0x5b, 0x23, 0x0b, 0xd9, 0xa8, 0xd7, 0x20, 0x95, 0xad, 0x11, 0xb6, 0xa0, 0xb6, 0x93, 0x09, + 0x6f, 0x1c, 0x51, 0xab, 0x62, 0xa3, 0x5e, 0x9d, 0x14, 0x21, 0xbe, 0x07, 0x4b, 0x5b, 0xc9, 0x3e, + 0xa7, 0x96, 0x21, 0xf7, 0x55, 0x80, 0x31, 0x54, 0x77, 0xc3, 0x8f, 0xd4, 0xaa, 0xda, 0xa8, 0x67, + 0x10, 0xb9, 0x76, 0x5e, 0x41, 0x6b, 0x97, 0x45, 0xb3, 0xe2, 0xfa, 0x78, 0x15, 0x0c, 0x42, 0x0f, + 0xf4, 0x21, 0xf9, 0x12, 0x77, 0x00, 0x46, 0xf4, 0x20, 0x4c, 0x42, 0x11, 0xb2, 0xc4, 0xaa, 0xd8, + 0x46, 0xaf, 0x45, 0x4a, 0x3b, 0xce, 0x0b, 0x58, 0xd6, 0x0a, 0x3a, 0xd9, 0x75, 0x30, 0x66, 0xe2, + 0x58, 0x67, 0x6a, 0x5d, 0xce, 0xf4, 0x1d, 0x4d, 0x05, 0x3d, 0x26, 0x39, 0xc8, 0x79, 0x08, 0xcb, + 0xbb, 0xc2, 0x13, 0x19, 0x5f, 0x78, 0xbe, 0xf3, 0x19, 0xc1, 0x4a, 0x81, 0xd1, 0x27, 0x3c, 0x83, + 0xfa, 0x4c, 0x8a, 0x50, 0xfe, 0xcf, 0x63, 0xe6, 0x48, 0xbc, 0x09, 0x75, 0x2e, 0x75, 0x28, 0x97, + 0x69, 0x34, 0x07, 0x9d, 0x45, 0x2c, 0x7d, 0xde, 0x1c, 0x8f, 0xfb, 0x50, 0x8d, 0x58, 0xc0, 0x2d, + 0x43, 0xf2, 0xee, 0x2f, 0xe2, 0xbd, 0x65, 0x01, 0x91, 0x40, 0xe7, 0xa4, 0x02, 0xa6, 0xda, 0xc3, + 0xdb, 0x60, 0x4e, 0xc2, 0x80, 0x72, 0xa1, 0xb2, 0x1a, 0x0e, 0x4e, 0x4e, 0xbb, 0x77, 0xbe, 0x9d, + 0x76, 0xd7, 0x4b, 0xd5, 0xc8, 0xa6, 0x34, 0xc9, 0xab, 0xd7, 0x0b, 0x13, 0x9a, 0xf2, 0x7e, 0xc0, + 0x36, 0x14, 0xc5, 0x1d, 0xc9, 0x1f, 0xa2, 0x15, 0x72, 0xad, 0x30, 0x99, 0x66, 0x42, 0x65, 0x70, + 0x43, 0x2d, 0xa5, 0x90, 0x97, 0x43, 0xe2, 0xc5, 0xaa, 0x46, 0x1a, 0x44, 0xae, 0xf1, 0x1a, 0x98, + 0xbe, 0xe7, 0x1f, 0xd2, 0x89, 0x2c, 0x92, 0x3a, 0xd1, 0x11, 0xde, 0x84, 0x1a, 0x17, 0x5e, 0x2a, + 0xe8, 0xc4, 0x5a, 0xb2, 0x51, 0xaf, 0x39, 0x68, 0xbb, 0xaa, 0x39, 0xdc, 0xa2, 0x39, 0xdc, 0xbd, + 0xa2, 0x39, 0x86, 0xd5, 0x4f, 0xdf, 0xbb, 0x88, 0x14, 0x04, 0xfc, 0x12, 0x1a, 0x3e, 0x8b, 0xa7, + 0x11, 0xcd, 0xd9, 0xe6, 0x35, 0xd9, 0xe7, 0x14, 0xe7, 0x67, 0x05, 0x5a, 0xe5, 0x67, 0xb9, 0xd4, + 0x07, 0xdb, 0x60, 0xaa, 0x47, 0x96, 0x6d, 0x70, 0x43, 0x53, 0x94, 0xc2, 0x95, 0xa6, 0x58, 0x50, + 0xf3, 0xb3, 0x34, 0xa5, 0x89, 0xd0, 0xad, 0x53, 0x84, 0x79, 0x9f, 0x09, 0x26, 0xbc, 0x48, 0x9a, + 0x62, 0x10, 0x15, 0xe0, 0x21, 0x34, 0xe6, 0x93, 0xe2, 0x1a, 0x09, 0xd7, 0xf3, 0xeb, 0xaa, 0xa4, + 0xe7, 0xb4, 0xb2, 0xe1, 0xb5, 0x5b, 0x19, 0x5e, 0xff, 0x7f, 0xc3, 0xbf, 0x20, 0x68, 0xcc, 0xeb, + 0xb9, 0xe4, 0x2e, 0xba, 0xb5, 0xbb, 0x7f, 0x38, 0x53, 0xb9, 0x99, 0x33, 0x6b, 0x60, 0x72, 0x91, + 0x52, 0x2f, 0x96, 0x6f, 0x64, 0x10, 0x1d, 0xe5, 0x93, 0x23, 0xe6, 0x81, 0x7c, 0xa1, 0x16, 0xc9, + 0x97, 0x83, 0x5f, 0x08, 0x6a, 0xaf, 0xd5, 0x27, 0x01, 0xef, 0x41, 0x63, 0x3e, 0x96, 0xb1, 0x73, + 0xb9, 0x7f, 0x2f, 0xce, 0xf1, 0xf6, 0xa3, 0xbf, 0x62, 0xf4, 0x20, 0x7a, 0x03, 0x4b, 0x72, 0xf6, + 0xe1, 0x2b, 0x26, 0x49, 0x79, 0xac, 0xb6, 0xbb, 0x0b, 0xff, 0xd7, 0x4a, 0x3b, 0x60, 0xea, 0xea, + 0xbe, 0x0a, 0x5a, 0x1e, 0x91, 0x6d, 0x7b, 0x31, 0x40, 0x89, 0x3d, 0x46, 0xc3, 0xd6, 0xc9, 0x59, + 0x07, 0x7d, 0x3d, 0xeb, 0xa0, 0x1f, 0x67, 0x1d, 0x34, 0x36, 0xa5, 0xb7, 0x4f, 0x7f, 0x07, 0x00, + 0x00, 0xff, 0xff, 0xea, 0xc1, 0xcc, 0x7c, 0x21, 0x07, 0x00, 0x00, } diff --git a/api/services/control/control.proto b/api/services/control/control.proto index 71e38655..db3a765d 100644 --- a/api/services/control/control.proto +++ b/api/services/control/control.proto @@ -63,7 +63,10 @@ message VertexStatus { string name = 3; int64 current = 4; int64 total = 5; + // TODO: add started, completed google.protobuf.Timestamp timestamp = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + google.protobuf.Timestamp started = 7 [(gogoproto.stdtime) = true ]; + google.protobuf.Timestamp completed = 8 [(gogoproto.stdtime) = true ]; } message VertexLog { diff --git a/client/graph.go b/client/graph.go index 0325ed40..08175129 100644 --- a/client/graph.go +++ b/client/graph.go @@ -20,9 +20,11 @@ type VertexStatus struct { ID string Vertex digest.Digest Name string - Total int - Current int + Total int64 + Current int64 Timestamp time.Time + Started *time.Time // StartedAt + Completed *time.Time // StartedAt } type VertexLog struct { diff --git a/control/control.go b/control/control.go index b6947e61..9442a61a 100644 --- a/control/control.go +++ b/control/control.go @@ -99,6 +99,18 @@ func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Con Completed: v.Completed, }) } + for _, v := range ss.Statuses { + sr.Statuses = append(sr.Statuses, &controlapi.VertexStatus{ + ID: v.ID, + Vertex: v.Vertex, + Name: v.Name, + Current: v.Current, + Total: v.Total, + Timestamp: v.Timestamp, + Started: v.Started, + Completed: v.Completed, + }) + } if err := stream.SendMsg(&sr); err != nil { return err } diff --git a/solver/jobs.go b/solver/jobs.go index 12d25309..1e2a21c8 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -2,6 +2,7 @@ package solver import ( "context" + "strings" "sync" "time" @@ -90,7 +91,7 @@ func (j *job) pipe(ctx context.Context, ch chan *client.SolveStatus) error { } } for { - p, err := pr.Read(ctx) // add cancelling + p, err := pr.Read(ctx) if err != nil { return err } @@ -105,6 +106,24 @@ func (j *job) pipe(ctx context.Context, ch chan *client.SolveStatus) error { return ctx.Err() case ch <- ss: } + case progress.Status: + i := strings.Index(p.ID, ".") + vs := &client.VertexStatus{ + ID: p.ID[i+1:], + Vertex: digest.Digest(p.ID[:i]), // TODO: this needs to be handled better + Name: v.Action, + Total: int64(v.Total), + Current: int64(v.Current), + Timestamp: p.Timestamp, + Started: v.Started, + Completed: v.Completed, + } + ss := &client.SolveStatus{Statuses: []*client.VertexStatus{vs}} + select { + case <-ctx.Done(): + return ctx.Err() + case ch <- ss: + } } } } diff --git a/solver/solver.go b/solver/solver.go index c2a7ffd8..279e1711 100644 --- a/solver/solver.go +++ b/solver/solver.go @@ -125,12 +125,14 @@ func (g *opVertex) solve(ctx context.Context, opt Opt) (retErr error) { eg, ctx := errgroup.WithContext(ctx) for _, in := range g.inputs { - eg.Go(func() error { - if err := in.solve(ctx, opt); err != nil { - return err - } - return nil - }) + func(in *opVertex) { + eg.Go(func() error { + if err := in.solve(ctx, opt); err != nil { + return err + } + return nil + }) + }(in) } err := eg.Wait() if err != nil { diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 0e4dced1..a82ec19b 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "net/http" + "sync" + "time" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" @@ -17,6 +19,7 @@ import ( "github.com/pkg/errors" "github.com/tonistiigi/buildkit_poc/cache" "github.com/tonistiigi/buildkit_poc/source" + "github.com/tonistiigi/buildkit_poc/util/progress" ) // TODO: break apart containerd specifics like contentstore so the resolver @@ -67,10 +70,18 @@ func (is *imageSource) Pull(ctx context.Context, id source.Identifier) (cache.Im return nil, errors.New("invalid identifier") } + resolveProgressDone := oneOffProgress(ctx, "resolve "+imageIdentifier.Reference.String()) ref, desc, err := is.resolver.Resolve(ctx, imageIdentifier.Reference.String()) if err != nil { - return nil, err + return nil, resolveProgressDone(err) } + resolveProgressDone(nil) + + ongoing := newJobs(ref) + pctx, stopProgress := context.WithCancel(ctx) + + go showProgress(pctx, ongoing, is.ContentStore) + fetcher, err := is.resolver.Fetcher(ctx, ref) if err != nil { return nil, err @@ -80,17 +91,24 @@ func (is *imageSource) Pull(ctx context.Context, id source.Identifier) (cache.Im // and snapshots as 1) buildkit shouldn't have a dependency on contentstore // or 2) cachemanager should manage the contentstore handlers := []images.Handler{ + images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ongoing.add(ctx, desc) + return nil, nil + }), remotes.FetchHandler(is.ContentStore, fetcher), images.ChildrenHandler(is.ContentStore), } if err := images.Dispatch(ctx, images.Handlers(handlers...), desc); err != nil { return nil, err } + stopProgress() + unpackProgressDone := oneOffProgress(ctx, "unpacking "+imageIdentifier.Reference.String()) chainid, err := is.unpack(ctx, desc) if err != nil { - return nil, err + return nil, unpackProgressDone(err) } + unpackProgressDone(nil) return is.CacheAccessor.Get(ctx, chainid) } @@ -153,3 +171,182 @@ func getLayers(ctx context.Context, provider content.Provider, desc ocispec.Desc } return layers, nil } + +func showProgress(ctx context.Context, ongoing *jobs, cs content.Store) { + var ( + ticker = time.NewTicker(100 * time.Millisecond) + // fw = progress.NewWriter(out) + statuses = map[string]statusInfo{} + done bool + ) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + case <-ctx.Done(): + done = true + } + // fw.Flush() + + // tw := tabwriter.NewWriter(fw, 1, 8, 1, ' ', 0) + + resolved := "resolved" + if !ongoing.isResolved() { + resolved = "resolving" + } + statuses[ongoing.name] = statusInfo{ + Ref: ongoing.name, + Status: resolved, + } + // keys := []string{ongoing.name} + + actives := make(map[string]statusInfo) + + if !done { + active, err := cs.Status(ctx, "") + if err != nil { + // log.G(ctx).WithError(err).Error("active check failed") + continue + } + // update status of active entries! + for _, active := range active { + actives[active.Ref] = statusInfo{ + Ref: active.Ref, + Status: "downloading", + Offset: active.Offset, + Total: active.Total, + StartedAt: active.StartedAt, + UpdatedAt: active.UpdatedAt, + } + } + } + + // now, update the items in jobs that are not in active + for _, j := range ongoing.jobs() { + refKey := remotes.MakeRefKey(ctx, j.Descriptor) + if a, ok := actives[refKey]; ok { + j.Write(progress.Status{ + Action: a.Status, + Total: int(a.Total), + Current: int(a.Offset), + Started: &j.started, + }) + continue + } + + if !j.done { + info, err := cs.Info(ctx, j.Digest) + if err != nil { + if content.IsNotFound(err) { + j.Write(progress.Status{ + Action: "waiting", + }) + continue + } + } else { + j.done = true + } + + if done || j.done { + j.Write(progress.Status{ + Action: "done", + Current: int(info.Size), + Total: int(info.Size), + Completed: &info.CommittedAt, + Started: &j.started, + }) + j.Done() + } + } + } + if done { + return + } + } +} + +// jobs provides a way of identifying the download keys for a particular task +// encountering during the pull walk. +// +// This is very minimal and will probably be replaced with something more +// featured. +type jobs struct { + name string + added map[digest.Digest]job + mu sync.Mutex + resolved bool +} + +type job struct { + ocispec.Descriptor + progress.ProgressWriter + done bool + started time.Time +} + +func newJobs(name string) *jobs { + return &jobs{ + name: name, + added: make(map[digest.Digest]job), + } +} + +func (j *jobs) add(ctx context.Context, desc ocispec.Descriptor) { + j.mu.Lock() + defer j.mu.Unlock() + + if _, ok := j.added[desc.Digest]; ok { + return + } + pw, _, _ := progress.FromContext(ctx, desc.Digest.String()) + j.added[desc.Digest] = job{ + Descriptor: desc, + ProgressWriter: pw, + started: time.Now(), + } +} + +func (j *jobs) jobs() []job { + j.mu.Lock() + defer j.mu.Unlock() + + descs := make([]job, 0, len(j.added)) + for _, j := range j.added { + descs = append(descs, j) + } + return descs +} + +func (j *jobs) isResolved() bool { + j.mu.Lock() + defer j.mu.Unlock() + return j.resolved +} + +type statusInfo struct { + Ref string + Status string + Offset int64 + Total int64 + StartedAt time.Time + UpdatedAt time.Time +} + +func oneOffProgress(ctx context.Context, id string) func(err error) error { + pw, _, _ := progress.FromContext(ctx, id) + now := time.Now() + st := progress.Status{ + Action: id, + Started: &now, + } + pw.Write(st) + return func(err error) error { + // TODO: set error on status + now := time.Now() + st.Completed = &now + pw.Write(st) + pw.Done() + return err + } +} diff --git a/util/progress/progress.go b/util/progress/progress.go index 8600c270..08ed5594 100644 --- a/util/progress/progress.go +++ b/util/progress/progress.go @@ -47,9 +47,11 @@ type Progress struct { type Status struct { // ...progress of an action - Action string - Current int - Total int + Action string + Current int + Total int + Started *time.Time + Completed *time.Time } type progressReader struct { From a8303438d0cd8f731036c460905d9e74fa7a7c15 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 15 Jun 2017 16:09:03 -0700 Subject: [PATCH 2/3] example: more complicated build example Signed-off-by: Tonis Tiigi --- client/llb/llb.go | 36 +++++++++++++++++++++++++++++------- examples/llbout/example.go | 7 +++++-- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/client/llb/llb.go b/client/llb/llb.go index 3db72bf1..783812db 100644 --- a/client/llb/llb.go +++ b/client/llb/llb.go @@ -22,8 +22,8 @@ type SourceOp struct { type ExecOp struct { meta Meta - mounts []*mount - root *mount + mounts []*Mount + root *Mount } type Meta struct { @@ -32,10 +32,10 @@ type Meta struct { Cwd string } -type mount struct { +type Mount struct { op *ExecOp dest string - mount *mount + mount *Mount src *SourceOp output bool } @@ -81,11 +81,11 @@ func Image(ref string) *SourceOp { return Source("docker-image://" + ref) // controversial } -func newExec(meta Meta, src *SourceOp, m *mount) *ExecOp { +func newExec(meta Meta, src *SourceOp, m *Mount) *ExecOp { exec := &ExecOp{ meta: meta, - mounts: []*mount{}, - root: &mount{ + mounts: []*Mount{}, + root: &Mount{ dest: "/", src: src, mount: m, @@ -97,6 +97,28 @@ func newExec(meta Meta, src *SourceOp, m *mount) *ExecOp { return exec } +func (eo *ExecOp) AddMount(dest string, src interface{}) *Mount { + var s *SourceOp + var m *Mount + switch v := src.(type) { + case *SourceOp: + s = v + case *Mount: + m = v + case *ExecOp: + m = v.root + default: + panic("invalid input") + } + eo.mounts = append(eo.mounts, &Mount{ + dest: dest, + src: s, + mount: m, + output: true, // TODO: should be set only if something inherits + }) + return m +} + func (eo *ExecOp) Validate() error { for _, m := range eo.mounts { if m.src != nil { diff --git a/examples/llbout/example.go b/examples/llbout/example.go index 5c53f6c0..e3cd7e88 100644 --- a/examples/llbout/example.go +++ b/examples/llbout/example.go @@ -10,9 +10,12 @@ func main() { busybox := llb.Image("docker.io/library/busybox:latest") mod1 := busybox.Run(llb.Meta{Args: []string{"/bin/sleep", "1"}, Cwd: "/"}) mod2 := mod1.Run(llb.Meta{Args: []string{"/bin/sh", "-c", "echo foo > /bar"}, Cwd: "/"}) - mod3 := mod2.Run(llb.Meta{Args: []string{"/bin/ls", "-l", "/"}, Cwd: "/"}) + alpine := llb.Image("docker.io/library/alpine:latest") + mod3 := mod2.Run(llb.Meta{Args: []string{"/bin/cp", "-a", "/alpine/etc/passwd", "baz"}, Cwd: "/"}) + mod3.AddMount("/alpine", alpine) + mod4 := mod3.Run(llb.Meta{Args: []string{"/bin/ls", "-l", "/"}, Cwd: "/"}) - res := mod3 + res := mod4 dt, err := res.Marshal() if err != nil { panic(err) From 8e7d0904ce97667f47e5bda03a53e285d2e90f9c Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 15 Jun 2017 16:34:42 -0700 Subject: [PATCH 3/3] client: handle status log in client Signed-off-by: Tonis Tiigi --- client/graph.go | 4 ++-- client/solve.go | 35 ++++++++++++++++++++++++++++++++--- cmd/buildctl/build.go | 28 ++++++++++++++++++++++++++-- source/containerimage/pull.go | 3 ++- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/client/graph.go b/client/graph.go index 08175129..2cad3140 100644 --- a/client/graph.go +++ b/client/graph.go @@ -23,8 +23,8 @@ type VertexStatus struct { Total int64 Current int64 Timestamp time.Time - Started *time.Time // StartedAt - Completed *time.Time // StartedAt + Started *time.Time + Completed *time.Time } type VertexLog struct { diff --git a/client/solve.go b/client/solve.go index db98b5c4..fcd50fbc 100644 --- a/client/solve.go +++ b/client/solve.go @@ -6,14 +6,13 @@ import ( "encoding/hex" "io" - "github.com/Sirupsen/logrus" "github.com/pkg/errors" controlapi "github.com/tonistiigi/buildkit_poc/api/services/control" "github.com/tonistiigi/buildkit_poc/client/llb" "golang.org/x/sync/errgroup" ) -func (c *Client) Solve(ctx context.Context, r io.Reader) error { +func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveStatus) error { def, err := llb.ReadFrom(r) if err != nil { return errors.Wrap(err, "failed to parse input") @@ -52,10 +51,40 @@ func (c *Client) Solve(ctx context.Context, r io.Reader) error { } return errors.Wrap(err, "failed to receive status") } - logrus.Debugf("status: %+v", resp) + s := SolveStatus{} + for _, v := range resp.Vertexes { + s.Vertexes = append(s.Vertexes, &Vertex{ + Digest: v.Digest, + Inputs: v.Inputs, + Name: v.Name, + Started: v.Started, + Completed: v.Completed, + }) + } + for _, v := range resp.Statuses { + s.Statuses = append(s.Statuses, &VertexStatus{ + ID: v.ID, + Vertex: v.Vertex, + Name: v.Name, + Total: v.Total, + Current: v.Current, + Timestamp: v.Timestamp, + Started: v.Started, + Completed: v.Completed, + }) + } + if statusChan != nil { + statusChan <- &s + } } }) + defer func() { + if statusChan != nil { + close(statusChan) + } + }() + return eg.Wait() } diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index 2a17cfe0..3d1400fb 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -2,9 +2,13 @@ package main import ( "context" + "log" "os" + "github.com/davecgh/go-spew/spew" + "github.com/tonistiigi/buildkit_poc/client" "github.com/urfave/cli" + "golang.org/x/sync/errgroup" ) var buildCommand = cli.Command{ @@ -14,9 +18,29 @@ var buildCommand = cli.Command{ } func build(clicontext *cli.Context) error { - client, err := resolveClient(clicontext) + c, err := resolveClient(clicontext) if err != nil { return err } - return client.Solve(context.TODO(), os.Stdin) + + ch := make(chan *client.SolveStatus) + eg, ctx := errgroup.WithContext(context.TODO()) // TODO: define appContext + + eg.Go(func() error { + return c.Solve(ctx, os.Stdin, ch) + }) + + eg.Go(func() error { + for s := range ch { + for _, v := range s.Vertexes { + log.Print(spew.Sdump(v)) + } + for _, v := range s.Statuses { + log.Print(spew.Sdump(v)) + } + } + return nil + }) + + return eg.Wait() } diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index a82ec19b..0e3ad63e 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -84,6 +84,7 @@ func (is *imageSource) Pull(ctx context.Context, id source.Identifier) (cache.Im fetcher, err := is.resolver.Fetcher(ctx, ref) if err != nil { + stopProgress() return nil, err } @@ -99,6 +100,7 @@ func (is *imageSource) Pull(ctx context.Context, id source.Identifier) (cache.Im images.ChildrenHandler(is.ContentStore), } if err := images.Dispatch(ctx, images.Handlers(handlers...), desc); err != nil { + stopProgress() return nil, err } stopProgress() @@ -337,7 +339,6 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error { pw, _, _ := progress.FromContext(ctx, id) now := time.Now() st := progress.Status{ - Action: id, Started: &now, } pw.Write(st)