From 45bfdd5e427ec31daf2f0bad84b6e88fad6c5a14 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 10 Jul 2017 13:03:38 -0700 Subject: [PATCH] exporter: add image exporter Signed-off-by: Tonis Tiigi --- api/services/control/control.pb.go | 297 ++++++++++++++++++++++++----- api/services/control/control.proto | 2 + cache/manager.go | 1 + cache/refs.go | 8 + client/client_test.go | 2 +- client/solve.go | 8 +- cmd/buildctl/build.go | 31 ++- control/control.go | 17 +- control/control_containerd.go | 5 +- control/control_default.go | 22 +++ control/control_standalone.go | 9 +- exporter/containerimage/export.go | 228 ++++++++++++++++++++++ exporter/exporter.go | 14 ++ gometalinter.json | 2 +- solver/exec.go | 7 +- solver/solver.go | 44 ++++- 16 files changed, 629 insertions(+), 68 deletions(-) create mode 100644 exporter/containerimage/export.go create mode 100644 exporter/exporter.go diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go index 607aa9de..45ca3194 100644 --- a/api/services/control/control.pb.go +++ b/api/services/control/control.pb.go @@ -117,8 +117,10 @@ func (m *UsageRecord) GetSize_() int64 { } type SolveRequest struct { - Ref string `protobuf:"bytes,1,opt,name=Ref,proto3" json:"Ref,omitempty"` - Definition [][]byte `protobuf:"bytes,2,rep,name=Definition" json:"Definition,omitempty"` + Ref string `protobuf:"bytes,1,opt,name=Ref,proto3" json:"Ref,omitempty"` + Definition [][]byte `protobuf:"bytes,2,rep,name=Definition" json:"Definition,omitempty"` + Exporter string `protobuf:"bytes,3,opt,name=Exporter,proto3" json:"Exporter,omitempty"` + ExporterAttrs map[string]string `protobuf:"bytes,4,rep,name=ExporterAttrs" json:"ExporterAttrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *SolveRequest) Reset() { *m = SolveRequest{} } @@ -140,6 +142,20 @@ func (m *SolveRequest) GetDefinition() [][]byte { return nil } +func (m *SolveRequest) GetExporter() string { + if m != nil { + return m.Exporter + } + return "" +} + +func (m *SolveRequest) GetExporterAttrs() map[string]string { + if m != nil { + return m.ExporterAttrs + } + return nil +} + type SolveResponse struct { Vtx []*Vertex `protobuf:"bytes,1,rep,name=vtx" json:"vtx,omitempty"` } @@ -658,6 +674,29 @@ func (m *SolveRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], b) } } + if len(m.Exporter) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintControl(dAtA, i, uint64(len(m.Exporter))) + i += copy(dAtA[i:], m.Exporter) + } + if len(m.ExporterAttrs) > 0 { + for k, _ := range m.ExporterAttrs { + dAtA[i] = 0x22 + i++ + v := m.ExporterAttrs[k] + mapSize := 1 + len(k) + sovControl(uint64(len(k))) + 1 + len(v) + sovControl(uint64(len(v))) + i = encodeVarintControl(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintControl(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x12 + i++ + i = encodeVarintControl(dAtA, i, uint64(len(v))) + i += copy(dAtA[i:], v) + } + } return i, nil } @@ -1044,6 +1083,18 @@ func (m *SolveRequest) Size() (n int) { n += 1 + l + sovControl(uint64(l)) } } + l = len(m.Exporter) + if l > 0 { + n += 1 + l + sovControl(uint64(l)) + } + if len(m.ExporterAttrs) > 0 { + for k, v := range m.ExporterAttrs { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovControl(uint64(len(k))) + 1 + len(v) + sovControl(uint64(len(v))) + n += mapEntrySize + 1 + sovControl(uint64(mapEntrySize)) + } + } return n } @@ -1550,6 +1601,151 @@ func (m *SolveRequest) Unmarshal(dAtA []byte) error { m.Definition = append(m.Definition, make([]byte, postIndex-iNdEx)) copy(m.Definition[len(m.Definition)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Exporter", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthControl + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Exporter = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExporterAttrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthControl + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthControl + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + if m.ExporterAttrs == nil { + m.ExporterAttrs = make(map[string]string) + } + if iNdEx < postIndex { + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthControl + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue := string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + m.ExporterAttrs[mapkey] = mapvalue + } else { + var mapvalue string + m.ExporterAttrs[mapkey] = mapvalue + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipControl(dAtA[iNdEx:]) @@ -2664,50 +2860,55 @@ var ( func init() { proto.RegisterFile("control.proto", fileDescriptorControl) } var fileDescriptorControl = []byte{ - // 711 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x55, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0x66, 0xe3, 0xd4, 0x49, 0x26, 0x69, 0x55, 0x56, 0xa8, 0xb2, 0x82, 0x48, 0x8c, 0xb9, 0x44, - 0x95, 0xea, 0x40, 0x80, 0x4b, 0x91, 0x10, 0x0a, 0x39, 0xd0, 0x8a, 0x5e, 0xb6, 0x2d, 0x77, 0xc7, - 0xd9, 0xba, 0x56, 0x6d, 0x6f, 0xf0, 0xae, 0xa3, 0xc2, 0x53, 0xf0, 0x2e, 0x3c, 0x03, 0x52, 0x8f, - 0x9c, 0x39, 0x14, 0xd4, 0x07, 0xe0, 0xcc, 0x11, 0x79, 0x77, 0x9d, 0x9a, 0xb6, 0x81, 0xd2, 0x9e, - 0xb2, 0xb3, 0xf9, 0xe6, 0xdb, 0x99, 0x6f, 0x7e, 0x0c, 0xcb, 0x3e, 0x4b, 0x44, 0xca, 0x22, 0x77, - 0x9a, 0x32, 0xc1, 0xf0, 0x6a, 0xcc, 0xc6, 0x1f, 0xdc, 0x71, 0x16, 0x46, 0x93, 0xa3, 0x50, 0xb8, - 0xb3, 0x27, 0xed, 0x8d, 0x20, 0x14, 0x87, 0xd9, 0xd8, 0xf5, 0x59, 0xdc, 0x0f, 0x58, 0xc0, 0xfa, - 0x12, 0x38, 0xce, 0x0e, 0xa4, 0x25, 0x0d, 0x79, 0x52, 0x04, 0xed, 0x6e, 0xc0, 0x58, 0x10, 0xd1, - 0x73, 0x94, 0x08, 0x63, 0xca, 0x85, 0x17, 0x4f, 0x15, 0xc0, 0xc1, 0xb0, 0x3a, 0x0a, 0xf9, 0xd1, - 0x3e, 0xf7, 0x02, 0x4a, 0xe8, 0xfb, 0x8c, 0x72, 0xe1, 0x6c, 0xc3, 0xdd, 0xd2, 0x1d, 0x9f, 0xb2, - 0x84, 0x53, 0xfc, 0x1c, 0xcc, 0x94, 0xfa, 0x2c, 0x9d, 0x58, 0xc8, 0x36, 0x7a, 0xcd, 0xc1, 0x03, - 0xf7, 0x62, 0x6c, 0xae, 0x76, 0xc8, 0x41, 0x44, 0x83, 0x1d, 0x0f, 0x9a, 0xa5, 0x6b, 0xbc, 0x02, - 0x95, 0xad, 0x91, 0x85, 0x6c, 0xd4, 0x6b, 0x90, 0xca, 0xd6, 0x08, 0x5b, 0x50, 0xdb, 0xc9, 0x84, - 0x37, 0x8e, 0xa8, 0x55, 0xb1, 0x51, 0xaf, 0x4e, 0x0a, 0x13, 0xdf, 0x83, 0xa5, 0xad, 0x64, 0x9f, - 0x53, 0xcb, 0x90, 0xf7, 0xca, 0xc0, 0x18, 0xaa, 0xbb, 0xe1, 0x47, 0x6a, 0x55, 0x6d, 0xd4, 0x33, - 0x88, 0x3c, 0x3b, 0xaf, 0xa0, 0xb5, 0xcb, 0xa2, 0x59, 0x11, 0x3e, 0x5e, 0x05, 0x83, 0xd0, 0x03, - 0xfd, 0x48, 0x7e, 0xc4, 0x1d, 0x80, 0x11, 0x3d, 0x08, 0x93, 0x50, 0x84, 0x2c, 0xb1, 0x2a, 0xb6, - 0xd1, 0x6b, 0x91, 0xd2, 0x8d, 0xf3, 0x02, 0x96, 0x35, 0x83, 0x4e, 0x76, 0x1d, 0x8c, 0x99, 0x38, - 0xd6, 0x99, 0x5a, 0x97, 0x33, 0x7d, 0x47, 0x53, 0x41, 0x8f, 0x49, 0x0e, 0x72, 0x1e, 0xc2, 0xf2, - 0xae, 0xf0, 0x44, 0xc6, 0x17, 0xbe, 0xef, 0x7c, 0x46, 0xb0, 0x52, 0x60, 0xf4, 0x0b, 0xcf, 0xa0, - 0x3e, 0x93, 0x24, 0x94, 0xff, 0xf3, 0x99, 0x39, 0x12, 0x6f, 0x42, 0x9d, 0x4b, 0x1e, 0xca, 0x65, - 0x1a, 0xcd, 0x41, 0x67, 0x91, 0x97, 0x7e, 0x6f, 0x8e, 0xc7, 0x7d, 0xa8, 0x46, 0x2c, 0xe0, 0x96, - 0x21, 0xfd, 0xee, 0x2f, 0xf2, 0x7b, 0xcb, 0x02, 0x22, 0x81, 0xce, 0x69, 0x05, 0x4c, 0x75, 0x87, - 0xb7, 0xc1, 0x9c, 0x84, 0x01, 0xe5, 0x42, 0x65, 0x35, 0x1c, 0x9c, 0x9c, 0x76, 0xef, 0x7c, 0x3b, - 0xed, 0xae, 0x97, 0xba, 0x91, 0x4d, 0x69, 0x92, 0x77, 0xaf, 0x17, 0x26, 0x34, 0xe5, 0xfd, 0x80, - 0x6d, 0x28, 0x17, 0x77, 0x24, 0x7f, 0x88, 0x66, 0xc8, 0xb9, 0xc2, 0x64, 0x9a, 0x09, 0x95, 0xc1, - 0x0d, 0xb9, 0x14, 0x43, 0xde, 0x0e, 0x89, 0x17, 0xab, 0x1e, 0x69, 0x10, 0x79, 0xc6, 0x6b, 0x60, - 0xfa, 0x9e, 0x7f, 0x48, 0x27, 0xb2, 0x49, 0xea, 0x44, 0x5b, 0x78, 0x13, 0x6a, 0x5c, 0x78, 0xa9, - 0xa0, 0x13, 0x6b, 0xc9, 0x46, 0xbd, 0xe6, 0xa0, 0xed, 0xaa, 0xe1, 0x70, 0x8b, 0xe1, 0x70, 0xf7, - 0x8a, 0xe1, 0x18, 0x56, 0x3f, 0x7d, 0xef, 0x22, 0x52, 0x38, 0xe0, 0x97, 0xd0, 0xf0, 0x59, 0x3c, - 0x8d, 0x68, 0xee, 0x6d, 0x5e, 0xd3, 0xfb, 0xdc, 0x25, 0x6f, 0x66, 0x9a, 0xa6, 0x2c, 0xb5, 0x6a, - 0x32, 0x50, 0x65, 0x38, 0x3f, 0x2b, 0xd0, 0x2a, 0x17, 0xeb, 0xd2, 0x74, 0x6c, 0x83, 0xa9, 0x4a, - 0x2f, 0x87, 0xe3, 0x86, 0x52, 0x29, 0x86, 0x2b, 0xa5, 0xb2, 0xa0, 0xe6, 0x67, 0x69, 0x4a, 0x13, - 0xa1, 0x07, 0xaa, 0x30, 0xf3, 0x80, 0x05, 0x13, 0x5e, 0x24, 0xa5, 0x32, 0x88, 0x32, 0xf0, 0x10, - 0x1a, 0xf3, 0xfd, 0x71, 0x0d, 0x19, 0xea, 0x79, 0xb8, 0x4a, 0x8a, 0xb9, 0x5b, 0xb9, 0x0c, 0xb5, - 0x5b, 0x95, 0xa1, 0xfe, 0xdf, 0x65, 0x70, 0xbe, 0x20, 0x68, 0xcc, 0xbb, 0xbc, 0xa4, 0x2e, 0xba, - 0xb5, 0xba, 0x7f, 0x28, 0x53, 0xb9, 0x99, 0x32, 0x6b, 0x60, 0x72, 0x91, 0x52, 0x2f, 0x96, 0x35, - 0x32, 0x88, 0xb6, 0xf2, 0x7d, 0x12, 0xf3, 0x40, 0x56, 0xa8, 0x45, 0xf2, 0xe3, 0xe0, 0x17, 0x82, - 0xda, 0x6b, 0xf5, 0xa1, 0xc0, 0x7b, 0xd0, 0x98, 0x2f, 0x6b, 0xec, 0x5c, 0x9e, 0xea, 0x8b, 0xdb, - 0xbd, 0xfd, 0xe8, 0xaf, 0x18, 0xbd, 0x9e, 0xde, 0xc0, 0x92, 0xdc, 0x88, 0xf8, 0x8a, 0xfd, 0x52, - 0x5e, 0xb6, 0xed, 0xee, 0xc2, 0xff, 0x35, 0xd3, 0x0e, 0x98, 0xba, 0xbb, 0xaf, 0x82, 0x96, 0x17, - 0x67, 0xdb, 0x5e, 0x0c, 0x50, 0x64, 0x8f, 0xd1, 0xb0, 0x75, 0x72, 0xd6, 0x41, 0x5f, 0xcf, 0x3a, - 0xe8, 0xc7, 0x59, 0x07, 0x8d, 0x4d, 0xa9, 0xed, 0xd3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc0, - 0x88, 0x4d, 0x15, 0x37, 0x07, 0x00, 0x00, + // 785 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x55, 0x4f, 0x6f, 0xe3, 0x44, + 0x14, 0x67, 0xec, 0xd4, 0x49, 0x5e, 0x92, 0x55, 0x19, 0xa1, 0x95, 0x65, 0x44, 0x12, 0xcc, 0x25, + 0x5a, 0x69, 0x1d, 0x36, 0x80, 0x84, 0x8a, 0x84, 0x20, 0x64, 0x25, 0x5a, 0xb1, 0x97, 0xe9, 0x16, + 0xce, 0x4e, 0x32, 0x75, 0xad, 0xd8, 0x9e, 0x30, 0x33, 0x8e, 0x1a, 0x3e, 0x05, 0xdf, 0x85, 0xcf, + 0x80, 0xd4, 0x23, 0x67, 0x0e, 0x05, 0xf5, 0x03, 0x70, 0x85, 0x23, 0xf2, 0xcc, 0x38, 0x75, 0x9b, + 0xa6, 0x94, 0xf6, 0x94, 0x79, 0x93, 0xdf, 0xfb, 0xbd, 0x37, 0xbf, 0xf7, 0xc7, 0xd0, 0x99, 0xb1, + 0x4c, 0x72, 0x96, 0x04, 0x4b, 0xce, 0x24, 0xc3, 0xfb, 0x29, 0x9b, 0xae, 0x83, 0x69, 0x1e, 0x27, + 0xf3, 0x45, 0x2c, 0x83, 0xd5, 0x2b, 0xef, 0x65, 0x14, 0xcb, 0xb3, 0x7c, 0x1a, 0xcc, 0x58, 0x3a, + 0x8c, 0x58, 0xc4, 0x86, 0x0a, 0x38, 0xcd, 0x4f, 0x95, 0xa5, 0x0c, 0x75, 0xd2, 0x04, 0x5e, 0x2f, + 0x62, 0x2c, 0x4a, 0xe8, 0x35, 0x4a, 0xc6, 0x29, 0x15, 0x32, 0x4c, 0x97, 0x1a, 0xe0, 0x63, 0xd8, + 0x9f, 0xc4, 0x62, 0x71, 0x22, 0xc2, 0x88, 0x12, 0xfa, 0x63, 0x4e, 0x85, 0xf4, 0x8f, 0xe0, 0xdd, + 0xca, 0x9d, 0x58, 0xb2, 0x4c, 0x50, 0xfc, 0x19, 0x38, 0x9c, 0xce, 0x18, 0x9f, 0xbb, 0xa8, 0x6f, + 0x0f, 0x5a, 0xa3, 0x0f, 0x82, 0xdb, 0xb9, 0x05, 0xc6, 0xa1, 0x00, 0x11, 0x03, 0xf6, 0x43, 0x68, + 0x55, 0xae, 0xf1, 0x33, 0xb0, 0x0e, 0x27, 0x2e, 0xea, 0xa3, 0x41, 0x93, 0x58, 0x87, 0x13, 0xec, + 0x42, 0xfd, 0x4d, 0x2e, 0xc3, 0x69, 0x42, 0x5d, 0xab, 0x8f, 0x06, 0x0d, 0x52, 0x9a, 0xf8, 0x3d, + 0xd8, 0x3b, 0xcc, 0x4e, 0x04, 0x75, 0x6d, 0x75, 0xaf, 0x0d, 0x8c, 0xa1, 0x76, 0x1c, 0xff, 0x44, + 0xdd, 0x5a, 0x1f, 0x0d, 0x6c, 0xa2, 0xce, 0xfe, 0xdf, 0x08, 0xda, 0xc7, 0x2c, 0x59, 0x95, 0xf9, + 0xe3, 0x7d, 0xb0, 0x09, 0x3d, 0x35, 0x51, 0x8a, 0x23, 0xee, 0x02, 0x4c, 0xe8, 0x69, 0x9c, 0xc5, + 0x32, 0x66, 0x99, 0x6b, 0xf5, 0xed, 0x41, 0x9b, 0x54, 0x6e, 0xb0, 0x07, 0x8d, 0xd7, 0xe7, 0x4b, + 0xc6, 0x25, 0xe5, 0x2a, 0x5e, 0x93, 0x6c, 0x6c, 0xfc, 0x03, 0x74, 0xca, 0xf3, 0xd7, 0x52, 0x72, + 0xe1, 0xd6, 0xd4, 0xfb, 0x5f, 0x6d, 0xbf, 0xbf, 0x9a, 0x44, 0x70, 0xc3, 0xe7, 0x75, 0x26, 0xf9, + 0x9a, 0xdc, 0xe4, 0xf1, 0xbe, 0x02, 0xbc, 0x0d, 0x2a, 0x92, 0x5f, 0xd0, 0x75, 0x99, 0xfc, 0x82, + 0xae, 0x0b, 0x25, 0x56, 0x61, 0x92, 0x6b, 0x85, 0x9a, 0x44, 0x1b, 0x07, 0xd6, 0xe7, 0xc8, 0xff, + 0x02, 0x3a, 0x26, 0xa6, 0x29, 0xd2, 0x0b, 0xb0, 0x57, 0xf2, 0xdc, 0x54, 0xc8, 0xdd, 0xce, 0xf0, + 0x7b, 0xca, 0x25, 0x3d, 0x27, 0x05, 0xc8, 0xff, 0x10, 0x3a, 0xc7, 0x32, 0x94, 0xb9, 0xd8, 0x29, + 0x9b, 0xff, 0x0b, 0x82, 0x67, 0x25, 0xc6, 0x44, 0xf8, 0x14, 0x1a, 0x2b, 0x45, 0x42, 0xc5, 0x7f, + 0x86, 0xd9, 0x20, 0xf1, 0x01, 0x34, 0x84, 0xe2, 0xa1, 0x42, 0xa9, 0xdf, 0x1a, 0x75, 0x77, 0x79, + 0x99, 0x78, 0x1b, 0x3c, 0x1e, 0x42, 0x2d, 0x61, 0x91, 0x70, 0x6d, 0xe5, 0xf7, 0xfe, 0x2e, 0xbf, + 0xef, 0x58, 0x44, 0x14, 0xd0, 0xbf, 0xb4, 0xc0, 0xd1, 0x77, 0xf8, 0x08, 0x9c, 0x79, 0x1c, 0x51, + 0x21, 0xf5, 0xab, 0xc6, 0xa3, 0x8b, 0xcb, 0xde, 0x3b, 0xbf, 0x5f, 0xf6, 0x5e, 0x54, 0xa6, 0x88, + 0x2d, 0x69, 0x56, 0x4c, 0x5d, 0x18, 0x67, 0x94, 0x8b, 0x61, 0xc4, 0x5e, 0x6a, 0x97, 0x60, 0xa2, + 0x7e, 0x88, 0x61, 0x28, 0xb8, 0xe2, 0x6c, 0x99, 0x4b, 0xfd, 0x82, 0x47, 0x72, 0x69, 0x86, 0xa2, + 0x8d, 0xb3, 0x30, 0xa5, 0xa6, 0xd7, 0xd4, 0x19, 0x3f, 0x07, 0x67, 0x16, 0xce, 0xce, 0xe8, 0x5c, + 0x35, 0x77, 0x83, 0x18, 0x0b, 0x1f, 0x40, 0x5d, 0xc8, 0x90, 0x4b, 0x3a, 0x77, 0xf7, 0xfa, 0x68, + 0xd0, 0x1a, 0x79, 0x81, 0x1e, 0xea, 0xa0, 0x1c, 0xea, 0xe0, 0x6d, 0x39, 0xd4, 0xe3, 0xda, 0xcf, + 0x7f, 0xf4, 0x10, 0x29, 0x1d, 0xf0, 0x97, 0xd0, 0x9c, 0xb1, 0x74, 0x99, 0xd0, 0xc2, 0xdb, 0x79, + 0xa0, 0xf7, 0xb5, 0x4b, 0xd1, 0x7a, 0x94, 0x73, 0xc6, 0xdd, 0xba, 0x6e, 0x3d, 0x65, 0xf8, 0x7f, + 0x59, 0xd0, 0xae, 0x16, 0x6b, 0x6b, 0xaa, 0x8f, 0xc0, 0xd1, 0xa5, 0xd7, 0x2d, 0xfb, 0x38, 0xa9, + 0x34, 0xc3, 0x9d, 0x52, 0xb9, 0x50, 0x9f, 0xe5, 0x9c, 0xd3, 0x4c, 0x9a, 0x45, 0x50, 0x9a, 0x45, + 0xc2, 0x92, 0xc9, 0x30, 0x51, 0x52, 0xd9, 0x44, 0x1b, 0x78, 0x0c, 0xcd, 0xcd, 0xde, 0x7b, 0x80, + 0x0c, 0x8d, 0x22, 0x5d, 0x2d, 0xc5, 0xc6, 0xad, 0x5a, 0x86, 0xfa, 0x93, 0xca, 0xd0, 0xf8, 0xdf, + 0x65, 0xf0, 0x7f, 0x45, 0xd0, 0xdc, 0x74, 0x79, 0x45, 0x5d, 0xf4, 0x64, 0x75, 0x6f, 0x28, 0x63, + 0x3d, 0x4e, 0x99, 0xe7, 0xe0, 0x08, 0xc9, 0x69, 0x98, 0xaa, 0x1a, 0xd9, 0xc4, 0x58, 0xc5, 0x3e, + 0x49, 0x45, 0xa4, 0x2a, 0xd4, 0x26, 0xc5, 0x71, 0xf4, 0x0f, 0x82, 0xfa, 0x37, 0xfa, 0x03, 0x87, + 0xdf, 0x42, 0x73, 0xf3, 0x91, 0xc1, 0xfe, 0xf6, 0x54, 0xdf, 0xfe, 0x2a, 0x79, 0x1f, 0xdd, 0x8b, + 0x31, 0xeb, 0xe9, 0x5b, 0xd8, 0x53, 0x1b, 0x11, 0x77, 0xef, 0x5f, 0xcf, 0x5e, 0x6f, 0xe7, 0xff, + 0x86, 0xe9, 0x0d, 0x38, 0xa6, 0xbb, 0xef, 0x82, 0x56, 0x17, 0xa7, 0xd7, 0xdf, 0x0d, 0xd0, 0x64, + 0x1f, 0xa3, 0x71, 0xfb, 0xe2, 0xaa, 0x8b, 0x7e, 0xbb, 0xea, 0xa2, 0x3f, 0xaf, 0xba, 0x68, 0xea, + 0x28, 0x6d, 0x3f, 0xf9, 0x37, 0x00, 0x00, 0xff, 0xff, 0x97, 0xc3, 0x70, 0xc2, 0xef, 0x07, 0x00, + 0x00, } diff --git a/api/services/control/control.proto b/api/services/control/control.proto index bafc4997..d4ad2074 100644 --- a/api/services/control/control.proto +++ b/api/services/control/control.proto @@ -32,6 +32,8 @@ message UsageRecord { message SolveRequest { string Ref = 1; repeated bytes Definition = 2; // TODO: remove repeated + string Exporter = 3; + map ExporterAttrs = 4; } message SolveResponse { diff --git a/cache/manager.go b/cache/manager.go index 6af1738e..2285b8b1 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -92,6 +92,7 @@ func (cm *cacheManager) Get(ctx context.Context, id string) (ImmutableRef, error defer cm.mu.Unlock() return cm.get(ctx, id) } + func (cm *cacheManager) get(ctx context.Context, id string) (ImmutableRef, error) { rec, err := cm.load(ctx, id) if err != nil { diff --git a/cache/refs.go b/cache/refs.go index 2e50ea78..67ac499f 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -16,6 +16,7 @@ type ImmutableRef interface { ID() string Release(context.Context) error Size(ctx context.Context) (int64, error) + Parent() ImmutableRef // Prepare() / ChainID() / Meta() } @@ -85,6 +86,13 @@ func (cr *cacheRecord) Size(ctx context.Context) (int64, error) { return s.(int64), err } +func (cr *cacheRecord) Parent() ImmutableRef { + if cr.parent == nil { + return nil + } + return cr.parent.(*immutableRef).ref() +} + func (cr *cacheRecord) Mount(ctx context.Context) ([]mount.Mount, error) { cr.mu.Lock() defer cr.mu.Unlock() diff --git a/client/client_test.go b/client/client_test.go index 89ff03a1..e882e27c 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -95,6 +95,6 @@ func testBuildMultiMount(t *testing.T, address string) { err = llb.WriteTo(dt, buf) assert.Nil(t, err) - err = c.Solve(context.TODO(), buf, nil) + err = c.Solve(context.TODO(), buf, nil, "", nil) assert.Nil(t, err) } diff --git a/client/solve.go b/client/solve.go index ddfb82fe..a9cbf144 100644 --- a/client/solve.go +++ b/client/solve.go @@ -13,7 +13,7 @@ import ( "golang.org/x/sync/errgroup" ) -func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveStatus) error { +func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveStatus, exporter string, exporterAttrs map[string]string) error { def, err := llb.ReadFrom(r) if err != nil { return errors.Wrap(err, "failed to parse input") @@ -36,8 +36,10 @@ func (c *Client) Solve(ctx context.Context, r io.Reader, statusChan chan *SolveS }() }() _, err = c.controlClient().Solve(ctx, &controlapi.SolveRequest{ - Ref: ref, - Definition: def, + Ref: ref, + Definition: def, + Exporter: exporter, + ExporterAttrs: exporterAttrs, }) if err != nil { return errors.Wrap(err, "failed to solve") diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index 6245925e..04e67f90 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -5,11 +5,13 @@ import ( "encoding/json" "io/ioutil" "os" + "strings" "github.com/Sirupsen/logrus" "github.com/moby/buildkit/client" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/progress/progressui" + "github.com/pkg/errors" "github.com/urfave/cli" "golang.org/x/sync/errgroup" ) @@ -18,6 +20,16 @@ var buildCommand = cli.Command{ Name: "build", Usage: "build", Action: build, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "exporter", + Usage: "Define exporter for build result", + }, + cli.StringSliceFlag{ + Name: "exporter-opt", + Usage: "Define custom options for exporter", + }, + }, } func build(clicontext *cli.Context) error { @@ -39,8 +51,13 @@ func build(clicontext *cli.Context) error { displayCh := make(chan *client.SolveStatus) eg, ctx := errgroup.WithContext(appcontext.Context()) + exporterAttrs, err := attrMap(clicontext.StringSlice("exporter-opt")) + if err != nil { + return errors.Wrap(err, "invalid exporter-opt") + } + eg.Go(func() error { - return c.Solve(ctx, os.Stdin, ch) + return c.Solve(ctx, os.Stdin, ch, clicontext.String("exporter"), exporterAttrs) }) eg.Go(func() error { @@ -61,3 +78,15 @@ func build(clicontext *cli.Context) error { return eg.Wait() } + +func attrMap(sl []string) (map[string]string, error) { + m := map[string]string{} + for _, v := range sl { + parts := strings.SplitN(v, "=", 2) + if len(parts) != 2 { + return nil, errors.Errorf("invalid value %s", v) + } + m[parts[0]] = parts[1] + } + return m, nil +} diff --git a/control/control.go b/control/control.go index 5832f20e..51e9618b 100644 --- a/control/control.go +++ b/control/control.go @@ -5,6 +5,7 @@ import ( controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/source" "github.com/moby/buildkit/worker" @@ -20,6 +21,7 @@ type Opt struct { Worker worker.Worker SourceManager *source.Manager InstructionCache solver.InstructionCache + Exporters map[string]exporter.Exporter } type Controller struct { // TODO: ControlService @@ -68,7 +70,20 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* if err != nil { return nil, errors.Wrap(err, "failed to load") } - if err := c.solver.Solve(ctx, req.Ref, v); err != nil { + + var expi exporter.ExporterInstance + if req.Exporter != "" { + exp, ok := c.opt.Exporters[req.Exporter] + if !ok { + return nil, errors.Errorf("exporter %q could not be found", req.Exporter) + } + expi, err = exp.Resolve(ctx, req.ExporterAttrs) + if err != nil { + return nil, err + } + } + + if err := c.solver.Solve(ctx, req.Ref, v, expi); err != nil { return nil, err } return &controlapi.SolveResponse{}, nil diff --git a/control/control_containerd.go b/control/control_containerd.go index f89c8e5d..e53bebe7 100644 --- a/control/control_containerd.go +++ b/control/control_containerd.go @@ -44,10 +44,13 @@ func NewContainerd(root, address string) (*Controller, error) { } func newContainerdPullDeps(client *containerd.Client) *pullDeps { + diff := client.DiffService() return &pullDeps{ Snapshotter: client.SnapshotService(), ContentStore: client.ContentStore(), - Applier: client.DiffService(), + Applier: diff, + Differ: diff, + Images: client.ImageService(), } } diff --git a/control/control_default.go b/control/control_default.go index 330e3fe8..a313fe0c 100644 --- a/control/control_default.go +++ b/control/control_default.go @@ -6,21 +6,28 @@ import ( "path/filepath" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/rootfs" ctdsnapshot "github.com/containerd/containerd/snapshot" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/instructioncache" "github.com/moby/buildkit/cache/metadata" + "github.com/moby/buildkit/exporter" + imageexporter "github.com/moby/buildkit/exporter/containerimage" "github.com/moby/buildkit/snapshot/blobmapping" "github.com/moby/buildkit/source" "github.com/moby/buildkit/source/containerimage" "github.com/moby/buildkit/source/git" ) +const keyImageExporter = "image" + type pullDeps struct { Snapshotter ctdsnapshot.Snapshotter ContentStore content.Store Applier rootfs.Applier + Differ rootfs.MountDiffer + Images images.Store } func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) { @@ -78,10 +85,25 @@ func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) { sm.Register(gs) + exporters := map[string]exporter.Exporter{} + + imageExporter, err := imageexporter.New(imageexporter.Opt{ + Snapshotter: snapshotter, + ContentStore: pd.ContentStore, + Differ: pd.Differ, + CacheAccessor: cm, + Images: pd.Images, + }) + if err != nil { + return nil, err + } + exporters[keyImageExporter] = imageExporter + return &Opt{ Snapshotter: snapshotter, CacheManager: cm, SourceManager: sm, InstructionCache: ic, + Exporters: exporters, }, nil } diff --git a/control/control_standalone.go b/control/control_standalone.go index ecf33cfa..7a974242 100644 --- a/control/control_standalone.go +++ b/control/control_standalone.go @@ -12,6 +12,7 @@ import ( "github.com/containerd/containerd/archive" "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/differ" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" ctdsnapshot "github.com/containerd/containerd/snapshot" @@ -61,12 +62,16 @@ func newStandalonePullDeps(root string) (*pullDeps, error) { return nil, err } - a := &localApplier{root: root, content: c} + differ, err := differ.NewBaseDiff(c) + if err != nil { + return nil, err + } return &pullDeps{ Snapshotter: &nsSnapshotter{s}, ContentStore: c, - Applier: a, + Applier: differ, + Differ: differ, }, nil } diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go new file mode 100644 index 00000000..a2aeae65 --- /dev/null +++ b/exporter/containerimage/export.go @@ -0,0 +1,228 @@ +package containerimage + +import ( + "bytes" + gocontext "context" + "encoding/json" + "runtime" + + "github.com/Sirupsen/logrus" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/rootfs" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/metadata" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/flightcontrol" + "github.com/moby/buildkit/util/system" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/net/context" + "golang.org/x/sync/errgroup" +) + +const ( + keyImageName = "name" +) + +type Opt struct { + Snapshotter snapshot.Snapshotter + ContentStore content.Store + Differ rootfs.MountDiffer + CacheAccessor cache.Accessor + MetadataStore metadata.Store + Images images.Store +} + +type imageExporter struct { + blobmap blobmapper + opt Opt + g flightcontrol.Group +} + +type diffPair struct { + diffID digest.Digest + blobsum digest.Digest +} + +type blobmapper interface { + GetBlob(ctx gocontext.Context, key string) (digest.Digest, error) + SetBlob(ctx gocontext.Context, key string, blob digest.Digest) error +} + +func New(opt Opt) (exporter.Exporter, error) { + blobmap, ok := opt.Snapshotter.(blobmapper) + if !ok { + return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support") + } + + im := &imageExporter{opt: opt, blobmap: blobmap} + return im, nil +} + +func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) { + i := &imageExporterInstance{imageExporter: e} + for k, v := range opt { + switch k { + case keyImageName: + i.targetName = v + default: + logrus.Warnf("unknown exporter option %s", k) + } + } + return i, nil +} + +func (e *imageExporter) getBlobs(ctx context.Context, ref cache.ImmutableRef) ([]diffPair, error) { + eg, ctx := errgroup.WithContext(ctx) + var diffPairs []diffPair + var currentPair diffPair + parent := ref.Parent() + if parent != nil { + defer parent.Release(context.TODO()) + eg.Go(func() error { + dp, err := e.getBlobs(ctx, parent) + if err != nil { + return err + } + diffPairs = dp + return nil + }) + } + eg.Go(func() error { + dp, err := e.g.Do(ctx, ref.ID(), func(ctx context.Context) (interface{}, error) { + blob, err := e.blobmap.GetBlob(ctx, ref.ID()) + if err != nil { + return nil, err + } + if blob != "" { + diffID, err := digest.Parse(ref.ID()) + if err != nil { + diffID = blob + } + return diffPair{diffID: diffID, blobsum: blob}, nil + } + // reference needs to be committed + parent := ref.Parent() + var lower []mount.Mount + if parent != nil { + defer parent.Release(context.TODO()) + lower, err = parent.Mount(ctx) + if err != nil { + return nil, err + } + } + upper, err := ref.Mount(ctx) + if err != nil { + return nil, err + } + descr, err := e.opt.Differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID()) + if err != nil { + return nil, err + } + if err := e.blobmap.SetBlob(ctx, ref.ID(), descr.Digest); err != nil { + return nil, err + } + return diffPair{diffID: descr.Digest, blobsum: descr.Digest}, nil + }) + if err != nil { + return err + } + currentPair = dp.(diffPair) + return nil + }) + err := eg.Wait() + if err != nil { + return nil, err + } + return append(diffPairs, currentPair), nil +} + +type imageExporterInstance struct { + *imageExporter + targetName string +} + +func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableRef) error { + diffPairs, err := e.getBlobs(ctx, ref) + if err != nil { + return err + } + + diffIDs := make([]digest.Digest, 0, len(diffPairs)) + for _, dp := range diffPairs { + diffIDs = append(diffIDs, dp.diffID) + } + + dt, err := json.Marshal(imageConfig(diffIDs)) + if err != nil { + return errors.Wrap(err, "failed to marshal image config") + } + + dgst := digest.FromBytes(dt) + + if err := content.WriteBlob(ctx, e.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst); err != nil { + return errors.Wrap(err, "error writing config blob") + } + + mfst := ocispec.Manifest{ + Config: ocispec.Descriptor{ + Digest: dgst, + Size: int64(len(dt)), + MediaType: ocispec.MediaTypeImageConfig, + }, + } + mfst.SchemaVersion = 2 + + for _, dp := range diffPairs { + info, err := e.opt.ContentStore.Info(ctx, dp.blobsum) + if err != nil { + return errors.Wrapf(err, "could not get blob %s", dp.blobsum) + } + mfst.Layers = append(mfst.Layers, ocispec.Descriptor{ + Digest: dp.blobsum, + Size: info.Size, + MediaType: ocispec.MediaTypeImageLayerGzip, + }) + } + + dt, err = json.Marshal(mfst) + if err != nil { + return errors.Wrap(err, "failed to marshal manifest") + } + + dgst = digest.FromBytes(dt) + + if err := content.WriteBlob(ctx, e.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst); err != nil { + return errors.Wrap(err, "error writing manifest blob") + } + + logrus.Debugf("wrote manifest %s", dgst) + + if e.opt.Images != nil && e.targetName != "" { + e.opt.Images.Update(ctx, e.targetName, ocispec.Descriptor{ + Digest: dgst, + Size: int64(len(dt)), + MediaType: ocispec.MediaTypeImageManifest, + }) + logrus.Debugf("updated tag for %s", e.targetName) + } + + return err +} + +// this is temporary: should move to dockerfile frontend +func imageConfig(diffIDs []digest.Digest) ocispec.Image { + img := ocispec.Image{ + Architecture: runtime.GOARCH, + OS: runtime.GOOS, + } + img.RootFS.Type = "layers" + img.RootFS.DiffIDs = diffIDs + img.Config.WorkingDir = "/" + img.Config.Env = []string{"PATH=" + system.DefaultPathEnv} + return img +} diff --git a/exporter/exporter.go b/exporter/exporter.go new file mode 100644 index 00000000..778d8b29 --- /dev/null +++ b/exporter/exporter.go @@ -0,0 +1,14 @@ +package exporter + +import ( + "github.com/moby/buildkit/cache" + "golang.org/x/net/context" +) + +type Exporter interface { + Resolve(context.Context, map[string]string) (ExporterInstance, error) +} + +type ExporterInstance interface { + Export(context.Context, cache.ImmutableRef) error +} diff --git a/gometalinter.json b/gometalinter.json index af1c0632..2b3efa7d 100644 --- a/gometalinter.json +++ b/gometalinter.json @@ -1,7 +1,7 @@ { "Vendor": true, "Deadline": "8m", - "Exclude": ["solver/pb/ops.pb.go"], + "Exclude": ["solver/pb/ops.pb.go", "api/services/control/control.pb.go"], "DisableAll": true, "Enable": [ "gofmt", diff --git a/solver/exec.go b/solver/exec.go index 5630f16d..7df8db2a 100644 --- a/solver/exec.go +++ b/solver/exec.go @@ -65,13 +65,8 @@ func (e *execOp) Run(ctx context.Context, inputs []Reference) ([]Reference, erro return nil, errors.Errorf("missing input %d", m.Input) } inp := inputs[int(m.Input)] - if sys, ok := inp.(interface { - Sys() Reference - }); ok { - inp = sys.Sys() - } var ok bool - ref, ok = inp.(cache.ImmutableRef) + ref, ok = toImmutableRef(inp) if !ok { return nil, errors.Errorf("invalid reference for exec %T", inputs[int(m.Input)]) } diff --git a/solver/solver.go b/solver/solver.go index f2bddc1a..85463272 100644 --- a/solver/solver.go +++ b/solver/solver.go @@ -4,6 +4,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/progress" @@ -63,7 +64,7 @@ func New(resolve ResolveOpFunc, cache InstructionCache) *Solver { return &Solver{resolve: resolve, jobs: newJobList(), cache: cache} } -func (s *Solver) Solve(ctx context.Context, id string, v Vertex) error { +func (s *Solver) Solve(ctx context.Context, id string, v Vertex, exp exporter.ExporterInstance) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -87,10 +88,22 @@ func (s *Solver) Solve(ctx context.Context, id string, v Vertex) error { return err } - for _, r := range refs { - r.Release(context.TODO()) + defer func() { + for _, r := range refs { + r.Release(context.TODO()) + } + }() + + if exp != nil { + immutable, ok := toImmutableRef(refs[0]) + if !ok { + return errors.Errorf("invalid reference for exporting: %T", refs[0]) + } + if err := exp.Export(ctx, immutable); err != nil { + return err + } } - // TODO: export final vertex state + return err } @@ -264,3 +277,26 @@ func toAny(in []Reference) []interface{} { } return out } + +func toImmutableRef(ref Reference) (cache.ImmutableRef, bool) { + sysRef := ref + if sys, ok := ref.(interface { + Sys() Reference + }); ok { + sysRef = sys.Sys() + } + immutable, ok := sysRef.(cache.ImmutableRef) + if !ok { + return nil, false + } + return &immutableRef{immutable, ref.Release}, true +} + +type immutableRef struct { + cache.ImmutableRef + release func(context.Context) error +} + +func (ir *immutableRef) Release(ctx context.Context) error { + return ir.release(ctx) +}