From 96b6a283128b030e53a030b90794265726bf496a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 29 Jul 2019 17:50:54 -0700 Subject: [PATCH] exporter: allow oci exporters visibility to response metadata Signed-off-by: Tonis Tiigi --- client/client_test.go | 18 ++++++--- client/solve.go | 4 +- cmd/buildctl/build/output.go | 11 ++++-- examples/build-using-dockerfile/main.go | 4 +- exporter/oci/export.go | 14 ++++++- exporter/tar/export.go | 2 +- frontend/dockerfile/dockerfile_test.go | 12 ++++-- session/filesync/diffcopy.go | 6 +++ session/filesync/filesync.go | 49 ++++++++++++++++++------- 9 files changed, 89 insertions(+), 31 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 78dce493..af693a00 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -740,7 +740,7 @@ func testFrontendImageNaming(t *testing.T, sb integration.Sandbox) { case ExporterDocker: outW, err := os.Create(out) require.NoError(t, err) - so.Exports[0].Output = outW + so.Exports[0].Output = fixedWriteCloser(outW) case ExporterImage: imageName = registry + "/" + imageName so.Exports[0].Attrs["push"] = "true" @@ -1304,7 +1304,7 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) { { Type: exp, Attrs: attrs, - Output: outW, + Output: fixedWriteCloser(outW), }, }, }, nil) @@ -1388,7 +1388,7 @@ func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) { { Type: ExporterOCI, Attrs: map[string]string{}, - Output: nopWriteCloser{ioutil.Discard}, + Output: fixedWriteCloser(nopWriteCloser{ioutil.Discard}), }, }, }, "", frontend, nil) @@ -2060,7 +2060,7 @@ func testDuplicateWhiteouts(t *testing.T, sb integration.Sandbox) { Exports: []ExportEntry{ { Type: ExporterOCI, - Output: outW, + Output: fixedWriteCloser(outW), }, }, }, nil) @@ -2130,7 +2130,7 @@ func testWhiteoutParentDir(t *testing.T, sb integration.Sandbox) { Exports: []ExportEntry{ { Type: ExporterOCI, - Output: outW, + Output: fixedWriteCloser(outW), }, }, }, nil) @@ -2465,7 +2465,7 @@ func testInvalidExporter(t *testing.T, sb integration.Sandbox) { { Type: ExporterLocal, Attrs: attrs, - Output: f, + Output: fixedWriteCloser(f), }, }, }, nil) @@ -2611,3 +2611,9 @@ func (*netModeDefault) UpdateConfigFile(in string) string { var hostNetwork integration.ConfigUpdater = &netModeHost{} var defaultNetwork integration.ConfigUpdater = &netModeDefault{} + +func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { + return func(map[string]string) (io.WriteCloser, error) { + return wc, nil + } +} diff --git a/client/solve.go b/client/solve.go index 17b3810c..d09c5f76 100644 --- a/client/solve.go +++ b/client/solve.go @@ -46,8 +46,8 @@ type SolveOpt struct { type ExportEntry struct { Type string Attrs map[string]string - Output io.WriteCloser // for ExporterOCI and ExporterDocker - OutputDir string // for ExporterLocal + Output func(map[string]string) (io.WriteCloser, error) // for ExporterOCI and ExporterDocker + OutputDir string // for ExporterLocal } type CacheOptionsEntry struct { diff --git a/cmd/buildctl/build/output.go b/cmd/buildctl/build/output.go index 993b6065..4ab7ee28 100644 --- a/cmd/buildctl/build/output.go +++ b/cmd/buildctl/build/output.go @@ -88,7 +88,12 @@ func ParseLegacyExporter(legacyExporter string, legacyExporterOpts []string) ([] } // resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path). -func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error) { +func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.WriteCloser, error), string, error) { + wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { + return func(m map[string]string) (io.WriteCloser, error) { + return wc, nil + } + } switch exporter { case client.ExporterLocal: if dest == "" { @@ -105,13 +110,13 @@ func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error) return nil, "", errors.Errorf("destination file is a directory") } w, err := os.Create(dest) - return w, "", err + return wrapWriter(w), "", err } // if no output file is specified, use stdout if _, err := console.ConsoleFromFile(os.Stdout); err == nil { return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter) } - return os.Stdout, "", nil + return wrapWriter(os.Stdout), "", nil default: // e.g. client.ExporterImage if dest != "" { return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter) diff --git a/examples/build-using-dockerfile/main.go b/examples/build-using-dockerfile/main.go index 68df8402..464b8dbc 100644 --- a/examples/build-using-dockerfile/main.go +++ b/examples/build-using-dockerfile/main.go @@ -166,7 +166,9 @@ func newSolveOpt(clicontext *cli.Context, w io.WriteCloser) (*client.SolveOpt, e Attrs: map[string]string{ "name": clicontext.String("tag"), }, - Output: w, + Output: func(_ map[string]string) (io.WriteCloser, error) { + return w, nil + }, }, }, LocalDirs: localDirs, diff --git a/exporter/oci/export.go b/exporter/oci/export.go index 1d20fa45..dac6d9a8 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -19,6 +19,8 @@ import ( "github.com/moby/buildkit/util/progress" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type ExporterVariant string @@ -135,6 +137,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) desc.Annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339) resp := make(map[string]string) + resp["containerimage.digest"] = desc.Digest.String() if n, ok := src.Metadata["image.name"]; e.name == "*" && ok { e.name = string(n) @@ -154,16 +157,23 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) return nil, err } - w, err := filesync.CopyFileWriter(ctx, e.caller) + w, err := filesync.CopyFileWriter(ctx, resp, e.caller) if err != nil { return nil, err } report := oneOffProgress(ctx, "sending tarball") if err := exp.Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil { w.Close() + if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.AlreadyExists { + return resp, report(nil) + } return nil, report(err) } - return resp, report(w.Close()) + err = w.Close() + if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.AlreadyExists { + return resp, report(nil) + } + return resp, report(err) } func oneOffProgress(ctx context.Context, id string) func(err error) error { diff --git a/exporter/tar/export.go b/exporter/tar/export.go index 12de8da9..365dc576 100644 --- a/exporter/tar/export.go +++ b/exporter/tar/export.go @@ -147,7 +147,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) fs = d.FS } - w, err := filesync.CopyFileWriter(ctx, e.caller) + w, err := filesync.CopyFileWriter(ctx, nil, e.caller) if err != nil { return nil, err } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 33eac255..9cec45da 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -385,7 +385,7 @@ FROM stage-$TARGETOS Exports: []client.ExportEntry{ { Type: client.ExporterTar, - Output: &nopWriteCloser{buf}, + Output: fixedWriteCloser(&nopWriteCloser{buf}), }, }, LocalDirs: map[string]string{ @@ -408,7 +408,7 @@ FROM stage-$TARGETOS Exports: []client.ExportEntry{ { Type: client.ExporterTar, - Output: &nopWriteCloser{buf}, + Output: fixedWriteCloser(&nopWriteCloser{buf}), }, }, FrontendAttrs: map[string]string{ @@ -1129,7 +1129,7 @@ COPY arch-$TARGETARCH whoami Exports: []client.ExportEntry{ { Type: client.ExporterOCI, - Output: outW, + Output: fixedWriteCloser(outW), }, }, }, nil) @@ -4244,3 +4244,9 @@ func (*secModeInsecure) UpdateConfigFile(in string) string { var securitySandbox integration.ConfigUpdater = &secModeSandbox{} var securityInsecure integration.ConfigUpdater = &secModeInsecure{} + +func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { + return func(map[string]string) (io.WriteCloser, error) { + return wc, nil + } +} diff --git a/session/filesync/diffcopy.go b/session/filesync/diffcopy.go index b82e3fc1..f1d7d78e 100644 --- a/session/filesync/diffcopy.go +++ b/session/filesync/diffcopy.go @@ -40,6 +40,12 @@ type streamWriterCloser struct { func (wc *streamWriterCloser) Write(dt []byte) (int, error) { if err := wc.ClientStream.SendMsg(&BytesMessage{Data: dt}); err != nil { + // SendMsg return EOF on remote errors + if errors.Cause(err) == io.EOF { + if err := errors.WithStack(wc.ClientStream.RecvMsg(struct{}{})); err != nil { + return 0, err + } + } return 0, errors.WithStack(err) } return len(dt), nil diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index b345569b..a45abe02 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -18,11 +18,12 @@ import ( ) const ( - keyOverrideExcludes = "override-excludes" - keyIncludePatterns = "include-patterns" - keyExcludePatterns = "exclude-patterns" - keyFollowPaths = "followpaths" - keyDirName = "dir-name" + keyOverrideExcludes = "override-excludes" + keyIncludePatterns = "include-patterns" + keyExcludePatterns = "exclude-patterns" + keyFollowPaths = "followpaths" + keyDirName = "dir-name" + keyExporterMetaPrefix = "exporter-md-" ) type fsSyncProvider struct { @@ -238,16 +239,16 @@ func NewFSSyncTargetDir(outdir string) session.Attachable { } // NewFSSyncTarget allows writing into an io.WriteCloser -func NewFSSyncTarget(w io.WriteCloser) session.Attachable { +func NewFSSyncTarget(f func(map[string]string) (io.WriteCloser, error)) session.Attachable { p := &fsSyncTarget{ - outfile: w, + f: f, } return p } type fsSyncTarget struct { - outdir string - outfile io.WriteCloser + outdir string + f func(map[string]string) (io.WriteCloser, error) } func (sp *fsSyncTarget) Register(server *grpc.Server) { @@ -258,11 +259,26 @@ func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) error { if sp.outdir != "" { return syncTargetDiffCopy(stream, sp.outdir) } - if sp.outfile == nil { + + if sp.f == nil { return errors.New("empty outfile and outdir") } - defer sp.outfile.Close() - return writeTargetFile(stream, sp.outfile) + opts, _ := metadata.FromIncomingContext(stream.Context()) // if no metadata continue with empty object + md := map[string]string{} + for k, v := range opts { + if strings.HasPrefix(k, keyExporterMetaPrefix) { + md[strings.TrimPrefix(k, keyExporterMetaPrefix)] = strings.Join(v, ",") + } + } + wc, err := sp.f(md) + if err != nil { + return err + } + if wc == nil { + return status.Errorf(codes.AlreadyExists, "target already exists") + } + defer wc.Close() + return writeTargetFile(stream, wc) } func CopyToCaller(ctx context.Context, fs fsutil.FS, c session.Caller, progress func(int, bool)) error { @@ -281,7 +297,7 @@ func CopyToCaller(ctx context.Context, fs fsutil.FS, c session.Caller, progress return sendDiffCopy(cc, fs, progress) } -func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) { +func CopyFileWriter(ctx context.Context, md map[string]string, c session.Caller) (io.WriteCloser, error) { method := session.MethodURL(_FileSend_serviceDesc.ServiceName, "diffcopy") if !c.Supports(method) { return nil, errors.Errorf("method %s not supported by the client", method) @@ -289,6 +305,13 @@ func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, erro client := NewFileSendClient(c.Conn()) + opts := make(map[string][]string, len(md)) + for k, v := range md { + opts[keyExporterMetaPrefix+k] = []string{v} + } + + ctx = metadata.NewOutgoingContext(ctx, opts) + cc, err := client.DiffCopy(ctx) if err != nil { return nil, errors.WithStack(err)