JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

exporter: allow oci exporters visibility to response metadata 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-19.03
Tonis Tiigi 2019-07-29 17:50:54 -07:00
parent 9ae7b298e1
commit 96b6a28312
9 changed files with 89 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
client/client_test.go
View File

@ -740,7 +740,7 @@ func testFrontendImageNaming(t *testing.T, sb integration.Sandbox) {
case ExporterDocker:
outW, err := os.Create(out)
require.NoError(t, err)
so.Exports[0].Output = outW
so.Exports[0].Output = fixedWriteCloser(outW)
case ExporterImage:
imageName = registry + "/" + imageName
so.Exports[0].Attrs["push"] = "true"
@ -1304,7 +1304,7 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) {
{
Type: exp,
Attrs: attrs,
Output: outW,
Output: fixedWriteCloser(outW),
},
},
}, nil)
@ -1388,7 +1388,7 @@ func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) {
{
Type: ExporterOCI,
Attrs: map[string]string{},
Output: nopWriteCloser{ioutil.Discard},
Output: fixedWriteCloser(nopWriteCloser{ioutil.Discard}),
},
},
}, "", frontend, nil)
@ -2060,7 +2060,7 @@ func testDuplicateWhiteouts(t *testing.T, sb integration.Sandbox) {
Exports: []ExportEntry{
{
Type: ExporterOCI,
Output: outW,
Output: fixedWriteCloser(outW),
},
},
}, nil)
@ -2130,7 +2130,7 @@ func testWhiteoutParentDir(t *testing.T, sb integration.Sandbox) {
Exports: []ExportEntry{
{
Type: ExporterOCI,
Output: outW,
Output: fixedWriteCloser(outW),
},
},
}, nil)
@ -2465,7 +2465,7 @@ func testInvalidExporter(t *testing.T, sb integration.Sandbox) {
{
Type: ExporterLocal,
Attrs: attrs,
Output: f,
Output: fixedWriteCloser(f),
},
},
}, nil)
@ -2611,3 +2611,9 @@ func (*netModeDefault) UpdateConfigFile(in string) string {
var hostNetwork integration.ConfigUpdater = &netModeHost{}
var defaultNetwork integration.ConfigUpdater = &netModeDefault{}
func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}

4
client/solve.go
View File

@ -46,8 +46,8 @@ type SolveOpt struct {
type ExportEntry struct {
Type string
Attrs map[string]string
Output io.WriteCloser // for ExporterOCI and ExporterDocker
OutputDir string // for ExporterLocal
Output func(map[string]string) (io.WriteCloser, error) // for ExporterOCI and ExporterDocker
OutputDir string // for ExporterLocal
}
type CacheOptionsEntry struct {

11
cmd/buildctl/build/output.go
View File

@ -88,7 +88,12 @@ func ParseLegacyExporter(legacyExporter string, legacyExporterOpts []string) ([]
}
// resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path).
func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error) {
func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.WriteCloser, error), string, error) {
wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(m map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}
switch exporter {
case client.ExporterLocal:
if dest == "" {
@ -105,13 +110,13 @@ func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error)
return nil, "", errors.Errorf("destination file is a directory")
}
w, err := os.Create(dest)
return w, "", err
return wrapWriter(w), "", err
}
// if no output file is specified, use stdout
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
}
return os.Stdout, "", nil
return wrapWriter(os.Stdout), "", nil
default: // e.g. client.ExporterImage
if dest != "" {
return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)

4
examples/build-using-dockerfile/main.go
View File

@ -166,7 +166,9 @@ func newSolveOpt(clicontext *cli.Context, w io.WriteCloser) (*client.SolveOpt, e
Attrs: map[string]string{
"name": clicontext.String("tag"),
},
Output: w,
Output: func(_ map[string]string) (io.WriteCloser, error) {
return w, nil
},
},
},
LocalDirs: localDirs,

14
exporter/oci/export.go
View File

@ -19,6 +19,8 @@ import (
"github.com/moby/buildkit/util/progress"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type ExporterVariant string
@ -135,6 +137,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
desc.Annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339)
resp := make(map[string]string)
resp["containerimage.digest"] = desc.Digest.String()
if n, ok := src.Metadata["image.name"]; e.name == "*" && ok {
e.name = string(n)
@ -154,16 +157,23 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
return nil, err
}
w, err := filesync.CopyFileWriter(ctx, e.caller)
w, err := filesync.CopyFileWriter(ctx, resp, e.caller)
if err != nil {
return nil, err
}
report := oneOffProgress(ctx, "sending tarball")
if err := exp.Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil {
w.Close()
if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.AlreadyExists {
return resp, report(nil)
}
return nil, report(err)
}
return resp, report(w.Close())
err = w.Close()
if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.AlreadyExists {
return resp, report(nil)
}
return resp, report(err)
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {

2
exporter/tar/export.go
View File

@ -147,7 +147,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
fs = d.FS
}
w, err := filesync.CopyFileWriter(ctx, e.caller)
w, err := filesync.CopyFileWriter(ctx, nil, e.caller)
if err != nil {
return nil, err
}

12
frontend/dockerfile/dockerfile_test.go
View File

@ -385,7 +385,7 @@ FROM stage-$TARGETOS
Exports: []client.ExportEntry{
{
Type: client.ExporterTar,
Output: &nopWriteCloser{buf},
Output: fixedWriteCloser(&nopWriteCloser{buf}),
},
},
LocalDirs: map[string]string{
@ -408,7 +408,7 @@ FROM stage-$TARGETOS
Exports: []client.ExportEntry{
{
Type: client.ExporterTar,
Output: &nopWriteCloser{buf},
Output: fixedWriteCloser(&nopWriteCloser{buf}),
},
},
FrontendAttrs: map[string]string{
@ -1129,7 +1129,7 @@ COPY arch-$TARGETARCH whoami
Exports: []client.ExportEntry{
{
Type: client.ExporterOCI,
Output: outW,
Output: fixedWriteCloser(outW),
},
},
}, nil)
@ -4244,3 +4244,9 @@ func (*secModeInsecure) UpdateConfigFile(in string) string {
var securitySandbox integration.ConfigUpdater = &secModeSandbox{}
var securityInsecure integration.ConfigUpdater = &secModeInsecure{}
func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}

6
session/filesync/diffcopy.go
View File

@ -40,6 +40,12 @@ type streamWriterCloser struct {
func (wc *streamWriterCloser) Write(dt []byte) (int, error) {
if err := wc.ClientStream.SendMsg(&BytesMessage{Data: dt}); err != nil {
// SendMsg return EOF on remote errors
if errors.Cause(err) == io.EOF {
if err := errors.WithStack(wc.ClientStream.RecvMsg(struct{}{})); err != nil {
return 0, err
}
}
return 0, errors.WithStack(err)
}
return len(dt), nil

49
session/filesync/filesync.go
View File

@ -18,11 +18,12 @@ import (
)
const (
keyOverrideExcludes = "override-excludes"
keyIncludePatterns = "include-patterns"
keyExcludePatterns = "exclude-patterns"
keyFollowPaths = "followpaths"
keyDirName = "dir-name"
keyOverrideExcludes = "override-excludes"
keyIncludePatterns = "include-patterns"
keyExcludePatterns = "exclude-patterns"
keyFollowPaths = "followpaths"
keyDirName = "dir-name"
keyExporterMetaPrefix = "exporter-md-"
)
type fsSyncProvider struct {
@ -238,16 +239,16 @@ func NewFSSyncTargetDir(outdir string) session.Attachable {
}
// NewFSSyncTarget allows writing into an io.WriteCloser
func NewFSSyncTarget(w io.WriteCloser) session.Attachable {
func NewFSSyncTarget(f func(map[string]string) (io.WriteCloser, error)) session.Attachable {
p := &fsSyncTarget{
outfile: w,
f: f,
}
return p
}
type fsSyncTarget struct {
outdir string
outfile io.WriteCloser
outdir string
f func(map[string]string) (io.WriteCloser, error)
}
func (sp *fsSyncTarget) Register(server *grpc.Server) {
@ -258,11 +259,26 @@ func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) error {
if sp.outdir != "" {
return syncTargetDiffCopy(stream, sp.outdir)
}
if sp.outfile == nil {
if sp.f == nil {
return errors.New("empty outfile and outdir")
}
defer sp.outfile.Close()
return writeTargetFile(stream, sp.outfile)
opts, _ := metadata.FromIncomingContext(stream.Context()) // if no metadata continue with empty object
md := map[string]string{}
for k, v := range opts {
if strings.HasPrefix(k, keyExporterMetaPrefix) {
md[strings.TrimPrefix(k, keyExporterMetaPrefix)] = strings.Join(v, ",")
}
}
wc, err := sp.f(md)
if err != nil {
return err
}
if wc == nil {
return status.Errorf(codes.AlreadyExists, "target already exists")
}
defer wc.Close()
return writeTargetFile(stream, wc)
}
func CopyToCaller(ctx context.Context, fs fsutil.FS, c session.Caller, progress func(int, bool)) error {
@ -281,7 +297,7 @@ func CopyToCaller(ctx context.Context, fs fsutil.FS, c session.Caller, progress
return sendDiffCopy(cc, fs, progress)
}
func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) {
func CopyFileWriter(ctx context.Context, md map[string]string, c session.Caller) (io.WriteCloser, error) {
method := session.MethodURL(_FileSend_serviceDesc.ServiceName, "diffcopy")
if !c.Supports(method) {
return nil, errors.Errorf("method %s not supported by the client", method)
@ -289,6 +305,13 @@ func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, erro
client := NewFileSendClient(c.Conn())
opts := make(map[string][]string, len(md))
for k, v := range md {
opts[keyExporterMetaPrefix+k] = []string{v}
}
ctx = metadata.NewOutgoingContext(ctx, opts)
cc, err := client.DiffCopy(ctx)
if err != nil {
return nil, errors.WithStack(err)