From c1a1d7033ddac2528a2ca29c3539d19f64560b1e Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 26 Mar 2019 23:10:21 -0700 Subject: [PATCH] exporter: add tar exporter Signed-off-by: Tonis Tiigi --- README.md | 8 + client/exporters.go | 1 + client/solve.go | 2 +- cmd/buildctl/build/output.go | 4 +- exporter/local/export.go | 7 +- exporter/tar/export.go | 155 ++++++++++++++++++ frontend/dockerfile/dockerfile_test.go | 86 ++++++++++ go.mod | 2 +- go.sum | 4 +- vendor/github.com/tonistiigi/fsutil/fs.go | 90 +++++++--- .../github.com/tonistiigi/fsutil/tarwriter.go | 72 ++++++++ vendor/modules.txt | 2 +- worker/base/worker.go | 5 + 13 files changed, 407 insertions(+), 31 deletions(-) create mode 100644 exporter/tar/export.go create mode 100644 vendor/github.com/tonistiigi/fsutil/tarwriter.go diff --git a/README.md b/README.md index 1db2e841..47da288a 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,14 @@ The local client will copy the files directly to the client. This is useful if B buildctl build ... --output type=local,dest=path/to/output-dir ``` +Tar exporter is similar to local exporter but transfers the files through a tarball. + +``` +buildctl build ... --output type=tar,dest=out.tar +buildctl build ... --output type=tar > out.tar +``` + + ##### Exporting built image to Docker ``` diff --git a/client/exporters.go b/client/exporters.go index 4160d92a..0f70d59c 100644 --- a/client/exporters.go +++ b/client/exporters.go @@ -3,6 +3,7 @@ package client const ( ExporterImage = "image" ExporterLocal = "local" + ExporterTar = "tar" ExporterOCI = "oci" ExporterDocker = "docker" ) diff --git a/client/solve.go b/client/solve.go index ce321f92..d772eaa3 100644 --- a/client/solve.go +++ b/client/solve.go @@ -124,7 +124,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG return nil, errors.New("output directory is required for local exporter") } s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir)) - case ExporterOCI, ExporterDocker: + case ExporterOCI, ExporterDocker, ExporterTar: if ex.OutputDir != "" { return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type) } diff --git a/cmd/buildctl/build/output.go b/cmd/buildctl/build/output.go index 61584077..993b6065 100644 --- a/cmd/buildctl/build/output.go +++ b/cmd/buildctl/build/output.go @@ -95,8 +95,8 @@ func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error) return nil, "", errors.New("output directory is required for local exporter") } return nil, dest, nil - case client.ExporterOCI, client.ExporterDocker: - if dest != "" { + case client.ExporterOCI, client.ExporterDocker, client.ExporterTar: + if dest != "" && dest != "-" { fi, err := os.Stat(dest) if err != nil && !os.IsNotExist(err) { return nil, "", errors.Wrapf(err, "invalid destination file: %s", dest) diff --git a/exporter/local/export.go b/exporter/local/export.go index 8140af64..24f0fcd8 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -93,10 +93,13 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) lbl := "copying files" if isMap { lbl += " " + k - fs = fsutil.SubDirFS(fs, fstypes.Stat{ + fs, err = fsutil.SubDirFS([]fsutil.Dir{{FS: fs, Stat: fstypes.Stat{ Mode: uint32(os.ModeDir | 0755), Path: strings.Replace(k, "/", "_", -1), - }) + }}}) + if err != nil { + return err + } } progress := newProgressHandler(ctx, lbl) diff --git a/exporter/tar/export.go b/exporter/tar/export.go new file mode 100644 index 00000000..7ea496aa --- /dev/null +++ b/exporter/tar/export.go @@ -0,0 +1,155 @@ +package local + +import ( + "context" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/filesync" + "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/progress" + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil" + fstypes "github.com/tonistiigi/fsutil/types" +) + +type Opt struct { + SessionManager *session.Manager +} + +type localExporter struct { + opt Opt + // session manager +} + +func New(opt Opt) (exporter.Exporter, error) { + le := &localExporter{opt: opt} + return le, nil +} + +func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) { + id := session.FromContext(ctx) + if id == "" { + return nil, errors.New("could not access local files without session") + } + + timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + caller, err := e.opt.SessionManager.Get(timeoutCtx, id) + if err != nil { + return nil, err + } + + li := &localExporterInstance{localExporter: e, caller: caller} + return li, nil +} + +type localExporterInstance struct { + *localExporter + caller session.Caller +} + +func (e *localExporterInstance) Name() string { + return "exporting to client" +} + +func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) (map[string]string, error) { + var defers []func() + + defer func() { + for i := len(defers) - 1; i >= 0; i-- { + defers[i]() + } + }() + + getDir := func(ctx context.Context, k string, ref cache.ImmutableRef) (*fsutil.Dir, error) { + var src string + var err error + if ref == nil { + src, err = ioutil.TempDir("", "buildkit") + if err != nil { + return nil, err + } + defers = append(defers, func() { os.RemoveAll(src) }) + } else { + mount, err := ref.Mount(ctx, true) + if err != nil { + return nil, err + } + + lm := snapshot.LocalMounter(mount) + + src, err = lm.Mount() + if err != nil { + return nil, err + } + defers = append(defers, func() { lm.Unmount() }) + } + + return &fsutil.Dir{ + FS: fsutil.NewFS(src, nil), + Stat: fstypes.Stat{ + Mode: uint32(os.ModeDir | 0755), + Path: strings.Replace(k, "/", "_", -1), + }, + }, nil + } + + var fs fsutil.FS + + if len(inp.Refs) > 0 { + dirs := make([]fsutil.Dir, 0, len(inp.Refs)) + for k, ref := range inp.Refs { + d, err := getDir(ctx, k, ref) + if err != nil { + return nil, err + } + dirs = append(dirs, *d) + } + var err error + fs, err = fsutil.SubDirFS(dirs) + if err != nil { + return nil, err + } + } else { + d, err := getDir(ctx, "", inp.Ref) + if err != nil { + return nil, err + } + fs = d.FS + } + + w, err := filesync.CopyFileWriter(ctx, e.caller) + if err != nil { + return nil, err + } + report := oneOffProgress(ctx, "sending tarball") + if err := fsutil.WriteTar(ctx, fs, w); err != nil { + w.Close() + return nil, report(err) + } + return nil, report(w.Close()) +} + +func oneOffProgress(ctx context.Context, id string) func(err error) error { + pw, _, _ := progress.FromContext(ctx) + now := time.Now() + st := progress.Status{ + Started: &now, + } + pw.Write(id, st) + return func(err error) error { + // TODO: set error on status + now := time.Now() + st.Completed = &now + pw.Write(id, st) + pw.Close() + return err + } +} diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 960c6c57..13ba5e0d 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/http/httptest" @@ -84,6 +85,7 @@ var allTests = []integration.Test{ testCopyChownExistingDir, testCopyWildcardCache, testDockerignoreOverride, + testTarExporter, } var fileOpTests = []integration.Test{ @@ -235,6 +237,84 @@ RUN [ "$(cat testfile)" == "contents0" ] require.NoError(t, err) } +func testTarExporter(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM scratch AS stage-linux +COPY foo forlinux + +FROM scratch AS stage-darwin +COPY bar fordarwin + +FROM stage-$TARGETOS +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("foo", []byte("data"), 0600), + fstest.CreateFile("bar", []byte("data2"), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + c, err := client.New(context.TODO(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + buf := &bytes.Buffer{} + _, err = f.Solve(context.TODO(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterTar, + Output: &nopWriteCloser{buf}, + }, + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + m, err := testutil.ReadTarToMap(buf.Bytes(), false) + require.NoError(t, err) + + mi, ok := m["forlinux"] + require.Equal(t, true, ok) + require.Equal(t, "data", string(mi.Data)) + + // repeat multi-platform + buf = &bytes.Buffer{} + _, err = f.Solve(context.TODO(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterTar, + Output: &nopWriteCloser{buf}, + }, + }, + FrontendAttrs: map[string]string{ + "platform": "linux/amd64,darwin/amd64", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + m, err = testutil.ReadTarToMap(buf.Bytes(), false) + require.NoError(t, err) + + mi, ok = m["linux_amd64/forlinux"] + require.Equal(t, true, ok) + require.Equal(t, "data", string(mi.Data)) + + mi, ok = m["darwin_amd64/fordarwin"] + require.Equal(t, true, ok) + require.Equal(t, "data2", string(mi.Data)) +} + func testWorkdirCreatesDir(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) @@ -3824,3 +3904,9 @@ func getFileOp(t *testing.T, sb integration.Sandbox) bool { require.True(t, ok) return vv } + +type nopWriteCloser struct { + io.Writer +} + +func (nopWriteCloser) Close() error { return nil } diff --git a/go.mod b/go.mod index ccde0773..ff7ce047 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/sirupsen/logrus v1.3.0 github.com/stretchr/testify v1.3.0 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect - github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49 + github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e github.com/uber/jaeger-lib v1.2.1 // indirect diff --git a/go.sum b/go.sum index 28e53e8c..f8eada62 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49 h1:UFQ7uDVXIH4fFfOb+fISgTl8Ukk0CkGQudHQh980l+0= -github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc= +github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76 h1:eGfgYrNUSD448sa4mxH6nQpyZfN39QH0mLB7QaKIjus= +github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc= github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe h1:pd7hrFSqUPxYS9IB+UMG1AB/8EXGXo17ssx0bSQ5L6Y= github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe/go.mod h1:/+MCh11CJf2oz0BXmlmqyopK/ad1rKkcOXPoYuPCJYU= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= diff --git a/vendor/github.com/tonistiigi/fsutil/fs.go b/vendor/github.com/tonistiigi/fsutil/fs.go index 132850c9..a9467e94 100644 --- a/vendor/github.com/tonistiigi/fsutil/fs.go +++ b/vendor/github.com/tonistiigi/fsutil/fs.go @@ -3,9 +3,11 @@ package fsutil import ( "context" "io" + "io/ioutil" "os" "path" "path/filepath" + "sort" "strings" "github.com/pkg/errors" @@ -37,36 +39,80 @@ func (fs *fs) Open(p string) (io.ReadCloser, error) { return os.Open(filepath.Join(fs.root, p)) } -func SubDirFS(fs FS, stat types.Stat) FS { - return &subDirFS{fs: fs, stat: stat} +type Dir struct { + Stat types.Stat + FS FS +} + +func SubDirFS(dirs []Dir) (FS, error) { + sort.Slice(dirs, func(i, j int) bool { + return dirs[i].Stat.Path < dirs[j].Stat.Path + }) + m := map[string]Dir{} + for _, d := range dirs { + if path.Base(d.Stat.Path) != d.Stat.Path { + return nil, errors.Errorf("subdir %s must be single file", d.Stat.Path) + } + if _, ok := m[d.Stat.Path]; ok { + return nil, errors.Errorf("invalid path %s", d.Stat.Path) + } + m[d.Stat.Path] = d + } + return &subDirFS{m: m, dirs: dirs}, nil } type subDirFS struct { - fs FS - stat types.Stat + m map[string]Dir + dirs []Dir } func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { - main := &StatInfo{Stat: &fs.stat} - if !main.IsDir() { - return errors.Errorf("fs subdir not mode directory") - } - if main.Name() != fs.stat.Path { - return errors.Errorf("subdir path must be single file") - } - if err := fn(fs.stat.Path, main, nil); err != nil { - return err - } - return fs.fs.Walk(ctx, func(p string, fi os.FileInfo, err error) error { - stat, ok := fi.Sys().(*types.Stat) - if !ok { - return errors.Wrapf(err, "invalid fileinfo without stat info: %s", p) + for _, d := range fs.dirs { + fi := &StatInfo{Stat: &d.Stat} + if !fi.IsDir() { + return errors.Errorf("fs subdir %s not mode directory", d.Stat.Path) } - stat.Path = path.Join(fs.stat.Path, stat.Path) - return fn(filepath.Join(fs.stat.Path, p), &StatInfo{stat}, nil) - }) + if err := fn(d.Stat.Path, fi, nil); err != nil { + return err + } + if err := d.FS.Walk(ctx, func(p string, fi os.FileInfo, err error) error { + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return errors.Wrapf(err, "invalid fileinfo without stat info: %s", p) + } + stat.Path = path.Join(d.Stat.Path, stat.Path) + if stat.Linkname != "" { + if fi.Mode()&os.ModeSymlink != 0 { + if strings.HasPrefix(stat.Linkname, "/") { + stat.Linkname = path.Join("/"+d.Stat.Path, stat.Linkname) + } + } else { + stat.Linkname = path.Join(d.Stat.Path, stat.Linkname) + } + } + return fn(filepath.Join(d.Stat.Path, p), &StatInfo{stat}, nil) + }); err != nil { + return err + } + } + return nil } func (fs *subDirFS) Open(p string) (io.ReadCloser, error) { - return fs.fs.Open(strings.TrimPrefix(p, fs.stat.Path+"/")) + parts := strings.SplitN(filepath.Clean(p), string(filepath.Separator), 2) + if len(parts) == 0 { + return ioutil.NopCloser(&emptyReader{}), nil + } + d, ok := fs.m[parts[0]] + if !ok { + return nil, os.ErrNotExist + } + return d.FS.Open(parts[1]) +} + +type emptyReader struct { +} + +func (*emptyReader) Read([]byte) (int, error) { + return 0, io.EOF } diff --git a/vendor/github.com/tonistiigi/fsutil/tarwriter.go b/vendor/github.com/tonistiigi/fsutil/tarwriter.go new file mode 100644 index 00000000..8dea63ba --- /dev/null +++ b/vendor/github.com/tonistiigi/fsutil/tarwriter.go @@ -0,0 +1,72 @@ +package fsutil + +import ( + "archive/tar" + "context" + "io" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil/types" +) + +func WriteTar(ctx context.Context, fs FS, w io.Writer) error { + tw := tar.NewWriter(w) + err := fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error { + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return errors.Wrapf(err, "invalid fileinfo without stat info: %s", path) + } + hdr, err := tar.FileInfoHeader(fi, stat.Linkname) + if err != nil { + return err + } + + name := filepath.ToSlash(path) + if fi.IsDir() && !strings.HasSuffix(name, "/") { + name += "/" + } + hdr.Name = name + + hdr.Uid = int(stat.Uid) + hdr.Gid = int(stat.Gid) + hdr.Devmajor = stat.Devmajor + hdr.Devminor = stat.Devminor + hdr.Linkname = stat.Linkname + if hdr.Linkname != "" { + hdr.Size = 0 + hdr.Typeflag = tar.TypeLink + } + + if len(stat.Xattrs) > 0 { + hdr.PAXRecords = map[string]string{} + } + for k, v := range stat.Xattrs { + hdr.PAXRecords["SCHILY.xattr."+k] = string(v) + } + + if err := tw.WriteHeader(hdr); err != nil { + return errors.Wrap(err, "failed to write file header") + } + + if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 && hdr.Linkname == "" { + rc, err := fs.Open(path) + if err != nil { + return err + } + if _, err := io.Copy(tw, rc); err != nil { + return err + } + if err := rc.Close(); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + return tw.Close() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b035a207..50fba3f9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -215,7 +215,7 @@ github.com/stretchr/testify/require github.com/stretchr/testify/assert # github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 github.com/syndtr/gocapability/capability -# github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49 +# github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76 github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/types github.com/tonistiigi/fsutil/copy diff --git a/worker/base/worker.go b/worker/base/worker.go index b051390d..125d2681 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -23,6 +23,7 @@ import ( imageexporter "github.com/moby/buildkit/exporter/containerimage" localexporter "github.com/moby/buildkit/exporter/local" ociexporter "github.com/moby/buildkit/exporter/oci" + tarexporter "github.com/moby/buildkit/exporter/tar" "github.com/moby/buildkit/frontend" gw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/identity" @@ -252,6 +253,10 @@ func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter, return localexporter.New(localexporter.Opt{ SessionManager: sm, }) + case client.ExporterTar: + return tarexporter.New(tarexporter.Opt{ + SessionManager: sm, + }) case client.ExporterOCI: return ociexporter.New(ociexporter.Opt{ SessionManager: sm,