JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

exporter: add tar exporter 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-19.03
Tonis Tiigi 2019-03-26 23:10:21 -07:00
parent 33bb70c810
commit c1a1d7033d
13 changed files with 407 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -184,6 +184,14 @@ The local client will copy the files directly to the client. This is useful if B
buildctl build ... --output type=local,dest=path/to/output-dir
```
Tar exporter is similar to local exporter but transfers the files through a tarball.
```
buildctl build ... --output type=tar,dest=out.tar
buildctl build ... --output type=tar > out.tar
```
##### Exporting built image to Docker
```

1
client/exporters.go
View File

@ -3,6 +3,7 @@ package client
const (
ExporterImage = "image"
ExporterLocal = "local"
ExporterTar = "tar"
ExporterOCI = "oci"
ExporterDocker = "docker"
)

2
client/solve.go
View File

@ -124,7 +124,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return nil, errors.New("output directory is required for local exporter")
}
s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
case ExporterOCI, ExporterDocker:
case ExporterOCI, ExporterDocker, ExporterTar:
if ex.OutputDir != "" {
return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
}

4
cmd/buildctl/build/output.go
View File

@ -95,8 +95,8 @@ func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error)
return nil, "", errors.New("output directory is required for local exporter")
}
return nil, dest, nil
case client.ExporterOCI, client.ExporterDocker:
if dest != "" {
case client.ExporterOCI, client.ExporterDocker, client.ExporterTar:
if dest != "" && dest != "-" {
fi, err := os.Stat(dest)
if err != nil && !os.IsNotExist(err) {
return nil, "", errors.Wrapf(err, "invalid destination file: %s", dest)

7
exporter/local/export.go
View File

@ -93,10 +93,13 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
lbl := "copying files"
if isMap {
lbl += " " + k
fs = fsutil.SubDirFS(fs, fstypes.Stat{
fs, err = fsutil.SubDirFS([]fsutil.Dir{{FS: fs, Stat: fstypes.Stat{
Mode: uint32(os.ModeDir | 0755),
Path: strings.Replace(k, "/", "_", -1),
})
}}})
if err != nil {
return err
}
}
progress := newProgressHandler(ctx, lbl)

155
exporter/tar/export.go Normal file
View File

@ -0,0 +1,155 @@
package local
import (
"context"
"io/ioutil"
"os"
"strings"
"time"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/progress"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
fstypes "github.com/tonistiigi/fsutil/types"
)
type Opt struct {
SessionManager *session.Manager
}
type localExporter struct {
opt Opt
// session manager
}
func New(opt Opt) (exporter.Exporter, error) {
le := &localExporter{opt: opt}
return le, nil
}
func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
id := session.FromContext(ctx)
if id == "" {
return nil, errors.New("could not access local files without session")
}
timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
caller, err := e.opt.SessionManager.Get(timeoutCtx, id)
if err != nil {
return nil, err
}
li := &localExporterInstance{localExporter: e, caller: caller}
return li, nil
}
type localExporterInstance struct {
*localExporter
caller session.Caller
}
func (e *localExporterInstance) Name() string {
return "exporting to client"
}
func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) (map[string]string, error) {
var defers []func()
defer func() {
for i := len(defers) - 1; i >= 0; i-- {
defers[i]()
}
}()
getDir := func(ctx context.Context, k string, ref cache.ImmutableRef) (*fsutil.Dir, error) {
var src string
var err error
if ref == nil {
src, err = ioutil.TempDir("", "buildkit")
if err != nil {
return nil, err
}
defers = append(defers, func() { os.RemoveAll(src) })
} else {
mount, err := ref.Mount(ctx, true)
if err != nil {
return nil, err
}
lm := snapshot.LocalMounter(mount)
src, err = lm.Mount()
if err != nil {
return nil, err
}
defers = append(defers, func() { lm.Unmount() })
}
return &fsutil.Dir{
FS: fsutil.NewFS(src, nil),
Stat: fstypes.Stat{
Mode: uint32(os.ModeDir | 0755),
Path: strings.Replace(k, "/", "_", -1),
},
}, nil
}
var fs fsutil.FS
if len(inp.Refs) > 0 {
dirs := make([]fsutil.Dir, 0, len(inp.Refs))
for k, ref := range inp.Refs {
d, err := getDir(ctx, k, ref)
if err != nil {
return nil, err
}
dirs = append(dirs, *d)
}
var err error
fs, err = fsutil.SubDirFS(dirs)
if err != nil {
return nil, err
}
} else {
d, err := getDir(ctx, "", inp.Ref)
if err != nil {
return nil, err
}
fs = d.FS
}
w, err := filesync.CopyFileWriter(ctx, e.caller)
if err != nil {
return nil, err
}
report := oneOffProgress(ctx, "sending tarball")
if err := fsutil.WriteTar(ctx, fs, w); err != nil {
w.Close()
return nil, report(err)
}
return nil, report(w.Close())
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {
pw, _, _ := progress.FromContext(ctx)
now := time.Now()
st := progress.Status{
Started: &now,
}
pw.Write(id, st)
return func(err error) error {
// TODO: set error on status
now := time.Now()
st.Completed = &now
pw.Write(id, st)
pw.Close()
return err
}
}

86
frontend/dockerfile/dockerfile_test.go
View File

@ -7,6 +7,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
@ -84,6 +85,7 @@ var allTests = []integration.Test{
testCopyChownExistingDir,
testCopyWildcardCache,
testDockerignoreOverride,
testTarExporter,
}
var fileOpTests = []integration.Test{
@ -235,6 +237,84 @@ RUN [ "$(cat testfile)" == "contents0" ]
require.NoError(t, err)
}
func testTarExporter(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM scratch AS stage-linux
COPY foo forlinux
FROM scratch AS stage-darwin
COPY bar fordarwin
FROM stage-$TARGETOS
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateFile("foo", []byte("data"), 0600),
fstest.CreateFile("bar", []byte("data2"), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
buf := &bytes.Buffer{}
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterTar,
Output: &nopWriteCloser{buf},
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
m, err := testutil.ReadTarToMap(buf.Bytes(), false)
require.NoError(t, err)
mi, ok := m["forlinux"]
require.Equal(t, true, ok)
require.Equal(t, "data", string(mi.Data))
// repeat multi-platform
buf = &bytes.Buffer{}
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterTar,
Output: &nopWriteCloser{buf},
},
},
FrontendAttrs: map[string]string{
"platform": "linux/amd64,darwin/amd64",
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
m, err = testutil.ReadTarToMap(buf.Bytes(), false)
require.NoError(t, err)
mi, ok = m["linux_amd64/forlinux"]
require.Equal(t, true, ok)
require.Equal(t, "data", string(mi.Data))
mi, ok = m["darwin_amd64/fordarwin"]
require.Equal(t, true, ok)
require.Equal(t, "data2", string(mi.Data))
}
func testWorkdirCreatesDir(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
@ -3824,3 +3904,9 @@ func getFileOp(t *testing.T, sb integration.Sandbox) bool {
require.True(t, ok)
return vv
}
type nopWriteCloser struct {
io.Writer
}
func (nopWriteCloser) Close() error { return nil }

2
go.mod
View File

@ -49,7 +49,7 @@ require (
github.com/sirupsen/logrus v1.3.0
github.com/stretchr/testify v1.3.0
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect
github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49
github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
github.com/uber/jaeger-lib v1.2.1 // indirect

4
go.sum
View File

@ -128,8 +128,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49 h1:UFQ7uDVXIH4fFfOb+fISgTl8Ukk0CkGQudHQh980l+0=
github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76 h1:eGfgYrNUSD448sa4mxH6nQpyZfN39QH0mLB7QaKIjus=
github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe h1:pd7hrFSqUPxYS9IB+UMG1AB/8EXGXo17ssx0bSQ5L6Y=
github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe/go.mod h1:/+MCh11CJf2oz0BXmlmqyopK/ad1rKkcOXPoYuPCJYU=
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=

90
vendor/github.com/tonistiigi/fsutil/fs.go generated vendored
View File

@ -3,9 +3,11 @@ package fsutil
import (
"context"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"sort"
"strings"
"github.com/pkg/errors"
@ -37,36 +39,80 @@ func (fs *fs) Open(p string) (io.ReadCloser, error) {
return os.Open(filepath.Join(fs.root, p))
}
func SubDirFS(fs FS, stat types.Stat) FS {
return &subDirFS{fs: fs, stat: stat}
type Dir struct {
Stat types.Stat
FS FS
}
func SubDirFS(dirs []Dir) (FS, error) {
sort.Slice(dirs, func(i, j int) bool {
return dirs[i].Stat.Path < dirs[j].Stat.Path
})
m := map[string]Dir{}
for _, d := range dirs {
if path.Base(d.Stat.Path) != d.Stat.Path {
return nil, errors.Errorf("subdir %s must be single file", d.Stat.Path)
}
if _, ok := m[d.Stat.Path]; ok {
return nil, errors.Errorf("invalid path %s", d.Stat.Path)
}
m[d.Stat.Path] = d
}
return &subDirFS{m: m, dirs: dirs}, nil
}
type subDirFS struct {
fs FS
stat types.Stat
m map[string]Dir
dirs []Dir
}
func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
main := &StatInfo{Stat: &fs.stat}
if !main.IsDir() {
return errors.Errorf("fs subdir not mode directory")
}
if main.Name() != fs.stat.Path {
return errors.Errorf("subdir path must be single file")
}
if err := fn(fs.stat.Path, main, nil); err != nil {
return err
}
return fs.fs.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.Wrapf(err, "invalid fileinfo without stat info: %s", p)
for _, d := range fs.dirs {
fi := &StatInfo{Stat: &d.Stat}
if !fi.IsDir() {
return errors.Errorf("fs subdir %s not mode directory", d.Stat.Path)
}
stat.Path = path.Join(fs.stat.Path, stat.Path)
return fn(filepath.Join(fs.stat.Path, p), &StatInfo{stat}, nil)
})
if err := fn(d.Stat.Path, fi, nil); err != nil {
return err
}
if err := d.FS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.Wrapf(err, "invalid fileinfo without stat info: %s", p)
}
stat.Path = path.Join(d.Stat.Path, stat.Path)
if stat.Linkname != "" {
if fi.Mode()&os.ModeSymlink != 0 {
if strings.HasPrefix(stat.Linkname, "/") {
stat.Linkname = path.Join("/"+d.Stat.Path, stat.Linkname)
}
} else {
stat.Linkname = path.Join(d.Stat.Path, stat.Linkname)
}
}
return fn(filepath.Join(d.Stat.Path, p), &StatInfo{stat}, nil)
}); err != nil {
return err
}
}
return nil
}
func (fs *subDirFS) Open(p string) (io.ReadCloser, error) {
return fs.fs.Open(strings.TrimPrefix(p, fs.stat.Path+"/"))
parts := strings.SplitN(filepath.Clean(p), string(filepath.Separator), 2)
if len(parts) == 0 {
return ioutil.NopCloser(&emptyReader{}), nil
}
d, ok := fs.m[parts[0]]
if !ok {
return nil, os.ErrNotExist
}
return d.FS.Open(parts[1])
}
type emptyReader struct {
}
func (*emptyReader) Read([]byte) (int, error) {
return 0, io.EOF
}

72
vendor/github.com/tonistiigi/fsutil/tarwriter.go generated vendored Normal file
View File

@ -0,0 +1,72 @@
package fsutil
import (
"archive/tar"
"context"
"io"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/types"
)
func WriteTar(ctx context.Context, fs FS, w io.Writer) error {
tw := tar.NewWriter(w)
err := fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error {
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.Wrapf(err, "invalid fileinfo without stat info: %s", path)
}
hdr, err := tar.FileInfoHeader(fi, stat.Linkname)
if err != nil {
return err
}
name := filepath.ToSlash(path)
if fi.IsDir() && !strings.HasSuffix(name, "/") {
name += "/"
}
hdr.Name = name
hdr.Uid = int(stat.Uid)
hdr.Gid = int(stat.Gid)
hdr.Devmajor = stat.Devmajor
hdr.Devminor = stat.Devminor
hdr.Linkname = stat.Linkname
if hdr.Linkname != "" {
hdr.Size = 0
hdr.Typeflag = tar.TypeLink
}
if len(stat.Xattrs) > 0 {
hdr.PAXRecords = map[string]string{}
}
for k, v := range stat.Xattrs {
hdr.PAXRecords["SCHILY.xattr."+k] = string(v)
}
if err := tw.WriteHeader(hdr); err != nil {
return errors.Wrap(err, "failed to write file header")
}
if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 && hdr.Linkname == "" {
rc, err := fs.Open(path)
if err != nil {
return err
}
if _, err := io.Copy(tw, rc); err != nil {
return err
}
if err := rc.Close(); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
return tw.Close()
}

2
vendor/modules.txt vendored
View File

@ -215,7 +215,7 @@ github.com/stretchr/testify/require
github.com/stretchr/testify/assert
# github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8
github.com/syndtr/gocapability/capability
# github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49
# github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
github.com/tonistiigi/fsutil
github.com/tonistiigi/fsutil/types
github.com/tonistiigi/fsutil/copy

5
worker/base/worker.go
View File

@ -23,6 +23,7 @@ import (
imageexporter "github.com/moby/buildkit/exporter/containerimage"
localexporter "github.com/moby/buildkit/exporter/local"
ociexporter "github.com/moby/buildkit/exporter/oci"
tarexporter "github.com/moby/buildkit/exporter/tar"
"github.com/moby/buildkit/frontend"
gw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/identity"
@ -252,6 +253,10 @@ func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter,
return localexporter.New(localexporter.Opt{
SessionManager: sm,
})
case client.ExporterTar:
return tarexporter.New(tarexporter.Opt{
SessionManager: sm,
})
case client.ExporterOCI:
return ociexporter.New(ociexporter.Opt{
SessionManager: sm,