From 85c0f99ba0b6d35b8bb171be8afb289400268d7f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 17 Dec 2017 16:20:19 -0800 Subject: [PATCH] exporter: add Docker compatible exporter Signed-off-by: Tonis Tiigi --- README.md | 9 +- client/client_test.go | 100 +++++++++++------ client/exporters.go | 7 +- client/solve.go | 4 +- exporter/oci/docker.go | 245 +++++++++++++++++++++++++++++++++++++++++ exporter/oci/export.go | 40 ++++++- worker/base/worker.go | 11 ++ 7 files changed, 373 insertions(+), 43 deletions(-) create mode 100644 exporter/oci/docker.go diff --git a/README.md b/README.md index 56a19734..31e65e7c 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,14 @@ If credentials are required, `buildctl` will attempt to read Docker configuratio buildctl build ... --exporter=local --exporter-opt output=path/to/output-dir ``` -#### Exporting OCI Image Format tarball to client +##### Exporting build result to Docker + +``` +# exported tarball is also compatible with OCI spec +buildctl build ... --exporter=docker --exporter-opt name=myimage | docker load +``` + +##### Exporting OCI Image Format tarball to client ``` buildctl build ... --exporter=oci --exporter-opt output=path/to/output.tar diff --git a/client/client_test.go b/client/client_test.go index efe34b79..ad5ba779 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -275,50 +275,80 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) { def, err := st.Marshal() require.NoError(t, err) - destDir, err := ioutil.TempDir("", "buildkit") - require.NoError(t, err) - defer os.RemoveAll(destDir) + for _, exp := range []string{ExporterOCI, ExporterDocker} { - out := filepath.Join(destDir, "out.tar") + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) - err = c.Solve(context.TODO(), def, SolveOpt{ - Exporter: ExporterOCI, - ExporterAttrs: map[string]string{ - "output": out, - }, - }, nil) - require.NoError(t, err) + out := filepath.Join(destDir, "out.tar") + target := "example.com/buildkit/testoci:latest" - dt, err := ioutil.ReadFile(out) - require.NoError(t, err) + err = c.Solve(context.TODO(), def, SolveOpt{ + Exporter: exp, + ExporterAttrs: map[string]string{ + "output": out, + "name": target, + }, + }, nil) + require.NoError(t, err) - m, err := readTarToMap(dt, false) - require.NoError(t, err) + dt, err := ioutil.ReadFile(out) + require.NoError(t, err) - _, ok := m["oci-layout"] - require.True(t, ok) + m, err := readTarToMap(dt, false) + require.NoError(t, err) - var index ocispec.Index - err = json.Unmarshal(m["index.json"].data, &index) - require.NoError(t, err) - require.Equal(t, 2, index.SchemaVersion) - require.Equal(t, 1, len(index.Manifests)) + _, ok := m["oci-layout"] + require.True(t, ok) - var mfst ocispec.Manifest - err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst) - require.NoError(t, err) - require.Equal(t, 2, len(mfst.Layers)) + var index ocispec.Index + err = json.Unmarshal(m["index.json"].data, &index) + require.NoError(t, err) + require.Equal(t, 2, index.SchemaVersion) + require.Equal(t, 1, len(index.Manifests)) - var ociimg ocispec.Image - err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].data, &ociimg) - require.NoError(t, err) - require.Equal(t, "layers", ociimg.RootFS.Type) - require.Equal(t, 2, len(ociimg.RootFS.DiffIDs)) + var mfst ocispec.Manifest + err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst) + require.NoError(t, err) + require.Equal(t, 2, len(mfst.Layers)) - _, ok = m["blobs/sha256/"+mfst.Layers[0].Digest.Hex()] - require.True(t, ok) - _, ok = m["blobs/sha256/"+mfst.Layers[1].Digest.Hex()] - require.True(t, ok) + var ociimg ocispec.Image + err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].data, &ociimg) + require.NoError(t, err) + require.Equal(t, "layers", ociimg.RootFS.Type) + require.Equal(t, 2, len(ociimg.RootFS.DiffIDs)) + + _, ok = m["blobs/sha256/"+mfst.Layers[0].Digest.Hex()] + require.True(t, ok) + _, ok = m["blobs/sha256/"+mfst.Layers[1].Digest.Hex()] + require.True(t, ok) + + if exp != ExporterDocker { + continue + } + + var dockerMfst []struct { + Config string + RepoTags []string + Layers []string + } + err = json.Unmarshal(m["manifest.json"].data, &dockerMfst) + require.NoError(t, err) + require.Equal(t, 1, len(dockerMfst)) + + _, ok = m[dockerMfst[0].Config] + require.True(t, ok) + require.Equal(t, 2, len(dockerMfst[0].Layers)) + require.Equal(t, 1, len(dockerMfst[0].RepoTags)) + require.Equal(t, target, dockerMfst[0].RepoTags[0]) + + for _, l := range dockerMfst[0].Layers { + _, ok := m[l] + require.True(t, ok) + } + + } } func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) { diff --git a/client/exporters.go b/client/exporters.go index 841696d8..2e9acb47 100644 --- a/client/exporters.go +++ b/client/exporters.go @@ -1,9 +1,10 @@ package client const ( - ExporterImage = "image" - ExporterLocal = "local" - ExporterOCI = "oci" + ExporterImage = "image" + ExporterLocal = "local" + ExporterOCI = "oci" + ExporterDocker = "docker" exporterLocalOutputDir = "output" exporterOCIDestination = "output" diff --git a/client/solve.go b/client/solve.go index a7f7521c..630ffbda 100644 --- a/client/solve.go +++ b/client/solve.go @@ -79,7 +79,7 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s return errors.Errorf("output directory is required for local exporter") } s.Allow(filesync.NewFSSyncTarget(outputDir)) - case ExporterOCI: + case ExporterOCI, ExporterDocker: outputFile, ok := opt.ExporterAttrs[exporterOCIDestination] if ok { fi, err := os.Stat(outputFile) @@ -91,7 +91,7 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s } } else { if _, err := console.ConsoleFromFile(os.Stdout); err == nil { - return errors.Errorf("output file is required for OCI exporter. refusing to write to console") + return errors.Errorf("output file is required for %s exporter. refusing to write to console", opt.Exporter) } outputFile = "" } diff --git a/exporter/oci/docker.go b/exporter/oci/docker.go new file mode 100644 index 00000000..ea0abf93 --- /dev/null +++ b/exporter/oci/docker.go @@ -0,0 +1,245 @@ +package oci + +import ( + "archive/tar" + "context" + "encoding/json" + "io" + "path" + "sort" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + ocispecs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// DockerExporter implements exporting to +// Docker Combined Image JSON + Filesystem Changeset Format v1.1 +// https://github.com/moby/moby/blob/master/image/spec/v1.1.md#combined-image-json--filesystem-changeset-format +// The outputed tarball is also compatible wih OCI Image Format Specification +type DockerExporter struct { + name string +} + +// Export exports tarball into writer. +func (de *DockerExporter) Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error { + tw := tar.NewWriter(writer) + defer tw.Close() + + dockerManifest, err := dockerManifestRecord(ctx, store, desc, de.name) + if err != nil { + return err + } + + records := []tarRecord{ + ociLayoutFile(""), + ociIndexRecord(desc), + *dockerManifest, + } + + algorithms := map[string]struct{}{} + exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + records = append(records, blobRecord(store, desc)) + algorithms[desc.Digest.Algorithm().String()] = struct{}{} + return nil, nil + } + + handlers := images.Handlers( + images.ChildrenHandler(store, platforms.Default()), + images.HandlerFunc(exportHandler), + ) + + // Walk sequentially since the number of fetchs is likely one and doing in + // parallel requires locking the export handler + if err := images.Walk(ctx, handlers, desc); err != nil { + return err + } + + if len(algorithms) > 0 { + records = append(records, directoryRecord("blobs/", 0755)) + for alg := range algorithms { + records = append(records, directoryRecord("blobs/"+alg+"/", 0755)) + } + } + + return writeTar(ctx, tw, records) +} + +type tarRecord struct { + Header *tar.Header + CopyTo func(context.Context, io.Writer) (int64, error) +} + +func dockerManifestRecord(ctx context.Context, provider content.Provider, desc ocispec.Descriptor, name string) (*tarRecord, error) { + switch desc.MediaType { + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + p, err := content.ReadBlob(ctx, provider, desc.Digest) + if err != nil { + return nil, err + } + var manifest ocispec.Manifest + if err := json.Unmarshal(p, &manifest); err != nil { + return nil, err + } + type mfstItem struct { + Config string + RepoTags []string + Layers []string + } + item := mfstItem{ + Config: path.Join("blobs", manifest.Config.Digest.Algorithm().String(), manifest.Config.Digest.Hex()), + } + + for _, l := range manifest.Layers { + item.Layers = append(item.Layers, path.Join("blobs", l.Digest.Algorithm().String(), l.Digest.Hex())) + } + + if name != "" { + item.RepoTags = append(item.RepoTags, name) + } + + dt, err := json.Marshal([]mfstItem{item}) + if err != nil { + return nil, err + } + + return &tarRecord{ + Header: &tar.Header{ + Name: "manifest.json", + Mode: 0444, + Size: int64(len(dt)), + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + n, err := w.Write(dt) + return int64(n), err + }, + }, nil + default: + return nil, errors.Errorf("%v not supported for Docker exporter", desc.MediaType) + } + +} + +func blobRecord(cs content.Store, desc ocispec.Descriptor) tarRecord { + path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex() + return tarRecord{ + Header: &tar.Header{ + Name: path, + Mode: 0444, + Size: desc.Size, + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + r, err := cs.ReaderAt(ctx, desc.Digest) + if err != nil { + return 0, err + } + defer r.Close() + + // Verify digest + dgstr := desc.Digest.Algorithm().Digester() + + n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) + if err != nil { + return 0, err + } + if dgstr.Digest() != desc.Digest { + return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest()) + } + return n, nil + }, + } +} + +func directoryRecord(name string, mode int64) tarRecord { + return tarRecord{ + Header: &tar.Header{ + Name: name, + Mode: mode, + Typeflag: tar.TypeDir, + }, + } +} + +func ociLayoutFile(version string) tarRecord { + if version == "" { + version = ocispec.ImageLayoutVersion + } + layout := ocispec.ImageLayout{ + Version: version, + } + + b, err := json.Marshal(layout) + if err != nil { + panic(err) + } + + return tarRecord{ + Header: &tar.Header{ + Name: ocispec.ImageLayoutFile, + Mode: 0444, + Size: int64(len(b)), + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + n, err := w.Write(b) + return int64(n), err + }, + } + +} + +func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord { + index := ocispec.Index{ + Versioned: ocispecs.Versioned{ + SchemaVersion: 2, + }, + Manifests: manifests, + } + + b, err := json.Marshal(index) + if err != nil { + panic(err) + } + + return tarRecord{ + Header: &tar.Header{ + Name: "index.json", + Mode: 0644, + Size: int64(len(b)), + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + n, err := w.Write(b) + return int64(n), err + }, + } +} + +func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { + sort.Slice(records, func(i, j int) bool { + return records[i].Header.Name < records[j].Header.Name + }) + + for _, record := range records { + if err := tw.WriteHeader(record.Header); err != nil { + return err + } + if record.CopyTo != nil { + n, err := record.CopyTo(ctx, tw) + if err != nil { + return err + } + if n != record.Header.Size { + return errors.Errorf("unexpected copy size for %s", record.Header.Name) + } + } else if record.Header.Size > 0 { + return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name) + } + } + return nil +} diff --git a/exporter/oci/export.go b/exporter/oci/export.go index b8aabc17..ac18170c 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -1,27 +1,36 @@ package oci import ( - "errors" "time" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/images/oci" + "github.com/docker/distribution/reference" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/exporter/containerimage" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/util/progress" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" ) +type ExporterVariant string + const ( exporterImageConfig = "containerimage.config" + keyImageName = "name" + VariantOCI = "oci" + VariantDocker = "docker" ) type Opt struct { SessionManager *session.Manager ImageWriter *containerimage.ImageWriter + Variant ExporterVariant } type imageExporter struct { @@ -52,6 +61,12 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp switch k { case exporterImageConfig: i.config = []byte(v) + case keyImageName: + parsed, err := reference.ParseNormalizedNamed(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", v) + } + i.name = reference.TagNameOnly(parsed).String() default: logrus.Warnf("oci exporter: unknown option %s", k) } @@ -63,6 +78,7 @@ type imageExporterInstance struct { *imageExporter config []byte caller session.Caller + name string } func (e *imageExporterInstance) Name() string { @@ -77,13 +93,22 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR if err != nil { return err } + if desc.Annotations == nil { + desc.Annotations = map[string]string{} + } + desc.Annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339) + + exp, err := getExporter(e.opt.Variant, e.name) + if err != nil { + return err + } w, err := filesync.CopyFileWriter(ctx, e.caller) if err != nil { return err } report := oneOffProgress(ctx, "sending tarball") - if err := (&oci.V1Exporter{}).Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil { + if err := exp.Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil { w.Close() return report(err) } @@ -106,3 +131,14 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error { return err } } + +func getExporter(variant ExporterVariant, name string) (images.Exporter, error) { + switch variant { + case VariantOCI: + return &oci.V1Exporter{}, nil + case VariantDocker: + return &DockerExporter{name: name}, nil + default: + return nil, errors.Errorf("invalid variant %q", variant) + } +} diff --git a/worker/base/worker.go b/worker/base/worker.go index c8f1a41f..3111af44 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -170,12 +170,23 @@ func NewWorker(opt WorkerOpt) (*Worker, error) { ociExporter, err := ociexporter.New(ociexporter.Opt{ SessionManager: opt.SessionManager, ImageWriter: iw, + Variant: ociexporter.VariantOCI, }) if err != nil { return nil, err } exporters[client.ExporterOCI] = ociExporter + dockerExporter, err := ociexporter.New(ociexporter.Opt{ + SessionManager: opt.SessionManager, + ImageWriter: iw, + Variant: ociexporter.VariantDocker, + }) + if err != nil { + return nil, err + } + exporters[client.ExporterDocker] = dockerExporter + ce := cacheimport.NewCacheExporter(cacheimport.ExporterOpt{ Snapshotter: bmSnapshotter, ContentStore: opt.ContentStore,