JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #228 from tonistiigi/docker-exporter 

Add Docker exporter
docker-18.09
Akihiro Suda 2017-12-19 11:59:26 +09:00 committed by GitHub
parent 1c0b8acf6d 85c0f99ba0
commit 28228468c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 373 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -123,7 +123,14 @@ If credentials are required, `buildctl` will attempt to read Docker configuratio
buildctl build ... --exporter=local --exporter-opt output=path/to/output-dir
```
#### Exporting OCI Image Format tarball to client
##### Exporting build result to Docker
```
# exported tarball is also compatible with OCI spec
buildctl build ... --exporter=docker --exporter-opt name=myimage | docker load
```
##### Exporting OCI Image Format tarball to client
```
buildctl build ... --exporter=oci --exporter-opt output=path/to/output.tar

100
client/client_test.go
View File

@ -275,50 +275,80 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) {
def, err := st.Marshal()
require.NoError(t, err)
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
for _, exp := range []string{ExporterOCI, ExporterDocker} {
out := filepath.Join(destDir, "out.tar")
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
err = c.Solve(context.TODO(), def, SolveOpt{
Exporter: ExporterOCI,
ExporterAttrs: map[string]string{
"output": out,
},
}, nil)
require.NoError(t, err)
out := filepath.Join(destDir, "out.tar")
target := "example.com/buildkit/testoci:latest"
dt, err := ioutil.ReadFile(out)
require.NoError(t, err)
err = c.Solve(context.TODO(), def, SolveOpt{
Exporter: exp,
ExporterAttrs: map[string]string{
"output": out,
"name": target,
},
}, nil)
require.NoError(t, err)
m, err := readTarToMap(dt, false)
require.NoError(t, err)
dt, err := ioutil.ReadFile(out)
require.NoError(t, err)
_, ok := m["oci-layout"]
require.True(t, ok)
m, err := readTarToMap(dt, false)
require.NoError(t, err)
var index ocispec.Index
err = json.Unmarshal(m["index.json"].data, &index)
require.NoError(t, err)
require.Equal(t, 2, index.SchemaVersion)
require.Equal(t, 1, len(index.Manifests))
_, ok := m["oci-layout"]
require.True(t, ok)
var mfst ocispec.Manifest
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst)
require.NoError(t, err)
require.Equal(t, 2, len(mfst.Layers))
var index ocispec.Index
err = json.Unmarshal(m["index.json"].data, &index)
require.NoError(t, err)
require.Equal(t, 2, index.SchemaVersion)
require.Equal(t, 1, len(index.Manifests))
var ociimg ocispec.Image
err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].data, &ociimg)
require.NoError(t, err)
require.Equal(t, "layers", ociimg.RootFS.Type)
require.Equal(t, 2, len(ociimg.RootFS.DiffIDs))
var mfst ocispec.Manifest
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst)
require.NoError(t, err)
require.Equal(t, 2, len(mfst.Layers))
_, ok = m["blobs/sha256/"+mfst.Layers[0].Digest.Hex()]
require.True(t, ok)
_, ok = m["blobs/sha256/"+mfst.Layers[1].Digest.Hex()]
require.True(t, ok)
var ociimg ocispec.Image
err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].data, &ociimg)
require.NoError(t, err)
require.Equal(t, "layers", ociimg.RootFS.Type)
require.Equal(t, 2, len(ociimg.RootFS.DiffIDs))
_, ok = m["blobs/sha256/"+mfst.Layers[0].Digest.Hex()]
require.True(t, ok)
_, ok = m["blobs/sha256/"+mfst.Layers[1].Digest.Hex()]
require.True(t, ok)
if exp != ExporterDocker {
continue
}
var dockerMfst []struct {
Config string
RepoTags []string
Layers []string
}
err = json.Unmarshal(m["manifest.json"].data, &dockerMfst)
require.NoError(t, err)
require.Equal(t, 1, len(dockerMfst))
_, ok = m[dockerMfst[0].Config]
require.True(t, ok)
require.Equal(t, 2, len(dockerMfst[0].Layers))
require.Equal(t, 1, len(dockerMfst[0].RepoTags))
require.Equal(t, target, dockerMfst[0].RepoTags[0])
for _, l := range dockerMfst[0].Layers {
_, ok := m[l]
require.True(t, ok)
}
}
}
func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {

7
client/exporters.go
View File

@ -1,9 +1,10 @@
package client
const (
ExporterImage = "image"
ExporterLocal = "local"
ExporterOCI = "oci"
ExporterImage = "image"
ExporterLocal = "local"
ExporterOCI = "oci"
ExporterDocker = "docker"
exporterLocalOutputDir = "output"
exporterOCIDestination = "output"

4
client/solve.go
View File

@ -79,7 +79,7 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s
return errors.Errorf("output directory is required for local exporter")
}
s.Allow(filesync.NewFSSyncTarget(outputDir))
case ExporterOCI:
case ExporterOCI, ExporterDocker:
outputFile, ok := opt.ExporterAttrs[exporterOCIDestination]
if ok {
fi, err := os.Stat(outputFile)
@ -91,7 +91,7 @@ func (c *Client) Solve(ctx context.Context, def *llb.Definition, opt SolveOpt, s
}
} else {
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
return errors.Errorf("output file is required for OCI exporter. refusing to write to console")
return errors.Errorf("output file is required for %s exporter. refusing to write to console", opt.Exporter)
}
outputFile = ""
}

245
exporter/oci/docker.go Normal file
View File

@ -0,0 +1,245 @@
package oci
import (
"archive/tar"
"context"
"encoding/json"
"io"
"path"
"sort"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
ocispecs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
// DockerExporter implements exporting to
// Docker Combined Image JSON + Filesystem Changeset Format v1.1
// https://github.com/moby/moby/blob/master/image/spec/v1.1.md#combined-image-json--filesystem-changeset-format
// The outputed tarball is also compatible wih OCI Image Format Specification
type DockerExporter struct {
name string
}
// Export exports tarball into writer.
func (de *DockerExporter) Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error {
tw := tar.NewWriter(writer)
defer tw.Close()
dockerManifest, err := dockerManifestRecord(ctx, store, desc, de.name)
if err != nil {
return err
}
records := []tarRecord{
ociLayoutFile(""),
ociIndexRecord(desc),
*dockerManifest,
}
algorithms := map[string]struct{}{}
exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
records = append(records, blobRecord(store, desc))
algorithms[desc.Digest.Algorithm().String()] = struct{}{}
return nil, nil
}
handlers := images.Handlers(
images.ChildrenHandler(store, platforms.Default()),
images.HandlerFunc(exportHandler),
)
// Walk sequentially since the number of fetchs is likely one and doing in
// parallel requires locking the export handler
if err := images.Walk(ctx, handlers, desc); err != nil {
return err
}
if len(algorithms) > 0 {
records = append(records, directoryRecord("blobs/", 0755))
for alg := range algorithms {
records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
}
}
return writeTar(ctx, tw, records)
}
type tarRecord struct {
Header *tar.Header
CopyTo func(context.Context, io.Writer) (int64, error)
}
func dockerManifestRecord(ctx context.Context, provider content.Provider, desc ocispec.Descriptor, name string) (*tarRecord, error) {
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
if err != nil {
return nil, err
}
var manifest ocispec.Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
return nil, err
}
type mfstItem struct {
Config string
RepoTags []string
Layers []string
}
item := mfstItem{
Config: path.Join("blobs", manifest.Config.Digest.Algorithm().String(), manifest.Config.Digest.Hex()),
}
for _, l := range manifest.Layers {
item.Layers = append(item.Layers, path.Join("blobs", l.Digest.Algorithm().String(), l.Digest.Hex()))
}
if name != "" {
item.RepoTags = append(item.RepoTags, name)
}
dt, err := json.Marshal([]mfstItem{item})
if err != nil {
return nil, err
}
return &tarRecord{
Header: &tar.Header{
Name: "manifest.json",
Mode: 0444,
Size: int64(len(dt)),
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
n, err := w.Write(dt)
return int64(n), err
},
}, nil
default:
return nil, errors.Errorf("%v not supported for Docker exporter", desc.MediaType)
}
}
func blobRecord(cs content.Store, desc ocispec.Descriptor) tarRecord {
path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex()
return tarRecord{
Header: &tar.Header{
Name: path,
Mode: 0444,
Size: desc.Size,
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
r, err := cs.ReaderAt(ctx, desc.Digest)
if err != nil {
return 0, err
}
defer r.Close()
// Verify digest
dgstr := desc.Digest.Algorithm().Digester()
n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
if err != nil {
return 0, err
}
if dgstr.Digest() != desc.Digest {
return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest())
}
return n, nil
},
}
}
func directoryRecord(name string, mode int64) tarRecord {
return tarRecord{
Header: &tar.Header{
Name: name,
Mode: mode,
Typeflag: tar.TypeDir,
},
}
}
func ociLayoutFile(version string) tarRecord {
if version == "" {
version = ocispec.ImageLayoutVersion
}
layout := ocispec.ImageLayout{
Version: version,
}
b, err := json.Marshal(layout)
if err != nil {
panic(err)
}
return tarRecord{
Header: &tar.Header{
Name: ocispec.ImageLayoutFile,
Mode: 0444,
Size: int64(len(b)),
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
n, err := w.Write(b)
return int64(n), err
},
}
}
func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord {
index := ocispec.Index{
Versioned: ocispecs.Versioned{
SchemaVersion: 2,
},
Manifests: manifests,
}
b, err := json.Marshal(index)
if err != nil {
panic(err)
}
return tarRecord{
Header: &tar.Header{
Name: "index.json",
Mode: 0644,
Size: int64(len(b)),
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
n, err := w.Write(b)
return int64(n), err
},
}
}
func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
sort.Slice(records, func(i, j int) bool {
return records[i].Header.Name < records[j].Header.Name
})
for _, record := range records {
if err := tw.WriteHeader(record.Header); err != nil {
return err
}
if record.CopyTo != nil {
n, err := record.CopyTo(ctx, tw)
if err != nil {
return err
}
if n != record.Header.Size {
return errors.Errorf("unexpected copy size for %s", record.Header.Name)
}
} else if record.Header.Size > 0 {
return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
}
}
return nil
}

40
exporter/oci/export.go
View File

@ -1,27 +1,36 @@
package oci
import (
"errors"
"time"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/images/oci"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/containerimage"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/util/progress"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
)
type ExporterVariant string
const (
exporterImageConfig = "containerimage.config"
keyImageName = "name"
VariantOCI = "oci"
VariantDocker = "docker"
)
type Opt struct {
SessionManager *session.Manager
ImageWriter *containerimage.ImageWriter
Variant ExporterVariant
}
type imageExporter struct {
@ -52,6 +61,12 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
switch k {
case exporterImageConfig:
i.config = []byte(v)
case keyImageName:
parsed, err := reference.ParseNormalizedNamed(v)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse %s", v)
}
i.name = reference.TagNameOnly(parsed).String()
default:
logrus.Warnf("oci exporter: unknown option %s", k)
}
@ -63,6 +78,7 @@ type imageExporterInstance struct {
*imageExporter
config []byte
caller session.Caller
name string
}
func (e *imageExporterInstance) Name() string {
@ -77,13 +93,22 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
if err != nil {
return err
}
if desc.Annotations == nil {
desc.Annotations = map[string]string{}
}
desc.Annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339)
exp, err := getExporter(e.opt.Variant, e.name)
if err != nil {
return err
}
w, err := filesync.CopyFileWriter(ctx, e.caller)
if err != nil {
return err
}
report := oneOffProgress(ctx, "sending tarball")
if err := (&oci.V1Exporter{}).Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil {
if err := exp.Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil {
w.Close()
return report(err)
}
@ -106,3 +131,14 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error {
return err
}
}
func getExporter(variant ExporterVariant, name string) (images.Exporter, error) {
switch variant {
case VariantOCI:
return &oci.V1Exporter{}, nil
case VariantDocker:
return &DockerExporter{name: name}, nil
default:
return nil, errors.Errorf("invalid variant %q", variant)
}
}

11
worker/base/worker.go
View File

@ -170,12 +170,23 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
ociExporter, err := ociexporter.New(ociexporter.Opt{
SessionManager: opt.SessionManager,
ImageWriter: iw,
Variant: ociexporter.VariantOCI,
})
if err != nil {
return nil, err
}
exporters[client.ExporterOCI] = ociExporter
dockerExporter, err := ociexporter.New(ociexporter.Opt{
SessionManager: opt.SessionManager,
ImageWriter: iw,
Variant: ociexporter.VariantDocker,
})
if err != nil {
return nil, err
}
exporters[client.ExporterDocker] = dockerExporter
ce := cacheimport.NewCacheExporter(cacheimport.ExporterOpt{
Snapshotter: bmSnapshotter,
ContentStore: opt.ContentStore,