JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #195 from tonistiigi/differ-test2 

client: add tests for differ and pushed data
docker-18.09
Akihiro Suda 2017-12-09 12:48:45 +09:00 committed by GitHub
parent bd9992f70f f5f3b39228
commit 8f8cf19266
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 328 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
client/client_test.go
View File

@ -1,19 +1,31 @@
package client
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/namespaces"
"github.com/docker/distribution/manifest/schema2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/testutil/httpserver"
"github.com/moby/buildkit/util/testutil/integration"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
@ -22,6 +34,7 @@ func TestClientIntegration(t *testing.T) {
testCallDiskUsage,
testBuildMultiMount,
testBuildHTTPSource,
testBuildPushAndValidate,
})
}
@ -143,8 +156,201 @@ func testBuildHTTPSource(t *testing.T, sb integration.Sandbox) {
// TODO: check that second request was marked as cached
}
func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
requiresLinux(t)
t.Parallel()
c, err := New(sb.Address())
require.NoError(t, err)
defer c.Close()
busybox := llb.Image("busybox:latest")
st := llb.Scratch()
run := func(cmd string) {
st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
}
run(`sh -c "mkdir -p foo/sub; echo -n first > foo/sub/bar; chmod 0741 foo;"`)
run(`sh -c "echo -n second > foo/sub/baz"`)
def, err := st.Marshal()
require.NoError(t, err)
registry, err := sb.NewRegistry()
if errors.Cause(err) == integration.ErrorRequirements {
t.Skip(err.Error())
}
require.NoError(t, err)
target := registry + "/buildkit/testpush:latest"
err = c.Solve(context.TODO(), def, SolveOpt{
Exporter: ExporterImage,
ExporterAttrs: map[string]string{
"name": target,
"push": "true",
},
}, nil)
require.NoError(t, err)
// test existence of the image with next build
firstBuild := llb.Image(target)
def, err = firstBuild.Marshal()
require.NoError(t, err)
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
err = c.Solve(context.TODO(), def, SolveOpt{
Exporter: ExporterLocal,
ExporterAttrs: map[string]string{
"output": destDir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo/sub/bar"))
require.NoError(t, err)
require.Equal(t, dt, []byte("first"))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo/sub/baz"))
require.NoError(t, err)
require.Equal(t, dt, []byte("second"))
fi, err := os.Stat(filepath.Join(destDir, "foo"))
require.NoError(t, err)
require.Equal(t, 0741, int(fi.Mode()&0777))
// examine contents of exported tars (requires containerd)
var cdAddress string
if cd, ok := sb.(interface {
ContainerdAddress() string
}); !ok {
return
} else {
cdAddress = cd.ContainerdAddress()
}
// TODO: make public pull helper function so this can be checked for standalone as well
client, err := containerd.New(cdAddress)
require.NoError(t, err)
defer client.Close()
ctx := namespaces.WithNamespace(context.Background(), "buildkit")
img, err := client.Pull(ctx, target)
require.NoError(t, err)
desc, err := img.Config(ctx)
require.NoError(t, err)
dt, err = content.ReadBlob(ctx, img.ContentStore(), desc.Digest)
require.NoError(t, err)
var ociimg ocispec.Image
err = json.Unmarshal(dt, &ociimg)
require.NoError(t, err)
require.NotEqual(t, "", ociimg.OS)
require.NotEqual(t, "", ociimg.Architecture)
require.NotEqual(t, "", ociimg.Config.WorkingDir)
require.Equal(t, "layers", ociimg.RootFS.Type)
require.Equal(t, 2, len(ociimg.RootFS.DiffIDs))
require.Condition(t, func() bool {
for _, env := range ociimg.Config.Env {
if strings.HasPrefix(env, "PATH=") {
return true
}
}
return false
})
dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target().Digest)
require.NoError(t, err)
var mfst schema2.Manifest
err = json.Unmarshal(dt, &mfst)
require.NoError(t, err)
require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
require.Equal(t, 2, len(mfst.Layers))
dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[0].Digest)
require.NoError(t, err)
m, err := readTarToMap(dt)
require.NoError(t, err)
item, ok := m["foo/"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
require.Equal(t, 0741, int(item.header.Mode&0777))
item, ok = m["foo/sub/"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
item, ok = m["foo/sub/bar"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
require.Equal(t, []byte("first"), item.data)
_, ok = m["foo/sub/baz"]
require.False(t, ok)
dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[1].Digest)
require.NoError(t, err)
m, err = readTarToMap(dt)
require.NoError(t, err)
item, ok = m["foo/sub/baz"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
require.Equal(t, []byte("second"), item.data)
// TODO: #154 check that the unmodified parents are still in tar
// item, ok = m["foo/"]
// require.True(t, ok)
// require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
}
func requiresLinux(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skipf("unsupported GOOS: %s", runtime.GOOS)
}
}
type tarItem struct {
header *tar.Header
data []byte
}
func readTarToMap(dt []byte) (map[string]*tarItem, error) {
m := map[string]*tarItem{}
gz, err := gzip.NewReader(bytes.NewBuffer(dt))
if err != nil {
return nil, err
}
defer gz.Close()
rdr := tar.NewReader(gz)
for {
h, err := rdr.Next()
if err != nil {
if err == io.EOF {
return m, nil
}
return nil, err
}
var dt []byte
if h.Typeflag == tar.TypeReg {
dt, err = ioutil.ReadAll(rdr)
if err != nil {
return nil, err
}
}
m[h.Name] = &tarItem{header: h, data: dt}
}
}

4
hack/dockerfiles/test.Dockerfile
View File

@ -1,6 +1,7 @@
ARG RUNC_VERSION=74a17296470088de3805e138d3d87c62e613dfc4
ARG CONTAINERD_VERSION=v1.0.0
ARG BUILDKIT_TARGET=standalone
ARG REGISTRY_VERSION=2.6
FROM golang:1.9-alpine AS gobuild-base
RUN apk add --no-cache g++ linux-headers
@ -44,10 +45,13 @@ FROM buildkit-base AS buildd-containerd
ENV CGO_ENABLED=0
RUN go build -ldflags '-d' -o /usr/bin/buildd-containerd -tags containerd ./cmd/buildd
FROM registry:$REGISTRY_VERSION AS registry
FROM unit-tests AS integration-tests
COPY --from=buildctl /usr/bin/buildctl /usr/bin/
COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin
COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin
COPY --from=registry /bin/registry /usr/bin
FROM gobuild-base AS cross-windows
ENV GOOS=windows

2
util/testutil/integration/containerd.go
View File

@ -70,7 +70,7 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) {
}
deferF.append(stop)
return &cdsandbox{address: address, sandbox: sandbox{address: builddSock, logs: logs}}, cl, nil
return &cdsandbox{address: address, sandbox: sandbox{address: builddSock, logs: logs, cleanup: deferF}}, cl, nil
}
type cdsandbox struct {

102
util/testutil/integration/registry.go Normal file
View File

@ -0,0 +1,102 @@
package integration
import (
"bufio"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"time"
"github.com/pkg/errors"
)
func newRegistry() (url string, cl func() error, err error) {
if err := lookupBinary("registry"); err != nil {
return "", nil, err
}
deferF := &multiCloser{}
cl = deferF.F()
defer func() {
if err != nil {
deferF.F()()
cl = nil
}
}()
tmpdir, err := ioutil.TempDir("", "test-registry")
if err != nil {
return "", nil, err
}
deferF.append(func() error { return os.RemoveAll(tmpdir) })
template := fmt.Sprintf(`version: 0.1
loglevel: debug
storage:
filesystem:
rootdirectory: %s
http:
addr: 127.0.0.1:0
`, filepath.Join(tmpdir, "data"))
if err := ioutil.WriteFile(filepath.Join(tmpdir, "config.yaml"), []byte(template), 0600); err != nil {
return "", nil, err
}
cmd := exec.Command("registry", "serve", filepath.Join(tmpdir, "config.yaml"))
rc, err := cmd.StdoutPipe()
if err != nil {
return "", nil, err
}
if stop, err := startCmd(cmd, nil); err != nil {
return "", nil, err
} else {
deferF.append(stop)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
url, err = detectPort(ctx, rc)
if err != nil {
return "", nil, err
}
return
}
func detectPort(ctx context.Context, rc io.ReadCloser) (string, error) {
r := regexp.MustCompile("listening on 127\\.0\\.0\\.1:(\\d+)")
s := bufio.NewScanner(rc)
found := make(chan struct{})
defer func() {
close(found)
go io.Copy(ioutil.Discard, rc)
}()
go func() {
select {
case <-ctx.Done():
select {
case <-found:
return
default:
rc.Close()
}
case <-found:
}
}()
for s.Scan() {
res := r.FindSubmatch(s.Bytes())
if len(res) > 1 {
return "localhost:" + string(res[1]), nil
}
}
return "", errors.Errorf("no listening address found")
}

1
util/testutil/integration/run.go
View File

@ -16,6 +16,7 @@ type Sandbox interface {
Address() string
PrintLogs(*testing.T)
Cmd(...string) *exec.Cmd
NewRegistry() (string, error)
}
type Worker interface {

15
util/testutil/integration/standalone.go
View File

@ -38,12 +38,16 @@ func (s *standalone) New() (Sandbox, func() error, error) {
return nil, nil, err
}
return &sandbox{address: builddSock, logs: logs}, stop, nil
deferF := &multiCloser{}
deferF.append(stop)
return &sandbox{address: builddSock, logs: logs, cleanup: deferF}, deferF.F(), nil
}
type sandbox struct {
address string
logs map[string]*bytes.Buffer
cleanup *multiCloser
}
func (sb *sandbox) Address() string {
@ -60,6 +64,15 @@ func (sb *sandbox) PrintLogs(t *testing.T) {
}
}
func (sb *sandbox) NewRegistry() (string, error) {
url, cl, err := newRegistry()
if err != nil {
return "", err
}
sb.cleanup.append(cl)
return url, nil
}
func (sb *sandbox) Cmd(args ...string) *exec.Cmd {
if len(args) == 1 {
if split, err := shlex.Split(args[0]); err == nil {