357 lines
8.8 KiB
Go
357 lines
8.8 KiB
Go
package client
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/containerd/containerd"
|
|
"github.com/containerd/containerd/content"
|
|
"github.com/containerd/containerd/namespaces"
|
|
"github.com/docker/distribution/manifest/schema2"
|
|
"github.com/moby/buildkit/client/llb"
|
|
"github.com/moby/buildkit/identity"
|
|
"github.com/moby/buildkit/util/testutil/httpserver"
|
|
"github.com/moby/buildkit/util/testutil/integration"
|
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
"github.com/pkg/errors"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestClientIntegration(t *testing.T) {
|
|
integration.Run(t, []integration.Test{
|
|
testCallDiskUsage,
|
|
testBuildMultiMount,
|
|
testBuildHTTPSource,
|
|
testBuildPushAndValidate,
|
|
})
|
|
}
|
|
|
|
func testCallDiskUsage(t *testing.T, sb integration.Sandbox) {
|
|
c, err := New(sb.Address())
|
|
require.NoError(t, err)
|
|
defer c.Close()
|
|
_, err = c.DiskUsage(context.TODO())
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func testBuildMultiMount(t *testing.T, sb integration.Sandbox) {
|
|
requiresLinux(t)
|
|
t.Parallel()
|
|
c, err := New(sb.Address())
|
|
require.NoError(t, err)
|
|
defer c.Close()
|
|
|
|
alpine := llb.Image("docker.io/library/alpine:latest")
|
|
ls := alpine.Run(llb.Shlex("/bin/ls -l"))
|
|
busybox := llb.Image("docker.io/library/busybox:latest")
|
|
cp := ls.Run(llb.Shlex("/bin/cp -a /busybox/etc/passwd baz"))
|
|
cp.AddMount("/busybox", busybox)
|
|
|
|
def, err := cp.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{}, nil)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func testBuildHTTPSource(t *testing.T, sb integration.Sandbox) {
|
|
t.Parallel()
|
|
|
|
c, err := New(sb.Address())
|
|
require.NoError(t, err)
|
|
defer c.Close()
|
|
|
|
modTime := time.Now().Add(-24 * time.Hour) // avoid falso positive with current time
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content1"),
|
|
LastModified: &modTime,
|
|
}
|
|
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/foo": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
// invalid URL first
|
|
st := llb.HTTP(server.URL + "/bar")
|
|
|
|
def, err := st.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{}, nil)
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "invalid response status 404")
|
|
|
|
// first correct request
|
|
st = llb.HTTP(server.URL + "/foo")
|
|
|
|
def, err = st.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{}, nil)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{
|
|
Exporter: ExporterLocal,
|
|
ExporterAttrs: map[string]string{
|
|
exporterLocalOutputDir: tmpdir,
|
|
},
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 2)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 1)
|
|
|
|
dt, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("content1"), dt)
|
|
|
|
// test extra options
|
|
st = llb.HTTP(server.URL+"/foo", llb.Filename("bar"), llb.Chmod(0741), llb.Chown(1000, 1000))
|
|
|
|
def, err = st.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{
|
|
Exporter: ExporterLocal,
|
|
ExporterAttrs: map[string]string{
|
|
exporterLocalOutputDir: tmpdir,
|
|
},
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 3)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 1)
|
|
|
|
dt, err = ioutil.ReadFile(filepath.Join(tmpdir, "bar"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("content1"), dt)
|
|
|
|
fi, err := os.Stat(filepath.Join(tmpdir, "bar"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, fi.ModTime().Format(http.TimeFormat), modTime.Format(http.TimeFormat))
|
|
require.Equal(t, int(fi.Mode()&0777), 0741)
|
|
|
|
// TODO: check that second request was marked as cached
|
|
}
|
|
|
|
func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
|
|
requiresLinux(t)
|
|
t.Parallel()
|
|
c, err := New(sb.Address())
|
|
require.NoError(t, err)
|
|
defer c.Close()
|
|
|
|
busybox := llb.Image("busybox:latest")
|
|
st := llb.Scratch()
|
|
|
|
run := func(cmd string) {
|
|
st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
|
|
}
|
|
|
|
run(`sh -c "mkdir -p foo/sub; echo -n first > foo/sub/bar; chmod 0741 foo;"`)
|
|
run(`sh -c "echo -n second > foo/sub/baz"`)
|
|
|
|
def, err := st.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
registry, err := sb.NewRegistry()
|
|
if errors.Cause(err) == integration.ErrorRequirements {
|
|
t.Skip(err.Error())
|
|
}
|
|
require.NoError(t, err)
|
|
|
|
target := registry + "/buildkit/testpush:latest"
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{
|
|
Exporter: ExporterImage,
|
|
ExporterAttrs: map[string]string{
|
|
"name": target,
|
|
"push": "true",
|
|
},
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
// test existence of the image with next build
|
|
firstBuild := llb.Image(target)
|
|
|
|
def, err = firstBuild.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
destDir, err := ioutil.TempDir("", "buildkit")
|
|
require.NoError(t, err)
|
|
|
|
err = c.Solve(context.TODO(), def, SolveOpt{
|
|
Exporter: ExporterLocal,
|
|
ExporterAttrs: map[string]string{
|
|
"output": destDir,
|
|
},
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo/sub/bar"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("first"))
|
|
|
|
dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo/sub/baz"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("second"))
|
|
|
|
fi, err := os.Stat(filepath.Join(destDir, "foo"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 0741, int(fi.Mode()&0777))
|
|
|
|
// examine contents of exported tars (requires containerd)
|
|
var cdAddress string
|
|
if cd, ok := sb.(interface {
|
|
ContainerdAddress() string
|
|
}); !ok {
|
|
return
|
|
} else {
|
|
cdAddress = cd.ContainerdAddress()
|
|
}
|
|
|
|
// TODO: make public pull helper function so this can be checked for standalone as well
|
|
|
|
client, err := containerd.New(cdAddress)
|
|
require.NoError(t, err)
|
|
defer client.Close()
|
|
|
|
ctx := namespaces.WithNamespace(context.Background(), "buildkit")
|
|
|
|
img, err := client.Pull(ctx, target)
|
|
require.NoError(t, err)
|
|
|
|
desc, err := img.Config(ctx)
|
|
require.NoError(t, err)
|
|
|
|
dt, err = content.ReadBlob(ctx, img.ContentStore(), desc.Digest)
|
|
require.NoError(t, err)
|
|
|
|
var ociimg ocispec.Image
|
|
err = json.Unmarshal(dt, &ociimg)
|
|
require.NoError(t, err)
|
|
|
|
require.NotEqual(t, "", ociimg.OS)
|
|
require.NotEqual(t, "", ociimg.Architecture)
|
|
require.NotEqual(t, "", ociimg.Config.WorkingDir)
|
|
require.Equal(t, "layers", ociimg.RootFS.Type)
|
|
require.Equal(t, 2, len(ociimg.RootFS.DiffIDs))
|
|
require.Condition(t, func() bool {
|
|
for _, env := range ociimg.Config.Env {
|
|
if strings.HasPrefix(env, "PATH=") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
|
|
dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target().Digest)
|
|
require.NoError(t, err)
|
|
|
|
var mfst schema2.Manifest
|
|
err = json.Unmarshal(dt, &mfst)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
|
|
require.Equal(t, 2, len(mfst.Layers))
|
|
|
|
dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[0].Digest)
|
|
require.NoError(t, err)
|
|
|
|
m, err := readTarToMap(dt)
|
|
require.NoError(t, err)
|
|
|
|
item, ok := m["foo/"]
|
|
require.True(t, ok)
|
|
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
|
|
require.Equal(t, 0741, int(item.header.Mode&0777))
|
|
|
|
item, ok = m["foo/sub/"]
|
|
require.True(t, ok)
|
|
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
|
|
|
|
item, ok = m["foo/sub/bar"]
|
|
require.True(t, ok)
|
|
require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
|
|
require.Equal(t, []byte("first"), item.data)
|
|
|
|
_, ok = m["foo/sub/baz"]
|
|
require.False(t, ok)
|
|
|
|
dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[1].Digest)
|
|
require.NoError(t, err)
|
|
|
|
m, err = readTarToMap(dt)
|
|
require.NoError(t, err)
|
|
|
|
item, ok = m["foo/sub/baz"]
|
|
require.True(t, ok)
|
|
require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
|
|
require.Equal(t, []byte("second"), item.data)
|
|
|
|
// TODO: #154 check that the unmodified parents are still in tar
|
|
// item, ok = m["foo/"]
|
|
// require.True(t, ok)
|
|
// require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
|
|
}
|
|
|
|
func requiresLinux(t *testing.T) {
|
|
if runtime.GOOS != "linux" {
|
|
t.Skipf("unsupported GOOS: %s", runtime.GOOS)
|
|
}
|
|
}
|
|
|
|
type tarItem struct {
|
|
header *tar.Header
|
|
data []byte
|
|
}
|
|
|
|
func readTarToMap(dt []byte) (map[string]*tarItem, error) {
|
|
m := map[string]*tarItem{}
|
|
gz, err := gzip.NewReader(bytes.NewBuffer(dt))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer gz.Close()
|
|
rdr := tar.NewReader(gz)
|
|
for {
|
|
h, err := rdr.Next()
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return m, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
var dt []byte
|
|
if h.Typeflag == tar.TypeReg {
|
|
dt, err = ioutil.ReadAll(rdr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
m[h.Name] = &tarItem{header: h, data: dt}
|
|
}
|
|
}
|