client: add tests for differ and pushed data

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

client/client_test.go

@@ -1,19 +1,31 @@
 package client
 
 import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
 	"context"
+	"encoding/json"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/docker/distribution/manifest/schema2"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/testutil/httpserver"
 	"github.com/moby/buildkit/util/testutil/integration"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
 )
 
@@ -22,6 +34,7 @@ func TestClientIntegration(t *testing.T) {
 		testCallDiskUsage,
 		testBuildMultiMount,
 		testBuildHTTPSource,
+		testBuildPushAndValidate,
 	})
 }
 
@@ -143,8 +156,201 @@ func testBuildHTTPSource(t *testing.T, sb integration.Sandbox) {
 	// TODO: check that second request was marked as cached
 }
 
+func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	t.Parallel()
+	c, err := New(sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	busybox := llb.Image("busybox:latest")
+	st := llb.Scratch()
+
+	run := func(cmd string) {
+		st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+	}
+
+	run(`sh -c "mkdir -p foo/sub; echo -n first > foo/sub/bar; chmod 0741 foo;"`)
+	run(`sh -c "echo -n second > foo/sub/baz"`)
+
+	def, err := st.Marshal()
+	require.NoError(t, err)
+
+	registry, err := sb.NewRegistry()
+	if errors.Cause(err) == integration.ErrorRequirements {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
+	target := registry + "/buildkit/testpush:latest"
+
+	err = c.Solve(context.TODO(), def, SolveOpt{
+		Exporter: ExporterImage,
+		ExporterAttrs: map[string]string{
+			"name": target,
+			"push": "true",
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	// test existence of the image with next build
+	firstBuild := llb.Image(target)
+
+	def, err = firstBuild.Marshal()
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+
+	err = c.Solve(context.TODO(), def, SolveOpt{
+		Exporter: ExporterLocal,
+		ExporterAttrs: map[string]string{
+			"output": destDir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo/sub/bar"))
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("first"))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo/sub/baz"))
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("second"))
+
+	fi, err := os.Stat(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, 0741, int(fi.Mode()&0777))
+
+	// examine contents of exported tars (requires containerd)
+	var cdAddress string
+	if cd, ok := sb.(interface {
+		ContainerdAddress() string
+	}); !ok {
+		return
+	} else {
+		cdAddress = cd.ContainerdAddress()
+	}
+
+	// TODO: make public pull helper function so this can be checked for standalone as well
+
+	client, err := containerd.New(cdAddress)
+	require.NoError(t, err)
+	defer client.Close()
+
+	ctx := namespaces.WithNamespace(context.Background(), "buildkit")
+
+	img, err := client.Pull(ctx, target)
+	require.NoError(t, err)
+
+	desc, err := img.Config(ctx)
+	require.NoError(t, err)
+
+	dt, err = content.ReadBlob(ctx, img.ContentStore(), desc.Digest)
+	require.NoError(t, err)
+
+	var ociimg ocispec.Image
+	err = json.Unmarshal(dt, &ociimg)
+	require.NoError(t, err)
+
+	require.NotEqual(t, "", ociimg.OS)
+	require.NotEqual(t, "", ociimg.Architecture)
+	require.NotEqual(t, "", ociimg.Config.WorkingDir)
+	require.Equal(t, "layers", ociimg.RootFS.Type)
+	require.Equal(t, 2, len(ociimg.RootFS.DiffIDs))
+	require.Condition(t, func() bool {
+		for _, env := range ociimg.Config.Env {
+			if strings.HasPrefix(env, "PATH=") {
+				return true
+			}
+		}
+		return false
+	})
+
+	dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target().Digest)
+	require.NoError(t, err)
+
+	var mfst schema2.Manifest
+	err = json.Unmarshal(dt, &mfst)
+	require.NoError(t, err)
+
+	require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
+	require.Equal(t, 2, len(mfst.Layers))
+
+	dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[0].Digest)
+	require.NoError(t, err)
+
+	m, err := readTarToMap(dt)
+	require.NoError(t, err)
+
+	item, ok := m["foo/"]
+	require.True(t, ok)
+	require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
+	require.Equal(t, 0741, int(item.header.Mode&0777))
+
+	item, ok = m["foo/sub/"]
+	require.True(t, ok)
+	require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
+
+	item, ok = m["foo/sub/bar"]
+	require.True(t, ok)
+	require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
+	require.Equal(t, []byte("first"), item.data)
+
+	_, ok = m["foo/sub/baz"]
+	require.False(t, ok)
+
+	dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[1].Digest)
+	require.NoError(t, err)
+
+	m, err = readTarToMap(dt)
+	require.NoError(t, err)
+
+	item, ok = m["foo/sub/baz"]
+	require.True(t, ok)
+	require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
+	require.Equal(t, []byte("second"), item.data)
+
+	// TODO: #154 check that the unmodified parents are still in tar
+	// item, ok = m["foo/"]
+	// require.True(t, ok)
+	// require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
+}
+
 func requiresLinux(t *testing.T) {
 	if runtime.GOOS != "linux" {
 		t.Skipf("unsupported GOOS: %s", runtime.GOOS)
 	}
 }
+
+type tarItem struct {
+	header *tar.Header
+	data   []byte
+}
+
+func readTarToMap(dt []byte) (map[string]*tarItem, error) {
+	m := map[string]*tarItem{}
+	gz, err := gzip.NewReader(bytes.NewBuffer(dt))
+	if err != nil {
+		return nil, err
+	}
+	defer gz.Close()
+	rdr := tar.NewReader(gz)
+	for {
+		h, err := rdr.Next()
+		if err != nil {
+			if err == io.EOF {
+				return m, nil
+			}
+			return nil, err
+		}
+		var dt []byte
+		if h.Typeflag == tar.TypeReg {
+			dt, err = ioutil.ReadAll(rdr)
+			if err != nil {
+				return nil, err
+			}
+		}
+		m[h.Name] = &tarItem{header: h, data: dt}
+	}
+}