From e1ced023dfaaf08c5362f315471b249e89acc145 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 27 Nov 2017 14:24:29 -0800 Subject: [PATCH 1/5] update intergration tests for reuse Signed-off-by: Tonis Tiigi --- client/client_containerd_test.go | 76 -------------- client/client_nocontainerd_test.go | 7 -- client/client_nostandalone_test.go | 7 -- client/client_standalone_test.go | 26 ----- client/client_test.go | 107 +++++++------------- control/control_standalone_test.go | 10 +- util/testutil/integration/containerd.go | 74 ++++++++++++++ util/testutil/integration/list.go | 27 +++++ util/testutil/integration/standalone.go | 99 ++++++++++++++++++ util/testutil/integration/util.go | 128 ++++++++++++++++++++++++ 10 files changed, 375 insertions(+), 186 deletions(-) delete mode 100644 client/client_containerd_test.go delete mode 100644 client/client_nocontainerd_test.go delete mode 100644 client/client_nostandalone_test.go delete mode 100644 client/client_standalone_test.go create mode 100644 util/testutil/integration/containerd.go create mode 100644 util/testutil/integration/list.go create mode 100644 util/testutil/integration/standalone.go create mode 100644 util/testutil/integration/util.go diff --git a/client/client_containerd_test.go b/client/client_containerd_test.go deleted file mode 100644 index c289e6b4..00000000 --- a/client/client_containerd_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// +build containerd - -package client - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "syscall" - "testing" - "time" -) - -func runContainerd() (string, func(), error) { - tmpdir, err := ioutil.TempDir("", "containerd") - if err != nil { - return "", nil, err - } - - address := filepath.Join(tmpdir, "containerd.sock") - - args := append([]string{}, "containerd", "--root", tmpdir, "--root", filepath.Join(tmpdir, "state"), "--address", address) - - cmd := exec.Command(args[0], args[1:]...) - // cmd.Stderr = os.Stdout - // cmd.Stdout = os.Stdout - - if err := cmd.Start(); err != nil { - os.RemoveAll(tmpdir) - return "", nil, err - } - - time.Sleep(200 * time.Millisecond) // TODO - - return address, func() { - os.RemoveAll(tmpdir) - - // tear down the daemon and resources created - if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { - fmt.Fprintln(os.Stderr, err) - } - if _, err := cmd.Process.Wait(); err != nil { - fmt.Fprintln(os.Stderr, err) - } - os.RemoveAll(tmpdir) - }, nil -} - -func setupContainerd() (func(), error) { - containerdSock, cleanupContainerd, err := runContainerd() - if err != nil { - return nil, err - } - sock, cleanup, err := runBuildd([]string{"buildd-containerd", "--containerd", containerdSock}) - if err != nil { - cleanupContainerd() - return nil, err - } - - clientAddressContainerd = sock - time.Sleep(100 * time.Millisecond) // TODO - return func() { - cleanup() - cleanupContainerd() - }, nil -} - -func TestCallDiskUsageContainerd(t *testing.T) { - testCallDiskUsage(t, clientAddressContainerd) -} - -func TestBuildMultiMountContainerd(t *testing.T) { - testBuildMultiMount(t, clientAddressContainerd) -} diff --git a/client/client_nocontainerd_test.go b/client/client_nocontainerd_test.go deleted file mode 100644 index ba5a9133..00000000 --- a/client/client_nocontainerd_test.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !standalone - -package client - -func setupStandalone() (func(), error) { - return func() {}, nil -} diff --git a/client/client_nostandalone_test.go b/client/client_nostandalone_test.go deleted file mode 100644 index dd482eff..00000000 --- a/client/client_nostandalone_test.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !containerd - -package client - -func setupContainerd() (func(), error) { - return func() {}, nil -} diff --git a/client/client_standalone_test.go b/client/client_standalone_test.go deleted file mode 100644 index d199046b..00000000 --- a/client/client_standalone_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build standalone - -package client - -import ( - "testing" - "time" -) - -func setupStandalone() (func(), error) { - sock, close, err := runBuildd([]string{"buildd-standalone"}) - if err != nil { - return nil, err - } - clientAddressStandalone = sock - time.Sleep(100 * time.Millisecond) // TODO - return close, nil -} - -func TestCallDiskUsageStandalone(t *testing.T) { - testCallDiskUsage(t, clientAddressStandalone) -} - -func TestBuildMultiMountStandalone(t *testing.T) { - testBuildMultiMount(t, clientAddressStandalone) -} diff --git a/client/client_test.go b/client/client_test.go index f9408401..cc70ae9c 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -2,96 +2,59 @@ package client import ( "context" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" "runtime" - "syscall" "testing" "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -var clientAddressStandalone string -var clientAddressContainerd string +var testCases = map[string]integration.Test{ + "TestCallDiskUsage": testCallDiskUsage, + "TestBuildMultiMount": testBuildMultiMount, +} -func TestMain(m *testing.M) { +func TestClientIntegration(t *testing.T) { if testing.Short() { - os.Exit(m.Run()) + t.Skip("skipping in short mode") } - - cleanup, err := setupStandalone() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - defer cleanup() - - cleanup, err = setupContainerd() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - defer cleanup() - - os.Exit(m.Run()) -} - -func runBuildd(args []string) (string, func(), error) { - tmpdir, err := ioutil.TempDir("", "buildd") - if err != nil { - return "", nil, err - } - defer os.RemoveAll(tmpdir) - - address := filepath.Join(tmpdir, "buildd.sock") - if runtime.GOOS == "windows" { - address = "//./pipe/buildd-" + filepath.Base(tmpdir) - } else { - address = "unix://" + address - } - - args = append(args, "--root", tmpdir, "--addr", address, "--debug") - - cmd := exec.Command(args[0], args[1:]...) - // cmd.Stderr = os.Stdout - // cmd.Stdout = os.Stdout - if err := cmd.Start(); err != nil { - return "", nil, err - } - - return address, func() { - // tear down the daemon and resources created - if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { - fmt.Fprintln(os.Stderr, err) + for _, br := range integration.List() { + for name, tc := range testCases { + ok := t.Run(name+"/worker="+br.Name(), func(t *testing.T) { + sb, close, err := br.New() + if err != nil { + if errors.Cause(err) == integration.ErrorRequirements { + t.Skip(err.Error()) + } + require.NoError(t, err) + } + defer func() { + assert.NoError(t, close()) + if t.Failed() { + sb.PrintLogs(t) + } + }() + tc(t, sb) + }) + require.True(t, ok) } - if _, err := cmd.Process.Wait(); err != nil { - fmt.Fprintln(os.Stderr, err) - } - os.RemoveAll(tmpdir) - }, nil -} - -func requiresLinux(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skipf("unsupported GOOS: %s", runtime.GOOS) } } -func testCallDiskUsage(t *testing.T, address string) { - c, err := New(address) +func testCallDiskUsage(t *testing.T, sb integration.Sandbox) { + c, err := New(sb.Address()) assert.Nil(t, err) _, err = c.DiskUsage(context.TODO()) assert.Nil(t, err) } -func testBuildMultiMount(t *testing.T, address string) { +func testBuildMultiMount(t *testing.T, sb integration.Sandbox) { requiresLinux(t) t.Parallel() - c, err := New(address) + c, err := New(sb.Address()) assert.Nil(t, err) alpine := llb.Image("docker.io/library/alpine:latest") @@ -106,3 +69,9 @@ func testBuildMultiMount(t *testing.T, address string) { err = c.Solve(context.TODO(), def, SolveOpt{}, nil) assert.Nil(t, err) } + +func requiresLinux(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skipf("unsupported GOOS: %s", runtime.GOOS) + } +} diff --git a/control/control_standalone_test.go b/control/control_standalone_test.go index 6ae99097..60acbb15 100644 --- a/control/control_standalone_test.go +++ b/control/control_standalone_test.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "os" + "os/exec" "path/filepath" "testing" @@ -24,7 +25,14 @@ import ( "golang.org/x/net/context" ) -func TestControl(t *testing.T) { +func TestControlStandalone(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("requires root") + } + if _, err := exec.LookPath("runc"); err != nil { + t.Skipf("no runc found: %s", err.Error()) + } + ctx := namespaces.WithNamespace(context.Background(), "buildkit-test") // this should be an example or e2e test diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go new file mode 100644 index 00000000..7589d0bb --- /dev/null +++ b/util/testutil/integration/containerd.go @@ -0,0 +1,74 @@ +package integration + +import ( + "bytes" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "time" +) + +func init() { + register(&containerd{}) +} + +type containerd struct { +} + +func (c *containerd) Name() string { + return "containerd" +} + +func (c *containerd) New() (sb Sandbox, cl func() error, err error) { + if err := lookupBinary("containerd"); err != nil { + return nil, nil, err + } + if err := lookupBinary("buildd-containerd"); err != nil { + return nil, nil, err + } + if err := requireRoot(); err != nil { + return nil, nil, err + } + + deferF := &multiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + tmpdir, err := ioutil.TempDir("", "bktest_containerd") + if err != nil { + return nil, nil, err + } + + deferF.append(func() error { return os.RemoveAll(tmpdir) }) + + address := filepath.Join(tmpdir, "containerd.sock") + args := append([]string{}, "containerd", "--root", filepath.Join(tmpdir, "root"), "--root", filepath.Join(tmpdir, "state"), "--address", address) + + cmd := exec.Command(args[0], args[1:]...) + + logs := map[string]*bytes.Buffer{} + + if stop, err := startCmd(cmd, logs); err != nil { + return nil, nil, err + } else { + deferF.append(stop) + } + if err := waitUnix(address, 5*time.Second); err != nil { + return nil, nil, err + } + + builddSock, stop, err := runBuildd([]string{"buildd-containerd", "--containerd", address}, logs) + if err != nil { + return nil, nil, err + } + deferF.append(stop) + + return &sandbox{address: builddSock, logs: logs}, cl, nil +} diff --git a/util/testutil/integration/list.go b/util/testutil/integration/list.go new file mode 100644 index 00000000..5ad3240d --- /dev/null +++ b/util/testutil/integration/list.go @@ -0,0 +1,27 @@ +package integration + +import ( + "testing" +) + +type Sandbox interface { + Address() string + PrintLogs(t *testing.T) +} + +type Worker interface { + New() (Sandbox, func() error, error) + Name() string +} + +type Test func(*testing.T, Sandbox) + +var defaultWorkers []Worker + +func register(w Worker) { + defaultWorkers = append(defaultWorkers, w) +} + +func List() []Worker { + return defaultWorkers +} diff --git a/util/testutil/integration/standalone.go b/util/testutil/integration/standalone.go new file mode 100644 index 00000000..1c369ff9 --- /dev/null +++ b/util/testutil/integration/standalone.go @@ -0,0 +1,99 @@ +package integration + +import ( + "bufio" + "bytes" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" +) + +func init() { + register(&standalone{}) +} + +type standalone struct { +} + +func (s *standalone) Name() string { + return "standalone" +} + +func (s *standalone) New() (Sandbox, func() error, error) { + if err := lookupBinary("buildd-standalone"); err != nil { + return nil, nil, err + } + if err := requireRoot(); err != nil { + return nil, nil, err + } + logs := map[string]*bytes.Buffer{} + builddSock, stop, err := runBuildd([]string{"buildd-standalone"}, logs) + if err != nil { + return nil, nil, err + } + + return &sandbox{address: builddSock, logs: logs}, stop, nil +} + +type sandbox struct { + address string + logs map[string]*bytes.Buffer +} + +func (sb *sandbox) Address() string { + return sb.address +} + +func (sb *sandbox) PrintLogs(t *testing.T) { + for name, l := range sb.logs { + t.Log(name) + s := bufio.NewScanner(l) + for s.Scan() { + t.Log(s.Text()) + } + } +} + +func runBuildd(args []string, logs map[string]*bytes.Buffer) (address string, cl func() error, err error) { + deferF := &multiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + tmpdir, err := ioutil.TempDir("", "bktest_buildd") + if err != nil { + return "", nil, err + } + deferF.append(func() error { return os.RemoveAll(tmpdir) }) + + address = filepath.Join(tmpdir, "buildd.sock") + if runtime.GOOS == "windows" { + address = "//./pipe/buildd-" + filepath.Base(tmpdir) + } else { + address = "unix://" + address + } + + args = append(args, "--root", tmpdir, "--addr", address, "--debug") + cmd := exec.Command(args[0], args[1:]...) + + if stop, err := startCmd(cmd, logs); err != nil { + return "", nil, err + } else { + deferF.append(stop) + } + + if err := waitUnix(address, 5*time.Second); err != nil { + return "", nil, err + } + + return +} diff --git a/util/testutil/integration/util.go b/util/testutil/integration/util.go new file mode 100644 index 00000000..136d57aa --- /dev/null +++ b/util/testutil/integration/util.go @@ -0,0 +1,128 @@ +package integration + +import ( + "bytes" + "context" + "net" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +func startCmd(cmd *exec.Cmd, logs map[string]*bytes.Buffer) (func() error, error) { + if logs != nil { + b := new(bytes.Buffer) + logs["stdout: "+cmd.Path] = b + cmd.Stdout = b + b = new(bytes.Buffer) + logs["stderr: "+cmd.Path] = b + cmd.Stderr = b + + } + + if err := cmd.Start(); err != nil { + return nil, err + } + eg, ctx := errgroup.WithContext(context.TODO()) + + stopped := make(chan struct{}) + stop := make(chan struct{}) + eg.Go(func() error { + _, err := cmd.Process.Wait() + close(stopped) + select { + case <-stop: + return nil + default: + return err + } + }) + + eg.Go(func() error { + select { + case <-ctx.Done(): + case <-stopped: + case <-stop: + cmd.Process.Signal(syscall.SIGTERM) + go func() { + select { + case <-stopped: + case <-time.After(20 * time.Second): + cmd.Process.Kill() + } + }() + } + return nil + }) + + return func() error { + close(stop) + return eg.Wait() + }, nil +} + +func waitUnix(address string, d time.Duration) error { + address = strings.TrimPrefix(address, "unix://") + addr, err := net.ResolveUnixAddr("unix", address) + if err != nil { + return err + } + + step := 50 * time.Millisecond + i := 0 + for { + if conn, err := net.DialUnix("unix", nil, addr); err == nil { + conn.Close() + break + } + i++ + if time.Duration(i)*step > d { + return errors.Errorf("failed dialing: %s", address) + } + time.Sleep(step) + } + return nil +} + +type multiCloser struct { + fns []func() error +} + +func (mc *multiCloser) F() func() error { + return func() error { + var err error + for i := range mc.fns { + if err1 := mc.fns[len(mc.fns)-1-i](); err == nil { + err = err1 + } + } + mc.fns = nil + return err + } +} + +func (mc *multiCloser) append(f func() error) { + mc.fns = append(mc.fns, f) +} + +var ErrorRequirements = errors.Errorf("missing requirements") + +func lookupBinary(name string) error { + _, err := exec.LookPath(name) + if err != nil { + return errors.Wrapf(ErrorRequirements, "failed to lookup %s binary", name) + } + return nil +} + +func requireRoot() error { + if os.Getuid() != 0 { + return errors.Wrap(ErrorRequirements, "requires root") + } + return nil +} From 9450a8b71469945e63ad4239c417d1ff677b4455 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 29 Nov 2017 17:41:44 -0800 Subject: [PATCH 2/5] hack: update test script Signed-off-by: Tonis Tiigi --- examples/buildkit0/buildkit.go | 1 + hack/dockerfiles/test.Dockerfile | 1 + hack/test | 8 ++++++-- hack/test-integration | 8 -------- hack/test-unit | 8 -------- 5 files changed, 8 insertions(+), 18 deletions(-) delete mode 100755 hack/test-integration delete mode 100755 hack/test-unit diff --git a/examples/buildkit0/buildkit.go b/examples/buildkit0/buildkit.go index 6b87eb34..22075bbf 100644 --- a/examples/buildkit0/buildkit.go +++ b/examples/buildkit0/buildkit.go @@ -1,3 +1,4 @@ +// +build ignore package main import ( diff --git a/hack/dockerfiles/test.Dockerfile b/hack/dockerfiles/test.Dockerfile index 73774f63..eed3d4b3 100644 --- a/hack/dockerfiles/test.Dockerfile +++ b/hack/dockerfiles/test.Dockerfile @@ -42,6 +42,7 @@ ENV CGO_ENABLED=0 RUN go build -ldflags '-d' -o /usr/bin/buildd-containerd -tags containerd ./cmd/buildd FROM unit-tests AS integration-tests +COPY --from=buildctl /usr/bin/buildctl /usr/bin/ COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin diff --git a/hack/test b/hack/test index c77a6a63..a3466745 100755 --- a/hack/test +++ b/hack/test @@ -2,5 +2,9 @@ set -eu -o pipefail -x -./hack/test-unit -./hack/test-integration +# update this to iidfile after 17.06 +docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . + +docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone ${TESTFLAGS:--v} ${TESTPKGS:-./...} + +docker run --rm buildkit:test go build ./frontend/gateway/client diff --git a/hack/test-integration b/hack/test-integration deleted file mode 100755 index 966cf2f5..00000000 --- a/hack/test-integration +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -# update this to iidfile after 17.06 -docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . -docker run --rm -v /tmp --privileged buildkit:test go test -tags 'containerd standalone' ./client -docker run --rm buildkit:test go build ./frontend/gateway/client diff --git a/hack/test-unit b/hack/test-unit deleted file mode 100755 index 546966b1..00000000 --- a/hack/test-unit +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -# update this to iidfile after 17.06 -docker build -t buildkit:test --target unit-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . -docker run --rm -v /tmp --privileged buildkit:test go test ${TESTFLAGS:--v} ${TESTPKGS:-./...} -docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone -v ./control From f343fc30ecd61722bf4453c4e6244b39a0902d00 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 29 Nov 2017 18:15:01 -0800 Subject: [PATCH 3/5] buildctl: add sample cli test Signed-off-by: Tonis Tiigi --- client/client_test.go | 27 +---------- cmd/buildctl/buildctl_test.go | 15 +++++++ cmd/buildctl/diskusage_test.go | 14 ++++++ examples/buildkit0/buildkit.go | 1 - util/testutil/integration/containerd.go | 2 +- util/testutil/integration/list.go | 27 ----------- util/testutil/integration/run.go | 60 +++++++++++++++++++++++++ util/testutil/integration/standalone.go | 11 +++-- 8 files changed, 99 insertions(+), 58 deletions(-) create mode 100644 cmd/buildctl/buildctl_test.go create mode 100644 cmd/buildctl/diskusage_test.go delete mode 100644 util/testutil/integration/list.go create mode 100644 util/testutil/integration/run.go diff --git a/client/client_test.go b/client/client_test.go index cc70ae9c..8f859c77 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -7,9 +7,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/util/testutil/integration" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var testCases = map[string]integration.Test{ @@ -18,30 +16,7 @@ var testCases = map[string]integration.Test{ } func TestClientIntegration(t *testing.T) { - if testing.Short() { - t.Skip("skipping in short mode") - } - for _, br := range integration.List() { - for name, tc := range testCases { - ok := t.Run(name+"/worker="+br.Name(), func(t *testing.T) { - sb, close, err := br.New() - if err != nil { - if errors.Cause(err) == integration.ErrorRequirements { - t.Skip(err.Error()) - } - require.NoError(t, err) - } - defer func() { - assert.NoError(t, close()) - if t.Failed() { - sb.PrintLogs(t) - } - }() - tc(t, sb) - }) - require.True(t, ok) - } - } + integration.Run(t, testCases) } func testCallDiskUsage(t *testing.T, sb integration.Sandbox) { diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go new file mode 100644 index 00000000..f347cfe5 --- /dev/null +++ b/cmd/buildctl/buildctl_test.go @@ -0,0 +1,15 @@ +package main + +import ( + "testing" + + "github.com/moby/buildkit/util/testutil/integration" +) + +var testCases = map[string]integration.Test{ + "TestDiskUsage": testDiskUsage, +} + +func TestCLIIntegration(t *testing.T) { + integration.Run(t, testCases) +} diff --git a/cmd/buildctl/diskusage_test.go b/cmd/buildctl/diskusage_test.go new file mode 100644 index 00000000..68f84591 --- /dev/null +++ b/cmd/buildctl/diskusage_test.go @@ -0,0 +1,14 @@ +package main + +import ( + "testing" + + "github.com/moby/buildkit/util/testutil/integration" + "github.com/stretchr/testify/assert" +) + +func testDiskUsage(t *testing.T, sb integration.Sandbox) { + cmd := sb.Cmd("du") + err := cmd.Run() + assert.NoError(t, err) +} diff --git a/examples/buildkit0/buildkit.go b/examples/buildkit0/buildkit.go index 22075bbf..6b87eb34 100644 --- a/examples/buildkit0/buildkit.go +++ b/examples/buildkit0/buildkit.go @@ -1,4 +1,3 @@ -// +build ignore package main import ( diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go index 7589d0bb..6b1c3172 100644 --- a/util/testutil/integration/containerd.go +++ b/util/testutil/integration/containerd.go @@ -49,7 +49,7 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) { deferF.append(func() error { return os.RemoveAll(tmpdir) }) address := filepath.Join(tmpdir, "containerd.sock") - args := append([]string{}, "containerd", "--root", filepath.Join(tmpdir, "root"), "--root", filepath.Join(tmpdir, "state"), "--address", address) + args := append([]string{}, "containerd", "--root", filepath.Join(tmpdir, "root"), "--log-level", "debug", "--root", filepath.Join(tmpdir, "state"), "--address", address) cmd := exec.Command(args[0], args[1:]...) diff --git a/util/testutil/integration/list.go b/util/testutil/integration/list.go deleted file mode 100644 index 5ad3240d..00000000 --- a/util/testutil/integration/list.go +++ /dev/null @@ -1,27 +0,0 @@ -package integration - -import ( - "testing" -) - -type Sandbox interface { - Address() string - PrintLogs(t *testing.T) -} - -type Worker interface { - New() (Sandbox, func() error, error) - Name() string -} - -type Test func(*testing.T, Sandbox) - -var defaultWorkers []Worker - -func register(w Worker) { - defaultWorkers = append(defaultWorkers, w) -} - -func List() []Worker { - return defaultWorkers -} diff --git a/util/testutil/integration/run.go b/util/testutil/integration/run.go new file mode 100644 index 00000000..e00a03f8 --- /dev/null +++ b/util/testutil/integration/run.go @@ -0,0 +1,60 @@ +package integration + +import ( + "os/exec" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type Sandbox interface { + Address() string + PrintLogs(*testing.T) + Cmd(...string) *exec.Cmd +} + +type Worker interface { + New() (Sandbox, func() error, error) + Name() string +} + +type Test func(*testing.T, Sandbox) + +var defaultWorkers []Worker + +func register(w Worker) { + defaultWorkers = append(defaultWorkers, w) +} + +func List() []Worker { + return defaultWorkers +} + +func Run(t *testing.T, testCases map[string]Test) { + if testing.Short() { + t.Skip("skipping in short mode") + } + for _, br := range List() { + for name, tc := range testCases { + ok := t.Run(name+"/worker="+br.Name(), func(t *testing.T) { + sb, close, err := br.New() + if err != nil { + if errors.Cause(err) == ErrorRequirements { + t.Skip(err.Error()) + } + require.NoError(t, err) + } + defer func() { + assert.NoError(t, close()) + if t.Failed() { + sb.PrintLogs(t) + } + }() + tc(t, sb) + }) + require.True(t, ok) + } + } +} diff --git a/util/testutil/integration/standalone.go b/util/testutil/integration/standalone.go index 1c369ff9..e3f19d89 100644 --- a/util/testutil/integration/standalone.go +++ b/util/testutil/integration/standalone.go @@ -58,6 +58,13 @@ func (sb *sandbox) PrintLogs(t *testing.T) { } } +func (sb *sandbox) Cmd(args ...string) *exec.Cmd { + cmd := exec.Command("buildctl", args...) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address()) + return cmd +} + func runBuildd(args []string, logs map[string]*bytes.Buffer) (address string, cl func() error, err error) { deferF := &multiCloser{} cl = deferF.F() @@ -75,11 +82,9 @@ func runBuildd(args []string, logs map[string]*bytes.Buffer) (address string, cl } deferF.append(func() error { return os.RemoveAll(tmpdir) }) - address = filepath.Join(tmpdir, "buildd.sock") + address = "unix://" + filepath.Join(tmpdir, "buildd.sock") if runtime.GOOS == "windows" { address = "//./pipe/buildd-" + filepath.Base(tmpdir) - } else { - address = "unix://" + address } args = append(args, "--root", tmpdir, "--addr", address, "--debug") From e3f85cc1ac1a9474b194e669410fab98a25bd2e3 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 29 Nov 2017 18:49:04 -0800 Subject: [PATCH 4/5] integration: detect test names with reflect Signed-off-by: Tonis Tiigi --- client/client_test.go | 10 ++++------ cmd/buildctl/buildctl_test.go | 8 +++----- util/testutil/integration/run.go | 15 ++++++++++++--- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 8f859c77..9aa0cbf4 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -10,13 +10,11 @@ import ( "github.com/stretchr/testify/assert" ) -var testCases = map[string]integration.Test{ - "TestCallDiskUsage": testCallDiskUsage, - "TestBuildMultiMount": testBuildMultiMount, -} - func TestClientIntegration(t *testing.T) { - integration.Run(t, testCases) + integration.Run(t, []integration.Test{ + testCallDiskUsage, + testBuildMultiMount, + }) } func testCallDiskUsage(t *testing.T, sb integration.Sandbox) { diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go index f347cfe5..b789c1d0 100644 --- a/cmd/buildctl/buildctl_test.go +++ b/cmd/buildctl/buildctl_test.go @@ -6,10 +6,8 @@ import ( "github.com/moby/buildkit/util/testutil/integration" ) -var testCases = map[string]integration.Test{ - "TestDiskUsage": testDiskUsage, -} - func TestCLIIntegration(t *testing.T) { - integration.Run(t, testCases) + integration.Run(t, []integration.Test{ + testDiskUsage, + }) } diff --git a/util/testutil/integration/run.go b/util/testutil/integration/run.go index e00a03f8..5ed363c3 100644 --- a/util/testutil/integration/run.go +++ b/util/testutil/integration/run.go @@ -2,6 +2,9 @@ package integration import ( "os/exec" + "reflect" + "runtime" + "strings" "testing" "github.com/pkg/errors" @@ -32,13 +35,13 @@ func List() []Worker { return defaultWorkers } -func Run(t *testing.T, testCases map[string]Test) { +func Run(t *testing.T, testCases []Test) { if testing.Short() { t.Skip("skipping in short mode") } for _, br := range List() { - for name, tc := range testCases { - ok := t.Run(name+"/worker="+br.Name(), func(t *testing.T) { + for _, tc := range testCases { + ok := t.Run(getFunctionName(tc)+"/worker="+br.Name(), func(t *testing.T) { sb, close, err := br.New() if err != nil { if errors.Cause(err) == ErrorRequirements { @@ -58,3 +61,9 @@ func Run(t *testing.T, testCases map[string]Test) { } } } + +func getFunctionName(i interface{}) string { + fullname := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() + dot := strings.LastIndex(fullname, ".") + 1 + return strings.Title(fullname[dot:]) +} From 0fbb6d22399a40a23cb529f6f13d57e83aacf162 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 29 Nov 2017 21:04:04 -0800 Subject: [PATCH 5/5] Update testing info in readme Signed-off-by: Tonis Tiigi --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 4f7b7cb6..b0e5379c 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,20 @@ Running tests: make test ``` +This runs all unit and integration tests in a containerized environment. Locally, every package can be tested separately with standard Go tools but integration tests are skipped if local user doesn't have enough permissions or worker binaries are not installed. + +``` +# test a specific package only +make test TESTPKGS=./client + +# run a specific test with all worker combinations +make test TESTPKGS=./client TESTFLAGS="--run /TestCallDiskUsage -v" + +# run all integration tests with a specific worker +# supported workers are standalone and containerd +make test TESTPKGS=./client TESTFLAGS="--run //worker=containerd -v" +``` + Updating vendored dependencies: ```bash