diff --git a/README.md b/README.md index 4f7b7cb6..b0e5379c 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,20 @@ Running tests: make test ``` +This runs all unit and integration tests in a containerized environment. Locally, every package can be tested separately with standard Go tools but integration tests are skipped if local user doesn't have enough permissions or worker binaries are not installed. + +``` +# test a specific package only +make test TESTPKGS=./client + +# run a specific test with all worker combinations +make test TESTPKGS=./client TESTFLAGS="--run /TestCallDiskUsage -v" + +# run all integration tests with a specific worker +# supported workers are standalone and containerd +make test TESTPKGS=./client TESTFLAGS="--run //worker=containerd -v" +``` + Updating vendored dependencies: ```bash diff --git a/client/client_containerd_test.go b/client/client_containerd_test.go deleted file mode 100644 index c289e6b4..00000000 --- a/client/client_containerd_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// +build containerd - -package client - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "syscall" - "testing" - "time" -) - -func runContainerd() (string, func(), error) { - tmpdir, err := ioutil.TempDir("", "containerd") - if err != nil { - return "", nil, err - } - - address := filepath.Join(tmpdir, "containerd.sock") - - args := append([]string{}, "containerd", "--root", tmpdir, "--root", filepath.Join(tmpdir, "state"), "--address", address) - - cmd := exec.Command(args[0], args[1:]...) - // cmd.Stderr = os.Stdout - // cmd.Stdout = os.Stdout - - if err := cmd.Start(); err != nil { - os.RemoveAll(tmpdir) - return "", nil, err - } - - time.Sleep(200 * time.Millisecond) // TODO - - return address, func() { - os.RemoveAll(tmpdir) - - // tear down the daemon and resources created - if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { - fmt.Fprintln(os.Stderr, err) - } - if _, err := cmd.Process.Wait(); err != nil { - fmt.Fprintln(os.Stderr, err) - } - os.RemoveAll(tmpdir) - }, nil -} - -func setupContainerd() (func(), error) { - containerdSock, cleanupContainerd, err := runContainerd() - if err != nil { - return nil, err - } - sock, cleanup, err := runBuildd([]string{"buildd-containerd", "--containerd", containerdSock}) - if err != nil { - cleanupContainerd() - return nil, err - } - - clientAddressContainerd = sock - time.Sleep(100 * time.Millisecond) // TODO - return func() { - cleanup() - cleanupContainerd() - }, nil -} - -func TestCallDiskUsageContainerd(t *testing.T) { - testCallDiskUsage(t, clientAddressContainerd) -} - -func TestBuildMultiMountContainerd(t *testing.T) { - testBuildMultiMount(t, clientAddressContainerd) -} diff --git a/client/client_nocontainerd_test.go b/client/client_nocontainerd_test.go deleted file mode 100644 index ba5a9133..00000000 --- a/client/client_nocontainerd_test.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !standalone - -package client - -func setupStandalone() (func(), error) { - return func() {}, nil -} diff --git a/client/client_nostandalone_test.go b/client/client_nostandalone_test.go deleted file mode 100644 index dd482eff..00000000 --- a/client/client_nostandalone_test.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !containerd - -package client - -func setupContainerd() (func(), error) { - return func() {}, nil -} diff --git a/client/client_standalone_test.go b/client/client_standalone_test.go deleted file mode 100644 index d199046b..00000000 --- a/client/client_standalone_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build standalone - -package client - -import ( - "testing" - "time" -) - -func setupStandalone() (func(), error) { - sock, close, err := runBuildd([]string{"buildd-standalone"}) - if err != nil { - return nil, err - } - clientAddressStandalone = sock - time.Sleep(100 * time.Millisecond) // TODO - return close, nil -} - -func TestCallDiskUsageStandalone(t *testing.T) { - testCallDiskUsage(t, clientAddressStandalone) -} - -func TestBuildMultiMountStandalone(t *testing.T) { - testBuildMultiMount(t, clientAddressStandalone) -} diff --git a/client/client_test.go b/client/client_test.go index f9408401..9aa0cbf4 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -2,96 +2,32 @@ package client import ( "context" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" "runtime" - "syscall" "testing" "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/util/testutil/integration" "github.com/stretchr/testify/assert" ) -var clientAddressStandalone string -var clientAddressContainerd string - -func TestMain(m *testing.M) { - if testing.Short() { - os.Exit(m.Run()) - } - - cleanup, err := setupStandalone() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - defer cleanup() - - cleanup, err = setupContainerd() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - defer cleanup() - - os.Exit(m.Run()) +func TestClientIntegration(t *testing.T) { + integration.Run(t, []integration.Test{ + testCallDiskUsage, + testBuildMultiMount, + }) } -func runBuildd(args []string) (string, func(), error) { - tmpdir, err := ioutil.TempDir("", "buildd") - if err != nil { - return "", nil, err - } - defer os.RemoveAll(tmpdir) - - address := filepath.Join(tmpdir, "buildd.sock") - if runtime.GOOS == "windows" { - address = "//./pipe/buildd-" + filepath.Base(tmpdir) - } else { - address = "unix://" + address - } - - args = append(args, "--root", tmpdir, "--addr", address, "--debug") - - cmd := exec.Command(args[0], args[1:]...) - // cmd.Stderr = os.Stdout - // cmd.Stdout = os.Stdout - if err := cmd.Start(); err != nil { - return "", nil, err - } - - return address, func() { - // tear down the daemon and resources created - if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { - fmt.Fprintln(os.Stderr, err) - } - if _, err := cmd.Process.Wait(); err != nil { - fmt.Fprintln(os.Stderr, err) - } - os.RemoveAll(tmpdir) - }, nil -} - -func requiresLinux(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skipf("unsupported GOOS: %s", runtime.GOOS) - } -} - -func testCallDiskUsage(t *testing.T, address string) { - c, err := New(address) +func testCallDiskUsage(t *testing.T, sb integration.Sandbox) { + c, err := New(sb.Address()) assert.Nil(t, err) _, err = c.DiskUsage(context.TODO()) assert.Nil(t, err) } -func testBuildMultiMount(t *testing.T, address string) { +func testBuildMultiMount(t *testing.T, sb integration.Sandbox) { requiresLinux(t) t.Parallel() - c, err := New(address) + c, err := New(sb.Address()) assert.Nil(t, err) alpine := llb.Image("docker.io/library/alpine:latest") @@ -106,3 +42,9 @@ func testBuildMultiMount(t *testing.T, address string) { err = c.Solve(context.TODO(), def, SolveOpt{}, nil) assert.Nil(t, err) } + +func requiresLinux(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skipf("unsupported GOOS: %s", runtime.GOOS) + } +} diff --git a/cmd/buildctl/buildctl_test.go b/cmd/buildctl/buildctl_test.go new file mode 100644 index 00000000..b789c1d0 --- /dev/null +++ b/cmd/buildctl/buildctl_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" + + "github.com/moby/buildkit/util/testutil/integration" +) + +func TestCLIIntegration(t *testing.T) { + integration.Run(t, []integration.Test{ + testDiskUsage, + }) +} diff --git a/cmd/buildctl/diskusage_test.go b/cmd/buildctl/diskusage_test.go new file mode 100644 index 00000000..68f84591 --- /dev/null +++ b/cmd/buildctl/diskusage_test.go @@ -0,0 +1,14 @@ +package main + +import ( + "testing" + + "github.com/moby/buildkit/util/testutil/integration" + "github.com/stretchr/testify/assert" +) + +func testDiskUsage(t *testing.T, sb integration.Sandbox) { + cmd := sb.Cmd("du") + err := cmd.Run() + assert.NoError(t, err) +} diff --git a/control/control_standalone_test.go b/control/control_standalone_test.go index 6ae99097..60acbb15 100644 --- a/control/control_standalone_test.go +++ b/control/control_standalone_test.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "os" + "os/exec" "path/filepath" "testing" @@ -24,7 +25,14 @@ import ( "golang.org/x/net/context" ) -func TestControl(t *testing.T) { +func TestControlStandalone(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("requires root") + } + if _, err := exec.LookPath("runc"); err != nil { + t.Skipf("no runc found: %s", err.Error()) + } + ctx := namespaces.WithNamespace(context.Background(), "buildkit-test") // this should be an example or e2e test diff --git a/hack/dockerfiles/test.Dockerfile b/hack/dockerfiles/test.Dockerfile index 73774f63..eed3d4b3 100644 --- a/hack/dockerfiles/test.Dockerfile +++ b/hack/dockerfiles/test.Dockerfile @@ -42,6 +42,7 @@ ENV CGO_ENABLED=0 RUN go build -ldflags '-d' -o /usr/bin/buildd-containerd -tags containerd ./cmd/buildd FROM unit-tests AS integration-tests +COPY --from=buildctl /usr/bin/buildctl /usr/bin/ COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin diff --git a/hack/test b/hack/test index c77a6a63..a3466745 100755 --- a/hack/test +++ b/hack/test @@ -2,5 +2,9 @@ set -eu -o pipefail -x -./hack/test-unit -./hack/test-integration +# update this to iidfile after 17.06 +docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . + +docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone ${TESTFLAGS:--v} ${TESTPKGS:-./...} + +docker run --rm buildkit:test go build ./frontend/gateway/client diff --git a/hack/test-integration b/hack/test-integration deleted file mode 100755 index 966cf2f5..00000000 --- a/hack/test-integration +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -# update this to iidfile after 17.06 -docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . -docker run --rm -v /tmp --privileged buildkit:test go test -tags 'containerd standalone' ./client -docker run --rm buildkit:test go build ./frontend/gateway/client diff --git a/hack/test-unit b/hack/test-unit deleted file mode 100755 index 546966b1..00000000 --- a/hack/test-unit +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -# update this to iidfile after 17.06 -docker build -t buildkit:test --target unit-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . -docker run --rm -v /tmp --privileged buildkit:test go test ${TESTFLAGS:--v} ${TESTPKGS:-./...} -docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone -v ./control diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go new file mode 100644 index 00000000..6b1c3172 --- /dev/null +++ b/util/testutil/integration/containerd.go @@ -0,0 +1,74 @@ +package integration + +import ( + "bytes" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "time" +) + +func init() { + register(&containerd{}) +} + +type containerd struct { +} + +func (c *containerd) Name() string { + return "containerd" +} + +func (c *containerd) New() (sb Sandbox, cl func() error, err error) { + if err := lookupBinary("containerd"); err != nil { + return nil, nil, err + } + if err := lookupBinary("buildd-containerd"); err != nil { + return nil, nil, err + } + if err := requireRoot(); err != nil { + return nil, nil, err + } + + deferF := &multiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + tmpdir, err := ioutil.TempDir("", "bktest_containerd") + if err != nil { + return nil, nil, err + } + + deferF.append(func() error { return os.RemoveAll(tmpdir) }) + + address := filepath.Join(tmpdir, "containerd.sock") + args := append([]string{}, "containerd", "--root", filepath.Join(tmpdir, "root"), "--log-level", "debug", "--root", filepath.Join(tmpdir, "state"), "--address", address) + + cmd := exec.Command(args[0], args[1:]...) + + logs := map[string]*bytes.Buffer{} + + if stop, err := startCmd(cmd, logs); err != nil { + return nil, nil, err + } else { + deferF.append(stop) + } + if err := waitUnix(address, 5*time.Second); err != nil { + return nil, nil, err + } + + builddSock, stop, err := runBuildd([]string{"buildd-containerd", "--containerd", address}, logs) + if err != nil { + return nil, nil, err + } + deferF.append(stop) + + return &sandbox{address: builddSock, logs: logs}, cl, nil +} diff --git a/util/testutil/integration/run.go b/util/testutil/integration/run.go new file mode 100644 index 00000000..5ed363c3 --- /dev/null +++ b/util/testutil/integration/run.go @@ -0,0 +1,69 @@ +package integration + +import ( + "os/exec" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type Sandbox interface { + Address() string + PrintLogs(*testing.T) + Cmd(...string) *exec.Cmd +} + +type Worker interface { + New() (Sandbox, func() error, error) + Name() string +} + +type Test func(*testing.T, Sandbox) + +var defaultWorkers []Worker + +func register(w Worker) { + defaultWorkers = append(defaultWorkers, w) +} + +func List() []Worker { + return defaultWorkers +} + +func Run(t *testing.T, testCases []Test) { + if testing.Short() { + t.Skip("skipping in short mode") + } + for _, br := range List() { + for _, tc := range testCases { + ok := t.Run(getFunctionName(tc)+"/worker="+br.Name(), func(t *testing.T) { + sb, close, err := br.New() + if err != nil { + if errors.Cause(err) == ErrorRequirements { + t.Skip(err.Error()) + } + require.NoError(t, err) + } + defer func() { + assert.NoError(t, close()) + if t.Failed() { + sb.PrintLogs(t) + } + }() + tc(t, sb) + }) + require.True(t, ok) + } + } +} + +func getFunctionName(i interface{}) string { + fullname := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() + dot := strings.LastIndex(fullname, ".") + 1 + return strings.Title(fullname[dot:]) +} diff --git a/util/testutil/integration/standalone.go b/util/testutil/integration/standalone.go new file mode 100644 index 00000000..e3f19d89 --- /dev/null +++ b/util/testutil/integration/standalone.go @@ -0,0 +1,104 @@ +package integration + +import ( + "bufio" + "bytes" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" +) + +func init() { + register(&standalone{}) +} + +type standalone struct { +} + +func (s *standalone) Name() string { + return "standalone" +} + +func (s *standalone) New() (Sandbox, func() error, error) { + if err := lookupBinary("buildd-standalone"); err != nil { + return nil, nil, err + } + if err := requireRoot(); err != nil { + return nil, nil, err + } + logs := map[string]*bytes.Buffer{} + builddSock, stop, err := runBuildd([]string{"buildd-standalone"}, logs) + if err != nil { + return nil, nil, err + } + + return &sandbox{address: builddSock, logs: logs}, stop, nil +} + +type sandbox struct { + address string + logs map[string]*bytes.Buffer +} + +func (sb *sandbox) Address() string { + return sb.address +} + +func (sb *sandbox) PrintLogs(t *testing.T) { + for name, l := range sb.logs { + t.Log(name) + s := bufio.NewScanner(l) + for s.Scan() { + t.Log(s.Text()) + } + } +} + +func (sb *sandbox) Cmd(args ...string) *exec.Cmd { + cmd := exec.Command("buildctl", args...) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address()) + return cmd +} + +func runBuildd(args []string, logs map[string]*bytes.Buffer) (address string, cl func() error, err error) { + deferF := &multiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + tmpdir, err := ioutil.TempDir("", "bktest_buildd") + if err != nil { + return "", nil, err + } + deferF.append(func() error { return os.RemoveAll(tmpdir) }) + + address = "unix://" + filepath.Join(tmpdir, "buildd.sock") + if runtime.GOOS == "windows" { + address = "//./pipe/buildd-" + filepath.Base(tmpdir) + } + + args = append(args, "--root", tmpdir, "--addr", address, "--debug") + cmd := exec.Command(args[0], args[1:]...) + + if stop, err := startCmd(cmd, logs); err != nil { + return "", nil, err + } else { + deferF.append(stop) + } + + if err := waitUnix(address, 5*time.Second); err != nil { + return "", nil, err + } + + return +} diff --git a/util/testutil/integration/util.go b/util/testutil/integration/util.go new file mode 100644 index 00000000..136d57aa --- /dev/null +++ b/util/testutil/integration/util.go @@ -0,0 +1,128 @@ +package integration + +import ( + "bytes" + "context" + "net" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +func startCmd(cmd *exec.Cmd, logs map[string]*bytes.Buffer) (func() error, error) { + if logs != nil { + b := new(bytes.Buffer) + logs["stdout: "+cmd.Path] = b + cmd.Stdout = b + b = new(bytes.Buffer) + logs["stderr: "+cmd.Path] = b + cmd.Stderr = b + + } + + if err := cmd.Start(); err != nil { + return nil, err + } + eg, ctx := errgroup.WithContext(context.TODO()) + + stopped := make(chan struct{}) + stop := make(chan struct{}) + eg.Go(func() error { + _, err := cmd.Process.Wait() + close(stopped) + select { + case <-stop: + return nil + default: + return err + } + }) + + eg.Go(func() error { + select { + case <-ctx.Done(): + case <-stopped: + case <-stop: + cmd.Process.Signal(syscall.SIGTERM) + go func() { + select { + case <-stopped: + case <-time.After(20 * time.Second): + cmd.Process.Kill() + } + }() + } + return nil + }) + + return func() error { + close(stop) + return eg.Wait() + }, nil +} + +func waitUnix(address string, d time.Duration) error { + address = strings.TrimPrefix(address, "unix://") + addr, err := net.ResolveUnixAddr("unix", address) + if err != nil { + return err + } + + step := 50 * time.Millisecond + i := 0 + for { + if conn, err := net.DialUnix("unix", nil, addr); err == nil { + conn.Close() + break + } + i++ + if time.Duration(i)*step > d { + return errors.Errorf("failed dialing: %s", address) + } + time.Sleep(step) + } + return nil +} + +type multiCloser struct { + fns []func() error +} + +func (mc *multiCloser) F() func() error { + return func() error { + var err error + for i := range mc.fns { + if err1 := mc.fns[len(mc.fns)-1-i](); err == nil { + err = err1 + } + } + mc.fns = nil + return err + } +} + +func (mc *multiCloser) append(f func() error) { + mc.fns = append(mc.fns, f) +} + +var ErrorRequirements = errors.Errorf("missing requirements") + +func lookupBinary(name string) error { + _, err := exec.LookPath(name) + if err != nil { + return errors.Wrapf(ErrorRequirements, "failed to lookup %s binary", name) + } + return nil +} + +func requireRoot() error { + if os.Getuid() != 0 { + return errors.Wrap(ErrorRequirements, "requires root") + } + return nil +}