JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

all: factor only required methods out of Sandbox 

Signed-off-by: Sam Whited <sam@samwhited.com>
v0.7
Sam Whited 2019-08-23 14:21:34 -07:00
parent df7e93a159
commit e90be385b4
5 changed files with 249 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
util/testutil/integration/containerd.go
View File

@ -49,12 +49,7 @@ func (c *containerd) Name() string {
return c.name
}
func (c *containerd) New(opt ...SandboxOpt) (sb Sandbox, cl func() error, err error) {
var conf SandboxConf
for _, o := range opt {
o(&conf)
}
func (c *containerd) New(cfg *BackendConfig) (b Backend, cl func() error, err error) {
if err := lookupBinary(c.containerd); err != nil {
return nil, nil, err
}
@ -110,15 +105,13 @@ disabled_plugins = ["cri"]
cmd := exec.Command(c.containerd, "--config", configFile)
logs := map[string]*bytes.Buffer{}
ctdStop, err := startCmd(cmd, logs)
ctdStop, err := startCmd(cmd, cfg.Logs)
if err != nil {
return nil, nil, err
}
if err := waitUnix(address, 5*time.Second); err != nil {
ctdStop()
return nil, nil, errors.Wrapf(err, "containerd did not start up: %s", formatLogs(logs))
return nil, nil, errors.Wrapf(err, "containerd did not start up: %s", formatLogs(cfg.Logs))
}
deferF.append(ctdStop)
@ -130,37 +123,19 @@ disabled_plugins = ["cri"]
"--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603
}
var upt []ConfigUpdater
for _, v := range conf.mv.values {
if u, ok := v.value.(ConfigUpdater); ok {
upt = append(upt, u)
}
}
if conf.mirror != "" {
upt = append(upt, withMirrorConfig(conf.mirror))
}
if len(upt) > 0 {
dir, err := writeConfig(upt)
if err != nil {
return nil, nil, err
}
deferF.append(func() error {
return os.RemoveAll(dir)
})
buildkitdArgs = append(buildkitdArgs, "--config="+filepath.Join(dir, "buildkitd.toml"))
}
buildkitdSock, stop, err := runBuildkitd(buildkitdArgs, logs, 0, 0)
buildkitdSock, stop, err := runBuildkitd(cfg, buildkitdArgs, cfg.Logs, 0, 0)
if err != nil {
printLogs(logs, log.Println)
printLogs(cfg.Logs, log.Println)
return nil, nil, err
}
deferF.append(stop)
return &cdsandbox{address: address, sandbox: sandbox{mv: conf.mv, address: buildkitdSock, logs: logs, cleanup: deferF, rootless: false}}, cl, nil
return cdbackend{
containerdAddress: address,
backend: backend{
address: buildkitdSock,
rootless: false,
}}, cl, nil
}
func formatLogs(m map[string]*bytes.Buffer) string {
@ -173,11 +148,11 @@ func formatLogs(m map[string]*bytes.Buffer) string {
return strings.Join(ss, ",")
}
type cdsandbox struct {
sandbox
address string
type cdbackend struct {
backend
containerdAddress string
}
func (s *cdsandbox) ContainerdAddress() string {
return s.address
func (s cdbackend) ContainerdAddress() string {
return s.containerdAddress
}

178
util/testutil/integration/oci.go
View File

@ -4,18 +4,9 @@ import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/google/shlex"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -48,65 +39,34 @@ func (s *oci) Name() string {
return "oci"
}
func (s *oci) New(opt ...SandboxOpt) (Sandbox, func() error, error) {
var c SandboxConf
for _, o := range opt {
o(&c)
}
func (s *oci) New(cfg *BackendConfig) (Backend, func() error, error) {
if err := lookupBinary("buildkitd"); err != nil {
return nil, nil, err
}
if err := requireRoot(); err != nil {
return nil, nil, err
}
logs := map[string]*bytes.Buffer{}
// Include use of --oci-worker-labels to trigger https://github.com/moby/buildkit/pull/603
buildkitdArgs := []string{"buildkitd", "--oci-worker=true", "--containerd-worker=false", "--oci-worker-gc=false", "--oci-worker-labels=org.mobyproject.buildkit.worker.sandbox=true"}
deferF := &multiCloser{}
var upt []ConfigUpdater
for _, v := range c.mv.values {
if u, ok := v.value.(ConfigUpdater); ok {
upt = append(upt, u)
}
}
if c.mirror != "" {
upt = append(upt, withMirrorConfig(c.mirror))
}
if len(upt) > 0 {
dir, err := writeConfig(upt)
if err != nil {
return nil, nil, err
}
deferF.append(func() error {
return os.RemoveAll(dir)
})
buildkitdArgs = append(buildkitdArgs, "--config="+filepath.Join(dir, "buildkitd.toml"))
}
if s.uid != 0 {
if s.gid == 0 {
deferF.F()()
return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.uid, s.gid)
}
// TODO: make sure the user exists and subuid/subgid are configured.
buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.uid), "-i", "--", "rootlesskit"}, buildkitdArgs...)
}
buildkitdSock, stop, err := runBuildkitd(buildkitdArgs, logs, s.uid, s.gid)
buildkitdSock, stop, err := runBuildkitd(cfg, buildkitdArgs, cfg.Logs, s.uid, s.gid)
if err != nil {
deferF.F()()
printLogs(logs, log.Println)
printLogs(cfg.Logs, log.Println)
return nil, nil, err
}
deferF.append(stop)
return &sandbox{address: buildkitdSock, mv: c.mv, logs: logs, cleanup: deferF, rootless: s.uid != 0}, deferF.F(), nil
return backend{
address: buildkitdSock,
rootless: s.uid != 0,
}, stop, nil
}
func printLogs(logs map[string]*bytes.Buffer, f func(args ...interface{})) {
@ -118,125 +78,3 @@ func printLogs(logs map[string]*bytes.Buffer, f func(args ...interface{})) {
}
}
}
type sandbox struct {
address string
logs map[string]*bytes.Buffer
cleanup *multiCloser
rootless bool
mv matrixValue
}
func (sb *sandbox) Address() string {
return sb.address
}
func (sb *sandbox) PrintLogs(t *testing.T) {
printLogs(sb.logs, t.Log)
}
func (sb *sandbox) NewRegistry() (string, error) {
url, cl, err := NewRegistry("")
if err != nil {
return "", err
}
sb.cleanup.append(cl)
return url, nil
}
func (sb *sandbox) Cmd(args ...string) *exec.Cmd {
if len(args) == 1 {
if split, err := shlex.Split(args[0]); err == nil {
args = split
}
}
cmd := exec.Command("buildctl", args...)
cmd.Env = append(cmd.Env, os.Environ()...)
cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address())
return cmd
}
func (sb *sandbox) Rootless() bool {
return sb.rootless
}
func (sb *sandbox) Value(k string) interface{} {
return sb.mv.values[k].value
}
func runBuildkitd(args []string, logs map[string]*bytes.Buffer, uid, gid int) (address string, cl func() error, err error) {
deferF := &multiCloser{}
cl = deferF.F()
defer func() {
if err != nil {
deferF.F()()
cl = nil
}
}()
tmpdir, err := ioutil.TempDir("", "bktest_buildkitd")
if err != nil {
return "", nil, err
}
if err := os.Chown(tmpdir, uid, gid); err != nil {
return "", nil, err
}
if err := os.MkdirAll(filepath.Join(tmpdir, "tmp"), 0711); err != nil {
return "", nil, err
}
if err := os.Chown(filepath.Join(tmpdir, "tmp"), uid, gid); err != nil {
return "", nil, err
}
deferF.append(func() error { return os.RemoveAll(tmpdir) })
address = "unix://" + filepath.Join(tmpdir, "buildkitd.sock")
if runtime.GOOS == "windows" {
address = "//./pipe/buildkitd-" + filepath.Base(tmpdir)
}
args = append(args, "--root", tmpdir, "--addr", address, "--debug")
cmd := exec.Command(args[0], args[1:]...)
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
cmd.SysProcAttr = &syscall.SysProcAttr{
Setsid: true, // stretch sudo needs this for sigterm
}
if stop, err := startCmd(cmd, logs); err != nil {
return "", nil, err
} else {
deferF.append(stop)
}
if err := waitUnix(address, 5*time.Second); err != nil {
return "", nil, err
}
deferF.append(func() error {
f, err := os.Open("/proc/self/mountinfo")
if err != nil {
return errors.Wrap(err, "failed to open mountinfo")
}
defer f.Close()
s := bufio.NewScanner(f)
for s.Scan() {
if strings.Contains(s.Text(), tmpdir) {
return errors.Errorf("leaked mountpoint for %s", tmpdir)
}
}
return s.Err()
})
return
}
func rootlessSupported(uid int) bool {
cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "unshare", "-U", "true")
b, err := cmd.CombinedOutput()
if err != nil {
logrus.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))
return false
}
return true
}

56
util/testutil/integration/run.go
View File

@ -1,6 +1,7 @@
package integration
import (
"bytes"
"context"
"fmt"
"io/ioutil"
@ -25,47 +26,32 @@ import (
"github.com/stretchr/testify/require"
)
type Sandbox interface {
// Backend is the minimal interface that describes a testing backend.
type Backend interface {
Address() string
PrintLogs(*testing.T)
Cmd(...string) *exec.Cmd
NewRegistry() (string, error)
Rootless() bool
}
type Sandbox interface {
Backend
Cmd(...string) *exec.Cmd
PrintLogs(*testing.T)
NewRegistry() (string, error)
Value(string) interface{} // chosen matrix value
}
// BackendConfig is used to configure backends created by a worker.
type BackendConfig struct {
Logs map[string]*bytes.Buffer
ConfigFile string
}
type Worker interface {
New(...SandboxOpt) (Sandbox, func() error, error)
New(*BackendConfig) (Backend, func() error, error)
Name() string
}
type SandboxConf struct {
mirror string
mv matrixValue
}
func (sc *SandboxConf) Mirror() string {
return sc.mirror
}
func (sc *SandboxConf) Value(k string) interface{} {
return sc.mv.values[k].value
}
type SandboxOpt func(*SandboxConf)
func WithMirror(h string) SandboxOpt {
return func(c *SandboxConf) {
c.mirror = h
}
}
func withMatrixValues(mv matrixValue) SandboxOpt {
return func(c *SandboxConf) {
c.mv = mv
}
}
type ConfigUpdater interface {
UpdateConfigFile(string) string
}
@ -162,7 +148,7 @@ func Run(t *testing.T, testCases []Test, opt ...TestOpt) {
if !strings.HasSuffix(fn, "NoParallel") {
t.Parallel()
}
sb, close, err := br.New(WithMirror(mirror), withMatrixValues(mv))
sb, closer, err := newSandbox(br, mirror, mv)
if err != nil {
if errors.Cause(err) == ErrorRequirements {
t.Skip(err.Error())
@ -170,7 +156,7 @@ func Run(t *testing.T, testCases []Test, opt ...TestOpt) {
require.NoError(t, err)
}
defer func() {
assert.NoError(t, close())
assert.NoError(t, closer())
if t.Failed() {
sb.PrintLogs(t)
}
@ -277,7 +263,7 @@ func writeConfig(updaters []ConfigUpdater) (string, error) {
s = upt.UpdateConfigFile(s)
}
if err := ioutil.WriteFile(filepath.Join(tmpdir, "buildkitd.toml"), []byte(s), 0644); err != nil {
if err := ioutil.WriteFile(filepath.Join(tmpdir, buildkitdConfigFile), []byte(s), 0644); err != nil {
return "", err
}
return tmpdir, nil

204
util/testutil/integration/sandbox.go Normal file
View File

@ -0,0 +1,204 @@
package integration
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/google/shlex"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const buildkitdConfigFile = "buildkitd.toml"
type backend struct {
address string
rootless bool
}
func (b backend) Address() string {
return b.address
}
func (b backend) Rootless() bool {
return b.rootless
}
type sandbox struct {
Backend
logs map[string]*bytes.Buffer
cleanup *multiCloser
mv matrixValue
}
func (sb *sandbox) PrintLogs(t *testing.T) {
printLogs(sb.logs, t.Log)
}
func (sb *sandbox) NewRegistry() (string, error) {
url, cl, err := NewRegistry("")
if err != nil {
return "", err
}
sb.cleanup.append(cl)
return url, nil
}
func (sb *sandbox) Cmd(args ...string) *exec.Cmd {
if len(args) == 1 {
if split, err := shlex.Split(args[0]); err == nil {
args = split
}
}
cmd := exec.Command("buildctl", args...)
cmd.Env = append(cmd.Env, os.Environ()...)
cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address())
return cmd
}
func (sb *sandbox) Value(k string) interface{} {
return sb.mv.values[k].value
}
func newSandbox(w Worker, mirror string, mv matrixValue) (s Sandbox, cl func() error, err error) {
cfg := &BackendConfig{
Logs: make(map[string]*bytes.Buffer),
}
var upt []ConfigUpdater
for _, v := range mv.values {
if u, ok := v.value.(ConfigUpdater); ok {
upt = append(upt, u)
}
}
if mirror != "" {
upt = append(upt, withMirrorConfig(mirror))
}
deferF := &multiCloser{}
cl = deferF.F()
defer func() {
if err != nil {
deferF.F()()
cl = nil
}
}()
if len(upt) > 0 {
dir, err := writeConfig(upt)
if err != nil {
return nil, nil, err
}
deferF.append(func() error {
return os.RemoveAll(dir)
})
cfg.ConfigFile = filepath.Join(dir, buildkitdConfigFile)
}
b, closer, err := w.New(cfg)
if err != nil {
return nil, nil, err
}
deferF.append(closer)
return &sandbox{
Backend: b,
logs: cfg.Logs,
cleanup: deferF,
mv: mv,
}, cl, nil
}
func runBuildkitd(conf *BackendConfig, args []string, logs map[string]*bytes.Buffer, uid, gid int) (address string, cl func() error, err error) {
deferF := &multiCloser{}
cl = deferF.F()
defer func() {
if err != nil {
deferF.F()()
cl = nil
}
}()
if conf.ConfigFile != "" {
args = append(args, "--config="+conf.ConfigFile)
}
tmpdir, err := ioutil.TempDir("", "bktest_buildkitd")
if err != nil {
return "", nil, err
}
if err := os.Chown(tmpdir, uid, gid); err != nil {
return "", nil, err
}
if err := os.MkdirAll(filepath.Join(tmpdir, "tmp"), 0711); err != nil {
return "", nil, err
}
if err := os.Chown(filepath.Join(tmpdir, "tmp"), uid, gid); err != nil {
return "", nil, err
}
deferF.append(func() error { return os.RemoveAll(tmpdir) })
address = "unix://" + filepath.Join(tmpdir, "buildkitd.sock")
if runtime.GOOS == "windows" {
address = "//./pipe/buildkitd-" + filepath.Base(tmpdir)
}
args = append(args, "--root", tmpdir, "--addr", address, "--debug")
cmd := exec.Command(args[0], args[1:]...)
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
cmd.SysProcAttr = &syscall.SysProcAttr{
Setsid: true, // stretch sudo needs this for sigterm
}
if stop, err := startCmd(cmd, logs); err != nil {
return "", nil, err
} else {
deferF.append(stop)
}
if err := waitUnix(address, 5*time.Second); err != nil {
return "", nil, err
}
deferF.append(func() error {
f, err := os.Open("/proc/self/mountinfo")
if err != nil {
return errors.Wrap(err, "failed to open mountinfo")
}
defer f.Close()
s := bufio.NewScanner(f)
for s.Scan() {
if strings.Contains(s.Text(), tmpdir) {
return errors.Errorf("leaked mountpoint for %s", tmpdir)
}
}
return s.Err()
})
return address, cl, err
}
func rootlessSupported(uid int) bool {
cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "unshare", "-U", "true")
b, err := cmd.CombinedOutput()
if err != nil {
logrus.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))
return false
}
return true
}

1
util/testutil/integration/util.go
View File

@ -22,7 +22,6 @@ func startCmd(cmd *exec.Cmd, logs map[string]*bytes.Buffer) (func() error, error
b = new(bytes.Buffer)
logs["stderr: "+cmd.Path] = b
cmd.Stderr = b
}
if err := cmd.Start(); err != nil {