From 75ea12b6c98c91b14723e974a99a1ad32c2d1740 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 10 Sep 2018 17:08:07 -0700 Subject: [PATCH 1/3] integration: use local registry mirrors Signed-off-by: Tonis Tiigi --- util/contentutil/copy.go | 38 ++++++++++ util/contentutil/fetcher.go | 3 + util/contentutil/refs.go | 57 +++++++++++++++ util/testutil/integration/containerd.go | 21 +++++- util/testutil/integration/oci.go | 19 ++++- util/testutil/integration/registry.go | 28 +++++--- util/testutil/integration/run.go | 93 ++++++++++++++++++++++++- 7 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 util/contentutil/refs.go diff --git a/util/contentutil/copy.go b/util/contentutil/copy.go index a03f16e6..04d46c4f 100644 --- a/util/contentutil/copy.go +++ b/util/contentutil/copy.go @@ -3,10 +3,13 @@ package contentutil import ( "context" "io" + "sync" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/remotes" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) func Copy(ctx context.Context, ingester content.Ingester, provider content.Provider, desc ocispec.Descriptor) error { @@ -41,3 +44,38 @@ func (r *rc) Read(b []byte) (int, error) { } return n, err } + +func CopyChain(ctx context.Context, ingester content.Ingester, provider content.Provider, desc ocispec.Descriptor) error { + var m sync.Mutex + manifestStack := []ocispec.Descriptor{} + + filterHandler := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + switch desc.MediaType { + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest, + images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + m.Lock() + manifestStack = append(manifestStack, desc) + m.Unlock() + return nil, images.ErrStopHandler + default: + return nil, nil + } + }) + handlers := []images.Handler{ + images.ChildrenHandler(provider), + filterHandler, + remotes.FetchHandler(ingester, &localFetcher{provider}), + } + + if err := images.Dispatch(ctx, images.Handlers(handlers...), desc); err != nil { + return errors.WithStack(err) + } + + for i := len(manifestStack) - 1; i >= 0; i-- { + if err := Copy(ctx, ingester, provider, manifestStack[i]); err != nil { + return errors.WithStack(err) + } + } + + return nil +} diff --git a/util/contentutil/fetcher.go b/util/contentutil/fetcher.go index 0c87e647..d55c1012 100644 --- a/util/contentutil/fetcher.go +++ b/util/contentutil/fetcher.go @@ -55,6 +55,9 @@ func (r *readerAt) ReadAt(b []byte, off int64) (int, error) { var totalN int for len(b) > 0 { n, err := r.Reader.Read(b) + if err == io.EOF && n == len(b) { + err = nil + } r.offset += int64(n) totalN += n b = b[n:] diff --git a/util/contentutil/refs.go b/util/contentutil/refs.go new file mode 100644 index 00000000..b3a10d15 --- /dev/null +++ b/util/contentutil/refs.go @@ -0,0 +1,57 @@ +package contentutil + +import ( + "context" + "net/http" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func ProviderFromRef(ref string) (ocispec.Descriptor, content.Provider, error) { + remote := docker.NewResolver(docker.ResolverOptions{ + Client: http.DefaultClient, + }) + + name, desc, err := remote.Resolve(context.TODO(), ref) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + + fetcher, err := remote.Fetcher(context.TODO(), name) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + return desc, FromFetcher(fetcher), nil +} + +func IngesterFromRef(ref string) (content.Ingester, error) { + remote := docker.NewResolver(docker.ResolverOptions{ + Client: http.DefaultClient, + }) + + pusher, err := remote.Pusher(context.TODO(), ref) + if err != nil { + return nil, err + } + + return &ingester{ + pusher: pusher, + }, nil +} + +type ingester struct { + pusher remotes.Pusher +} + +func (w *ingester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + var wo content.WriterOpts + for _, o := range opts { + if err := o(&wo); err != nil { + return nil, err + } + } + return w.pusher.Push(ctx, wo.Desc) +} diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go index d892b9de..cb45d630 100644 --- a/util/testutil/integration/containerd.go +++ b/util/testutil/integration/containerd.go @@ -48,7 +48,12 @@ func (c *containerd) Name() string { return c.name } -func (c *containerd) New() (sb Sandbox, cl func() error, err error) { +func (c *containerd) New(opt ...SandboxOpt) (sb Sandbox, cl func() error, err error) { + var conf SandboxConf + for _, o := range opt { + o(&conf) + } + if err := lookupBinary(c.containerd); err != nil { return nil, nil, err } @@ -115,12 +120,22 @@ disabled_plugins = ["cri"] } deferF.append(ctdStop) - buildkitdSock, stop, err := runBuildkitd([]string{"buildkitd", + buildkitdArgs := []string{"buildkitd", "--oci-worker=false", "--containerd-worker=true", "--containerd-worker-addr", address, "--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603 - }, logs, 0, 0) + } + + if conf.mirror != "" { + dir, err := configWithMirror(conf.mirror) + if err != nil { + return nil, nil, err + } + buildkitdArgs = append(buildkitdArgs, "--config="+filepath.Join(dir, "buildkitd.toml")) + } + + buildkitdSock, stop, err := runBuildkitd(buildkitdArgs, logs, 0, 0) if err != nil { return nil, nil, err } diff --git a/util/testutil/integration/oci.go b/util/testutil/integration/oci.go index 7bc61446..e41283ad 100644 --- a/util/testutil/integration/oci.go +++ b/util/testutil/integration/oci.go @@ -30,6 +30,7 @@ func init() { register(&oci{uid: uid, gid: gid}) } } + } type oci struct { @@ -44,7 +45,12 @@ func (s *oci) Name() string { return "oci" } -func (s *oci) New() (Sandbox, func() error, error) { +func (s *oci) New(opt ...SandboxOpt) (Sandbox, func() error, error) { + var c SandboxConf + for _, o := range opt { + o(&c) + } + if err := lookupBinary("buildkitd"); err != nil { return nil, nil, err } @@ -54,6 +60,15 @@ func (s *oci) New() (Sandbox, func() error, error) { logs := map[string]*bytes.Buffer{} // Include use of --oci-worker-labels to trigger https://github.com/moby/buildkit/pull/603 buildkitdArgs := []string{"buildkitd", "--oci-worker=true", "--containerd-worker=false", "--oci-worker-labels=org.mobyproject.buildkit.worker.sandbox=true"} + + if c.mirror != "" { + dir, err := configWithMirror(c.mirror) + if err != nil { + return nil, nil, err + } + buildkitdArgs = append(buildkitdArgs, "--config="+filepath.Join(dir, "buildkitd.toml")) + } + if s.uid != 0 { if s.gid == 0 { return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.uid, s.gid) @@ -94,7 +109,7 @@ func (sb *sandbox) PrintLogs(t *testing.T) { } func (sb *sandbox) NewRegistry() (string, error) { - url, cl, err := newRegistry() + url, cl, err := newRegistry("") if err != nil { return "", err } diff --git a/util/testutil/integration/registry.go b/util/testutil/integration/registry.go index fa772dd2..cee72830 100644 --- a/util/testutil/integration/registry.go +++ b/util/testutil/integration/registry.go @@ -15,7 +15,7 @@ import ( "github.com/pkg/errors" ) -func newRegistry() (url string, cl func() error, err error) { +func newRegistry(dir string) (url string, cl func() error, err error) { if err := lookupBinary("registry"); err != nil { return "", nil, err } @@ -30,26 +30,34 @@ func newRegistry() (url string, cl func() error, err error) { } }() - tmpdir, err := ioutil.TempDir("", "test-registry") - if err != nil { - return "", nil, err + if dir == "" { + tmpdir, err := ioutil.TempDir("", "test-registry") + if err != nil { + return "", nil, err + } + deferF.append(func() error { return os.RemoveAll(tmpdir) }) + dir = tmpdir } - deferF.append(func() error { return os.RemoveAll(tmpdir) }) - template := fmt.Sprintf(`version: 0.1 + if _, err := os.Stat(filepath.Join(dir, "config.yaml")); err != nil { + if !os.IsNotExist(err) { + return "", nil, err + } + template := fmt.Sprintf(`version: 0.1 loglevel: debug storage: filesystem: rootdirectory: %s http: addr: 127.0.0.1:0 -`, filepath.Join(tmpdir, "data")) +`, filepath.Join(dir, "data")) - if err := ioutil.WriteFile(filepath.Join(tmpdir, "config.yaml"), []byte(template), 0600); err != nil { - return "", nil, err + if err := ioutil.WriteFile(filepath.Join(dir, "config.yaml"), []byte(template), 0600); err != nil { + return "", nil, err + } } - cmd := exec.Command("registry", "serve", filepath.Join(tmpdir, "config.yaml")) + cmd := exec.Command("registry", "serve", filepath.Join(dir, "config.yaml")) rc, err := cmd.StdoutPipe() if err != nil { return "", nil, err diff --git a/util/testutil/integration/run.go b/util/testutil/integration/run.go index 2d3803e7..e82a4ad9 100644 --- a/util/testutil/integration/run.go +++ b/util/testutil/integration/run.go @@ -1,12 +1,19 @@ package integration import ( + "context" + "fmt" + "io/ioutil" + "os" "os/exec" + "path/filepath" "reflect" "runtime" "strings" + "syscall" "testing" + "github.com/moby/buildkit/util/contentutil" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,10 +28,22 @@ type Sandbox interface { } type Worker interface { - New() (Sandbox, func() error, error) + New(...SandboxOpt) (Sandbox, func() error, error) Name() string } +type SandboxConf struct { + mirror string +} + +type SandboxOpt func(*SandboxConf) + +func WithMirror(h string) SandboxOpt { + return func(c *SandboxConf) { + c.mirror = h + } +} + type Test func(*testing.T, Sandbox) var defaultWorkers []Worker @@ -41,10 +60,37 @@ func Run(t *testing.T, testCases []Test) { if testing.Short() { t.Skip("skipping in short mode") } + + mirrorDir := os.Getenv("BUILDKIT_REGISTRY_MIRROR_DIR") + + var f *os.File + if mirrorDir != "" { + var err error + f, err = os.Create(filepath.Join(mirrorDir, "lock")) + require.NoError(t, err) + // defer f.Close() // this defer runs after subtest, cleanup on exit + + err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX) + require.NoError(t, err) + } + + mirror, cleanup, err := newRegistry(mirrorDir) + require.NoError(t, err) + _ = cleanup + // defer cleanup() // this defer runs after subtest, cleanup on exit + + err = copyImagesLocal(t, mirror) + require.NoError(t, err) + + if mirrorDir != "" { + err = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + require.NoError(t, err) + } + for _, br := range List() { for _, tc := range testCases { ok := t.Run(getFunctionName(tc)+"/worker="+br.Name(), func(t *testing.T) { - sb, close, err := br.New() + sb, close, err := br.New(WithMirror(mirror)) if err != nil { if errors.Cause(err) == ErrorRequirements { t.Skip(err.Error()) @@ -69,3 +115,46 @@ func getFunctionName(i interface{}) string { dot := strings.LastIndex(fullname, ".") + 1 return strings.Title(fullname[dot:]) } + +func copyImagesLocal(t *testing.T, host string) error { + for to, from := range offlineImages() { + desc, provider, err := contentutil.ProviderFromRef(from) + if err != nil { + return err + } + ingester, err := contentutil.IngesterFromRef(host + "/" + to) + if err != nil { + return err + } + if err := contentutil.CopyChain(context.TODO(), ingester, provider, desc); err != nil { + return err + } + t.Logf("copied %s to local mirror %s", from, host+"/"+to) + } + return nil +} + +func offlineImages() map[string]string { + arch := runtime.GOARCH + if arch == "arm64" { + arch = "arm64v8" + } + return map[string]string{ + "library/busybox:latest": "docker.io/" + arch + "/busybox:latest", + "library/alpine:latest": "docker.io/" + arch + "/alpine:latest", + } +} + +func configWithMirror(mirror string) (string, error) { + tmpdir, err := ioutil.TempDir("", "bktest_config") + if err != nil { + return "", err + } + if err := ioutil.WriteFile(filepath.Join(tmpdir, "buildkitd.toml"), []byte(fmt.Sprintf(` +[registry."docker.io"] +mirrors=["%s"] +`, mirror)), 0600); err != nil { + return "", err + } + return tmpdir, nil +} From fe8f2c9910cc385927d447fc6a03209b1212094a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 11 Sep 2018 18:28:11 -0700 Subject: [PATCH 2/3] integration: fix cleanup of registry data Signed-off-by: Tonis Tiigi --- util/testutil/integration/containerd.go | 3 + util/testutil/integration/oci.go | 8 ++- util/testutil/integration/run.go | 82 ++++++++++++++++++------- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go index cb45d630..b534c68c 100644 --- a/util/testutil/integration/containerd.go +++ b/util/testutil/integration/containerd.go @@ -132,6 +132,9 @@ disabled_plugins = ["cri"] if err != nil { return nil, nil, err } + deferF.append(func() error { + return os.RemoveAll(dir) + }) buildkitdArgs = append(buildkitdArgs, "--config="+filepath.Join(dir, "buildkitd.toml")) } diff --git a/util/testutil/integration/oci.go b/util/testutil/integration/oci.go index e41283ad..79d2550b 100644 --- a/util/testutil/integration/oci.go +++ b/util/testutil/integration/oci.go @@ -61,16 +61,22 @@ func (s *oci) New(opt ...SandboxOpt) (Sandbox, func() error, error) { // Include use of --oci-worker-labels to trigger https://github.com/moby/buildkit/pull/603 buildkitdArgs := []string{"buildkitd", "--oci-worker=true", "--containerd-worker=false", "--oci-worker-labels=org.mobyproject.buildkit.worker.sandbox=true"} + deferF := &multiCloser{} + if c.mirror != "" { dir, err := configWithMirror(c.mirror) if err != nil { return nil, nil, err } + deferF.append(func() error { + return os.RemoveAll(dir) + }) buildkitdArgs = append(buildkitdArgs, "--config="+filepath.Join(dir, "buildkitd.toml")) } if s.uid != 0 { if s.gid == 0 { + deferF.F()() return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.uid, s.gid) } // TODO: make sure the user exists and subuid/subgid are configured. @@ -78,10 +84,10 @@ func (s *oci) New(opt ...SandboxOpt) (Sandbox, func() error, error) { } buildkitdSock, stop, err := runBuildkitd(buildkitdArgs, logs, s.uid, s.gid) if err != nil { + deferF.F()() return nil, nil, err } - deferF := &multiCloser{} deferF.append(stop) return &sandbox{address: buildkitdSock, logs: logs, cleanup: deferF, rootless: s.uid != 0}, deferF.F(), nil diff --git a/util/testutil/integration/run.go b/util/testutil/integration/run.go index e82a4ad9..9c6c6ca6 100644 --- a/util/testutil/integration/run.go +++ b/util/testutil/integration/run.go @@ -10,9 +10,11 @@ import ( "reflect" "runtime" "strings" + "sync" "syscall" "testing" + "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" "github.com/moby/buildkit/util/contentutil" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -61,35 +63,28 @@ func Run(t *testing.T, testCases []Test) { t.Skip("skipping in short mode") } - mirrorDir := os.Getenv("BUILDKIT_REGISTRY_MIRROR_DIR") - - var f *os.File - if mirrorDir != "" { - var err error - f, err = os.Create(filepath.Join(mirrorDir, "lock")) - require.NoError(t, err) - // defer f.Close() // this defer runs after subtest, cleanup on exit - - err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX) - require.NoError(t, err) - } - - mirror, cleanup, err := newRegistry(mirrorDir) - require.NoError(t, err) - _ = cleanup - // defer cleanup() // this defer runs after subtest, cleanup on exit - - err = copyImagesLocal(t, mirror) + mirror, cleanup, err := runMirror(t) require.NoError(t, err) - if mirrorDir != "" { - err = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) - require.NoError(t, err) + var mu sync.Mutex + var count int + cleanOnComplete := func() func() { + count++ + return func() { + mu.Lock() + count-- + if count == 0 { + cleanup() + } + mu.Unlock() + } } + defer cleanOnComplete()() for _, br := range List() { for _, tc := range testCases { ok := t.Run(getFunctionName(tc)+"/worker="+br.Name(), func(t *testing.T) { + defer cleanOnComplete()() sb, close, err := br.New(WithMirror(mirror)) if err != nil { if errors.Cause(err) == ErrorRequirements { @@ -142,6 +137,7 @@ func offlineImages() map[string]string { return map[string]string{ "library/busybox:latest": "docker.io/" + arch + "/busybox:latest", "library/alpine:latest": "docker.io/" + arch + "/alpine:latest", + "tonistiigi/copy:v0.1.4": "docker.io/" + dockerfile2llb.DefaultCopyImage, } } @@ -158,3 +154,45 @@ mirrors=["%s"] } return tmpdir, nil } + +func runMirror(t *testing.T) (host string, cleanup func() error, err error) { + mirrorDir := os.Getenv("BUILDKIT_REGISTRY_MIRROR_DIR") + + var f *os.File + if mirrorDir != "" { + f, err = os.Create(filepath.Join(mirrorDir, "lock")) + if err != nil { + return "", nil, err + } + defer func() { + if err != nil { + f.Close() + } + }() + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return "", nil, err + } + } + + mirror, cleanup, err := newRegistry(mirrorDir) + if err != nil { + return "", nil, err + } + defer func() { + if err != nil { + cleanup() + } + }() + + if err := copyImagesLocal(t, mirror); err != nil { + return "", nil, err + } + + if mirrorDir != "" { + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil { + return "", nil, err + } + } + + return mirror, cleanup, err +} From 087e11eedd07d71b6164fc1409a4d470e4365abf Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 11 Sep 2018 21:53:19 -0700 Subject: [PATCH 3/3] integration: fix config permissions for rootless Signed-off-by: Tonis Tiigi --- util/testutil/integration/run.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util/testutil/integration/run.go b/util/testutil/integration/run.go index 9c6c6ca6..dd00a8fb 100644 --- a/util/testutil/integration/run.go +++ b/util/testutil/integration/run.go @@ -146,10 +146,13 @@ func configWithMirror(mirror string) (string, error) { if err != nil { return "", err } + if err := os.Chmod(tmpdir, 0711); err != nil { + return "", err + } if err := ioutil.WriteFile(filepath.Join(tmpdir, "buildkitd.toml"), []byte(fmt.Sprintf(` [registry."docker.io"] mirrors=["%s"] -`, mirror)), 0600); err != nil { +`, mirror)), 0644); err != nil { return "", err } return tmpdir, nil