JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1594 from TBBle/generate_oci_specs_on_all_platforms 

Unify OCI Spec generation across all host platforms
v0.8
Tõnis Tiigi 2020-07-30 20:05:13 -07:00 committed by GitHub
parent 7b6dbdb715 603e1d7100
commit 3f3957d16c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 409 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/windows.yaml vendored
View File

@ -39,5 +39,5 @@ jobs:
- name: Unit test
env:
SKIP_INTEGRATION_TESTS: 1
run: go test -mod=vendor -v ./cache/... ./client/... ./frontend/dockerfile/... ./session/... ./solver/... ./source/... ./util/...
run: go test -mod=vendor -v ./...
working-directory: src/github.com/moby/buildkit

2
cmd/buildkitd/main_containerd_worker.go
View File

@ -1,4 +1,4 @@
// +build linux,!no_containerd_worker
// +build linux,!no_containerd_worker windows,!no_containerd_worker
package main

115
executor/oci/mounts.go
View File

@ -5,75 +5,50 @@ import (
"path/filepath"
"strings"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/oci"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)
// MountOpts sets oci spec specific info for mount points
type MountOpts func([]specs.Mount) ([]specs.Mount, error)
func withRemovedMount(destination string) oci.SpecOpts {
return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
newMounts := []specs.Mount{}
for _, o := range s.Mounts {
if o.Destination != destination {
newMounts = append(newMounts, o)
}
}
s.Mounts = newMounts
//GetMounts returns default required for buildkit
// https://github.com/moby/buildkit/issues/429
func GetMounts(ctx context.Context, mountOpts ...MountOpts) ([]specs.Mount, error) {
mounts := []specs.Mount{
{
Destination: "/proc",
Type: "proc",
Source: "proc",
},
{
Destination: "/dev",
Type: "tmpfs",
Source: "tmpfs",
Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"},
},
{
Destination: "/dev/pts",
Type: "devpts",
Source: "devpts",
Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", "gid=5"},
},
{
Destination: "/dev/shm",
Type: "tmpfs",
Source: "shm",
Options: []string{"nosuid", "noexec", "nodev", "mode=1777", "size=65536k"},
},
{
Destination: "/dev/mqueue",
Type: "mqueue",
Source: "mqueue",
Options: []string{"nosuid", "noexec", "nodev"},
},
{
Destination: "/sys",
Type: "sysfs",
Source: "sysfs",
Options: []string{"nosuid", "noexec", "nodev", "ro"},
},
return nil
}
var err error
for _, o := range mountOpts {
mounts, err = o(mounts)
if err != nil {
return nil, err
}
}
return mounts, nil
}
func withROBind(src, dest string) func(m []specs.Mount) ([]specs.Mount, error) {
return func(m []specs.Mount) ([]specs.Mount, error) {
m = append(m, specs.Mount{
func withROBind(src, dest string) oci.SpecOpts {
return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
s.Mounts = append(s.Mounts, specs.Mount{
Destination: dest,
Type: "bind",
Source: src,
Options: []string{"nosuid", "noexec", "nodev", "rbind", "ro"},
})
return m, nil
return nil
}
}
func withCGroup() oci.SpecOpts {
return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
s.Mounts = append(s.Mounts, specs.Mount{
Destination: "/sys/fs/cgroup",
Type: "cgroup",
Source: "cgroup",
Options: []string{"ro", "nosuid", "noexec", "nodev"},
})
return nil
}
}
func hasPrefix(p, prefixDir string) bool {
prefixDir = filepath.Clean(prefixDir)
if filepath.Base(prefixDir) == string(filepath.Separator) {
@ -93,13 +68,9 @@ func removeMountsWithPrefix(mounts []specs.Mount, prefixDir string) []specs.Moun
return ret
}
func withProcessMode(processMode ProcessMode) func([]specs.Mount) ([]specs.Mount, error) {
return func(m []specs.Mount) ([]specs.Mount, error) {
switch processMode {
case ProcessSandbox:
// keep the default
case NoProcessSandbox:
m = removeMountsWithPrefix(m, "/proc")
func withBoundProc() oci.SpecOpts {
return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
s.Mounts = removeMountsWithPrefix(s.Mounts, "/proc")
procMount := specs.Mount{
Destination: "/proc",
Type: "bind",
@ -108,10 +79,24 @@ func withProcessMode(processMode ProcessMode) func([]specs.Mount) ([]specs.Mount
// So we keep maskedPath and readonlyPaths (although not mandatory for rootless mode)
Options: []string{"rbind"},
}
m = append([]specs.Mount{procMount}, m...)
default:
return nil, errors.Errorf("unknown process mode: %v", processMode)
s.Mounts = append([]specs.Mount{procMount}, s.Mounts...)
var maskedPaths []string
for _, s := range s.Linux.MaskedPaths {
if !hasPrefix(s, "/proc") {
maskedPaths = append(maskedPaths, s)
}
return m, nil
}
s.Linux.MaskedPaths = maskedPaths
var readonlyPaths []string
for _, s := range s.Linux.ReadonlyPaths {
if !hasPrefix(s, "/proc") {
readonlyPaths = append(readonlyPaths, s)
}
}
s.Linux.ReadonlyPaths = readonlyPaths
return nil
}
}

58
executor/oci/mounts_test.go
View File

@ -4,6 +4,9 @@ import (
"runtime"
"testing"
"github.com/containerd/containerd/oci"
"github.com/moby/buildkit/util/appcontext"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
)
@ -94,3 +97,58 @@ func TestHasPrefix(t *testing.T) {
assert.Equal(t, tc.expected, actual, "#%d: under(%q,%q)", i, tc.path, tc.prefix)
}
}
func TestWithRemovedMounts(t *testing.T) {
// The default mount-list from containerd
s := oci.Spec{
Mounts: []specs.Mount{
{
Destination: "/proc",
Type: "proc",
Source: "proc",
Options: []string{"nosuid", "noexec", "nodev"},
},
{
Destination: "/dev",
Type: "tmpfs",
Source: "tmpfs",
Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"},
},
{
Destination: "/dev/pts",
Type: "devpts",
Source: "devpts",
Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", "gid=5"},
},
{
Destination: "/dev/shm",
Type: "tmpfs",
Source: "shm",
Options: []string{"nosuid", "noexec", "nodev", "mode=1777", "size=65536k"},
},
{
Destination: "/dev/mqueue",
Type: "mqueue",
Source: "mqueue",
Options: []string{"nosuid", "noexec", "nodev"},
},
{
Destination: "/sys",
Type: "sysfs",
Source: "sysfs",
Options: []string{"nosuid", "noexec", "nodev", "ro"},
},
{
Destination: "/run",
Type: "tmpfs",
Source: "tmpfs",
Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"},
},
},
}
oldLen := len(s.Mounts)
err := withRemovedMount("/run")(appcontext.Context(), nil, nil, &s)
assert.NoError(t, err)
assert.Equal(t, oldLen-1, len(s.Mounts))
}

216
executor/oci/spec.go
View File

@ -1,6 +1,25 @@
package oci
// ProcMode configures PID namespaces
import (
"context"
"path"
"sync"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/mitchellh/hashstructure"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/network"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)
// ProcessMode configures PID namespaces
type ProcessMode int
const (
@ -11,3 +30,198 @@ const (
// NoProcessSandbox should be enabled only when the BuildKit is running in a container as an unprivileged user.
NoProcessSandbox
)
// Ideally we don't have to import whole containerd just for the default spec
// GenerateSpec generates spec using containerd functionality.
// opts are ignored for s.Process, s.Hostname, and s.Mounts .
func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mount, id, resolvConf, hostsFile string, namespace network.Namespace, processMode ProcessMode, idmap *idtools.IdentityMapping, opts ...oci.SpecOpts) (*specs.Spec, func(), error) {
c := &containers.Container{
ID: id,
}
// containerd/oci.GenerateSpec requires a namespace, which
// will be used to namespace specs.Linux.CgroupsPath if generated
if _, ok := namespaces.Namespace(ctx); !ok {
ctx = namespaces.WithNamespace(ctx, "buildkit")
}
if mountOpts, err := generateMountOpts(resolvConf, hostsFile); err == nil {
opts = append(opts, mountOpts...)
} else {
return nil, nil, err
}
if securityOpts, err := generateSecurityOpts(meta.SecurityMode); err == nil {
opts = append(opts, securityOpts...)
} else {
return nil, nil, err
}
if processModeOpts, err := generateProcessModeOpts(processMode); err == nil {
opts = append(opts, processModeOpts...)
} else {
return nil, nil, err
}
if idmapOpts, err := generateIDmapOpts(idmap); err == nil {
opts = append(opts, idmapOpts...)
} else {
return nil, nil, err
}
opts = append(opts,
oci.WithProcessArgs(meta.Args...),
oci.WithEnv(meta.Env),
oci.WithProcessCwd(meta.Cwd),
oci.WithNewPrivileges,
oci.WithHostname("buildkitsandbox"),
)
s, err := oci.GenerateSpec(ctx, nil, c, opts...)
if err != nil {
return nil, nil, err
}
// set the networking information on the spec
if err := namespace.Set(s); err != nil {
return nil, nil, err
}
s.Process.Rlimits = nil // reset open files limit
sm := &submounts{}
var releasers []func() error
releaseAll := func() {
sm.cleanup()
for _, f := range releasers {
f()
}
}
for _, m := range mounts {
if m.Src == nil {
return nil, nil, errors.Errorf("mount %s has no source", m.Dest)
}
mountable, err := m.Src.Mount(ctx, m.Readonly)
if err != nil {
releaseAll()
return nil, nil, errors.Wrapf(err, "failed to mount %s", m.Dest)
}
mounts, release, err := mountable.Mount()
if err != nil {
releaseAll()
return nil, nil, errors.WithStack(err)
}
releasers = append(releasers, release)
for _, mount := range mounts {
mount, err = sm.subMount(mount, m.Selector)
if err != nil {
releaseAll()
return nil, nil, err
}
s.Mounts = append(s.Mounts, specs.Mount{
Destination: m.Dest,
Type: mount.Type,
Source: mount.Source,
Options: mount.Options,
})
}
}
return s, releaseAll, nil
}
type mountRef struct {
mount mount.Mount
unmount func() error
}
type submounts struct {
m map[uint64]mountRef
}
func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error) {
if path.Join("/", subPath) == "/" {
return m, nil
}
if s.m == nil {
s.m = map[uint64]mountRef{}
}
h, err := hashstructure.Hash(m, nil)
if err != nil {
return mount.Mount{}, nil
}
if mr, ok := s.m[h]; ok {
sm, err := sub(mr.mount, subPath)
if err != nil {
return mount.Mount{}, nil
}
return sm, nil
}
lm := snapshot.LocalMounterWithMounts([]mount.Mount{m})
mp, err := lm.Mount()
if err != nil {
return mount.Mount{}, err
}
opts := []string{"rbind"}
for _, opt := range m.Options {
if opt == "ro" {
opts = append(opts, opt)
}
}
s.m[h] = mountRef{
mount: mount.Mount{
Source: mp,
Type: "bind",
Options: opts,
},
unmount: lm.Unmount,
}
sm, err := sub(s.m[h].mount, subPath)
if err != nil {
return mount.Mount{}, err
}
return sm, nil
}
func (s *submounts) cleanup() {
var wg sync.WaitGroup
wg.Add(len(s.m))
for _, m := range s.m {
func(m mountRef) {
go func() {
m.unmount()
wg.Done()
}()
}(m)
}
wg.Wait()
}
func sub(m mount.Mount, subPath string) (mount.Mount, error) {
src, err := fs.RootPath(m.Source, subPath)
if err != nil {
return mount.Mount{}, err
}
m.Source = src
return m, nil
}
func specMapping(s []idtools.IDMap) []specs.LinuxIDMapping {
var ids []specs.LinuxIDMapping
for _, item := range s {
ids = append(ids, specs.LinuxIDMapping{
HostID: uint32(item.HostID),
ContainerID: uint32(item.ContainerID),
Size: uint32(item.Size),
})
}
return ids
}

264
executor/oci/spec_unix.go
View File

@ -3,252 +3,56 @@
package oci
import (
"context"
"path"
"sync"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/contrib/seccomp"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/mitchellh/hashstructure"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/entitlements/security"
"github.com/moby/buildkit/util/network"
"github.com/moby/buildkit/util/system"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)
// Ideally we don't have to import whole containerd just for the default spec
// GenerateSpec generates spec using containerd functionality.
// opts are ignored for s.Process, s.Hostname, and s.Mounts .
func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mount, id, resolvConf, hostsFile string, namespace network.Namespace, processMode ProcessMode, idmap *idtools.IdentityMapping, opts ...oci.SpecOpts) (*specs.Spec, func(), error) {
c := &containers.Container{
ID: id,
}
_, ok := namespaces.Namespace(ctx)
if !ok {
ctx = namespaces.WithNamespace(ctx, "buildkit")
}
if meta.SecurityMode == pb.SecurityMode_INSECURE {
opts = append(opts, security.WithInsecureSpec())
} else if system.SeccompSupported() && meta.SecurityMode == pb.SecurityMode_SANDBOX {
opts = append(opts, seccomp.WithDefaultProfile())
}
switch processMode {
case NoProcessSandbox:
// Mount for /proc is replaced in GetMounts()
opts = append(opts,
oci.WithHostNamespace(specs.PIDNamespace))
// TODO(AkihiroSuda): Configure seccomp to disable ptrace (and prctl?) explicitly
}
// Note that containerd.GenerateSpec is namespaced so as to make
// specs.Linux.CgroupsPath namespaced
s, err := oci.GenerateSpec(ctx, nil, c, opts...)
if err != nil {
return nil, nil, err
}
// set the networking information on the spec
namespace.Set(s)
s.Process.Args = meta.Args
s.Process.Env = meta.Env
s.Process.Cwd = meta.Cwd
s.Process.Rlimits = nil // reset open files limit
s.Process.NoNewPrivileges = false // reset nonewprivileges
s.Hostname = "buildkitsandbox"
s.Mounts, err = GetMounts(ctx,
withProcessMode(processMode),
func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
return []oci.SpecOpts{
// https://github.com/moby/buildkit/issues/429
withRemovedMount("/run"),
withROBind(resolvConf, "/etc/resolv.conf"),
withROBind(hostsFile, "/etc/hosts"),
)
if err != nil {
return nil, nil, err
}
s.Mounts = append(s.Mounts, specs.Mount{
Destination: "/sys/fs/cgroup",
Type: "cgroup",
Source: "cgroup",
Options: []string{"ro", "nosuid", "noexec", "nodev"},
})
if processMode == NoProcessSandbox {
var maskedPaths []string
for _, s := range s.Linux.MaskedPaths {
if !hasPrefix(s, "/proc") {
maskedPaths = append(maskedPaths, s)
}
}
s.Linux.MaskedPaths = maskedPaths
var readonlyPaths []string
for _, s := range s.Linux.ReadonlyPaths {
if !hasPrefix(s, "/proc") {
readonlyPaths = append(readonlyPaths, s)
}
}
s.Linux.ReadonlyPaths = readonlyPaths
}
if meta.SecurityMode == pb.SecurityMode_INSECURE {
if err = oci.WithWriteableCgroupfs(ctx, nil, c, s); err != nil {
return nil, nil, err
}
if err = oci.WithWriteableSysfs(ctx, nil, c, s); err != nil {
return nil, nil, err
}
}
if idmap != nil {
s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{
Type: specs.UserNamespace,
})
s.Linux.UIDMappings = specMapping(idmap.UIDs())
s.Linux.GIDMappings = specMapping(idmap.GIDs())
}
sm := &submounts{}
var releasers []func() error
releaseAll := func() {
sm.cleanup()
for _, f := range releasers {
f()
}
}
for _, m := range mounts {
if m.Src == nil {
return nil, nil, errors.Errorf("mount %s has no source", m.Dest)
}
mountable, err := m.Src.Mount(ctx, m.Readonly)
if err != nil {
releaseAll()
return nil, nil, errors.Wrapf(err, "failed to mount %s", m.Dest)
}
mounts, release, err := mountable.Mount()
if err != nil {
releaseAll()
return nil, nil, errors.WithStack(err)
}
releasers = append(releasers, release)
for _, mount := range mounts {
mount, err = sm.subMount(mount, m.Selector)
if err != nil {
releaseAll()
return nil, nil, err
}
s.Mounts = append(s.Mounts, specs.Mount{
Destination: m.Dest,
Type: mount.Type,
Source: mount.Source,
Options: mount.Options,
})
}
}
return s, releaseAll, nil
withCGroup(),
}, nil
}
type mountRef struct {
mount mount.Mount
unmount func() error
// generateSecurityOpts may affect mounts, so must be called after generateMountOpts
func generateSecurityOpts(mode pb.SecurityMode) ([]oci.SpecOpts, error) {
if mode == pb.SecurityMode_INSECURE {
return []oci.SpecOpts{
security.WithInsecureSpec(),
oci.WithWriteableCgroupfs,
oci.WithWriteableSysfs,
}, nil
} else if system.SeccompSupported() && mode == pb.SecurityMode_SANDBOX {
return []oci.SpecOpts{seccomp.WithDefaultProfile()}, nil
}
return nil, nil
}
type submounts struct {
m map[uint64]mountRef
// generateProcessModeOpts may affect mounts, so must be called after generateMountOpts
func generateProcessModeOpts(mode ProcessMode) ([]oci.SpecOpts, error) {
if mode == NoProcessSandbox {
return []oci.SpecOpts{
oci.WithHostNamespace(specs.PIDNamespace),
withBoundProc(),
}, nil
// TODO(AkihiroSuda): Configure seccomp to disable ptrace (and prctl?) explicitly
}
return nil, nil
}
func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error) {
if path.Join("/", subPath) == "/" {
return m, nil
func generateIDmapOpts(idmap *idtools.IdentityMapping) ([]oci.SpecOpts, error) {
if idmap == nil {
return nil, nil
}
if s.m == nil {
s.m = map[uint64]mountRef{}
}
h, err := hashstructure.Hash(m, nil)
if err != nil {
return mount.Mount{}, nil
}
if mr, ok := s.m[h]; ok {
sm, err := sub(mr.mount, subPath)
if err != nil {
return mount.Mount{}, nil
}
return sm, nil
}
lm := snapshot.LocalMounterWithMounts([]mount.Mount{m})
mp, err := lm.Mount()
if err != nil {
return mount.Mount{}, err
}
opts := []string{"rbind"}
for _, opt := range m.Options {
if opt == "ro" {
opts = append(opts, opt)
}
}
s.m[h] = mountRef{
mount: mount.Mount{
Source: mp,
Type: "bind",
Options: opts,
},
unmount: lm.Unmount,
}
sm, err := sub(s.m[h].mount, subPath)
if err != nil {
return mount.Mount{}, err
}
return sm, nil
}
func (s *submounts) cleanup() {
var wg sync.WaitGroup
wg.Add(len(s.m))
for _, m := range s.m {
func(m mountRef) {
go func() {
m.unmount()
wg.Done()
}()
}(m)
}
wg.Wait()
}
func sub(m mount.Mount, subPath string) (mount.Mount, error) {
src, err := fs.RootPath(m.Source, subPath)
if err != nil {
return mount.Mount{}, err
}
m.Source = src
return m, nil
}
func specMapping(s []idtools.IDMap) []specs.LinuxIDMapping {
var ids []specs.LinuxIDMapping
for _, item := range s {
ids = append(ids, specs.LinuxIDMapping{
HostID: uint32(item.HostID),
ContainerID: uint32(item.ContainerID),
Size: uint32(item.Size),
})
}
return ids
return []oci.SpecOpts{
oci.WithUserNamespace(specMapping(idmap.UIDs()), specMapping(idmap.GIDs())),
}, nil
}

42
executor/oci/spec_windows.go Normal file
View File

@ -0,0 +1,42 @@
// +build windows
package oci
import (
"github.com/containerd/containerd/contrib/seccomp"
"github.com/containerd/containerd/oci"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/system"
"github.com/pkg/errors"
)
func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
return nil, nil
}
// generateSecurityOpts may affect mounts, so must be called after generateMountOpts
func generateSecurityOpts(mode pb.SecurityMode) ([]oci.SpecOpts, error) {
if mode == pb.SecurityMode_INSECURE {
return nil, errors.New("no support for running in insecure mode on Windows")
} else if system.SeccompSupported() && mode == pb.SecurityMode_SANDBOX {
// TODO: Can LCOW support seccomp? Does that even make sense?
return []oci.SpecOpts{seccomp.WithDefaultProfile()}, nil
}
return nil, nil
}
// generateProcessModeOpts may affect mounts, so must be called after generateMountOpts
func generateProcessModeOpts(mode ProcessMode) ([]oci.SpecOpts, error) {
if mode == NoProcessSandbox {
return nil, errors.New("no support for NoProcessSandbox on Windows")
}
return nil, nil
}
func generateIDmapOpts(idmap *idtools.IdentityMapping) ([]oci.SpecOpts, error) {
if idmap == nil {
return nil, nil
}
return nil, errors.New("no support for IdentityMapping on Windows")
}