Merge pull request #897 from tonistiigi/userns

userns remapping support base
docker-19.03
Akihiro Suda 2019-04-03 13:33:47 +09:00 committed by GitHub
commit a2dcdf4277
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 364 additions and 84 deletions

View File

@ -10,6 +10,7 @@ import (
"path/filepath" "path/filepath"
"sync" "sync"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/locker"
iradix "github.com/hashicorp/go-immutable-radix" iradix "github.com/hashicorp/go-immutable-radix"
"github.com/hashicorp/golang-lru/simplelru" "github.com/hashicorp/golang-lru/simplelru"
@ -51,8 +52,8 @@ func ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, path string,
return getDefaultManager().ChecksumWildcard(ctx, ref, path, followLinks) return getDefaultManager().ChecksumWildcard(ctx, ref, path, followLinks)
} }
func GetCacheContext(ctx context.Context, md *metadata.StorageItem) (CacheContext, error) { func GetCacheContext(ctx context.Context, md *metadata.StorageItem, idmap *idtools.IdentityMapping) (CacheContext, error) {
return getDefaultManager().GetCacheContext(ctx, md) return getDefaultManager().GetCacheContext(ctx, md, idmap)
} }
func SetCacheContext(ctx context.Context, md *metadata.StorageItem, cc CacheContext) error { func SetCacheContext(ctx context.Context, md *metadata.StorageItem, cc CacheContext) error {
@ -81,7 +82,7 @@ type cacheManager struct {
} }
func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool) (digest.Digest, error) { func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool) (digest.Digest, error) {
cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata())) cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata()), ref.IdentityMapping())
if err != nil { if err != nil {
return "", nil return "", nil
} }
@ -89,14 +90,14 @@ func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p
} }
func (cm *cacheManager) ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool) (digest.Digest, error) { func (cm *cacheManager) ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool) (digest.Digest, error) {
cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata())) cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata()), ref.IdentityMapping())
if err != nil { if err != nil {
return "", nil return "", nil
} }
return cc.ChecksumWildcard(ctx, ref, p, followLinks) return cc.ChecksumWildcard(ctx, ref, p, followLinks)
} }
func (cm *cacheManager) GetCacheContext(ctx context.Context, md *metadata.StorageItem) (CacheContext, error) { func (cm *cacheManager) GetCacheContext(ctx context.Context, md *metadata.StorageItem, idmap *idtools.IdentityMapping) (CacheContext, error) {
cm.locker.Lock(md.ID()) cm.locker.Lock(md.ID())
cm.lruMu.Lock() cm.lruMu.Lock()
v, ok := cm.lru.Get(md.ID()) v, ok := cm.lru.Get(md.ID())
@ -106,7 +107,7 @@ func (cm *cacheManager) GetCacheContext(ctx context.Context, md *metadata.Storag
v.(*cacheContext).linkMap = map[string][][]byte{} v.(*cacheContext).linkMap = map[string][][]byte{}
return v.(*cacheContext), nil return v.(*cacheContext), nil
} }
cc, err := newCacheContext(md) cc, err := newCacheContext(md, idmap)
if err != nil { if err != nil {
cm.locker.Unlock(md.ID()) cm.locker.Unlock(md.ID())
return nil, err return nil, err
@ -152,6 +153,7 @@ type cacheContext struct {
node *iradix.Node node *iradix.Node
dirtyMap map[string]struct{} dirtyMap map[string]struct{}
linkMap map[string][][]byte linkMap map[string][][]byte
idmap *idtools.IdentityMapping
} }
type mount struct { type mount struct {
@ -191,12 +193,13 @@ func (m *mount) clean() error {
return nil return nil
} }
func newCacheContext(md *metadata.StorageItem) (*cacheContext, error) { func newCacheContext(md *metadata.StorageItem, idmap *idtools.IdentityMapping) (*cacheContext, error) {
cc := &cacheContext{ cc := &cacheContext{
md: md, md: md,
tree: iradix.New(), tree: iradix.New(),
dirtyMap: map[string]struct{}{}, dirtyMap: map[string]struct{}{},
linkMap: map[string][][]byte{}, linkMap: map[string][][]byte{},
idmap: idmap,
} }
if err := cc.load(); err != nil { if err := cc.load(); err != nil {
return nil, err return nil, err

View File

@ -49,7 +49,7 @@ func TestChecksumHardlinks(t *testing.T) {
ref := createRef(t, cm, ch) ref := createRef(t, cm, ch)
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
dgst, err := cc.Checksum(context.TODO(), ref, "abc/foo", false) dgst, err := cc.Checksum(context.TODO(), ref, "abc/foo", false)
@ -67,7 +67,7 @@ func TestChecksumHardlinks(t *testing.T) {
// validate same results with handleChange // validate same results with handleChange
ref2 := createRef(t, cm, nil) ref2 := createRef(t, cm, nil)
cc2, err := newCacheContext(ref2.Metadata()) cc2, err := newCacheContext(ref2.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
err = emit(cc2.HandleChange, changeStream(ch)) err = emit(cc2.HandleChange, changeStream(ch))
@ -138,7 +138,7 @@ func TestChecksumWildcard(t *testing.T) {
ref := createRef(t, cm, ch) ref := createRef(t, cm, ch)
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
dgst, err := cc.ChecksumWildcard(context.TODO(), ref, "f*o", false) dgst, err := cc.ChecksumWildcard(context.TODO(), ref, "f*o", false)
@ -189,7 +189,7 @@ func TestSymlinksNoFollow(t *testing.T) {
ref := createRef(t, cm, ch) ref := createRef(t, cm, ch)
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
expectedSym := digest.Digest("sha256:a2ba571981f48ec34eb79c9a3ab091b6491e825c2f7e9914ea86e8e958be7fae") expectedSym := digest.Digest("sha256:a2ba571981f48ec34eb79c9a3ab091b6491e825c2f7e9914ea86e8e958be7fae")
@ -251,7 +251,7 @@ func TestChecksumBasicFile(t *testing.T) {
// for the digest values, the actual values are not important in development // for the digest values, the actual values are not important in development
// phase but consistency is // phase but consistency is
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
_, err = cc.Checksum(context.TODO(), ref, "nosuch", true) _, err = cc.Checksum(context.TODO(), ref, "nosuch", true)
@ -312,7 +312,7 @@ func TestChecksumBasicFile(t *testing.T) {
ref = createRef(t, cm, ch) ref = createRef(t, cm, ch)
cc, err = newCacheContext(ref.Metadata()) cc, err = newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
dgst, err = cc.Checksum(context.TODO(), ref, "/", true) dgst, err = cc.Checksum(context.TODO(), ref, "/", true)
@ -331,7 +331,7 @@ func TestChecksumBasicFile(t *testing.T) {
ref = createRef(t, cm, ch) ref = createRef(t, cm, ch)
cc, err = newCacheContext(ref.Metadata()) cc, err = newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
dgst, err = cc.Checksum(context.TODO(), ref, "/", true) dgst, err = cc.Checksum(context.TODO(), ref, "/", true)
@ -357,7 +357,7 @@ func TestChecksumBasicFile(t *testing.T) {
ref = createRef(t, cm, ch) ref = createRef(t, cm, ch)
cc, err = newCacheContext(ref.Metadata()) cc, err = newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
dgst, err = cc.Checksum(context.TODO(), ref, "abc/aa/foo", true) dgst, err = cc.Checksum(context.TODO(), ref, "abc/aa/foo", true)
@ -401,7 +401,7 @@ func TestHandleChange(t *testing.T) {
// for the digest values, the actual values are not important in development // for the digest values, the actual values are not important in development
// phase but consistency is // phase but consistency is
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
err = emit(cc.HandleChange, changeStream(ch)) err = emit(cc.HandleChange, changeStream(ch))
@ -477,7 +477,7 @@ func TestHandleRecursiveDir(t *testing.T) {
ref := createRef(t, cm, nil) ref := createRef(t, cm, nil)
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
err = emit(cc.HandleChange, changeStream(ch)) err = emit(cc.HandleChange, changeStream(ch))
@ -524,7 +524,7 @@ func TestChecksumUnorderedFiles(t *testing.T) {
ref := createRef(t, cm, nil) ref := createRef(t, cm, nil)
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
err = emit(cc.HandleChange, changeStream(ch)) err = emit(cc.HandleChange, changeStream(ch))
@ -544,7 +544,7 @@ func TestChecksumUnorderedFiles(t *testing.T) {
ref = createRef(t, cm, nil) ref = createRef(t, cm, nil)
cc, err = newCacheContext(ref.Metadata()) cc, err = newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
err = emit(cc.HandleChange, changeStream(ch)) err = emit(cc.HandleChange, changeStream(ch))
@ -731,7 +731,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) {
ref := createRef(t, cm, nil) ref := createRef(t, cm, nil)
cc, err := newCacheContext(ref.Metadata()) cc, err := newCacheContext(ref.Metadata(), nil)
require.NoError(t, err) require.NoError(t, err)
err = emit(cc.HandleChange, changeStream(ch)) err = emit(cc.HandleChange, changeStream(ch))
@ -848,7 +848,7 @@ func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapsh
require.NoError(t, err) require.NoError(t, err)
cm, err := cache.NewManager(cache.ManagerOpt{ cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
MetadataStore: md, MetadataStore: md,
}) })
require.NoError(t, err) require.NoError(t, err)

7
cache/manager.go vendored
View File

@ -8,6 +8,7 @@ import (
"github.com/containerd/containerd/filters" "github.com/containerd/containerd/filters"
"github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/client" "github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity" "github.com/moby/buildkit/identity"
@ -34,6 +35,7 @@ type Accessor interface {
GetFromSnapshotter(ctx context.Context, id string, opts ...RefOption) (ImmutableRef, error) GetFromSnapshotter(ctx context.Context, id string, opts ...RefOption) (ImmutableRef, error)
New(ctx context.Context, s ImmutableRef, opts ...RefOption) (MutableRef, error) New(ctx context.Context, s ImmutableRef, opts ...RefOption) (MutableRef, error)
GetMutable(ctx context.Context, id string) (MutableRef, error) // Rebase? GetMutable(ctx context.Context, id string) (MutableRef, error) // Rebase?
IdentityMapping() *idtools.IdentityMapping
} }
type Controller interface { type Controller interface {
@ -96,6 +98,11 @@ func (cm *cacheManager) init(ctx context.Context) error {
return nil return nil
} }
// IdentityMapping returns the userns remapping used for refs
func (cm *cacheManager) IdentityMapping() *idtools.IdentityMapping {
return cm.Snapshotter.IdentityMapping()
}
// Close closes the manager and releases the metadata database lock. No other // Close closes the manager and releases the metadata database lock. No other
// method should be called after Close. // method should be called after Close.
func (cm *cacheManager) Close() error { func (cm *cacheManager) Close() error {

View File

@ -374,7 +374,7 @@ func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshot
require.NoError(t, err) require.NoError(t, err)
cm, err := NewManager(ManagerOpt{ cm, err := NewManager(ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
MetadataStore: md, MetadataStore: md,
}) })
require.NoError(t, err, fmt.Sprintf("error: %+v", err)) require.NoError(t, err, fmt.Sprintf("error: %+v", err))

6
cache/refs.go vendored
View File

@ -5,6 +5,7 @@ import (
"sync" "sync"
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/identity" "github.com/moby/buildkit/identity"
"github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/snapshot"
@ -20,6 +21,7 @@ type Ref interface {
Release(context.Context) error Release(context.Context) error
Size(ctx context.Context) (int64, error) Size(ctx context.Context) (int64, error)
Metadata() *metadata.StorageItem Metadata() *metadata.StorageItem
IdentityMapping() *idtools.IdentityMapping
} }
type ImmutableRef interface { type ImmutableRef interface {
@ -83,6 +85,10 @@ func (cr *cacheRecord) isDead() bool {
return cr.dead || (cr.equalImmutable != nil && cr.equalImmutable.dead) || (cr.equalMutable != nil && cr.equalMutable.dead) return cr.dead || (cr.equalImmutable != nil && cr.equalImmutable.dead) || (cr.equalMutable != nil && cr.equalMutable.dead)
} }
func (cr *cacheRecord) IdentityMapping() *idtools.IdentityMapping {
return cr.cm.IdentityMapping()
}
func (cr *cacheRecord) Size(ctx context.Context) (int64, error) { func (cr *cacheRecord) Size(ctx context.Context) (int64, error) {
// this expects that usage() is implemented lazily // this expects that usage() is implemented lazily
s, err := cr.sizeG.Do(ctx, cr.ID(), func(ctx context.Context) (interface{}, error) { s, err := cr.sizeG.Do(ctx, cr.ID(), func(ctx context.Context) (interface{}, error) {

View File

@ -64,6 +64,9 @@ type OCIConfig struct {
Rootless bool `toml:"rootless"` Rootless bool `toml:"rootless"`
NoProcessSandbox bool `toml:"noProcessSandbox"` NoProcessSandbox bool `toml:"noProcessSandbox"`
GCConfig GCConfig
// UserRemapUnsupported is unsupported key for testing. The feature is
// incomplete and the intention is to make it default without config.
UserRemapUnsupported string `toml:"userRemapUnsupported"`
} }
type ContainerdConfig struct { type ContainerdConfig struct {

View File

@ -168,6 +168,12 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
return nil, nil return nil, nil
} }
// TODO: this should never change the existing state dir
idmapping, err := parseIdentityMapping(cfg.UserRemapUnsupported)
if err != nil {
return nil, err
}
snFactory, err := snapshotterFactory(common.config.Root, cfg.Snapshotter) snFactory, err := snapshotterFactory(common.config.Root, cfg.Snapshotter)
if err != nil { if err != nil {
return nil, err return nil, err
@ -186,7 +192,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
processMode = oci.NoProcessSandbox processMode = oci.NoProcessSandbox
} }
opt, err := runc.NewWorkerOpt(common.config.Root, snFactory, cfg.Rootless, processMode, cfg.Labels) opt, err := runc.NewWorkerOpt(common.config.Root, snFactory, cfg.Rootless, processMode, cfg.Labels, idmapping)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -1,8 +1,13 @@
package main package main
import ( import (
"runtime"
"strconv" "strconv"
"strings" "strings"
"github.com/docker/docker/pkg/idtools"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
) )
// parseBoolOrAuto returns (nil, nil) if s is "auto" // parseBoolOrAuto returns (nil, nil) if s is "auto"
@ -13,3 +18,32 @@ func parseBoolOrAuto(s string) (*bool, error) {
b, err := strconv.ParseBool(s) b, err := strconv.ParseBool(s)
return &b, err return &b, err
} }
func parseIdentityMapping(str string) (*idtools.IdentityMapping, error) {
if str == "" {
return nil, nil
}
if runtime.GOOS != "linux" && str != "" {
return nil, errors.Errorf("user namespaces are only supported on linux")
}
idparts := strings.Split(str, ":")
if len(idparts) > 2 {
return nil, errors.Errorf("invalid userns remap specification in %q", str)
}
username := idparts[0]
groupname := username
if len(idparts) == 2 {
groupname = idparts[1]
}
logrus.Debugf("user namespaces: ID ranges will be mapped to subuid/subgid ranges of: %s:%s", username, groupname)
mappings, err := idtools.NewIdentityMapping(username, groupname)
if err != nil {
return nil, errors.Wrap(err, "failed to create ID mappings")
}
return mappings, nil
}

View File

@ -117,7 +117,7 @@ func (w containerdExecutor) Exec(ctx context.Context, meta executor.Meta, root c
opts = append(opts, containerdoci.WithCgroup(cgroupsPath)) opts = append(opts, containerdoci.WithCgroup(cgroupsPath))
} }
processMode := oci.ProcessSandbox // FIXME(AkihiroSuda) processMode := oci.ProcessSandbox // FIXME(AkihiroSuda)
spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, processMode, opts...) spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, processMode, nil, opts...)
if err != nil { if err != nil {
return err return err
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci" "github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs" "github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/mitchellh/hashstructure" "github.com/mitchellh/hashstructure"
"github.com/moby/buildkit/executor" "github.com/moby/buildkit/executor"
"github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/snapshot"
@ -40,7 +41,7 @@ const (
// GenerateSpec generates spec using containerd functionality. // GenerateSpec generates spec using containerd functionality.
// opts are ignored for s.Process, s.Hostname, and s.Mounts . // opts are ignored for s.Process, s.Hostname, and s.Mounts .
func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mount, id, resolvConf, hostsFile string, namespace network.Namespace, processMode ProcessMode, opts ...oci.SpecOpts) (*specs.Spec, func(), error) { func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mount, id, resolvConf, hostsFile string, namespace network.Namespace, processMode ProcessMode, idmap *idtools.IdentityMapping, opts ...oci.SpecOpts) (*specs.Spec, func(), error) {
c := &containers.Container{ c := &containers.Container{
ID: id, ID: id,
} }
@ -102,7 +103,14 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
} }
} }
} }
// TODO: User
if idmap != nil {
s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{
Type: specs.UserNamespace,
})
s.Linux.UIDMappings = specMapping(idmap.UIDs())
s.Linux.GIDMappings = specMapping(idmap.GIDs())
}
sm := &submounts{} sm := &submounts{}
@ -227,3 +235,15 @@ func sub(m mount.Mount, subPath string) (mount.Mount, error) {
m.Source = src m.Source = src
return m, nil return m, nil
} }
func specMapping(s []idtools.IDMap) []specs.LinuxIDMapping {
var ids []specs.LinuxIDMapping
for _, item := range s {
ids = append(ids, specs.LinuxIDMapping{
HostID: uint32(item.HostID),
ContainerID: uint32(item.ContainerID),
Size: uint32(item.Size),
})
}
return ids
}

View File

@ -17,6 +17,7 @@ import (
containerdoci "github.com/containerd/containerd/oci" containerdoci "github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs" "github.com/containerd/continuity/fs"
runc "github.com/containerd/go-runc" runc "github.com/containerd/go-runc"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/executor" "github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/oci" "github.com/moby/buildkit/executor/oci"
@ -39,6 +40,7 @@ type Opt struct {
DefaultCgroupParent string DefaultCgroupParent string
// ProcessMode // ProcessMode
ProcessMode oci.ProcessMode ProcessMode oci.ProcessMode
IdentityMapping *idtools.IdentityMapping
} }
var defaultCommandCandidates = []string{"buildkit-runc", "runc"} var defaultCommandCandidates = []string{"buildkit-runc", "runc"}
@ -51,6 +53,7 @@ type runcExecutor struct {
rootless bool rootless bool
networkProviders map[pb.NetMode]network.Provider networkProviders map[pb.NetMode]network.Provider
processMode oci.ProcessMode processMode oci.ProcessMode
idmap *idtools.IdentityMapping
} }
func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Executor, error) { func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Executor, error) {
@ -107,6 +110,7 @@ func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Ex
rootless: opt.Rootless, rootless: opt.Rootless,
networkProviders: networkProviders, networkProviders: networkProviders,
processMode: opt.ProcessMode, processMode: opt.ProcessMode,
idmap: opt.IdentityMapping,
} }
return w, nil return w, nil
} }
@ -157,8 +161,14 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
return err return err
} }
defer os.RemoveAll(bundle) defer os.RemoveAll(bundle)
identity := idtools.Identity{}
if w.idmap != nil {
identity = w.idmap.RootPair()
}
rootFSPath := filepath.Join(bundle, "rootfs") rootFSPath := filepath.Join(bundle, "rootfs")
if err := os.Mkdir(rootFSPath, 0700); err != nil { if err := idtools.MkdirAllAndChown(rootFSPath, 0700, identity); err != nil {
return err return err
} }
if err := mount.All(rootMount, rootFSPath); err != nil { if err := mount.All(rootMount, rootFSPath); err != nil {
@ -193,7 +203,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
} }
opts = append(opts, containerdoci.WithCgroup(cgroupsPath)) opts = append(opts, containerdoci.WithCgroup(cgroupsPath))
} }
spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, w.processMode, opts...) spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, w.processMode, w.idmap, opts...)
if err != nil { if err != nil {
return err return err
} }
@ -208,7 +218,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
if err != nil { if err != nil {
return errors.Wrapf(err, "working dir %s points to invalid target", newp) return errors.Wrapf(err, "working dir %s points to invalid target", newp)
} }
if err := os.MkdirAll(newp, 0755); err != nil { if err := idtools.MkdirAllAndChown(newp, 0755, identity); err != nil {
return errors.Wrapf(err, "failed to create working directory %s", newp) return errors.Wrapf(err, "failed to create working directory %s", newp)
} }

View File

@ -7,6 +7,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter" "github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session" "github.com/moby/buildkit/session"
@ -68,6 +69,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
return func() error { return func() error {
var src string var src string
var err error var err error
var idmap *idtools.IdentityMapping
if ref == nil { if ref == nil {
src, err = ioutil.TempDir("", "buildkit") src, err = ioutil.TempDir("", "buildkit")
if err != nil { if err != nil {
@ -86,10 +88,30 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
if err != nil { if err != nil {
return err return err
} }
idmap = mount.IdentityMapping()
defer lm.Unmount() defer lm.Unmount()
} }
fs := fsutil.NewFS(src, nil) walkOpt := &fsutil.WalkOpt{}
if idmap != nil {
walkOpt.Map = func(p string, st *fstypes.Stat) bool {
uid, gid, err := idmap.ToContainer(idtools.Identity{
UID: int(st.Uid),
GID: int(st.Gid),
})
if err != nil {
return false
}
st.Uid = uint32(uid)
st.Gid = uint32(gid)
return true
}
}
fs := fsutil.NewFS(src, walkOpt)
lbl := "copying files" lbl := "copying files"
if isMap { if isMap {
lbl += " " + k lbl += " " + k

View File

@ -7,6 +7,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter" "github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session" "github.com/moby/buildkit/session"
@ -71,6 +72,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
getDir := func(ctx context.Context, k string, ref cache.ImmutableRef) (*fsutil.Dir, error) { getDir := func(ctx context.Context, k string, ref cache.ImmutableRef) (*fsutil.Dir, error) {
var src string var src string
var err error var err error
var idmap *idtools.IdentityMapping
if ref == nil { if ref == nil {
src, err = ioutil.TempDir("", "buildkit") src, err = ioutil.TempDir("", "buildkit")
if err != nil { if err != nil {
@ -89,11 +91,31 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
if err != nil { if err != nil {
return nil, err return nil, err
} }
idmap = mount.IdentityMapping()
defers = append(defers, func() { lm.Unmount() }) defers = append(defers, func() { lm.Unmount() })
} }
walkOpt := &fsutil.WalkOpt{}
if idmap != nil {
walkOpt.Map = func(p string, st *fstypes.Stat) bool {
uid, gid, err := idmap.ToContainer(idtools.Identity{
UID: int(st.Uid),
GID: int(st.Gid),
})
if err != nil {
return false
}
st.Uid = uint32(uid)
st.Gid = uint32(gid)
return true
}
}
return &fsutil.Dir{ return &fsutil.Dir{
FS: fsutil.NewFS(src, nil), FS: fsutil.NewFS(src, walkOpt),
Stat: fstypes.Stat{ Stat: fstypes.Stat{
Mode: uint32(os.ModeDir | 0755), Mode: uint32(os.ModeDir | 0755),
Path: strings.Replace(k, "/", "_", -1), Path: strings.Replace(k, "/", "_", -1),

View File

@ -57,7 +57,7 @@ func (wc *streamWriterCloser) Close() error {
return nil return nil
} }
func recvDiffCopy(ds grpc.Stream, dest string, cu CacheUpdater, progress progressCb) error { func recvDiffCopy(ds grpc.Stream, dest string, cu CacheUpdater, progress progressCb, filter func(string, *fstypes.Stat) bool) error {
st := time.Now() st := time.Now()
defer func() { defer func() {
logrus.Debugf("diffcopy took: %v", time.Since(st)) logrus.Debugf("diffcopy took: %v", time.Since(st))
@ -73,6 +73,7 @@ func recvDiffCopy(ds grpc.Stream, dest string, cu CacheUpdater, progress progres
NotifyHashed: cf, NotifyHashed: cf,
ContentHasher: ch, ContentHasher: ch,
ProgressCb: progress, ProgressCb: progress,
Filter: fsutil.FilterFunc(filter),
}) })
} }

View File

@ -129,7 +129,7 @@ type progressCb func(int, bool)
type protocol struct { type protocol struct {
name string name string
sendFn func(stream grpc.Stream, fs fsutil.FS, progress progressCb) error sendFn func(stream grpc.Stream, fs fsutil.FS, progress progressCb) error
recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb) error recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb, mapFunc func(string, *fstypes.Stat) bool) error
} }
func isProtoSupported(p string) bool { func isProtoSupported(p string) bool {
@ -158,6 +158,7 @@ type FSSendRequestOpt struct {
DestDir string DestDir string
CacheUpdater CacheUpdater CacheUpdater CacheUpdater
ProgressCb func(int, bool) ProgressCb func(int, bool)
Filter func(string, *fstypes.Stat) bool
} }
// CacheUpdater is an object capable of sending notifications for the cache hash changes // CacheUpdater is an object capable of sending notifications for the cache hash changes
@ -225,7 +226,7 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
panic(fmt.Sprintf("invalid protocol: %q", pr.name)) panic(fmt.Sprintf("invalid protocol: %q", pr.name))
} }
return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb) return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Filter)
} }
// NewFSSyncTargetDir allows writing into a directory // NewFSSyncTargetDir allows writing into a directory

View File

@ -8,15 +8,16 @@ import (
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
ctdsnapshot "github.com/containerd/containerd/snapshots" ctdsnapshot "github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/snapshot/blobmapping" "github.com/moby/buildkit/snapshot/blobmapping"
) )
func NewSnapshotter(snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error) snapshot.Snapshotter { func NewSnapshotter(snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error, idmap *idtools.IdentityMapping) snapshot.Snapshotter {
return blobmapping.NewSnapshotter(blobmapping.Opt{ return blobmapping.NewSnapshotter(blobmapping.Opt{
Content: store, Content: store,
Snapshotter: snapshot.FromContainerdSnapshotter(&nsSnapshotter{ns, snapshotter, gc}), Snapshotter: snapshot.FromContainerdSnapshotter(&nsSnapshotter{ns, snapshotter, gc}, idmap),
MetadataStore: mdstore, MetadataStore: mdstore,
}) })
} }

View File

@ -6,6 +6,7 @@ import (
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
digest "github.com/opencontainers/go-digest" digest "github.com/opencontainers/go-digest"
) )
@ -13,6 +14,7 @@ type Mountable interface {
// ID() string // ID() string
Mount() ([]mount.Mount, error) Mount() ([]mount.Mount, error)
Release() error Release() error
IdentityMapping() *idtools.IdentityMapping
} }
type SnapshotterBase interface { type SnapshotterBase interface {
@ -27,6 +29,7 @@ type SnapshotterBase interface {
Remove(ctx context.Context, key string) error Remove(ctx context.Context, key string) error
Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error
Close() error Close() error
IdentityMapping() *idtools.IdentityMapping
} }
// Snapshotter defines interface that any snapshot implementation should satisfy // Snapshotter defines interface that any snapshot implementation should satisfy
@ -40,12 +43,13 @@ type Blobmapper interface {
SetBlob(ctx context.Context, key string, diffID, blob digest.Digest) error SetBlob(ctx context.Context, key string, diffID, blob digest.Digest) error
} }
func FromContainerdSnapshotter(s snapshots.Snapshotter) SnapshotterBase { func FromContainerdSnapshotter(s snapshots.Snapshotter, idmap *idtools.IdentityMapping) SnapshotterBase {
return &fromContainerd{Snapshotter: s} return &fromContainerd{Snapshotter: s, idmap: idmap}
} }
type fromContainerd struct { type fromContainerd struct {
snapshots.Snapshotter snapshots.Snapshotter
idmap *idtools.IdentityMapping
} }
func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, error) { func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, error) {
@ -53,7 +57,7 @@ func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, err
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &staticMountable{mounts}, nil return &staticMountable{mounts, s.idmap}, nil
} }
func (s *fromContainerd) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error { func (s *fromContainerd) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
_, err := s.Snapshotter.Prepare(ctx, key, parent, opts...) _, err := s.Snapshotter.Prepare(ctx, key, parent, opts...)
@ -64,11 +68,15 @@ func (s *fromContainerd) View(ctx context.Context, key, parent string, opts ...s
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &staticMountable{mounts}, nil return &staticMountable{mounts, s.idmap}, nil
}
func (s *fromContainerd) IdentityMapping() *idtools.IdentityMapping {
return s.idmap
} }
type staticMountable struct { type staticMountable struct {
mounts []mount.Mount mounts []mount.Mount
idmap *idtools.IdentityMapping
} }
func (m *staticMountable) Mount() ([]mount.Mount, error) { func (m *staticMountable) Mount() ([]mount.Mount, error) {
@ -79,6 +87,10 @@ func (cm *staticMountable) Release() error {
return nil return nil
} }
func (cm *staticMountable) IdentityMapping() *idtools.IdentityMapping {
return cm.idmap
}
// NewContainerdSnapshotter converts snapshotter to containerd snapshotter // NewContainerdSnapshotter converts snapshotter to containerd snapshotter
func NewContainerdSnapshotter(s Snapshotter) (snapshots.Snapshotter, func() error) { func NewContainerdSnapshotter(s Snapshotter) (snapshots.Snapshotter, func() error) {
cs := &containerdSnapshotter{Snapshotter: s} cs := &containerdSnapshotter{Snapshotter: s}

View File

@ -10,6 +10,7 @@ import (
"time" "time"
"github.com/containerd/continuity/fs" "github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes" "github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes"
"github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/solver/pb"
@ -25,12 +26,35 @@ func timestampToTime(ts int64) *time.Time {
return &tm return &tm
} }
func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *copy.ChownOpt) error { func mapUser(user *copy.ChownOpt, idmap *idtools.IdentityMapping) (*copy.ChownOpt, error) {
if idmap == nil {
return user, nil
}
if user == nil {
identity := idmap.RootPair()
return &copy.ChownOpt{Uid: identity.UID, Gid: identity.GID}, nil
}
identity, err := idmap.ToHost(idtools.Identity{
UID: user.Uid,
GID: user.Gid,
})
if err != nil {
return nil, err
}
return &copy.ChownOpt{Uid: identity.UID, Gid: identity.GID}, nil
}
func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *copy.ChownOpt, idmap *idtools.IdentityMapping) error {
p, err := fs.RootPath(d, filepath.Join(filepath.Join("/", action.Path))) p, err := fs.RootPath(d, filepath.Join(filepath.Join("/", action.Path)))
if err != nil { if err != nil {
return err return err
} }
user, err = mapUser(user, idmap)
if err != nil {
return err
}
if action.MakeParents { if action.MakeParents {
if err := copy.MkdirAll(p, os.FileMode(action.Mode)&0777, user, timestampToTime(action.Timestamp)); err != nil { if err := copy.MkdirAll(p, os.FileMode(action.Mode)&0777, user, timestampToTime(action.Timestamp)); err != nil {
return err return err
@ -53,12 +77,17 @@ func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *copy.
return nil return nil
} }
func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *copy.ChownOpt) error { func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *copy.ChownOpt, idmap *idtools.IdentityMapping) error {
p, err := fs.RootPath(d, filepath.Join(filepath.Join("/", action.Path))) p, err := fs.RootPath(d, filepath.Join(filepath.Join("/", action.Path)))
if err != nil { if err != nil {
return err return err
} }
user, err = mapUser(user, idmap)
if err != nil {
return err
}
if err := ioutil.WriteFile(p, action.Data, os.FileMode(action.Mode)&0777); err != nil { if err := ioutil.WriteFile(p, action.Data, os.FileMode(action.Mode)&0777); err != nil {
return err return err
} }
@ -90,7 +119,7 @@ func rm(ctx context.Context, d string, action pb.FileActionRm) error {
return nil return nil
} }
func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *copy.ChownOpt) error { func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *copy.ChownOpt, idmap *idtools.IdentityMapping) error {
srcPath := cleanPath(action.Src) srcPath := cleanPath(action.Src)
destPath := cleanPath(action.Dest) destPath := cleanPath(action.Dest)
@ -109,6 +138,12 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *
return nil return nil
} }
// TODO(tonistiigi): this is wrong. fsutil.Copy can't handle non-forced user
u, err := mapUser(u, idmap)
if err != nil {
return err
}
opt := []copy.Opt{ opt := []copy.Opt{
func(ci *copy.CopyInfo) { func(ci *copy.CopyInfo) {
ci.Chown = u ci.Chown = u
@ -195,7 +230,7 @@ func (fb *Backend) Mkdir(ctx context.Context, m, user, group fileoptypes.Mount,
return err return err
} }
return mkdir(ctx, dir, action, u) return mkdir(ctx, dir, action, u, mnt.m.IdentityMapping())
} }
func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount, action pb.FileActionMkFile) error { func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount, action pb.FileActionMkFile) error {
@ -216,7 +251,7 @@ func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount,
return err return err
} }
return mkfile(ctx, dir, action, u) return mkfile(ctx, dir, action, u, mnt.m.IdentityMapping())
} }
func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error { func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error {
mnt, ok := m.(*Mount) mnt, ok := m.(*Mount)
@ -262,5 +297,5 @@ func (fb *Backend) Copy(ctx context.Context, m1, m2, user, group fileoptypes.Mou
return err return err
} }
return docopy(ctx, src, dest, action, u) return docopy(ctx, src, dest, action, u, mnt2.m.IdentityMapping())
} }

View File

@ -17,6 +17,7 @@ import (
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/platforms" "github.com/containerd/containerd/platforms"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/locker"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
@ -329,30 +330,47 @@ func (e *execOp) getSSHMountable(ctx context.Context, m *pb.Mount) (cache.Mounta
return nil, err return nil, err
} }
return &sshMount{mount: m, caller: caller}, nil return &sshMount{mount: m, caller: caller, idmap: e.cm.IdentityMapping()}, nil
} }
type sshMount struct { type sshMount struct {
mount *pb.Mount mount *pb.Mount
caller session.Caller caller session.Caller
idmap *idtools.IdentityMapping
} }
func (sm *sshMount) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) { func (sm *sshMount) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
return &sshMountInstance{sm: sm}, nil return &sshMountInstance{sm: sm, idmap: sm.idmap}, nil
} }
type sshMountInstance struct { type sshMountInstance struct {
sm *sshMount sm *sshMount
cleanup func() error cleanup func() error
idmap *idtools.IdentityMapping
} }
func (sm *sshMountInstance) Mount() ([]mount.Mount, error) { func (sm *sshMountInstance) Mount() ([]mount.Mount, error) {
ctx, cancel := context.WithCancel(context.TODO()) ctx, cancel := context.WithCancel(context.TODO())
uid := int(sm.sm.mount.SSHOpt.Uid)
gid := int(sm.sm.mount.SSHOpt.Gid)
if sm.idmap != nil {
identity, err := sm.idmap.ToHost(idtools.Identity{
UID: uid,
GID: gid,
})
if err != nil {
return nil, err
}
uid = identity.UID
gid = identity.GID
}
sock, cleanup, err := sshforward.MountSSHSocket(ctx, sm.sm.caller, sshforward.SocketOpt{ sock, cleanup, err := sshforward.MountSSHSocket(ctx, sm.sm.caller, sshforward.SocketOpt{
ID: sm.sm.mount.SSHOpt.ID, ID: sm.sm.mount.SSHOpt.ID,
UID: int(sm.sm.mount.SSHOpt.Uid), UID: uid,
GID: int(sm.sm.mount.SSHOpt.Gid), GID: gid,
Mode: int(sm.sm.mount.SSHOpt.Mode & 0777), Mode: int(sm.sm.mount.SSHOpt.Mode & 0777),
}) })
if err != nil { if err != nil {
@ -384,6 +402,10 @@ func (sm *sshMountInstance) Release() error {
return nil return nil
} }
func (sm *sshMountInstance) IdentityMapping() *idtools.IdentityMapping {
return sm.idmap
}
func (e *execOp) getSecretMountable(ctx context.Context, m *pb.Mount) (cache.Mountable, error) { func (e *execOp) getSecretMountable(ctx context.Context, m *pb.Mount) (cache.Mountable, error) {
if m.SecretOpt == nil { if m.SecretOpt == nil {
return nil, errors.Errorf("invalid sercet mount options") return nil, errors.Errorf("invalid sercet mount options")
@ -416,21 +438,23 @@ func (e *execOp) getSecretMountable(ctx context.Context, m *pb.Mount) (cache.Mou
return nil, err return nil, err
} }
return &secretMount{mount: m, data: dt}, nil return &secretMount{mount: m, data: dt, idmap: e.cm.IdentityMapping()}, nil
} }
type secretMount struct { type secretMount struct {
mount *pb.Mount mount *pb.Mount
data []byte data []byte
idmap *idtools.IdentityMapping
} }
func (sm *secretMount) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) { func (sm *secretMount) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
return &secretMountInstance{sm: sm}, nil return &secretMountInstance{sm: sm, idmap: sm.idmap}, nil
} }
type secretMountInstance struct { type secretMountInstance struct {
sm *secretMount sm *secretMount
root string root string
idmap *idtools.IdentityMapping
} }
func (sm *secretMountInstance) Mount() ([]mount.Mount, error) { func (sm *secretMountInstance) Mount() ([]mount.Mount, error) {
@ -465,7 +489,22 @@ func (sm *secretMountInstance) Mount() ([]mount.Mount, error) {
return nil, err return nil, err
} }
if err := os.Chown(fp, int(sm.sm.mount.SecretOpt.Uid), int(sm.sm.mount.SecretOpt.Gid)); err != nil { uid := int(sm.sm.mount.SecretOpt.Uid)
gid := int(sm.sm.mount.SecretOpt.Gid)
if sm.idmap != nil {
identity, err := sm.idmap.ToHost(idtools.Identity{
UID: uid,
GID: gid,
})
if err != nil {
return nil, err
}
uid = identity.UID
gid = identity.GID
}
if err := os.Chown(fp, uid, gid); err != nil {
return nil, err return nil, err
} }
@ -490,6 +529,10 @@ func (sm *secretMountInstance) Release() error {
return nil return nil
} }
func (sm *secretMountInstance) IdentityMapping() *idtools.IdentityMapping {
return sm.idmap
}
func addDefaultEnvvar(env []string, k, v string) []string { func addDefaultEnvvar(env []string, k, v string) []string {
for _, e := range env { for _, e := range env {
if strings.HasPrefix(e, k+"=") { if strings.HasPrefix(e, k+"=") {
@ -585,7 +628,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res
} }
case pb.MountType_TMPFS: case pb.MountType_TMPFS:
mountable = newTmpfs() mountable = newTmpfs(e.cm.IdentityMapping())
case pb.MountType_SECRET: case pb.MountType_SECRET:
secretMount, err := e.getSecretMountable(ctx, m) secretMount, err := e.getSecretMountable(ctx, m)
@ -702,19 +745,21 @@ func proxyEnvList(p *pb.ProxyEnv) []string {
return out return out
} }
func newTmpfs() cache.Mountable { func newTmpfs(idmap *idtools.IdentityMapping) cache.Mountable {
return &tmpfs{} return &tmpfs{idmap: idmap}
} }
type tmpfs struct { type tmpfs struct {
idmap *idtools.IdentityMapping
} }
func (f *tmpfs) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) { func (f *tmpfs) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
return &tmpfsMount{readonly: readonly}, nil return &tmpfsMount{readonly: readonly, idmap: f.idmap}, nil
} }
type tmpfsMount struct { type tmpfsMount struct {
readonly bool readonly bool
idmap *idtools.IdentityMapping
} }
func (m *tmpfsMount) Mount() ([]mount.Mount, error) { func (m *tmpfsMount) Mount() ([]mount.Mount, error) {
@ -732,6 +777,10 @@ func (m *tmpfsMount) Release() error {
return nil return nil
} }
func (m *tmpfsMount) IdentityMapping() *idtools.IdentityMapping {
return m.idmap
}
var cacheRefsLocker = locker.New() var cacheRefsLocker = locker.New()
var sharedCacheRefs = &cacheRefs{} var sharedCacheRefs = &cacheRefs{}

View File

@ -335,6 +335,16 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote) return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote)
} }
if idmap := mount.IdentityMapping(); idmap != nil {
u := idmap.RootPair()
err := filepath.Walk(gitDir, func(p string, f os.FileInfo, err error) error {
return os.Lchown(p, u.UID, u.GID)
})
if err != nil {
return nil, errors.Wrap(err, "failed to remap git checkout")
}
}
lm.Unmount() lm.Unmount()
lm = nil lm = nil

View File

@ -297,7 +297,7 @@ func setupGitSource(t *testing.T, tmpdir string) source.Source {
assert.NoError(t, err) assert.NoError(t, err)
cm, err := cache.NewManager(cache.ManagerOpt{ cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
MetadataStore: md, MetadataStore: md,
}) })
assert.NoError(t, err) assert.NoError(t, err)

View File

@ -15,6 +15,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/locker"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
@ -278,8 +279,22 @@ func (hs *httpSourceHandler) save(ctx context.Context, resp *http.Response) (ref
} }
f = nil f = nil
if hs.src.UID != 0 || hs.src.GID != 0 { uid := hs.src.UID
if err := os.Chown(fp, hs.src.UID, hs.src.GID); err != nil { gid := hs.src.GID
if idmap := mount.IdentityMapping(); idmap != nil {
identity, err := idmap.ToHost(idtools.Identity{
UID: int(uid),
GID: int(gid),
})
if err != nil {
return nil, "", err
}
uid = identity.UID
gid = identity.GID
}
if gid != 0 || uid != 0 {
if err := os.Chown(fp, uid, gid); err != nil {
return nil, "", err return nil, "", err
} }
} }

View File

@ -315,7 +315,7 @@ func newHTTPSource(tmpdir string) (source.Source, error) {
} }
cm, err := cache.NewManager(cache.ManagerOpt{ cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
MetadataStore: md, MetadataStore: md,
}) })
if err != nil { if err != nil {

View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"time" "time"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/contenthash" "github.com/moby/buildkit/cache/contenthash"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
@ -19,6 +20,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/tonistiigi/fsutil" "github.com/tonistiigi/fsutil"
fstypes "github.com/tonistiigi/fsutil/types"
bolt "go.etcd.io/bbolt" bolt "go.etcd.io/bbolt"
"golang.org/x/time/rate" "golang.org/x/time/rate"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
@ -153,7 +155,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
} }
}() }()
cc, err := contenthash.GetCacheContext(ctx, mutable.Metadata()) cc, err := contenthash.GetCacheContext(ctx, mutable.Metadata(), mount.IdentityMapping())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -165,10 +167,25 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
FollowPaths: ls.src.FollowPaths, FollowPaths: ls.src.FollowPaths,
OverrideExcludes: false, OverrideExcludes: false,
DestDir: dest, DestDir: dest,
CacheUpdater: &cacheUpdater{cc}, CacheUpdater: &cacheUpdater{cc, mount.IdentityMapping()},
ProgressCb: newProgressHandler(ctx, "transferring "+ls.src.Name+":"), ProgressCb: newProgressHandler(ctx, "transferring "+ls.src.Name+":"),
} }
if idmap := mount.IdentityMapping(); idmap != nil {
opt.Filter = func(p string, stat *fstypes.Stat) bool {
identity, err := idmap.ToHost(idtools.Identity{
UID: int(stat.Uid),
GID: int(stat.Gid),
})
if err != nil {
return false
}
stat.Uid = uint32(identity.UID)
stat.Gid = uint32(identity.GID)
return true
}
}
if err := filesync.FSSync(ctx, caller, opt); err != nil { if err := filesync.FSSync(ctx, caller, opt); err != nil {
if status.Code(err) == codes.NotFound { if status.Code(err) == codes.NotFound {
return nil, errors.Errorf("local source %s not enabled from the client", ls.src.Name) return nil, errors.Errorf("local source %s not enabled from the client", ls.src.Name)
@ -245,6 +262,7 @@ func newProgressHandler(ctx context.Context, id string) func(int, bool) {
type cacheUpdater struct { type cacheUpdater struct {
contenthash.CacheContext contenthash.CacheContext
idmap *idtools.IdentityMapping
} }
func (cu *cacheUpdater) MarkSupported(bool) { func (cu *cacheUpdater) MarkSupported(bool) {

6
vendor/modules.txt vendored
View File

@ -94,9 +94,9 @@ github.com/containerd/continuity/fs
github.com/containerd/continuity/syscallx github.com/containerd/continuity/syscallx
github.com/containerd/continuity github.com/containerd/continuity
github.com/containerd/continuity/fs/fstest github.com/containerd/continuity/fs/fstest
github.com/containerd/continuity/pathdriver
github.com/containerd/continuity/devices github.com/containerd/continuity/devices
github.com/containerd/continuity/driver github.com/containerd/continuity/driver
github.com/containerd/continuity/pathdriver
github.com/containerd/continuity/proto github.com/containerd/continuity/proto
# github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 # github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260
github.com/containerd/fifo github.com/containerd/fifo
@ -116,6 +116,7 @@ github.com/docker/cli/cli/config/types
github.com/docker/distribution/reference github.com/docker/distribution/reference
github.com/docker/distribution/digestset github.com/docker/distribution/digestset
# github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c # github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c
github.com/docker/docker/pkg/idtools
github.com/docker/docker/pkg/locker github.com/docker/docker/pkg/locker
github.com/docker/docker/pkg/reexec github.com/docker/docker/pkg/reexec
github.com/docker/docker/builder/dockerignore github.com/docker/docker/builder/dockerignore
@ -124,15 +125,14 @@ github.com/docker/docker/pkg/signal
github.com/docker/docker/api/types/container github.com/docker/docker/api/types/container
github.com/docker/docker/pkg/archive github.com/docker/docker/pkg/archive
github.com/docker/docker/pkg/chrootarchive github.com/docker/docker/pkg/chrootarchive
github.com/docker/docker/pkg/system
github.com/docker/docker/pkg/fileutils github.com/docker/docker/pkg/fileutils
github.com/docker/docker/pkg/ioutils github.com/docker/docker/pkg/ioutils
github.com/docker/docker/api/types/blkiodev github.com/docker/docker/api/types/blkiodev
github.com/docker/docker/api/types/mount github.com/docker/docker/api/types/mount
github.com/docker/docker/pkg/homedir github.com/docker/docker/pkg/homedir
github.com/docker/docker/pkg/idtools
github.com/docker/docker/pkg/longpath github.com/docker/docker/pkg/longpath
github.com/docker/docker/pkg/pools github.com/docker/docker/pkg/pools
github.com/docker/docker/pkg/system
github.com/docker/docker/pkg/mount github.com/docker/docker/pkg/mount
# github.com/docker/docker-credential-helpers v0.6.0 # github.com/docker/docker-credential-helpers v0.6.0
github.com/docker/docker-credential-helpers/client github.com/docker/docker-credential-helpers/client

View File

@ -14,6 +14,7 @@ import (
"github.com/containerd/containerd/images" "github.com/containerd/containerd/images"
"github.com/containerd/containerd/rootfs" "github.com/containerd/containerd/rootfs"
cdsnapshot "github.com/containerd/containerd/snapshots" cdsnapshot "github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/blobs" "github.com/moby/buildkit/cache/blobs"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
@ -70,6 +71,7 @@ type WorkerOpt struct {
Differ diff.Comparer Differ diff.Comparer
ImageStore images.Store // optional ImageStore images.Store // optional
ResolveOptionsFunc resolver.ResolveOptionsFunc ResolveOptionsFunc resolver.ResolveOptionsFunc
IdentityMapping *idtools.IdentityMapping
} }
// Worker is a local worker instance with dedicated snapshotter, cache, and so on. // Worker is a local worker instance with dedicated snapshotter, cache, and so on.

View File

@ -106,7 +106,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s
Labels: xlabels, Labels: xlabels,
MetadataStore: md, MetadataStore: md,
Executor: containerdexecutor.New(client, root, "", network.Default()), Executor: containerdexecutor.New(client, root, "", network.Default()),
Snapshotter: containerdsnapshot.NewSnapshotter(client.SnapshotService(snapshotterName), cs, md, ns, gc), Snapshotter: containerdsnapshot.NewSnapshotter(client.SnapshotService(snapshotterName), cs, md, ns, gc, nil),
ContentStore: cs, ContentStore: cs,
Applier: winlayers.NewFileSystemApplierWithWindows(cs, df), Applier: winlayers.NewFileSystemApplierWithWindows(cs, df),
Differ: winlayers.NewWalkingDiffWithWindows(cs, df), Differ: winlayers.NewWalkingDiffWithWindows(cs, df),

View File

@ -12,6 +12,7 @@ import (
ctdmetadata "github.com/containerd/containerd/metadata" ctdmetadata "github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/platforms" "github.com/containerd/containerd/platforms"
ctdsnapshot "github.com/containerd/containerd/snapshots" ctdsnapshot "github.com/containerd/containerd/snapshots"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/executor/oci" "github.com/moby/buildkit/executor/oci"
"github.com/moby/buildkit/executor/runcexecutor" "github.com/moby/buildkit/executor/runcexecutor"
@ -32,7 +33,7 @@ type SnapshotterFactory struct {
} }
// NewWorkerOpt creates a WorkerOpt. // NewWorkerOpt creates a WorkerOpt.
func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string) (base.WorkerOpt, error) { func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *idtools.IdentityMapping) (base.WorkerOpt, error) {
var opt base.WorkerOpt var opt base.WorkerOpt
name := "runc-" + snFactory.Name name := "runc-" + snFactory.Name
root = filepath.Join(root, name) root = filepath.Join(root, name)
@ -49,6 +50,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc
// without root privileges // without root privileges
Rootless: rootless, Rootless: rootless,
ProcessMode: processMode, ProcessMode: processMode,
IdentityMapping: idmap,
}, network.Default()) }, network.Default())
if err != nil { if err != nil {
return opt, err return opt, err
@ -101,12 +103,13 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc
Labels: xlabels, Labels: xlabels,
MetadataStore: md, MetadataStore: md,
Executor: exe, Executor: exe,
Snapshotter: containerdsnapshot.NewSnapshotter(mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc), Snapshotter: containerdsnapshot.NewSnapshotter(mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc, idmap),
ContentStore: c, ContentStore: c,
Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)), Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)),
Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)), Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)),
ImageStore: nil, // explicitly ImageStore: nil, // explicitly
Platforms: []specs.Platform{platforms.Normalize(platforms.DefaultSpec())}, Platforms: []specs.Platform{platforms.Normalize(platforms.DefaultSpec())},
IdentityMapping: idmap,
} }
return opt, nil return opt, nil
} }

View File

@ -39,7 +39,7 @@ func newWorkerOpt(t *testing.T, processMode oci.ProcessMode) (base.WorkerOpt, fu
}, },
} }
rootless := false rootless := false
workerOpt, err := NewWorkerOpt(tmpdir, snFactory, rootless, processMode, nil) workerOpt, err := NewWorkerOpt(tmpdir, snFactory, rootless, processMode, nil, nil)
require.NoError(t, err) require.NoError(t, err)
return workerOpt, cleanup return workerOpt, cleanup