From ff951eecd9afbe35a3e6d1844c55f328a3a34b68 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 17 Jul 2017 23:08:22 -0700 Subject: [PATCH] llb: add readonly mounts support Signed-off-by: Tonis Tiigi --- cache/manager_test.go | 2 +- cache/refs.go | 30 ++++++++++++++++++++++++-- client/llb/llb.go | 7 +++--- client/llb/state.go | 15 ++++++++++--- control/control_standalone_test.go | 4 ++-- exporter/containerimage/export.go | 4 ++-- solver/exec.go | 30 ++++++++++++++++---------- solver/pb/ops.pb.go | 34 ++++++++++++++++++++++++++++++ solver/pb/ops.proto | 1 + source/git/gitsource.go | 4 ++-- source/git/gitsource_test.go | 10 ++++----- source/local/local.go | 2 +- worker/oci/spec_unix.go | 2 +- worker/runcworker/worker.go | 2 +- worker/worker.go | 2 +- 15 files changed, 114 insertions(+), 35 deletions(-) diff --git a/cache/manager_test.go b/cache/manager_test.go index e5b190e0..4b783d7f 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -33,7 +33,7 @@ func TestManager(t *testing.T) { active, err := cm.New(ctx, nil) require.NoError(t, err) - m, err := active.Mount(ctx) + m, err := active.Mount(ctx, false) require.NoError(t, err) lm := snapshot.LocalMounter(m) diff --git a/cache/refs.go b/cache/refs.go index 263243d7..c4d6a8b4 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -30,7 +30,7 @@ type MutableRef interface { } type Mountable interface { - Mount(ctx context.Context) ([]mount.Mount, error) + Mount(ctx context.Context, readonly bool) ([]mount.Mount, error) } type cacheRecord struct { @@ -101,7 +101,7 @@ func (cr *cacheRecord) Parent() ImmutableRef { return cr.parent.(*immutableRef).ref() } -func (cr *cacheRecord) Mount(ctx context.Context) ([]mount.Mount, error) { +func (cr *cacheRecord) Mount(ctx context.Context, readonly bool) ([]mount.Mount, error) { cr.mu.Lock() defer cr.mu.Unlock() @@ -110,8 +110,20 @@ func (cr *cacheRecord) Mount(ctx context.Context) ([]mount.Mount, error) { if err != nil { return nil, errors.Wrapf(err, "failed to mount %s", cr.ID()) } + if readonly { + m = setReadonly(m) + } return m, nil } + + if cr.equalMutable != nil && readonly { + m, err := cr.cm.Snapshotter.Mounts(ctx, cr.equalMutable.ID()) + if err != nil { + return nil, errors.Wrapf(err, "failed to mount %s", cr.equalMutable.ID()) + } + return setReadonly(m), nil + } + if err := cr.finalize(ctx); err != nil { return nil, err } @@ -268,3 +280,17 @@ func (sr *mutableRef) release(ctx context.Context) error { // } return nil } + +func setReadonly(mounts []mount.Mount) []mount.Mount { + for i, m := range mounts { + opts := make([]string, 0, len(m.Options)) + for _, opt := range m.Options { + if opt != "rw" { + opts = append(opts, opt) + } + } + opts = append(opts, "ro") + mounts[i].Options = opts + } + return mounts +} diff --git a/client/llb/llb.go b/client/llb/llb.go index 28e46e11..461a8530 100644 --- a/client/llb/llb.go +++ b/client/llb/llb.go @@ -167,8 +167,9 @@ func (eo *exec) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (dige } pm := &pb.Mount{ - Input: int64(inputIndex), - Dest: m.dest, + Input: int64(inputIndex), + Dest: m.dest, + Readonly: m.readonly, } if m.hasOutput { pm.Output = outputIndex @@ -186,7 +187,7 @@ func (eo *exec) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (dige type mount struct { execState *ExecState dest string - // ro bool + readonly bool // either parent or source has to be set parent *mount source *source diff --git a/client/llb/state.go b/client/llb/state.go index ba13018a..4dc557a0 100644 --- a/client/llb/state.go +++ b/client/llb/state.go @@ -138,7 +138,7 @@ type ExecState struct { State } -func (s *ExecState) AddMount(dest string, mountState *State) *State { +func (s *ExecState) AddMount(dest string, mountState *State, opts ...MountOption) *State { m := &mount{ dest: dest, source: mountState.source, @@ -146,6 +146,9 @@ func (s *ExecState) AddMount(dest string, mountState *State) *State { execState: s, hasOutput: true, // TODO: should be set only if something inherits } + for _, opt := range opts { + opt(m) + } var newState State newState.meta = s.meta newState.metaNext = s.metaNext @@ -182,9 +185,15 @@ func (s *ExecState) updateMeta(fn metaOption) *ExecState { type RunOption func(es *ExecState) *ExecState -func AddMount(dest string, mountState *State) RunOption { +type MountOption func(*mount) + +func Readonly(m *mount) { + m.readonly = true +} + +func AddMount(dest string, mountState *State, opts ...MountOption) RunOption { return func(es *ExecState) *ExecState { - es.AddMount(dest, mountState) + es.AddMount(dest, mountState, opts...) return nil } } diff --git a/control/control_standalone_test.go b/control/control_standalone_test.go index 7f0be46c..80831868 100644 --- a/control/control_standalone_test.go +++ b/control/control_standalone_test.go @@ -72,7 +72,7 @@ func TestControl(t *testing.T) { snap, err := src.Snapshot(ctx) assert.NoError(t, err) - mounts, err := snap.Mount(ctx) + mounts, err := snap.Mount(ctx, false) assert.NoError(t, err) lm := snapshot.LocalMounter(mounts) @@ -127,7 +127,7 @@ func TestControl(t *testing.T) { rf, err := root.Commit(ctx) assert.NoError(t, err) - mounts, err = rf.Mount(ctx) + mounts, err = rf.Mount(ctx, false) assert.NoError(t, err) lm = snapshot.LocalMounter(mounts) diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 79e0c913..3dfe6ab6 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -113,12 +113,12 @@ func (e *imageExporter) getBlobs(ctx context.Context, ref cache.ImmutableRef) ([ var lower []mount.Mount if parent != nil { defer parent.Release(context.TODO()) - lower, err = parent.Mount(ctx) + lower, err = parent.Mount(ctx, true) if err != nil { return nil, err } } - upper, err := ref.Mount(ctx) + upper, err := ref.Mount(ctx, true) if err != nil { return nil, err } diff --git a/solver/exec.go b/solver/exec.go index b7ceef56..96279837 100644 --- a/solver/exec.go +++ b/solver/exec.go @@ -50,7 +50,7 @@ func (e *execOp) CacheKey(ctx context.Context, inputs []string) (string, int, er func (e *execOp) Run(ctx context.Context, inputs []Reference) ([]Reference, error) { var mounts []worker.Mount - var outputs []cache.MutableRef + var outputs []Reference var root cache.Mountable defer func() { @@ -77,17 +77,21 @@ func (e *execOp) Run(ctx context.Context, inputs []Reference) ([]Reference, erro mountable = ref } if m.Output != pb.SkipOutput { - active, err := e.cm.New(ctx, ref) // TODO: should be method - if err != nil { - return nil, err + if m.Readonly && ref != nil { + outputs = append(outputs, newSharedRef(ref).Clone()) + } else { + active, err := e.cm.New(ctx, ref) // TODO: should be method + if err != nil { + return nil, err + } + outputs = append(outputs, active) + mountable = active } - outputs = append(outputs, active) - mountable = active } if m.Dest == pb.RootMount { root = mountable } else { - mounts = append(mounts, worker.Mount{Src: mountable, Dest: m.Dest}) + mounts = append(mounts, worker.Mount{Src: mountable, Dest: m.Dest, Readonly: m.Readonly}) } } @@ -111,11 +115,15 @@ func (e *execOp) Run(ctx context.Context, inputs []Reference) ([]Reference, erro refs := []Reference{} for i, o := range outputs { - ref, err := o.Commit(ctx) - if err != nil { - return nil, errors.Wrapf(err, "error committing %s", o.ID()) + if mutable, ok := o.(cache.MutableRef); ok { + ref, err := mutable.Commit(ctx) + if err != nil { + return nil, errors.Wrapf(err, "error committing %s", mutable.ID()) + } + refs = append(refs, ref) + } else { + refs = append(refs, o) } - refs = append(refs, ref) outputs[i] = nil } return refs, nil diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go index 3f60292c..147be7da 100644 --- a/solver/pb/ops.pb.go +++ b/solver/pb/ops.pb.go @@ -216,6 +216,7 @@ type Mount struct { Selector string `protobuf:"bytes,2,opt,name=selector,proto3" json:"selector,omitempty"` Dest string `protobuf:"bytes,3,opt,name=dest,proto3" json:"dest,omitempty"` Output int64 `protobuf:"varint,4,opt,name=output,proto3" json:"output,omitempty"` + Readonly bool `protobuf:"varint,5,opt,name=readonly,proto3" json:"readonly,omitempty"` } func (m *Mount) Reset() { *m = Mount{} } @@ -513,6 +514,16 @@ func (m *Mount) MarshalTo(data []byte) (int, error) { i++ i = encodeVarintOps(data, i, uint64(m.Output)) } + if m.Readonly { + data[i] = 0x28 + i++ + if m.Readonly { + data[i] = 1 + } else { + data[i] = 0 + } + i++ + } return i, nil } @@ -759,6 +770,9 @@ func (m *Mount) Size() (n int) { if m.Output != 0 { n += 1 + sovOps(uint64(m.Output)) } + if m.Readonly { + n += 2 + } return n } @@ -1473,6 +1487,26 @@ func (m *Mount) Unmarshal(data []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Readonly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Readonly = bool(v != 0) default: iNdEx = preIndex skippy, err := skipOps(data[iNdEx:]) diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto index ed5b069d..9fc18ca7 100644 --- a/solver/pb/ops.proto +++ b/solver/pb/ops.proto @@ -34,6 +34,7 @@ message Mount { string selector = 2; string dest = 3; int64 output = 4; + bool readonly = 5; } message CopyOp { diff --git a/source/git/gitsource.go b/source/git/gitsource.go index 74382686..d0cf15cd 100644 --- a/source/git/gitsource.go +++ b/source/git/gitsource.go @@ -95,7 +95,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str } }() - mount, err := remoteRef.Mount(ctx) + mount, err := remoteRef.Mount(ctx, false) if err != nil { return "", nil, err } @@ -271,7 +271,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe } }() - mount, err := checkoutRef.Mount(ctx) + mount, err := checkoutRef.Mount(ctx, false) if err != nil { return nil, err } diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index f9d8da6e..576e5509 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -55,7 +55,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { require.NoError(t, err) defer ref1.Release(context.TODO()) - mount, err := ref1.Mount(ctx) + mount, err := ref1.Mount(ctx, false) require.NoError(t, err) lm := snapshot.LocalMounter(mount) @@ -102,7 +102,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { require.NoError(t, err) defer ref3.Release(context.TODO()) - mount, err = ref3.Mount(ctx) + mount, err = ref3.Mount(ctx, false) require.NoError(t, err) lm = snapshot.LocalMounter(mount) @@ -161,7 +161,7 @@ func testFetchBySHA(t *testing.T, keepGitDir bool) { require.NoError(t, err) defer ref1.Release(context.TODO()) - mount, err := ref1.Mount(ctx) + mount, err := ref1.Mount(ctx, false) require.NoError(t, err) lm := snapshot.LocalMounter(mount) @@ -234,7 +234,7 @@ func testMultipleRepos(t *testing.T, keepGitDir bool) { require.NoError(t, err) defer ref1.Release(context.TODO()) - mount, err := ref1.Mount(ctx) + mount, err := ref1.Mount(ctx, false) require.NoError(t, err) lm := snapshot.LocalMounter(mount) @@ -246,7 +246,7 @@ func testMultipleRepos(t *testing.T, keepGitDir bool) { require.NoError(t, err) defer ref2.Release(context.TODO()) - mount, err = ref2.Mount(ctx) + mount, err = ref2.Mount(ctx, false) require.NoError(t, err) lm = snapshot.LocalMounter(mount) diff --git a/source/local/local.go b/source/local/local.go index bded3fad..85cb6ec9 100644 --- a/source/local/local.go +++ b/source/local/local.go @@ -118,7 +118,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable } }() - mount, err := mutable.Mount(ctx) + mount, err := mutable.Mount(ctx, false) if err != nil { return nil, err } diff --git a/worker/oci/spec_unix.go b/worker/oci/spec_unix.go index 8df6b832..2d12d4df 100644 --- a/worker/oci/spec_unix.go +++ b/worker/oci/spec_unix.go @@ -22,7 +22,7 @@ func GenerateSpec(ctx context.Context, meta worker.Meta, mounts []worker.Mount) // TODO: User for _, m := range mounts { - mounts, err := m.Src.Mount(ctx) + mounts, err := m.Src.Mount(ctx, m.Readonly) if err != nil { return nil, errors.Wrapf(err, "failed to mount %s", m.Dest) } diff --git a/worker/runcworker/worker.go b/worker/runcworker/worker.go index d6a6ed34..b61443c7 100644 --- a/worker/runcworker/worker.go +++ b/worker/runcworker/worker.go @@ -56,7 +56,7 @@ func New(root string) (worker.Worker, error) { } func (w *runcworker) Exec(ctx context.Context, meta worker.Meta, root cache.Mountable, mounts []worker.Mount, stdout, stderr io.WriteCloser) error { - rootMount, err := root.Mount(ctx) + rootMount, err := root.Mount(ctx, false) if err != nil { return err } diff --git a/worker/worker.go b/worker/worker.go index 7deb0f4b..3bd9ab52 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -19,7 +19,7 @@ type Meta struct { type Mount struct { Src cache.Mountable Dest string - ReadOnly bool + Readonly bool } type Worker interface {