JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #113 from tonistiigi/share-resolver 

imagesource: share config resolver with frontend
docker-18.09
Akihiro Suda 2017-09-06 18:28:33 +09:00 committed by GitHub
parent 659e899c3e 6d335c3c86
commit 2f3ecb50dd
10 changed files with 222 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
client/llb/resolver.go
View File

@ -3,9 +3,6 @@ package llb
import (
"bytes"
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"sync"
"time"
@ -13,9 +10,9 @@ import (
"github.com/BurntSushi/locker"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/util/imageutil"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@ -35,7 +32,7 @@ func WithDefaultMetaResolver(ii *ImageInfo) {
}
type ImageMetaResolver interface {
Resolve(ctx context.Context, ref string) (*ocispec.Image, error)
ResolveImageConfig(ctx context.Context, ref string) (*ocispec.Image, error)
}
func NewImageMetaResolver() ImageMetaResolver {
@ -63,7 +60,7 @@ type imageMetaResolver struct {
cache map[string]*ocispec.Image
}
func (imr *imageMetaResolver) Resolve(ctx context.Context, ref string) (*ocispec.Image, error) {
func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string) (*ocispec.Image, error) {
imr.locker.Lock(ref)
defer imr.locker.Unlock(ref)
@ -71,87 +68,13 @@ func (imr *imageMetaResolver) Resolve(ctx context.Context, ref string) (*ocispec
return img, nil
}
ref, desc, err := imr.resolver.Resolve(ctx, ref)
img, err := imageutil.Config(ctx, ref, imr.resolver, imr.ingester)
if err != nil {
return nil, err
}
fetcher, err := imr.resolver.Fetcher(ctx, ref)
if err != nil {
return nil, err
}
handlers := []images.Handler{
remotes.FetchHandler(imr.ingester, fetcher),
childrenConfigHandler(imr.ingester),
}
if err := images.Dispatch(ctx, images.Handlers(handlers...), desc); err != nil {
return nil, err
}
config, err := images.Config(ctx, imr.ingester, desc)
if err != nil {
return nil, err
}
var ociimage ocispec.Image
r, err := imr.ingester.Reader(ctx, config.Digest)
if err != nil {
return nil, err
}
defer r.Close()
dec := json.NewDecoder(r)
if err := dec.Decode(&ociimage); err != nil {
return nil, err
}
if dec.More() {
return nil, errors.New("invalid image config")
}
imr.cache[ref] = &ociimage
return &ociimage, nil
}
func childrenConfigHandler(provider content.Provider) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var descs []ocispec.Descriptor
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
if err != nil {
return nil, err
}
// TODO(stevvooe): We just assume oci manifest, for now. There may be
// subtle differences from the docker version.
var manifest ocispec.Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
return nil, err
}
descs = append(descs, manifest.Config)
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
if err != nil {
return nil, err
}
var index ocispec.Index
if err := json.Unmarshal(p, &index); err != nil {
return nil, err
}
descs = append(descs, index.Manifests...)
case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
// childless data types.
return nil, nil
default:
return nil, errors.Errorf("encountered unknown type %v; children may not be fetched", desc.MediaType)
}
return descs, nil
}
imr.cache[ref] = img
return img, nil
}
type inMemoryIngester struct {
@ -184,20 +107,6 @@ func (i *inMemoryIngester) Writer(ctx context.Context, ref string, size int64, e
return w, nil
}
func (i *inMemoryIngester) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) {
rdr, err := i.reader(ctx, dgst)
if err != nil {
return nil, err
}
return ioutil.NopCloser(rdr), nil
}
type ReaderAt interface {
io.ReaderAt
io.Closer
Size() int64
}
func (i *inMemoryIngester) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
r, err := i.reader(ctx, dgst)
if err != nil {

2
client/llb/source.go
View File

@ -85,7 +85,7 @@ func Image(ref string, opts ...ImageOption) State {
opt(&info)
}
if info.metaResolver != nil {
img, err := info.metaResolver.Resolve(context.TODO(), ref)
img, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref)
if err != nil {
src.err = err
} else {

2
control/control.go
View File

@ -28,6 +28,7 @@ type Opt struct {
Exporters map[string]exporter.Exporter
SessionManager *session.Manager
Frontends map[string]frontend.Frontend
ImageSource source.Source
}
type Controller struct { // TODO: ControlService
@ -43,6 +44,7 @@ func NewController(opt Opt) (*Controller, error) {
CacheManager: opt.CacheManager,
Worker: opt.Worker,
InstructionCache: opt.InstructionCache,
ImageSource: opt.ImageSource,
}),
}
return c, nil

1
control/control_default.go
View File

@ -137,5 +137,6 @@ func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) {
Exporters: exporters,
SessionManager: sessm,
Frontends: frontends,
ImageSource: is,
}, nil
}

2
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -79,7 +79,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
d.stage.BaseName = reference.TagNameOnly(ref).String()
if opt.MetaResolver != nil {
img, err := metaResolver.Resolve(ctx, d.stage.BaseName)
img, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName)
if err != nil {
return err
}

3
frontend/frontend.go
View File

@ -2,6 +2,7 @@ package frontend
import (
"github.com/moby/buildkit/cache"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/net/context"
)
@ -12,5 +13,5 @@ type Frontend interface {
type FrontendLLBBridge interface {
Solve(ctx context.Context, vtx [][]byte) (cache.ImmutableRef, error)
// ImageConfigResolver
ResolveImageConfig(ctx context.Context, ref string) (*ocispec.Image, error)
}

17
solver/solver.go
View File

@ -14,6 +14,7 @@ import (
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
@ -25,6 +26,7 @@ type LLBOpt struct {
CacheManager cache.Manager // TODO: this shouldn't be needed before instruction cache
Worker worker.Worker
InstructionCache InstructionCache
ImageSource source.Source
}
func NewLLBSolver(opt LLBOpt) *Solver {
@ -40,7 +42,7 @@ func NewLLBSolver(opt LLBOpt) *Solver {
default:
return nil, errors.Errorf("invalid op type %T", op)
}
}, opt.InstructionCache)
}, opt.InstructionCache, opt.ImageSource)
return s
}
@ -71,13 +73,19 @@ type Solver struct {
jobs *jobList
activeState activeState
cache InstructionCache
imageSource source.Source
}
func New(resolve ResolveOpFunc, cache InstructionCache) *Solver {
return &Solver{resolve: resolve, jobs: newJobList(), cache: cache}
func New(resolve ResolveOpFunc, cache InstructionCache, imageSource source.Source) *Solver {
return &Solver{resolve: resolve, jobs: newJobList(), cache: cache, imageSource: imageSource}
}
type resolveImageConfig interface {
ResolveImageConfig(ctx context.Context, ref string) (*ocispec.Image, error)
}
type llbBridge struct {
resolveImageConfig
solver func(ctx context.Context, v *vertex, i Index) (Reference, error)
}
@ -147,7 +155,8 @@ func (s *Solver) Solve(ctx context.Context, id string, f frontend.Frontend, v Ve
ref, err = s.getRef(ctx, solveVertex, index)
} else {
ref, err = f.Solve(ctx, &llbBridge{
solver: s.getRef,
solver: s.getRef,
resolveImageConfig: s.imageSource.(resolveImageConfig),
}, frontendOpt)
}
s.activeState.cancel(j)

89
source/containerimage/pull.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"time"
"github.com/BurntSushi/locker"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
@ -17,6 +18,7 @@ import (
"github.com/containerd/containerd/snapshot"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/progress"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
@ -40,17 +42,24 @@ type blobmapper interface {
SetBlob(ctx gocontext.Context, key string, blob digest.Digest) error
}
type resolveRecord struct {
desc ocispec.Descriptor
ts time.Time
}
type imageSource struct {
SourceOpt
resolver remotes.Resolver
lru map[string]resolveRecord
}
func NewSource(opt SourceOpt) (source.Source, error) {
is := &imageSource{
SourceOpt: opt,
resolver: docker.NewResolver(docker.ResolverOptions{
lru: map[string]resolveRecord{},
resolver: newCachedResolver(docker.NewResolver(docker.ResolverOptions{
Client: http.DefaultClient,
}),
}), 5*time.Second),
}
if _, ok := opt.Snapshotter.(blobmapper); !ok {
@ -64,13 +73,8 @@ func (is *imageSource) ID() string {
return source.DockerImageScheme
}
type puller struct {
is *imageSource
resolveOnce sync.Once
src *source.ImageIdentifier
desc ocispec.Descriptor
ref string
resolveErr error
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (*ocispec.Image, error) {
return imageutil.Config(ctx, ref, is.resolver, is.ContentStore)
}
func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
@ -86,9 +90,34 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
return p, nil
}
type puller struct {
is *imageSource
resolveOnce sync.Once
src *source.ImageIdentifier
desc ocispec.Descriptor
ref string
resolveErr error
}
func (p *puller) resolve(ctx context.Context) error {
p.resolveOnce.Do(func() {
resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String())
dgst := p.src.Reference.Digest()
if dgst != "" {
info, err := p.is.ContentStore.Info(ctx, dgst)
if err == nil {
p.ref = p.src.Reference.String()
p.desc = ocispec.Descriptor{
Size: info.Size,
Digest: dgst,
MediaType: ocispec.MediaTypeImageManifest, // TODO: detect schema1/manifest-list
}
resolveProgressDone(nil)
return
}
}
ref, desc, err := p.is.resolver.Resolve(ctx, p.src.Reference.String())
if err != nil {
p.resolveErr = err
@ -383,3 +412,45 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error {
return err
}
}
func newCachedResolver(r remotes.Resolver, timeout time.Duration) remotes.Resolver {
return &cachedResolver{
Resolver: r,
cache: map[string]cachedResult{},
timeout: timeout,
locker: locker.NewLocker(),
}
}
type cachedResolver struct {
remotes.Resolver
cache map[string]cachedResult
timeout time.Duration
locker *locker.Locker
}
func (cr *cachedResolver) Resolve(ctx gocontext.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
cr.locker.Lock(ref)
defer cr.locker.Unlock(ref)
r, ok := cr.cache[ref]
if ok && time.Since(r.ts) < cr.timeout {
return r.name, r.desc, nil
}
delete(cr.cache, ref)
n, d, err := cr.Resolver.Resolve(ctx, ref)
if err != nil {
return "", d, err
}
cr.cache[ref] = cachedResult{
name: n,
desc: d,
ts: time.Now(),
}
return n, d, nil
}
type cachedResult struct {
name string
desc ocispec.Descriptor
ts time.Time
}

115
util/imageutil/config.go Normal file
View File

@ -0,0 +1,115 @@
package imageutil
import (
"context"
"encoding/json"
"io"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/remotes"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
type IngesterProvider interface {
content.Ingester
content.Provider
}
func Config(ctx context.Context, ref string, resolver remotes.Resolver, ingester IngesterProvider) (*ocispec.Image, error) {
ref, desc, err := resolver.Resolve(ctx, ref)
if err != nil {
return nil, err
}
fetcher, err := resolver.Fetcher(ctx, ref)
if err != nil {
return nil, err
}
handlers := []images.Handler{
remotes.FetchHandler(ingester, fetcher),
childrenConfigHandler(ingester),
}
if err := images.Dispatch(ctx, images.Handlers(handlers...), desc); err != nil {
return nil, err
}
config, err := images.Config(ctx, ingester, desc)
if err != nil {
return nil, err
}
var ociimage ocispec.Image
r, err := ingester.ReaderAt(ctx, config.Digest)
if err != nil {
return nil, err
}
defer r.Close()
dec := json.NewDecoder(readerAtToReader(r))
if err := dec.Decode(&ociimage); err != nil {
return nil, err
}
if dec.More() {
return nil, errors.New("invalid image config")
}
return &ociimage, nil
}
func childrenConfigHandler(provider content.Provider) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var descs []ocispec.Descriptor
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
if err != nil {
return nil, err
}
// TODO(stevvooe): We just assume oci manifest, for now. There may be
// subtle differences from the docker version.
var manifest ocispec.Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
return nil, err
}
descs = append(descs, manifest.Config)
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
if err != nil {
return nil, err
}
var index ocispec.Index
if err := json.Unmarshal(p, &index); err != nil {
return nil, err
}
descs = append(descs, index.Manifests...)
case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
// childless data types.
return nil, nil
default:
return nil, errors.Errorf("encountered unknown type %v; children may not be fetched", desc.MediaType)
}
return descs, nil
}
}
func readerAtToReader(r io.ReaderAt) io.Reader {
return &reader{ra: r}
}
type reader struct {
ra io.ReaderAt
offset int64
}
func (r *reader) Read(b []byte) (int, error) {
n, err := r.ra.ReadAt(b, r.offset)
r.offset += int64(n)
return n, err
}

2
vendor.conf
View File

@ -33,7 +33,7 @@ github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716
golang.org/x/time 8be79e1e0910c292df4e79c241bb7e8f7e725959
github.com/BurntSushi/locker 392720b78f44e9d0249fcac6c43b111b47a370b8
github.com/docker/docker 6f723db8c6f0c7f0b252674a9673a25b5978db04 https://github.com/simonferquel/docker.git
github.com/docker/docker 6f723db8c6f0c7f0b252674a9673a25b5978db04 https://github.com/tonistiigi/docker.git
github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f
github.com/tonistiigi/fsutil d49833a9a6fa5b41f63e7e338038633d10276b57