JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

resolver: add config support for mirrors/plainhttp 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
docker-18.09
Tonis Tiigi 2018-09-07 13:45:59 -07:00
parent a9fe50acf1
commit ecd4a22546
12 changed files with 130 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cache/remotecache/registry/registry.go vendored
View File

@ -10,17 +10,17 @@ import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/tracing"
"github.com/moby/buildkit/util/resolver"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExporterFunc {
func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc {
return func(ctx context.Context, typ, ref string) (remotecache.Exporter, error) {
if typ != "" {
return nil, errors.Errorf("unsupported cache exporter type: %s", typ)
}
remote := newRemoteResolver(ctx, sm)
remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
pusher, err := remote.Pusher(ctx, ref)
if err != nil {
return nil, err
@ -29,12 +29,12 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor
}
}
func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImporterFunc {
func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
return func(ctx context.Context, typ, ref string) (remotecache.Importer, specs.Descriptor, error) {
if typ != "" {
return nil, specs.Descriptor{}, errors.Errorf("unsupported cache importer type: %s", typ)
}
remote := newRemoteResolver(ctx, sm)
remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
xref, desc, err := remote.Resolve(ctx, ref)
if err != nil {
return nil, specs.Descriptor{}, err
@ -47,11 +47,10 @@ func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImpor
}
}
func newRemoteResolver(ctx context.Context, sm *session.Manager) remotes.Resolver {
return docker.NewResolver(docker.ResolverOptions{
Client: tracing.DefaultClient,
Credentials: getCredentialsFunc(ctx, sm),
})
func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
opt := resolverOpt(ref)
opt.Credentials = getCredentialsFunc(ctx, sm)
return docker.NewResolver(opt)
}
func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {

7
cmd/buildkitd/config/config.go
View File

@ -22,6 +22,8 @@ type Config struct {
OCI OCIConfig `toml:"oci"`
Containerd ContainerdConfig `toml:"containerd"`
} `toml:"worker"`
Registries map[string]RegistryConfig `toml:"registry"`
}
type GRPCConfig struct {
@ -35,6 +37,11 @@ type GRPCConfig struct {
// MaxSendMsgSize int `toml:"max_send_message_size"`
}
type RegistryConfig struct {
Mirrors []string `toml:"mirrors"`
PlainHTTP bool `toml:"http"`
}
type TLSConfig struct {
Cert string `toml:"cert"`
Key string `toml:"key"`

7
cmd/buildkitd/config/config_test.go
View File

@ -39,6 +39,10 @@ keepDuration=3600
[[worker.containerd.gcpolicy]]
keepBytes=40
keepDuration=7200
[registry."docker.io"]
mirrors=["hub.docker.io"]
http=true
`
cfg, md, err := Load(bytes.NewBuffer([]byte(testConfig)))
@ -78,4 +82,7 @@ keepDuration=7200
require.Equal(t, int64(7200), cfg.Workers.Containerd.GCPolicy[1].KeepDuration)
require.Equal(t, 1, len(cfg.Workers.Containerd.GCPolicy[0].Filters))
require.Equal(t, 0, len(cfg.Workers.Containerd.GCPolicy[1].Filters))
require.Equal(t, cfg.Registries["docker.io"].PlainHTTP, true)
require.Equal(t, cfg.Registries["docker.io"].Mirrors[0], "hub.docker.io")
}

18
cmd/buildkitd/main.go
View File

@ -35,6 +35,7 @@ import (
"github.com/moby/buildkit/util/appcontext"
"github.com/moby/buildkit/util/appdefaults"
"github.com/moby/buildkit/util/profiler"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/version"
"github.com/moby/buildkit/worker"
specs "github.com/opencontainers/image-spec/specs-go/v1"
@ -504,17 +505,30 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
return nil, err
}
resolverFn := resolverFunc(cfg)
return control.NewController(control.Opt{
SessionManager: sessionManager,
WorkerController: wc,
Frontends: frontends,
// TODO: support non-registry remote cache
ResolveCacheExporterFunc: registryremotecache.ResolveCacheExporterFunc(sessionManager),
ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(sessionManager),
ResolveCacheExporterFunc: registryremotecache.ResolveCacheExporterFunc(sessionManager, resolverFn),
ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(sessionManager, resolverFn),
CacheKeyStorage: cacheStorage,
})
}
func resolverFunc(cfg *config.Config) resolver.ResolveOptionsFunc {
m := map[string]resolver.RegistryConf{}
for k, v := range cfg.Registries {
m[k] = resolver.RegistryConf{
Mirrors: v.Mirrors,
PlainHTTP: v.PlainHTTP,
}
}
return resolver.NewResolveOptionsFunc(m)
}
func newWorkerController(c *cli.Context, wiOpt workerInitializerOpt) (*worker.Controller, error) {
wc := &worker.Controller{}
nWorkers := 0

1
cmd/buildkitd/main_containerd_worker.go
View File

@ -125,6 +125,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
}
opt.SessionManager = common.sessionManager
opt.GCPolicy = getGCPolicy(cfg.GCPolicy, common.config.Root)
opt.ResolveOptionsFunc = resolverFunc(common.config)
if platformsStr := cfg.Platforms; len(platformsStr) != 0 {
platforms, err := parsePlatforms(platformsStr)

1
cmd/buildkitd/main_oci_worker.go
View File

@ -142,6 +142,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
}
opt.SessionManager = common.sessionManager
opt.GCPolicy = getGCPolicy(cfg.GCPolicy, common.config.Root)
opt.ResolveOptionsFunc = resolverFunc(common.config)
if platformsStr := cfg.Platforms; len(platformsStr) != 0 {
platforms, err := parsePlatforms(platformsStr)

4
exporter/containerimage/export.go
View File

@ -11,6 +11,7 @@ import (
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/push"
"github.com/moby/buildkit/util/resolver"
"github.com/pkg/errors"
)
@ -25,6 +26,7 @@ type Opt struct {
SessionManager *session.Manager
ImageWriter *ImageWriter
Images images.Store
ResolverOpt resolver.ResolveOptionsFunc
}
type imageExporter struct {
@ -144,7 +146,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
tagDone(nil)
}
if e.push {
if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure); err != nil {
if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt); err != nil {
return nil, err
}
}

6
source/containerimage/pull.go
View File

@ -18,6 +18,7 @@ import (
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/pull"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/util/winlayers"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
@ -35,6 +36,7 @@ type SourceOpt struct {
Applier diff.Applier
CacheAccessor cache.Accessor
ImageStore images.Store // optional
ResolverOpt resolver.ResolveOptionsFunc
}
type imageSource struct {
@ -70,7 +72,7 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt g
}
res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) {
dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.SessionManager, is.ImageStore, rm), is.ContentStore, opt.Platform)
dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.ResolverOpt, is.SessionManager, is.ImageStore, rm, ref), is.ContentStore, opt.Platform)
if err != nil {
return nil, err
}
@ -99,7 +101,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
ContentStore: is.ContentStore,
Applier: is.Applier,
Src: imageIdentifier.Reference,
Resolver: pull.NewResolver(ctx, is.SessionManager, is.ImageStore, imageIdentifier.ResolveMode),
Resolver: pull.NewResolver(ctx, is.ResolverOpt, is.SessionManager, is.ImageStore, imageIdentifier.ResolveMode, imageIdentifier.Reference.String()),
Platform: &platform,
}
p := &puller{

17
util/pull/resolver.go
View File

@ -2,6 +2,7 @@ package pull
import (
"context"
"net/http"
"time"
"github.com/containerd/containerd/images"
@ -10,15 +11,21 @@ import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/util/tracing"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func NewResolver(ctx context.Context, sm *session.Manager, imageStore images.Store, mode source.ResolveMode) remotes.Resolver {
r := docker.NewResolver(docker.ResolverOptions{
Client: tracing.DefaultClient,
Credentials: getCredentialsFromSession(ctx, sm),
})
func NewResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, sm *session.Manager, imageStore images.Store, mode source.ResolveMode, ref string) remotes.Resolver {
opt := docker.ResolverOptions{
Client: http.DefaultClient,
}
if rfn != nil {
opt = rfn(ref)
}
opt.Credentials = getCredentialsFromSession(ctx, sm)
r := docker.NewResolver(opt)
if imageStore == nil || mode == source.ResolveModeForcePull {
return r

16
util/push/push.go
View File

@ -16,7 +16,7 @@ import (
"github.com/moby/buildkit/session/auth"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/tracing"
"github.com/moby/buildkit/util/resolver"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
@ -40,7 +40,7 @@ func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (
}
}
func Push(ctx context.Context, sm *session.Manager, cs content.Provider, dgst digest.Digest, ref string, insecure bool) error {
func Push(ctx context.Context, sm *session.Manager, cs content.Provider, dgst digest.Digest, ref string, insecure bool, rfn resolver.ResolveOptionsFunc) error {
desc := ocispec.Descriptor{
Digest: dgst,
}
@ -50,11 +50,13 @@ func Push(ctx context.Context, sm *session.Manager, cs content.Provider, dgst di
}
ref = reference.TagNameOnly(parsed).String()
resolver := docker.NewResolver(docker.ResolverOptions{
Client: tracing.DefaultClient,
Credentials: getCredentialsFunc(ctx, sm),
PlainHTTP: insecure,
})
opt := rfn(ref)
opt.Credentials = getCredentialsFunc(ctx, sm)
if insecure {
opt.PlainHTTP = insecure
}
resolver := docker.NewResolver(opt)
pusher, err := resolver.Pusher(ctx, ref)
if err != nil {

45
util/resolver/resolver.go Normal file
View File

@ -0,0 +1,45 @@
package resolver
import (
"math/rand"
"github.com/containerd/containerd/remotes/docker"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/tracing"
)
type RegistryConf struct {
Mirrors []string
PlainHTTP bool
}
type ResolveOptionsFunc func(string) docker.ResolverOptions
func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
return func(ref string) docker.ResolverOptions {
def := docker.ResolverOptions{
Client: tracing.DefaultClient,
}
parsed, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return def
}
host := reference.Domain(parsed)
c, ok := m[host]
if !ok {
return def
}
if len(c.Mirrors) > 0 {
def.Host = func(string) (string, error) {
return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
}
}
def.PlainHTTP = c.PlainHTTP
return def
}
}

28
worker/base/worker.go
View File

@ -39,6 +39,7 @@ import (
"github.com/moby/buildkit/source/local"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/resolver"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
ociidentity "github.com/opencontainers/image-spec/identity"
@ -55,18 +56,19 @@ const labelCreatedAt = "buildkit/createdat"
// WorkerOpt is specific to a worker.
// See also CommonOpt.
type WorkerOpt struct {
ID string
Labels map[string]string
Platforms []specs.Platform
GCPolicy []client.PruneInfo
SessionManager *session.Manager
MetadataStore *metadata.Store
Executor executor.Executor
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Applier diff.Applier
Differ diff.Comparer
ImageStore images.Store // optional
ID string
Labels map[string]string
Platforms []specs.Platform
GCPolicy []client.PruneInfo
SessionManager *session.Manager
MetadataStore *metadata.Store
Executor executor.Executor
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Applier diff.Applier
Differ diff.Comparer
ImageStore images.Store // optional
ResolveOptionsFunc resolver.ResolveOptionsFunc
}
// Worker is a local worker instance with dedicated snapshotter, cache, and so on.
@ -108,6 +110,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
Applier: opt.Applier,
ImageStore: opt.ImageStore,
CacheAccessor: cm,
ResolverOpt: opt.ResolveOptionsFunc,
})
if err != nil {
return nil, err
@ -160,6 +163,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
Images: opt.ImageStore,
SessionManager: opt.SessionManager,
ImageWriter: iw,
ResolverOpt: opt.ResolveOptionsFunc,
})
if err != nil {
return nil, err