resolver: add config support for mirrors/plainhttp

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

cache/remotecache/registry/registry.go

@@ -10,17 +10,17 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/auth"
 	"github.com/moby/buildkit/util/contentutil"
-	"github.com/moby/buildkit/util/tracing"
+	"github.com/moby/buildkit/util/resolver"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
-func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExporterFunc {
+func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc {
 	return func(ctx context.Context, typ, ref string) (remotecache.Exporter, error) {
 		if typ != "" {
 			return nil, errors.Errorf("unsupported cache exporter type: %s", typ)
 		}
-		remote := newRemoteResolver(ctx, sm)
+		remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
 		pusher, err := remote.Pusher(ctx, ref)
 		if err != nil {
 			return nil, err
@@ -29,12 +29,12 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor
 	}
 }
 
-func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImporterFunc {
+func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
 	return func(ctx context.Context, typ, ref string) (remotecache.Importer, specs.Descriptor, error) {
 		if typ != "" {
 			return nil, specs.Descriptor{}, errors.Errorf("unsupported cache importer type: %s", typ)
 		}
-		remote := newRemoteResolver(ctx, sm)
+		remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
 		xref, desc, err := remote.Resolve(ctx, ref)
 		if err != nil {
 			return nil, specs.Descriptor{}, err
@@ -47,11 +47,10 @@ func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImpor
 	}
 }
 
-func newRemoteResolver(ctx context.Context, sm *session.Manager) remotes.Resolver {
+func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
-	return docker.NewResolver(docker.ResolverOptions{
+	opt := resolverOpt(ref)
-		Client:      tracing.DefaultClient,
+	opt.Credentials = getCredentialsFunc(ctx, sm)
-		Credentials: getCredentialsFunc(ctx, sm),
+	return docker.NewResolver(opt)
-	})
 }
 
 func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {

cmd/buildkitd/config/config.go

@@ -22,6 +22,8 @@ type Config struct {
 		OCI        OCIConfig        `toml:"oci"`
 		Containerd ContainerdConfig `toml:"containerd"`
 	} `toml:"worker"`
+
+	Registries map[string]RegistryConfig `toml:"registry"`
 }
 
 type GRPCConfig struct {
@@ -35,6 +37,11 @@ type GRPCConfig struct {
 	// MaxSendMsgSize int    `toml:"max_send_message_size"`
 }
 
+type RegistryConfig struct {
+	Mirrors   []string `toml:"mirrors"`
+	PlainHTTP bool     `toml:"http"`
+}
+
 type TLSConfig struct {
 	Cert string `toml:"cert"`
 	Key  string `toml:"key"`

cmd/buildkitd/config/config_test.go

@@ -39,6 +39,10 @@ keepDuration=3600
 [[worker.containerd.gcpolicy]]
 keepBytes=40
 keepDuration=7200
+
+[registry."docker.io"]
+mirrors=["hub.docker.io"]
+http=true
 `
 
 	cfg, md, err := Load(bytes.NewBuffer([]byte(testConfig)))
@@ -78,4 +82,7 @@ keepDuration=7200
 	require.Equal(t, int64(7200), cfg.Workers.Containerd.GCPolicy[1].KeepDuration)
 	require.Equal(t, 1, len(cfg.Workers.Containerd.GCPolicy[0].Filters))
 	require.Equal(t, 0, len(cfg.Workers.Containerd.GCPolicy[1].Filters))
+
+	require.Equal(t, cfg.Registries["docker.io"].PlainHTTP, true)
+	require.Equal(t, cfg.Registries["docker.io"].Mirrors[0], "hub.docker.io")
 }

cmd/buildkitd/main.go

@@ -35,6 +35,7 @@ import (
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/appdefaults"
 	"github.com/moby/buildkit/util/profiler"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/version"
 	"github.com/moby/buildkit/worker"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -504,17 +505,30 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 		return nil, err
 	}
 
+	resolverFn := resolverFunc(cfg)
+
 	return control.NewController(control.Opt{
 		SessionManager:   sessionManager,
 		WorkerController: wc,
 		Frontends:        frontends,
 		// TODO: support non-registry remote cache
-		ResolveCacheExporterFunc: registryremotecache.ResolveCacheExporterFunc(sessionManager),
+		ResolveCacheExporterFunc: registryremotecache.ResolveCacheExporterFunc(sessionManager, resolverFn),
-		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(sessionManager),
+		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(sessionManager, resolverFn),
 		CacheKeyStorage:          cacheStorage,
 	})
 }
 
+func resolverFunc(cfg *config.Config) resolver.ResolveOptionsFunc {
+	m := map[string]resolver.RegistryConf{}
+	for k, v := range cfg.Registries {
+		m[k] = resolver.RegistryConf{
+			Mirrors:   v.Mirrors,
+			PlainHTTP: v.PlainHTTP,
+		}
+	}
+	return resolver.NewResolveOptionsFunc(m)
+}
+
 func newWorkerController(c *cli.Context, wiOpt workerInitializerOpt) (*worker.Controller, error) {
 	wc := &worker.Controller{}
 	nWorkers := 0

cmd/buildkitd/main_containerd_worker.go

@@ -125,6 +125,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 	}
 	opt.SessionManager = common.sessionManager
 	opt.GCPolicy = getGCPolicy(cfg.GCPolicy, common.config.Root)
+	opt.ResolveOptionsFunc = resolverFunc(common.config)
 
 	if platformsStr := cfg.Platforms; len(platformsStr) != 0 {
 		platforms, err := parsePlatforms(platformsStr)

cmd/buildkitd/main_oci_worker.go

@@ -142,6 +142,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 	}
 	opt.SessionManager = common.sessionManager
 	opt.GCPolicy = getGCPolicy(cfg.GCPolicy, common.config.Root)
+	opt.ResolveOptionsFunc = resolverFunc(common.config)
 
 	if platformsStr := cfg.Platforms; len(platformsStr) != 0 {
 		platforms, err := parsePlatforms(platformsStr)

exporter/containerimage/export.go

@@ -11,6 +11,7 @@ import (
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/util/push"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/pkg/errors"
 )
 
@@ -25,6 +26,7 @@ type Opt struct {
 	SessionManager *session.Manager
 	ImageWriter    *ImageWriter
 	Images         images.Store
+	ResolverOpt    resolver.ResolveOptionsFunc
 }
 
 type imageExporter struct {
@@ -144,7 +146,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
 				tagDone(nil)
 			}
 			if e.push {
-				if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure); err != nil {
+				if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt); err != nil {
 					return nil, err
 				}
 			}

source/containerimage/pull.go

@@ -18,6 +18,7 @@ import (
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/pull"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/winlayers"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
@@ -35,6 +36,7 @@ type SourceOpt struct {
 	Applier        diff.Applier
 	CacheAccessor  cache.Accessor
 	ImageStore     images.Store // optional
+	ResolverOpt    resolver.ResolveOptionsFunc
 }
 
 type imageSource struct {
@@ -70,7 +72,7 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt g
 	}
 
 	res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) {
-		dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.SessionManager, is.ImageStore, rm), is.ContentStore, opt.Platform)
+		dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.ResolverOpt, is.SessionManager, is.ImageStore, rm, ref), is.ContentStore, opt.Platform)
 		if err != nil {
 			return nil, err
 		}
@@ -99,7 +101,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
 		ContentStore: is.ContentStore,
 		Applier:      is.Applier,
 		Src:          imageIdentifier.Reference,
-		Resolver:     pull.NewResolver(ctx, is.SessionManager, is.ImageStore, imageIdentifier.ResolveMode),
+		Resolver:     pull.NewResolver(ctx, is.ResolverOpt, is.SessionManager, is.ImageStore, imageIdentifier.ResolveMode, imageIdentifier.Reference.String()),
 		Platform:     &platform,
 	}
 	p := &puller{

util/pull/resolver.go

@@ -2,6 +2,7 @@ package pull
 
 import (
 	"context"
+	"net/http"
 	"time"
 
 	"github.com/containerd/containerd/images"
@@ -10,15 +11,21 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/auth"
 	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-func NewResolver(ctx context.Context, sm *session.Manager, imageStore images.Store, mode source.ResolveMode) remotes.Resolver {
+func NewResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, sm *session.Manager, imageStore images.Store, mode source.ResolveMode, ref string) remotes.Resolver {
-	r := docker.NewResolver(docker.ResolverOptions{
+	opt := docker.ResolverOptions{
-		Client:      tracing.DefaultClient,
+		Client: http.DefaultClient,
-		Credentials: getCredentialsFromSession(ctx, sm),
+	}
-	})
+	if rfn != nil {
+		opt = rfn(ref)
+	}
+	opt.Credentials = getCredentialsFromSession(ctx, sm)
+
+	r := docker.NewResolver(opt)
 
 	if imageStore == nil || mode == source.ResolveModeForcePull {
 		return r

util/push/push.go

@@ -16,7 +16,7 @@ import (
 	"github.com/moby/buildkit/session/auth"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/progress"
-	"github.com/moby/buildkit/util/tracing"
+	"github.com/moby/buildkit/util/resolver"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
@@ -40,7 +40,7 @@ func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (
 	}
 }
 
-func Push(ctx context.Context, sm *session.Manager, cs content.Provider, dgst digest.Digest, ref string, insecure bool) error {
+func Push(ctx context.Context, sm *session.Manager, cs content.Provider, dgst digest.Digest, ref string, insecure bool, rfn resolver.ResolveOptionsFunc) error {
 	desc := ocispec.Descriptor{
 		Digest: dgst,
 	}
@@ -50,11 +50,13 @@ func Push(ctx context.Context, sm *session.Manager, cs content.Provider, dgst di
 	}
 	ref = reference.TagNameOnly(parsed).String()
 
-	resolver := docker.NewResolver(docker.ResolverOptions{
+	opt := rfn(ref)
-		Client:      tracing.DefaultClient,
+	opt.Credentials = getCredentialsFunc(ctx, sm)
-		Credentials: getCredentialsFunc(ctx, sm),
+	if insecure {
-		PlainHTTP:   insecure,
+		opt.PlainHTTP = insecure
-	})
+	}
+
+	resolver := docker.NewResolver(opt)
 
 	pusher, err := resolver.Pusher(ctx, ref)
 	if err != nil {

util/resolver/resolver.go

@@ -0,0 +1,45 @@
+package resolver
+
+import (
+	"math/rand"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/util/tracing"
+)
+
+type RegistryConf struct {
+	Mirrors   []string
+	PlainHTTP bool
+}
+
+type ResolveOptionsFunc func(string) docker.ResolverOptions
+
+func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
+	return func(ref string) docker.ResolverOptions {
+		def := docker.ResolverOptions{
+			Client: tracing.DefaultClient,
+		}
+
+		parsed, err := reference.ParseNormalizedNamed(ref)
+		if err != nil {
+			return def
+		}
+		host := reference.Domain(parsed)
+
+		c, ok := m[host]
+		if !ok {
+			return def
+		}
+
+		if len(c.Mirrors) > 0 {
+			def.Host = func(string) (string, error) {
+				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
+			}
+		}
+
+		def.PlainHTTP = c.PlainHTTP
+
+		return def
+	}
+}

worker/base/worker.go

@@ -39,6 +39,7 @@ import (
 	"github.com/moby/buildkit/source/local"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/worker"
 	digest "github.com/opencontainers/go-digest"
 	ociidentity "github.com/opencontainers/image-spec/identity"
@@ -67,6 +68,7 @@ type WorkerOpt struct {
 	Applier            diff.Applier
 	Differ             diff.Comparer
 	ImageStore         images.Store // optional
+	ResolveOptionsFunc resolver.ResolveOptionsFunc
 }
 
 // Worker is a local worker instance with dedicated snapshotter, cache, and so on.
@@ -108,6 +110,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
 		Applier:        opt.Applier,
 		ImageStore:     opt.ImageStore,
 		CacheAccessor:  cm,
+		ResolverOpt:    opt.ResolveOptionsFunc,
 	})
 	if err != nil {
 		return nil, err
@@ -160,6 +163,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
 		Images:         opt.ImageStore,
 		SessionManager: opt.SessionManager,
 		ImageWriter:    iw,
+		ResolverOpt:    opt.ResolveOptionsFunc,
 	})
 	if err != nil {
 		return nil, err