Merge pull request #2521 from tonistiigi/dockerfile-named-contexts

dockerfile: add support for named contexts

frontend/dockerfile/builder/build.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/go-units"
 	controlapi "github.com/moby/buildkit/api/services/control"
 	"github.com/moby/buildkit/client/llb"
@@ -461,6 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 						}
 						c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
 					},
+					ContextByName: contextByNameFunc(c, tp),
 				})
 
 				if err != nil {
@@ -802,6 +804,100 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
 	return opts
 }
 
+func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) {
+	return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) {
+		named, err := reference.ParseNormalizedNamed(name)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "invalid context name %s", name)
+		}
+		name = strings.TrimSuffix(reference.FamiliarString(named), ":latest")
+
+		if p != nil {
+			name := name + "::" + platforms.Format(platforms.Normalize(*p))
+			st, img, err := contextByName(ctx, c, name, p)
+			if err != nil {
+				return nil, nil, err
+			}
+			if st != nil {
+				return st, img, nil
+			}
+		}
+		return contextByName(ctx, c, name, p)
+	}
+}
+
+func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) {
+	opts := c.BuildOpts().Opts
+	v, ok := opts["context:"+name]
+	if !ok {
+		return nil, nil, nil
+	}
+
+	vv := strings.SplitN(v, ":", 2)
+	if len(vv) != 2 {
+		return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
+	}
+	switch vv[0] {
+	case "docker-image":
+		imgOpt := []llb.ImageOption{
+			llb.WithCustomName("[context " + name + "] " + vv[1]),
+			llb.WithMetaResolver(c),
+		}
+		if platform != nil {
+			imgOpt = append(imgOpt, llb.Platform(*platform))
+		}
+		st := llb.Image(strings.TrimPrefix(vv[1], "//"), imgOpt...)
+		return &st, nil, nil
+	case "git":
+		st, ok := detectGitContext(v, "1")
+		if !ok {
+			return nil, nil, errors.Errorf("invalid git context %s", v)
+		}
+		return st, nil, nil
+	case "http", "https":
+		st, ok := detectGitContext(v, "1")
+		if !ok {
+			httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
+			st = &httpst
+		}
+		return st, nil, nil
+	case "local":
+		st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name))
+		return &st, nil, nil
+	case "input":
+		inputs, err := c.Inputs(ctx)
+		if err != nil {
+			return nil, nil, err
+		}
+		st, ok := inputs[vv[1]]
+		if !ok {
+			return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
+		}
+		md, ok := opts["input-metadata:"+vv[1]]
+		if ok {
+			m := make(map[string][]byte)
+			if err := json.Unmarshal([]byte(md), &m); err != nil {
+				return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
+			}
+			dt, ok := m["containerimage.config"]
+			if ok {
+				st, err = st.WithImageConfig([]byte(dt))
+				if err != nil {
+					return nil, nil, err
+				}
+				var img dockerfile2llb.Image
+				if err := json.Unmarshal(dt, &img); err != nil {
+					return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
+				}
+				return &st, &img, nil
+			}
+		}
+		return &st, nil, nil
+	default:
+		return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
+	}
+}
+
 func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error {
 	if sm == nil {
 		return err

frontend/dockerfile/builder/caps.go

@@ -12,6 +12,7 @@ import (
 var enabledCaps = map[string]struct{}{
 	"moby.buildkit.frontend.inputs":      {},
 	"moby.buildkit.frontend.subrequests": {},
+	"moby.buildkit.frontend.contexts":    {},
 }
 
 func validateCaps(req string) (forward bool, err error) {

frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile

@@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \
 
 FROM scratch AS release
 LABEL moby.buildkit.frontend.network.none="true"
-LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests"
+LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts"
 COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend
 ENTRYPOINT ["/bin/dockerfile-frontend"]
 

frontend/dockerfile/dockerfile2llb/convert.go

@@ -69,9 +69,20 @@ type ConvertOpt struct {
 	SourceMap         *llb.SourceMap
 	Hostname          string
 	Warn              func(short, url string, detail [][]byte, location *parser.Range)
+	ContextByName     func(context.Context, string) (*llb.State, *Image, error)
 }
 
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) {
+	contextByName := opt.ContextByName
+	opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) {
+		if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
+			if contextByName != nil {
+				return contextByName(ctx, name)
+			}
+		}
+		return nil, nil, nil
+	}
+
 	if len(dt) == 0 {
 		return nil, nil, errors.Errorf("the Dockerfile cannot be empty")
 	}
@@ -133,13 +144,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		st.BaseName = name
 
 		ds := &dispatchState{
-			stage:          st,
 			deps:           make(map[*dispatchState]struct{}),
 			ctxPaths:       make(map[string]struct{}),
 			stageName:      st.Name,
 			prefixPlatform: opt.PrefixPlatform,
 		}
 
+		if st.Name != "" {
+			s, img, err := opt.ContextByName(ctx, st.Name)
+			if err != nil {
+				return nil, nil, err
+			}
+			if s != nil {
+				ds.noinit = true
+				ds.state = *s
+				if img != nil {
+					ds.image = *img
+				}
+				allDispatchStates.addState(ds)
+				continue
+			}
+		}
+
+		ds.stage = st
+
 		if st.Name == "" {
 			ds.stageName = fmt.Sprintf("stage-%d", i)
 		}
@@ -237,7 +265,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	for i, d := range allDispatchStates.states {
 		reachable := isReachable(target, d)
 		// resolve image config for every stage
-		if d.base == nil {
+		if d.base == nil && !d.noinit {
 			if d.stage.BaseName == emptyImageName {
 				d.state = llb.Scratch()
 				d.image = emptyImage(platformOpt.targetPlatform)
@@ -260,8 +288,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 						platform = &platformOpt.targetPlatform
 					}
 					d.stage.BaseName = reference.TagNameOnly(ref).String()
+
 					var isScratch bool
-					if metaResolver != nil && reachable {
+					st, img, err := opt.ContextByName(ctx, d.stage.BaseName)
+					if err != nil {
+						return err
+					}
+					if st != nil {
+						if img != nil {
+							d.image = *img
+						} else {
+							d.image = emptyImage(platformOpt.targetPlatform)
+						}
+						d.state = *st
+						d.platform = platform
+						return nil
+					}
+					if reachable {
 						prefix := "["
 						if opt.PrefixPlatform && platform != nil {
 							prefix += platforms.Format(*platform) + " "
@@ -615,6 +658,7 @@ type dispatchState struct {
 	platform       *ocispecs.Platform
 	stage          instructions.Stage
 	base           *dispatchState
+	noinit         bool
 	deps           map[*dispatchState]struct{}
 	buildArgs      []instructions.KeyValuePairOptional
 	commands       []command

frontend/dockerfile/dockerfile_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
 	"github.com/moby/buildkit/solver/errdefs"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/testutil"
 	"github.com/moby/buildkit/util/testutil/httpserver"
@@ -118,6 +119,10 @@ var allTests = integration.TestFuncs(
 	testShmSize,
 	testUlimit,
 	testCgroupParent,
+	testNamedImageContext,
+	testNamedLocalContext,
+	testNamedInputContext,
+	testNamedMultiplatformInputContext,
 )
 
 var fileOpTests = integration.TestFuncs(
@@ -159,6 +164,7 @@ var securityOpts []integration.TestOpt
 
 type frontend interface {
 	Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error)
+	SolveGateway(context.Context, gateway.Client, gateway.SolveRequest) (*gateway.Result, error)
 	DFCmdArgs(string, string) (string, string)
 	RequiresBuildctl(t *testing.T)
 }
@@ -5406,6 +5412,371 @@ COPY --from=base /out /
 	require.Contains(t, strings.TrimSpace(string(dt)), `/foocgroup/buildkit/`)
 }
 
+func testNamedImageContext(t *testing.T, sb integration.Sandbox) {
+	ctx := sb.Context()
+
+	c, err := client.New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dockerfile := []byte(`
+FROM busybox AS base
+RUN cat /etc/alpine-release > /out
+FROM scratch
+COPY --from=base /out /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	f := getFrontend(t, sb)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+		FrontendAttrs: map[string]string{
+			"context:busybox": "docker-image://alpine",
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
+	require.NoError(t, err)
+	require.True(t, len(dt) > 0)
+}
+
+func testNamedLocalContext(t *testing.T, sb integration.Sandbox) {
+	ctx := sb.Context()
+
+	c, err := client.New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dockerfile := []byte(`
+FROM busybox AS base
+RUN cat /etc/alpine-release > /out
+FROM scratch
+COPY --from=base /out /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	outf := []byte(`dummy-result`)
+
+	dir2, err := tmpdir(
+		fstest.CreateFile("out", outf, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir2)
+
+	f := getFrontend(t, sb)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+		FrontendAttrs: map[string]string{
+			"context:base": "local:basedir",
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+			"basedir":                          dir2,
+		},
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
+	require.NoError(t, err)
+	require.True(t, len(dt) > 0)
+}
+
+func testNamedInputContext(t *testing.T, sb integration.Sandbox) {
+	ctx := sb.Context()
+
+	c, err := client.New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dockerfile := []byte(`
+FROM alpine
+ENV FOO=bar
+RUN echo first > /out
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	dockerfile2 := []byte(`
+FROM base AS build
+RUN echo "foo is $FOO" > /foo
+FROM scratch
+COPY --from=build /foo /out /
+`)
+
+	dir2, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile2, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	f := getFrontend(t, sb)
+
+	b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{})
+		if err != nil {
+			return nil, err
+		}
+		ref, err := res.SingleRef()
+		if err != nil {
+			return nil, err
+		}
+		st, err := ref.ToState()
+		if err != nil {
+			return nil, err
+		}
+
+		def, err := st.Marshal(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		dt, ok := res.Metadata["containerimage.config"]
+		if !ok {
+			return nil, errors.Errorf("no containerimage.config in metadata")
+		}
+
+		dt, err = json.Marshal(map[string][]byte{
+			"containerimage.config": dt,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{
+			FrontendOpt: map[string]string{
+				"dockerfilekey":       builder.DefaultLocalNameDockerfile + "2",
+				"context:base":        "input:base",
+				"input-metadata:base": string(dt),
+			},
+			FrontendInputs: map[string]*pb.Definition{
+				"base": def.ToPB(),
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		return res, nil
+	}
+
+	product := "buildkit_test"
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Build(ctx, client.SolveOpt{
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile:       dir,
+			builder.DefaultLocalNameContext:          dir,
+			builder.DefaultLocalNameDockerfile + "2": dir2,
+		},
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, product, b, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
+	require.NoError(t, err)
+	require.Equal(t, "first\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, "foo is bar\n", string(dt))
+}
+
+func testNamedMultiplatformInputContext(t *testing.T, sb integration.Sandbox) {
+	ctx := sb.Context()
+
+	c, err := client.New(ctx, sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dockerfile := []byte(`
+FROM --platform=$BUILDPLATFORM alpine
+ARG TARGETARCH
+ENV FOO=bar-$TARGETARCH
+RUN echo "foo $TARGETARCH" > /out
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	dockerfile2 := []byte(`
+FROM base AS build
+RUN echo "foo is $FOO" > /foo
+FROM scratch
+COPY --from=build /foo /out /
+`)
+
+	dir2, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile2, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	f := getFrontend(t, sb)
+
+	b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{
+			FrontendOpt: map[string]string{
+				"platform": "linux/amd64,linux/arm64",
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		if len(res.Refs) != 2 {
+			return nil, errors.Errorf("expected 2 refs, got %d", len(res.Refs))
+		}
+
+		inputs := map[string]*pb.Definition{}
+		st, err := res.Refs["linux/amd64"].ToState()
+		if err != nil {
+			return nil, err
+		}
+		def, err := st.Marshal(ctx)
+		if err != nil {
+			return nil, err
+		}
+		inputs["base::linux/amd64"] = def.ToPB()
+
+		st, err = res.Refs["linux/arm64"].ToState()
+		if err != nil {
+			return nil, err
+		}
+		def, err = st.Marshal(ctx)
+		if err != nil {
+			return nil, err
+		}
+		inputs["base::linux/arm64"] = def.ToPB()
+
+		frontendOpt := map[string]string{
+			"dockerfilekey":             builder.DefaultLocalNameDockerfile + "2",
+			"context:base::linux/amd64": "input:base::linux/amd64",
+			"context:base::linux/arm64": "input:base::linux/arm64",
+			"platform":                  "linux/amd64,linux/arm64",
+		}
+
+		dt, ok := res.Metadata["containerimage.config/linux/amd64"]
+		if !ok {
+			return nil, errors.Errorf("no containerimage.config in metadata")
+		}
+		dt, err = json.Marshal(map[string][]byte{
+			"containerimage.config": dt,
+		})
+		if err != nil {
+			return nil, err
+		}
+		frontendOpt["input-metadata:base::linux/amd64"] = string(dt)
+
+		dt, ok = res.Metadata["containerimage.config/linux/arm64"]
+		if !ok {
+			return nil, errors.Errorf("no containerimage.config in metadata")
+		}
+		dt, err = json.Marshal(map[string][]byte{
+			"containerimage.config": dt,
+		})
+		if err != nil {
+			return nil, err
+		}
+		frontendOpt["input-metadata:base::linux/arm64"] = string(dt)
+
+		res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{
+			FrontendOpt:    frontendOpt,
+			FrontendInputs: inputs,
+		})
+		if err != nil {
+			return nil, err
+		}
+		return res, nil
+	}
+
+	product := "buildkit_test"
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Build(ctx, client.SolveOpt{
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile:       dir,
+			builder.DefaultLocalNameContext:          dir,
+			builder.DefaultLocalNameDockerfile + "2": dir2,
+		},
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, product, b, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/out"))
+	require.NoError(t, err)
+	require.Equal(t, "foo amd64\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/foo"))
+	require.NoError(t, err)
+	require.Equal(t, "foo is bar-amd64\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/out"))
+	require.NoError(t, err)
+	require.Equal(t, "foo arm64\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/foo"))
+	require.NoError(t, err)
+	require.Equal(t, "foo is bar-arm64\n", string(dt))
+}
+
 func tmpdir(appliers ...fstest.Applier) (string, error) {
 	tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile")
 	if err != nil {
@@ -5542,6 +5913,11 @@ func (f *builtinFrontend) Solve(ctx context.Context, c *client.Client, opt clien
 	return c.Solve(ctx, nil, opt, statusChan)
 }
 
+func (f *builtinFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) {
+	req.Frontend = "dockerfile.v0"
+	return c.Solve(ctx, req)
+}
+
 func (f *builtinFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) {
 	return dfCmdArgs(ctx, dockerfile, "--frontend dockerfile.v0")
 }
@@ -5556,6 +5932,13 @@ func (f *clientFrontend) Solve(ctx context.Context, c *client.Client, opt client
 	return c.Build(ctx, opt, "", builder.Build, statusChan)
 }
 
+func (f *clientFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) {
+	if req.Frontend == "" && req.Definition == nil {
+		req.Frontend = "dockerfile.v0"
+	}
+	return c.Solve(ctx, req)
+}
+
 func (f *clientFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) {
 	return "", ""
 }
@@ -5578,6 +5961,15 @@ func (f *gatewayFrontend) Solve(ctx context.Context, c *client.Client, opt clien
 	return c.Solve(ctx, nil, opt, statusChan)
 }
 
+func (f *gatewayFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) {
+	req.Frontend = "gateway.v0"
+	if req.FrontendOpt == nil {
+		req.FrontendOpt = make(map[string]string)
+	}
+	req.FrontendOpt["source"] = f.gw
+	return c.Solve(ctx, req)
+}
+
 func (f *gatewayFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) {
 	return dfCmdArgs(ctx, dockerfile, "--frontend gateway.v0 --opt=source="+f.gw)
 }