JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2521 from tonistiigi/dockerfile-named-contexts 

dockerfile: add support for named contexts
master
Akihiro Suda 2021-12-27 20:24:02 +09:00 committed by GitHub
parent d6b191e33a 23709ef316
commit a8278dd166
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 537 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
frontend/dockerfile/builder/build.go
View File

@ -14,6 +14,7 @@ import (
"strings"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/docker/go-units"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/client/llb"
@ -461,6 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
}
c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
},
ContextByName: contextByNameFunc(c, tp),
})
if err != nil {
@ -802,6 +804,100 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
return opts
}
func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) {
return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) {
named, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, nil, errors.Wrapf(err, "invalid context name %s", name)
}
name = strings.TrimSuffix(reference.FamiliarString(named), ":latest")
if p != nil {
name := name + "::" + platforms.Format(platforms.Normalize(*p))
st, img, err := contextByName(ctx, c, name, p)
if err != nil {
return nil, nil, err
}
if st != nil {
return st, img, nil
}
}
return contextByName(ctx, c, name, p)
}
}
func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) {
opts := c.BuildOpts().Opts
v, ok := opts["context:"+name]
if !ok {
return nil, nil, nil
}
vv := strings.SplitN(v, ":", 2)
if len(vv) != 2 {
return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
}
switch vv[0] {
case "docker-image":
imgOpt := []llb.ImageOption{
llb.WithCustomName("[context " + name + "] " + vv[1]),
llb.WithMetaResolver(c),
}
if platform != nil {
imgOpt = append(imgOpt, llb.Platform(*platform))
}
st := llb.Image(strings.TrimPrefix(vv[1], "//"), imgOpt...)
return &st, nil, nil
case "git":
st, ok := detectGitContext(v, "1")
if !ok {
return nil, nil, errors.Errorf("invalid git context %s", v)
}
return st, nil, nil
case "http", "https":
st, ok := detectGitContext(v, "1")
if !ok {
httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
st = &httpst
}
return st, nil, nil
case "local":
st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name))
return &st, nil, nil
case "input":
inputs, err := c.Inputs(ctx)
if err != nil {
return nil, nil, err
}
st, ok := inputs[vv[1]]
if !ok {
return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
}
md, ok := opts["input-metadata:"+vv[1]]
if ok {
m := make(map[string][]byte)
if err := json.Unmarshal([]byte(md), &m); err != nil {
return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
}
dt, ok := m["containerimage.config"]
if ok {
st, err = st.WithImageConfig([]byte(dt))
if err != nil {
return nil, nil, err
}
var img dockerfile2llb.Image
if err := json.Unmarshal(dt, &img); err != nil {
return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
}
return &st, &img, nil
}
}
return &st, nil, nil
default:
return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
}
}
func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error {
if sm == nil {
return err

1
frontend/dockerfile/builder/caps.go
View File

@ -12,6 +12,7 @@ import (
var enabledCaps = map[string]struct{}{
"moby.buildkit.frontend.inputs": {},
"moby.buildkit.frontend.subrequests": {},
"moby.buildkit.frontend.contexts": {},
}
func validateCaps(req string) (forward bool, err error) {

2
frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
View File

@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \
FROM scratch AS release
LABEL moby.buildkit.frontend.network.none="true"
LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests"
LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts"
COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend
ENTRYPOINT ["/bin/dockerfile-frontend"]

50
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -69,9 +69,20 @@ type ConvertOpt struct {
SourceMap *llb.SourceMap
Hostname string
Warn func(short, url string, detail [][]byte, location *parser.Range)
ContextByName func(context.Context, string) (*llb.State, *Image, error)
}
func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) {
contextByName := opt.ContextByName
opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) {
if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
if contextByName != nil {
return contextByName(ctx, name)
}
}
return nil, nil, nil
}
if len(dt) == 0 {
return nil, nil, errors.Errorf("the Dockerfile cannot be empty")
}
@ -133,13 +144,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
st.BaseName = name
ds := &dispatchState{
stage: st,
deps: make(map[*dispatchState]struct{}),
ctxPaths: make(map[string]struct{}),
stageName: st.Name,
prefixPlatform: opt.PrefixPlatform,
}
if st.Name != "" {
s, img, err := opt.ContextByName(ctx, st.Name)
if err != nil {
return nil, nil, err
}
if s != nil {
ds.noinit = true
ds.state = *s
if img != nil {
ds.image = *img
}
allDispatchStates.addState(ds)
continue
}
}
ds.stage = st
if st.Name == "" {
ds.stageName = fmt.Sprintf("stage-%d", i)
}
@ -237,7 +265,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
for i, d := range allDispatchStates.states {
reachable := isReachable(target, d)
// resolve image config for every stage
if d.base == nil {
if d.base == nil && !d.noinit {
if d.stage.BaseName == emptyImageName {
d.state = llb.Scratch()
d.image = emptyImage(platformOpt.targetPlatform)
@ -260,8 +288,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
platform = &platformOpt.targetPlatform
}
d.stage.BaseName = reference.TagNameOnly(ref).String()
var isScratch bool
if metaResolver != nil && reachable {
st, img, err := opt.ContextByName(ctx, d.stage.BaseName)
if err != nil {
return err
}
if st != nil {
if img != nil {
d.image = *img
} else {
d.image = emptyImage(platformOpt.targetPlatform)
}
d.state = *st
d.platform = platform
return nil
}
if reachable {
prefix := "["
if opt.PrefixPlatform && platform != nil {
prefix += platforms.Format(*platform) + " "
@ -615,6 +658,7 @@ type dispatchState struct {
platform *ocispecs.Platform
stage instructions.Stage
base *dispatchState
noinit bool
deps map[*dispatchState]struct{}
buildArgs []instructions.KeyValuePairOptional
commands []command

392
frontend/dockerfile/dockerfile_test.go
View File

@ -39,6 +39,7 @@ import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/testutil"
"github.com/moby/buildkit/util/testutil/httpserver"
@ -118,6 +119,10 @@ var allTests = integration.TestFuncs(
testShmSize,
testUlimit,
testCgroupParent,
testNamedImageContext,
testNamedLocalContext,
testNamedInputContext,
testNamedMultiplatformInputContext,
)
var fileOpTests = integration.TestFuncs(
@ -159,6 +164,7 @@ var securityOpts []integration.TestOpt
type frontend interface {
Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error)
SolveGateway(context.Context, gateway.Client, gateway.SolveRequest) (*gateway.Result, error)
DFCmdArgs(string, string) (string, string)
RequiresBuildctl(t *testing.T)
}
@ -5406,6 +5412,371 @@ COPY --from=base /out /
require.Contains(t, strings.TrimSpace(string(dt)), `/foocgroup/buildkit/`)
}
func testNamedImageContext(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()
c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()
dockerfile := []byte(`
FROM busybox AS base
RUN cat /etc/alpine-release > /out
FROM scratch
COPY --from=base /out /
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
f := getFrontend(t, sb)
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"context:busybox": "docker-image://alpine",
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
require.NoError(t, err)
require.True(t, len(dt) > 0)
}
func testNamedLocalContext(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()
c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()
dockerfile := []byte(`
FROM busybox AS base
RUN cat /etc/alpine-release > /out
FROM scratch
COPY --from=base /out /
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
outf := []byte(`dummy-result`)
dir2, err := tmpdir(
fstest.CreateFile("out", outf, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir2)
f := getFrontend(t, sb)
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"context:base": "local:basedir",
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
"basedir": dir2,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
require.NoError(t, err)
require.True(t, len(dt) > 0)
}
func testNamedInputContext(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()
c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()
dockerfile := []byte(`
FROM alpine
ENV FOO=bar
RUN echo first > /out
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
dockerfile2 := []byte(`
FROM base AS build
RUN echo "foo is $FOO" > /foo
FROM scratch
COPY --from=build /foo /out /
`)
dir2, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile2, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
f := getFrontend(t, sb)
b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
st, err := ref.ToState()
if err != nil {
return nil, err
}
def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}
dt, ok := res.Metadata["containerimage.config"]
if !ok {
return nil, errors.Errorf("no containerimage.config in metadata")
}
dt, err = json.Marshal(map[string][]byte{
"containerimage.config": dt,
})
if err != nil {
return nil, err
}
res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{
FrontendOpt: map[string]string{
"dockerfilekey": builder.DefaultLocalNameDockerfile + "2",
"context:base": "input:base",
"input-metadata:base": string(dt),
},
FrontendInputs: map[string]*pb.Definition{
"base": def.ToPB(),
},
})
if err != nil {
return nil, err
}
return res, nil
}
product := "buildkit_test"
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = c.Build(ctx, client.SolveOpt{
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
builder.DefaultLocalNameDockerfile + "2": dir2,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
}, product, b, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
require.NoError(t, err)
require.Equal(t, "first\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo"))
require.NoError(t, err)
require.Equal(t, "foo is bar\n", string(dt))
}
func testNamedMultiplatformInputContext(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()
c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()
dockerfile := []byte(`
FROM --platform=$BUILDPLATFORM alpine
ARG TARGETARCH
ENV FOO=bar-$TARGETARCH
RUN echo "foo $TARGETARCH" > /out
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
dockerfile2 := []byte(`
FROM base AS build
RUN echo "foo is $FOO" > /foo
FROM scratch
COPY --from=build /foo /out /
`)
dir2, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile2, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
f := getFrontend(t, sb)
b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{
FrontendOpt: map[string]string{
"platform": "linux/amd64,linux/arm64",
},
})
if err != nil {
return nil, err
}
if len(res.Refs) != 2 {
return nil, errors.Errorf("expected 2 refs, got %d", len(res.Refs))
}
inputs := map[string]*pb.Definition{}
st, err := res.Refs["linux/amd64"].ToState()
if err != nil {
return nil, err
}
def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}
inputs["base::linux/amd64"] = def.ToPB()
st, err = res.Refs["linux/arm64"].ToState()
if err != nil {
return nil, err
}
def, err = st.Marshal(ctx)
if err != nil {
return nil, err
}
inputs["base::linux/arm64"] = def.ToPB()
frontendOpt := map[string]string{
"dockerfilekey": builder.DefaultLocalNameDockerfile + "2",
"context:base::linux/amd64": "input:base::linux/amd64",
"context:base::linux/arm64": "input:base::linux/arm64",
"platform": "linux/amd64,linux/arm64",
}
dt, ok := res.Metadata["containerimage.config/linux/amd64"]
if !ok {
return nil, errors.Errorf("no containerimage.config in metadata")
}
dt, err = json.Marshal(map[string][]byte{
"containerimage.config": dt,
})
if err != nil {
return nil, err
}
frontendOpt["input-metadata:base::linux/amd64"] = string(dt)
dt, ok = res.Metadata["containerimage.config/linux/arm64"]
if !ok {
return nil, errors.Errorf("no containerimage.config in metadata")
}
dt, err = json.Marshal(map[string][]byte{
"containerimage.config": dt,
})
if err != nil {
return nil, err
}
frontendOpt["input-metadata:base::linux/arm64"] = string(dt)
res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{
FrontendOpt: frontendOpt,
FrontendInputs: inputs,
})
if err != nil {
return nil, err
}
return res, nil
}
product := "buildkit_test"
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = c.Build(ctx, client.SolveOpt{
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
builder.DefaultLocalNameDockerfile + "2": dir2,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
}, product, b, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/out"))
require.NoError(t, err)
require.Equal(t, "foo amd64\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/foo"))
require.NoError(t, err)
require.Equal(t, "foo is bar-amd64\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/out"))
require.NoError(t, err)
require.Equal(t, "foo arm64\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/foo"))
require.NoError(t, err)
require.Equal(t, "foo is bar-arm64\n", string(dt))
}
func tmpdir(appliers ...fstest.Applier) (string, error) {
tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile")
if err != nil {
@ -5542,6 +5913,11 @@ func (f *builtinFrontend) Solve(ctx context.Context, c *client.Client, opt clien
return c.Solve(ctx, nil, opt, statusChan)
}
func (f *builtinFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) {
req.Frontend = "dockerfile.v0"
return c.Solve(ctx, req)
}
func (f *builtinFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) {
return dfCmdArgs(ctx, dockerfile, "--frontend dockerfile.v0")
}
@ -5556,6 +5932,13 @@ func (f *clientFrontend) Solve(ctx context.Context, c *client.Client, opt client
return c.Build(ctx, opt, "", builder.Build, statusChan)
}
func (f *clientFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) {
if req.Frontend == "" && req.Definition == nil {
req.Frontend = "dockerfile.v0"
}
return c.Solve(ctx, req)
}
func (f *clientFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) {
return "", ""
}
@ -5578,6 +5961,15 @@ func (f *gatewayFrontend) Solve(ctx context.Context, c *client.Client, opt clien
return c.Solve(ctx, nil, opt, statusChan)
}
func (f *gatewayFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) {
req.Frontend = "gateway.v0"
if req.FrontendOpt == nil {
req.FrontendOpt = make(map[string]string)
}
req.FrontendOpt["source"] = f.gw
return c.Solve(ctx, req)
}
func (f *gatewayFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) {
return dfCmdArgs(ctx, dockerfile, "--frontend gateway.v0 --opt=source="+f.gw)
}