From cd110f6b1086663bef7935319cffd1837e6118e7 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 21 Feb 2018 16:33:02 -0800 Subject: [PATCH] dockerfile: fix copy from implicit stage Signed-off-by: Tonis Tiigi --- frontend/dockerfile/dockerfile2llb/convert.go | 91 ++++++++++++------- frontend/dockerfile/dockerfile_test.go | 78 ++++++++++++++++ 2 files changed, 134 insertions(+), 35 deletions(-) diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index b15ee188..4f9aed34 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -104,22 +104,17 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, // fill dependencies to stages so unreachable ones can avoid loading image configs for _, d := range allDispatchStates { - for _, cmd := range d.stage.Commands { - if c, ok := cmd.(*instructions.CopyCommand); ok { - if c.From != "" { - index, err := strconv.Atoi(c.From) - if err != nil { - stn, ok := dispatchStatesByName[strings.ToLower(c.From)] - if !ok { - return nil, nil, errors.Errorf("stage %s not found", c.From) - } - d.deps[stn] = struct{}{} - } else { - if index < 0 || index >= len(allDispatchStates) { - return nil, nil, errors.Errorf("invalid stage index %d", index) - } - d.deps[allDispatchStates[index]] = struct{}{} - } + d.commands = make([]command, len(d.stage.Commands)) + for i, cmd := range d.stage.Commands { + newCmd, created, err := toCommand(cmd, dispatchStatesByName, allDispatchStates) + if err != nil { + return nil, nil, err + } + d.commands[i] = newCmd + if newCmd.copySource != nil { + d.deps[newCmd.copySource] = struct{}{} + if created { + allDispatchStates = append(allDispatchStates, newCmd.copySource) } } } @@ -218,7 +213,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, return nil, nil, err } - for _, cmd := range d.stage.Commands { + for _, cmd := range d.commands { if err := dispatch(d, cmd, opt); err != nil { return nil, nil, err } @@ -228,6 +223,34 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, return &target.state, &target.image, nil } +func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, bool, error) { + cmd := command{Command: ic} + created := false + if c, ok := ic.(*instructions.CopyCommand); ok { + if c.From != "" { + var stn *dispatchState + index, err := strconv.Atoi(c.From) + if err != nil { + stn, ok = dispatchStatesByName[strings.ToLower(c.From)] + if !ok { + stn = &dispatchState{ + stage: instructions.Stage{BaseName: c.From}, + deps: make(map[*dispatchState]struct{}), + } + created = true + } + } else { + if index < 0 || index >= len(allDispatchStates) { + return command{}, false, errors.Errorf("invalid stage index %d", index) + } + stn = allDispatchStates[index] + } + cmd.copySource = stn + } + } + return cmd, created, nil +} + type dispatchOpt struct { allDispatchStates []*dispatchState dispatchStatesByName map[string]*dispatchState @@ -238,8 +261,8 @@ type dispatchOpt struct { buildContext llb.State } -func dispatch(d *dispatchState, cmd instructions.Command, opt dispatchOpt) error { - if ex, ok := cmd.(instructions.SupportsSingleWordExpansion); ok { +func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { + if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansion); ok { err := ex.Expand(func(word string) (string, error) { return opt.shlex.ProcessWord(word, toEnvList(d.buildArgs, d.image.Config.Env)) }) @@ -249,7 +272,7 @@ func dispatch(d *dispatchState, cmd instructions.Command, opt dispatchOpt) error } var err error - switch c := cmd.(type) { + switch c := cmd.Command.(type) { case *instructions.MaintainerCommand: err = dispatchMaintainer(d, c) case *instructions.EnvCommand: @@ -284,20 +307,8 @@ func dispatch(d *dispatchState, cmd instructions.Command, opt dispatchOpt) error err = dispatchArg(d, c, opt.metaArgs, opt.buildArgValues) case *instructions.CopyCommand: l := opt.buildContext - if c.From != "" { - index, err := strconv.Atoi(c.From) - if err != nil { - stn, ok := opt.dispatchStatesByName[strings.ToLower(c.From)] - if !ok { - return errors.Errorf("stage %s not found", c.From) - } - l = stn.state - } else { - if index >= len(opt.allDispatchStates) { - return errors.Errorf("invalid stage index %d", index) - } - l = opt.allDispatchStates[index].state - } + if cmd.copySource != nil { + l = cmd.copySource.state } err = dispatchCopy(d, c.SourcesAndDest, l, false, c, c.Chown) default: @@ -312,6 +323,12 @@ type dispatchState struct { base *dispatchState deps map[*dispatchState]struct{} buildArgs []instructions.ArgCommand + commands []command +} + +type command struct { + instructions.Command + copySource *dispatchState } func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error { @@ -323,7 +340,11 @@ func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error if len(ast.AST.Children) != 1 { return errors.New("onbuild trigger should be a single expression") } - cmd, err := instructions.ParseCommand(ast.AST.Children[0]) + ic, err := instructions.ParseCommand(ast.AST.Children[0]) + if err != nil { + return err + } + cmd, _, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates) if err != nil { return err } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 65db3186..bf06480a 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -48,6 +48,7 @@ func TestIntegration(t *testing.T) { testCopyChown, testCopyWildcards, testCopyOverrideFiles, + testMultiStageImplicitFrom, }) } @@ -1047,6 +1048,83 @@ COPY --from=build foo bar2 require.Equal(t, "fromgit", string(dt)) } +func testMultiStageImplicitFrom(t *testing.T, sb integration.Sandbox) { + t.Parallel() + + dockerfile := []byte(` +FROM scratch +COPY --from=busybox /etc/passwd test +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + c, err := client.New(sb.Address()) + require.NoError(t, err) + defer c.Close() + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + err = c.Solve(context.TODO(), nil, client.SolveOpt{ + Frontend: "dockerfile.v0", + Exporter: client.ExporterLocal, + ExporterAttrs: map[string]string{ + "output": destDir, + }, + LocalDirs: map[string]string{ + builder.LocalNameDockerfile: dir, + builder.LocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "test")) + require.NoError(t, err) + require.Contains(t, string(dt), "root") + + // testing masked image will load actual stage + + dockerfile = []byte(` +FROM busybox AS golang +RUN mkdir /usr/bin && echo -n foo > /usr/bin/go + +FROM scratch +COPY --from=golang /usr/bin/go go +`) + + dir, err = tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + destDir, err = ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + err = c.Solve(context.TODO(), nil, client.SolveOpt{ + Frontend: "dockerfile.v0", + Exporter: client.ExporterLocal, + ExporterAttrs: map[string]string{ + "output": destDir, + }, + LocalDirs: map[string]string{ + builder.LocalNameDockerfile: dir, + builder.LocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "go")) + require.NoError(t, err) + require.Contains(t, string(dt), "foo") +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil {