JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2596 from tonistiigi/copy-link 

dockerfile: add COPY --link for copy via merging layers
master
Tõnis Tiigi 2022-02-14 14:23:23 -08:00 committed by GitHub
parent 0fab75a169 8441d0a45d
commit 61027554b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 144 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -22,6 +22,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
@ -616,7 +617,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
case *instructions.WorkdirCommand:
err = dispatchWorkdir(d, c, true, &opt)
case *instructions.AddCommand:
err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, c.Chown, c.Chmod, c.Location(), opt)
err = dispatchCopy(d, copyConfig{
params: c.SourcesAndDest,
source: opt.buildContext,
isAddCommand: true,
cmdToPrint: c,
chown: c.Chown,
chmod: c.Chmod,
link: c.Link,
location: c.Location(),
opt: opt,
})
if err == nil {
for _, src := range c.SourcePaths {
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
@ -651,7 +662,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
if len(cmd.sources) != 0 {
l = cmd.sources[0].state
}
err = dispatchCopy(d, c.SourcesAndDest, l, false, c, c.Chown, c.Chmod, c.Location(), opt)
err = dispatchCopy(d, copyConfig{
params: c.SourcesAndDest,
source: l,
isAddCommand: false,
cmdToPrint: c,
chown: c.Chown,
chmod: c.Chmod,
link: c.Link,
location: c.Location(),
opt: opt,
})
if err == nil && len(cmd.sources) == 0 {
for _, src := range c.SourcePaths {
d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
@ -922,25 +943,25 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
return nil
}
func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, chmod string, loc []parser.Range, opt dispatchOpt) error {
pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
func dispatchCopyFileOp(d *dispatchState, cfg copyConfig) error {
pp, err := pathRelativeToWorkingDir(d.state, cfg.params.DestPath)
if err != nil {
return err
}
dest := path.Join("/", pp)
if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
if cfg.params.DestPath == "." || cfg.params.DestPath == "" || cfg.params.DestPath[len(cfg.params.DestPath)-1] == filepath.Separator {
dest += string(filepath.Separator)
}
var copyOpt []llb.CopyOption
if chown != "" {
copyOpt = append(copyOpt, llb.WithUser(chown))
if cfg.chown != "" {
copyOpt = append(copyOpt, llb.WithUser(cfg.chown))
}
var mode *os.FileMode
if chmod != "" {
p, err := strconv.ParseUint(chmod, 8, 32)
if cfg.chmod != "" {
p, err := strconv.ParseUint(cfg.chmod, 8, 32)
if err == nil {
perm := os.FileMode(p)
mode = &perm
@ -948,7 +969,7 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
}
commitMessage := bytes.NewBufferString("")
if isAddCommand {
if cfg.isAddCommand {
commitMessage.WriteString("ADD")
} else {
commitMessage.WriteString("COPY")
@ -956,10 +977,10 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
var a *llb.FileAction
for _, src := range c.SourcePaths {
for _, src := range cfg.params.SourcePaths {
commitMessage.WriteString(" " + src)
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !isAddCommand {
if !cfg.isAddCommand {
return errors.New("source can't be a URL for COPY")
}
@ -976,7 +997,7 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
}
}
st := llb.HTTP(src, llb.Filename(f), dfCmd(c))
st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params))
opts := append([]llb.CopyOption{&llb.CopyInfo{
Mode: mode,
@ -993,21 +1014,21 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
Mode: mode,
FollowSymlinks: true,
CopyDirContentsOnly: true,
AttemptUnpack: isAddCommand,
AttemptUnpack: cfg.isAddCommand,
CreateDestPath: true,
AllowWildcard: true,
AllowEmptyWildcard: true,
}}, copyOpt...)
if a == nil {
a = llb.Copy(sourceState, filepath.Join("/", src), dest, opts...)
a = llb.Copy(cfg.source, filepath.Join("/", src), dest, opts...)
} else {
a = a.Copy(sourceState, filepath.Join("/", src), dest, opts...)
a = a.Copy(cfg.source, filepath.Join("/", src), dest, opts...)
}
}
}
for _, src := range c.SourceContents {
for _, src := range cfg.params.SourceContents {
commitMessage.WriteString(" <<" + src.Path)
data := src.Data
@ -1029,9 +1050,9 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
}
}
commitMessage.WriteString(" " + c.DestPath)
commitMessage.WriteString(" " + cfg.params.DestPath)
platform := opt.targetPlatform
platform := cfg.opt.targetPlatform
if d.platform != nil {
platform = *d.platform
}
@ -1041,50 +1062,72 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
return err
}
name := uppercaseCmd(processCmdEnv(cfg.opt.shlex, cfg.cmdToPrint.String(), env))
fileOpt := []llb.ConstraintsOpt{
llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform, env)),
location(opt.sourceMap, loc),
llb.WithCustomName(prefixCommand(d, name, d.prefixPlatform, &platform, env)),
location(cfg.opt.sourceMap, cfg.location),
}
if d.ignoreCache {
fileOpt = append(fileOpt, llb.IgnoreCache)
}
d.state = d.state.File(a, fileOpt...)
if cfg.opt.llbCaps.Supports(pb.CapMergeOp) == nil && cfg.link && cfg.chmod == "" {
d.cmdIndex-- // prefixCommand increases it
fileOpt = append(fileOpt, llb.ProgressGroup(identity.NewID(), prefixCommand(d, name, d.prefixPlatform, &platform, env)))
d.cmdIndex--
mergeOpt := append(fileOpt, llb.WithCustomName(prefixCommand(d, "LINK "+name, d.prefixPlatform, &platform, env)))
d.state = llb.Merge([]llb.State{d.state, llb.Scratch().File(a, fileOpt...)}, mergeOpt...)
} else {
d.state = d.state.File(a, fileOpt...)
}
return commitToHistory(&d.image, commitMessage.String(), true, &d.state)
}
func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, chmod string, loc []parser.Range, opt dispatchOpt) error {
if useFileOp(opt.buildArgValues, opt.llbCaps) {
return dispatchCopyFileOp(d, c, sourceState, isAddCommand, cmdToPrint, chown, chmod, loc, opt)
type copyConfig struct {
params instructions.SourcesAndDest
source llb.State
isAddCommand bool
cmdToPrint fmt.Stringer
chown string
chmod string
link bool
location []parser.Range
opt dispatchOpt
}
func dispatchCopy(d *dispatchState, cfg copyConfig) error {
if useFileOp(cfg.opt.buildArgValues, cfg.opt.llbCaps) {
return dispatchCopyFileOp(d, cfg)
}
if len(c.SourceContents) > 0 {
if len(cfg.params.SourceContents) > 0 {
return errors.New("inline content copy is not supported")
}
if chmod != "" {
if opt.llbCaps != nil && opt.llbCaps.Supports(pb.CapFileBase) != nil {
return errors.Wrap(opt.llbCaps.Supports(pb.CapFileBase), "chmod is not supported")
if cfg.chmod != "" {
if cfg.opt.llbCaps != nil && cfg.opt.llbCaps.Supports(pb.CapFileBase) != nil {
return errors.Wrap(cfg.opt.llbCaps.Supports(pb.CapFileBase), "chmod is not supported")
}
return errors.New("chmod is not supported")
}
img := llb.Image(opt.copyImage, llb.MarkImageInternal, llb.Platform(opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
img := llb.Image(cfg.opt.copyImage, llb.MarkImageInternal, llb.Platform(cfg.opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
pp, err := pathRelativeToWorkingDir(d.state, cfg.params.DestPath)
if err != nil {
return err
}
dest := path.Join(".", pp)
if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
if cfg.params.DestPath == "." || cfg.params.DestPath == "" || cfg.params.DestPath[len(cfg.params.DestPath)-1] == filepath.Separator {
dest += string(filepath.Separator)
}
args := []string{"copy"}
unpack := isAddCommand
unpack := cfg.isAddCommand
mounts := make([]llb.RunOption, 0, len(c.SourcePaths))
if chown != "" {
args = append(args, fmt.Sprintf("--chown=%s", chown))
_, _, err := parseUser(chown)
mounts := make([]llb.RunOption, 0, len(cfg.params.SourcePaths))
if cfg.chown != "" {
args = append(args, fmt.Sprintf("--chown=%s", cfg.chown))
_, _, err := parseUser(cfg.chown)
if err != nil {
mounts = append(mounts, llb.AddMount("/etc/passwd", d.state, llb.SourcePath("/etc/passwd"), llb.Readonly))
mounts = append(mounts, llb.AddMount("/etc/group", d.state, llb.SourcePath("/etc/group"), llb.Readonly))
@ -1092,16 +1135,16 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
}
commitMessage := bytes.NewBufferString("")
if isAddCommand {
if cfg.isAddCommand {
commitMessage.WriteString("ADD")
} else {
commitMessage.WriteString("COPY")
}
for i, src := range c.SourcePaths {
for i, src := range cfg.params.SourcePaths {
commitMessage.WriteString(" " + src)
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !isAddCommand {
if !cfg.isAddCommand {
return errors.New("source can't be a URL for COPY")
}
@ -1120,7 +1163,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
}
target := path.Join(fmt.Sprintf("/src-%d", i), f)
args = append(args, target)
mounts = append(mounts, llb.AddMount(path.Dir(target), llb.HTTP(src, llb.Filename(f), dfCmd(c)), llb.Readonly))
mounts = append(mounts, llb.AddMount(path.Dir(target), llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params)), llb.Readonly))
} else {
d, f := splitWildcards(src)
targetCmd := fmt.Sprintf("/src-%d", i)
@ -1131,18 +1174,18 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
}
targetCmd = path.Join(targetCmd, f)
args = append(args, targetCmd)
mounts = append(mounts, llb.AddMount(targetMount, sourceState, llb.SourcePath(d), llb.Readonly))
mounts = append(mounts, llb.AddMount(targetMount, cfg.source, llb.SourcePath(d), llb.Readonly))
}
}
commitMessage.WriteString(" " + c.DestPath)
commitMessage.WriteString(" " + cfg.params.DestPath)
args = append(args, dest)
if unpack {
args = append(args[:1], append([]string{"--unpack"}, args[1:]...)...)
}
platform := opt.targetPlatform
platform := cfg.opt.targetPlatform
if d.platform != nil {
platform = *d.platform
}
@ -1156,16 +1199,16 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
llb.Args(args),
llb.Dir("/dest"),
llb.ReadonlyRootFS(),
dfCmd(cmdToPrint),
llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform, env)),
location(opt.sourceMap, loc),
dfCmd(cfg.cmdToPrint),
llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(cfg.opt.shlex, cfg.cmdToPrint.String(), env)), d.prefixPlatform, &platform, env)),
location(cfg.opt.sourceMap, cfg.location),
}
if d.ignoreCache {
runOpt = append(runOpt, llb.IgnoreCache)
}
if opt.llbCaps != nil {
if err := opt.llbCaps.Supports(pb.CapExecMetaNetwork); err == nil {
if cfg.opt.llbCaps != nil {
if err := cfg.opt.llbCaps.Supports(pb.CapExecMetaNetwork); err == nil {
runOpt = append(runOpt, llb.Network(llb.NetModeNone))
}
}

46
frontend/dockerfile/docs/syntax.md
View File

@ -25,6 +25,52 @@ incrementing the major component of a version and you may want to pin the image
change in between releases on labs channel, the old versions are guaranteed to be backward compatible.
## Linked copies `COPY --link`, `ADD --link`
To use this flag set Dockerfile version to at least `1.4`.
```dockerfile
# syntax=docker/dockerfile:1.4
```
Enabling this flag in `COPY` or `ADD` commands allows you to copy files with enhanced semantics where your files remain independent on their own layer and don't get invalidated when commands on previous layers are changed.
When `--link` is used your source files are copied into an empty destination directory. That directory is turned into a layer that is linked on top of your previous state.
```dockerfile
# syntax=docker/dockerfile:1.4
FROM alpine
COPY --link /foo /bar
```
Is equivalent of doing two builds:
```dockerfile
FROM alpine
```
and
```dockerfile
FROM scratch
COPY /foo /bar
```
and merging all the layers of both images together.
#### Benefits of using `--link`
Using `--link` allows to reuse already built layers in subsequent builds with `--cache-from` even if the previous layers have changed. This is especially important for multi-stage builds where a `COPY --from` statement would previously get invalidated if any previous commands in the same stage changed, causing the need to rebuild the intermediate stages again. With `--link` the layer the previous build generated is reused and merged on top of the new layers. This also means you can easily rebase your images when the base images receive updates, without having to execute the whole build again. In backends that support it, BuildKit can do this rebase action without the need to push or pull any layers between the client and the registry. BuildKit will detect this case and only create new image manifest that contains the new layers and old layers in correct order.
The same behavior where BuildKit can avoid pulling down the base image can also happen when using `--link` and no other commands that would require access to the files in the base image. In that case BuildKit will only build the layers for the `COPY` commands and push them to the registry directly on top of the layers of the base image.
#### Incompatibilities with `--link=false`
When using `--link` the `COPY/ADD` commands are not allowed to read any files from the previous state. This means that if in previous state the destination directory was a path that contained a symlink, `COPY/ADD` can not follow it. In the final image the destination path created with `--link` will always be a path containing only directories.
If you don't rely on the behavior of following symlinks in the destination path, using `--link` is always recommended. The performance of `--link` is equivalent or better than the default behavior and it creates much better conditions for cache reuse.
## Build Mounts `RUN --mount=...`
To use this flag set Dockerfile version to at least `1.2`

2
frontend/dockerfile/instructions/commands.go
View File

@ -226,6 +226,7 @@ type AddCommand struct {
SourcesAndDest
Chown string
Chmod string
Link bool
}
// Expand variables
@ -249,6 +250,7 @@ type CopyCommand struct {
From string
Chown string
Chmod string
Link bool
}
// Expand variables

4
frontend/dockerfile/instructions/parse.go
View File

@ -280,6 +280,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
}
flChown := req.flags.AddString("chown", "")
flChmod := req.flags.AddString("chmod", "")
flLink := req.flags.AddBool("link", false)
if err := req.flags.Parse(); err != nil {
return nil, err
}
@ -294,6 +295,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
SourcesAndDest: *sourcesAndDest,
Chown: flChown.Value,
Chmod: flChmod.Value,
Link: flLink.Value == "true",
}, nil
}
@ -304,6 +306,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
flChown := req.flags.AddString("chown", "")
flFrom := req.flags.AddString("from", "")
flChmod := req.flags.AddString("chmod", "")
flLink := req.flags.AddBool("link", false)
if err := req.flags.Parse(); err != nil {
return nil, err
}
@ -319,6 +322,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
From: flFrom.Value,
Chown: flChown.Value,
Chmod: flChmod.Value,
Link: flLink.Value == "true",
}, nil
}