dockerfile: add support for COPY/ADD —link
Enables smarter file copy logic using the mergeop implementation. Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>master
Tonis Tiigi
parent
2f996517bc
commit
6f64543249
|
|
@ -621,7 +621,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
|||
case *instructions.WorkdirCommand:
|
||||
err = dispatchWorkdir(d, c, true, &opt)
|
||||
case *instructions.AddCommand:
|
||||
err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, c.Chown, c.Chmod, c.Location(), opt)
|
||||
err = dispatchCopy(d, copyConfig{
|
||||
params: c.SourcesAndDest,
|
||||
source: opt.buildContext,
|
||||
isAddCommand: true,
|
||||
cmdToPrint: c,
|
||||
chown: c.Chown,
|
||||
chmod: c.Chmod,
|
||||
link: c.Link,
|
||||
location: c.Location(),
|
||||
opt: opt,
|
||||
})
|
||||
if err == nil {
|
||||
for _, src := range c.SourcePaths {
|
||||
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
|
||||
|
|
@ -656,7 +666,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
|||
if len(cmd.sources) != 0 {
|
||||
l = cmd.sources[0].state
|
||||
}
|
||||
err = dispatchCopy(d, c.SourcesAndDest, l, false, c, c.Chown, c.Chmod, c.Location(), opt)
|
||||
err = dispatchCopy(d, copyConfig{
|
||||
params: c.SourcesAndDest,
|
||||
source: l,
|
||||
isAddCommand: false,
|
||||
cmdToPrint: c,
|
||||
chown: c.Chown,
|
||||
chmod: c.Chmod,
|
||||
link: c.Link,
|
||||
location: c.Location(),
|
||||
opt: opt,
|
||||
})
|
||||
if err == nil && len(cmd.sources) == 0 {
|
||||
for _, src := range c.SourcePaths {
|
||||
d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
|
||||
|
|
@ -927,25 +947,25 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
|
|||
return nil
|
||||
}
|
||||
|
||||
func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, chmod string, loc []parser.Range, opt dispatchOpt) error {
|
||||
pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
|
||||
func dispatchCopyFileOp(d *dispatchState, cfg copyConfig) error {
|
||||
pp, err := pathRelativeToWorkingDir(d.state, cfg.params.DestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest := path.Join("/", pp)
|
||||
if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
|
||||
if cfg.params.DestPath == "." || cfg.params.DestPath == "" || cfg.params.DestPath[len(cfg.params.DestPath)-1] == filepath.Separator {
|
||||
dest += string(filepath.Separator)
|
||||
}
|
||||
|
||||
var copyOpt []llb.CopyOption
|
||||
|
||||
if chown != "" {
|
||||
copyOpt = append(copyOpt, llb.WithUser(chown))
|
||||
if cfg.chown != "" {
|
||||
copyOpt = append(copyOpt, llb.WithUser(cfg.chown))
|
||||
}
|
||||
|
||||
var mode *os.FileMode
|
||||
if chmod != "" {
|
||||
p, err := strconv.ParseUint(chmod, 8, 32)
|
||||
if cfg.chmod != "" {
|
||||
p, err := strconv.ParseUint(cfg.chmod, 8, 32)
|
||||
if err == nil {
|
||||
perm := os.FileMode(p)
|
||||
mode = &perm
|
||||
|
|
@ -953,7 +973,7 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
|
|||
}
|
||||
|
||||
commitMessage := bytes.NewBufferString("")
|
||||
if isAddCommand {
|
||||
if cfg.isAddCommand {
|
||||
commitMessage.WriteString("ADD")
|
||||
} else {
|
||||
commitMessage.WriteString("COPY")
|
||||
|
|
@ -961,10 +981,10 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
|
|||
|
||||
var a *llb.FileAction
|
||||
|
||||
for _, src := range c.SourcePaths {
|
||||
for _, src := range cfg.params.SourcePaths {
|
||||
commitMessage.WriteString(" " + src)
|
||||
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
|
||||
if !isAddCommand {
|
||||
if !cfg.isAddCommand {
|
||||
return errors.New("source can't be a URL for COPY")
|
||||
}
|
||||
|
||||
|
|
@ -981,7 +1001,7 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
|
|||
}
|
||||
}
|
||||
|
||||
st := llb.HTTP(src, llb.Filename(f), dfCmd(c))
|
||||
st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params))
|
||||
|
||||
opts := append([]llb.CopyOption{&llb.CopyInfo{
|
||||
Mode: mode,
|
||||
|
|
@ -998,21 +1018,21 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
|
|||
Mode: mode,
|
||||
FollowSymlinks: true,
|
||||
CopyDirContentsOnly: true,
|
||||
AttemptUnpack: isAddCommand,
|
||||
AttemptUnpack: cfg.isAddCommand,
|
||||
CreateDestPath: true,
|
||||
AllowWildcard: true,
|
||||
AllowEmptyWildcard: true,
|
||||
}}, copyOpt...)
|
||||
|
||||
if a == nil {
|
||||
a = llb.Copy(sourceState, filepath.Join("/", src), dest, opts...)
|
||||
a = llb.Copy(cfg.source, filepath.Join("/", src), dest, opts...)
|
||||
} else {
|
||||
a = a.Copy(sourceState, filepath.Join("/", src), dest, opts...)
|
||||
a = a.Copy(cfg.source, filepath.Join("/", src), dest, opts...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, src := range c.SourceContents {
|
||||
for _, src := range cfg.params.SourceContents {
|
||||
commitMessage.WriteString(" <<" + src.Path)
|
||||
|
||||
data := src.Data
|
||||
|
|
@ -1034,9 +1054,9 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
|
|||
}
|
||||
}
|
||||
|
||||
commitMessage.WriteString(" " + c.DestPath)
|
||||
commitMessage.WriteString(" " + cfg.params.DestPath)
|
||||
|
||||
platform := opt.targetPlatform
|
||||
platform := cfg.opt.targetPlatform
|
||||
if d.platform != nil {
|
||||
platform = *d.platform
|
||||
}
|
||||
|
|
@ -1046,50 +1066,69 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
|
|||
return err
|
||||
}
|
||||
|
||||
name := uppercaseCmd(processCmdEnv(cfg.opt.shlex, cfg.cmdToPrint.String(), env))
|
||||
fileOpt := []llb.ConstraintsOpt{
|
||||
llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform, env)),
|
||||
location(opt.sourceMap, loc),
|
||||
llb.WithCustomName(prefixCommand(d, name, d.prefixPlatform, &platform, env)),
|
||||
location(cfg.opt.sourceMap, cfg.location),
|
||||
}
|
||||
if d.ignoreCache {
|
||||
fileOpt = append(fileOpt, llb.IgnoreCache)
|
||||
}
|
||||
|
||||
if cfg.opt.llbCaps.Supports(pb.CapMergeOp) == nil && cfg.link && cfg.chmod == "" {
|
||||
mergeOpt := append(fileOpt, llb.WithCustomName(prefixCommand(d, "LINK "+name, d.prefixPlatform, &platform, env)))
|
||||
d.state = llb.Merge([]llb.State{d.state, llb.Scratch().File(a, fileOpt...)}, mergeOpt...)
|
||||
} else {
|
||||
d.state = d.state.File(a, fileOpt...)
|
||||
}
|
||||
|
||||
return commitToHistory(&d.image, commitMessage.String(), true, &d.state)
|
||||
}
|
||||
|
||||
func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, chmod string, loc []parser.Range, opt dispatchOpt) error {
|
||||
if useFileOp(opt.buildArgValues, opt.llbCaps) {
|
||||
return dispatchCopyFileOp(d, c, sourceState, isAddCommand, cmdToPrint, chown, chmod, loc, opt)
|
||||
type copyConfig struct {
|
||||
params instructions.SourcesAndDest
|
||||
source llb.State
|
||||
isAddCommand bool
|
||||
cmdToPrint fmt.Stringer
|
||||
chown string
|
||||
chmod string
|
||||
link bool
|
||||
location []parser.Range
|
||||
opt dispatchOpt
|
||||
}
|
||||
|
||||
func dispatchCopy(d *dispatchState, cfg copyConfig) error {
|
||||
if useFileOp(cfg.opt.buildArgValues, cfg.opt.llbCaps) {
|
||||
return dispatchCopyFileOp(d, cfg)
|
||||
}
|
||||
|
||||
if len(c.SourceContents) > 0 {
|
||||
if len(cfg.params.SourceContents) > 0 {
|
||||
return errors.New("inline content copy is not supported")
|
||||
}
|
||||
|
||||
if chmod != "" {
|
||||
if opt.llbCaps != nil && opt.llbCaps.Supports(pb.CapFileBase) != nil {
|
||||
return errors.Wrap(opt.llbCaps.Supports(pb.CapFileBase), "chmod is not supported")
|
||||
if cfg.chmod != "" {
|
||||
if cfg.opt.llbCaps != nil && cfg.opt.llbCaps.Supports(pb.CapFileBase) != nil {
|
||||
return errors.Wrap(cfg.opt.llbCaps.Supports(pb.CapFileBase), "chmod is not supported")
|
||||
}
|
||||
return errors.New("chmod is not supported")
|
||||
}
|
||||
|
||||
img := llb.Image(opt.copyImage, llb.MarkImageInternal, llb.Platform(opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
|
||||
pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
|
||||
img := llb.Image(cfg.opt.copyImage, llb.MarkImageInternal, llb.Platform(cfg.opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
|
||||
pp, err := pathRelativeToWorkingDir(d.state, cfg.params.DestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest := path.Join(".", pp)
|
||||
if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
|
||||
if cfg.params.DestPath == "." || cfg.params.DestPath == "" || cfg.params.DestPath[len(cfg.params.DestPath)-1] == filepath.Separator {
|
||||
dest += string(filepath.Separator)
|
||||
}
|
||||
args := []string{"copy"}
|
||||
unpack := isAddCommand
|
||||
unpack := cfg.isAddCommand
|
||||
|
||||
mounts := make([]llb.RunOption, 0, len(c.SourcePaths))
|
||||
if chown != "" {
|
||||
args = append(args, fmt.Sprintf("--chown=%s", chown))
|
||||
_, _, err := parseUser(chown)
|
||||
mounts := make([]llb.RunOption, 0, len(cfg.params.SourcePaths))
|
||||
if cfg.chown != "" {
|
||||
args = append(args, fmt.Sprintf("--chown=%s", cfg.chown))
|
||||
_, _, err := parseUser(cfg.chown)
|
||||
if err != nil {
|
||||
mounts = append(mounts, llb.AddMount("/etc/passwd", d.state, llb.SourcePath("/etc/passwd"), llb.Readonly))
|
||||
mounts = append(mounts, llb.AddMount("/etc/group", d.state, llb.SourcePath("/etc/group"), llb.Readonly))
|
||||
|
|
@ -1097,16 +1136,16 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
|
|||
}
|
||||
|
||||
commitMessage := bytes.NewBufferString("")
|
||||
if isAddCommand {
|
||||
if cfg.isAddCommand {
|
||||
commitMessage.WriteString("ADD")
|
||||
} else {
|
||||
commitMessage.WriteString("COPY")
|
||||
}
|
||||
|
||||
for i, src := range c.SourcePaths {
|
||||
for i, src := range cfg.params.SourcePaths {
|
||||
commitMessage.WriteString(" " + src)
|
||||
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
|
||||
if !isAddCommand {
|
||||
if !cfg.isAddCommand {
|
||||
return errors.New("source can't be a URL for COPY")
|
||||
}
|
||||
|
||||
|
|
@ -1125,7 +1164,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
|
|||
}
|
||||
target := path.Join(fmt.Sprintf("/src-%d", i), f)
|
||||
args = append(args, target)
|
||||
mounts = append(mounts, llb.AddMount(path.Dir(target), llb.HTTP(src, llb.Filename(f), dfCmd(c)), llb.Readonly))
|
||||
mounts = append(mounts, llb.AddMount(path.Dir(target), llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params)), llb.Readonly))
|
||||
} else {
|
||||
d, f := splitWildcards(src)
|
||||
targetCmd := fmt.Sprintf("/src-%d", i)
|
||||
|
|
@ -1136,18 +1175,18 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
|
|||
}
|
||||
targetCmd = path.Join(targetCmd, f)
|
||||
args = append(args, targetCmd)
|
||||
mounts = append(mounts, llb.AddMount(targetMount, sourceState, llb.SourcePath(d), llb.Readonly))
|
||||
mounts = append(mounts, llb.AddMount(targetMount, cfg.source, llb.SourcePath(d), llb.Readonly))
|
||||
}
|
||||
}
|
||||
|
||||
commitMessage.WriteString(" " + c.DestPath)
|
||||
commitMessage.WriteString(" " + cfg.params.DestPath)
|
||||
|
||||
args = append(args, dest)
|
||||
if unpack {
|
||||
args = append(args[:1], append([]string{"--unpack"}, args[1:]...)...)
|
||||
}
|
||||
|
||||
platform := opt.targetPlatform
|
||||
platform := cfg.opt.targetPlatform
|
||||
if d.platform != nil {
|
||||
platform = *d.platform
|
||||
}
|
||||
|
|
@ -1161,16 +1200,16 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
|
|||
llb.Args(args),
|
||||
llb.Dir("/dest"),
|
||||
llb.ReadonlyRootFS(),
|
||||
dfCmd(cmdToPrint),
|
||||
llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform, env)),
|
||||
location(opt.sourceMap, loc),
|
||||
dfCmd(cfg.cmdToPrint),
|
||||
llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(cfg.opt.shlex, cfg.cmdToPrint.String(), env)), d.prefixPlatform, &platform, env)),
|
||||
location(cfg.opt.sourceMap, cfg.location),
|
||||
}
|
||||
if d.ignoreCache {
|
||||
runOpt = append(runOpt, llb.IgnoreCache)
|
||||
}
|
||||
|
||||
if opt.llbCaps != nil {
|
||||
if err := opt.llbCaps.Supports(pb.CapExecMetaNetwork); err == nil {
|
||||
if cfg.opt.llbCaps != nil {
|
||||
if err := cfg.opt.llbCaps.Supports(pb.CapExecMetaNetwork); err == nil {
|
||||
runOpt = append(runOpt, llb.Network(llb.NetModeNone))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,52 @@ incrementing the major component of a version and you may want to pin the image
|
|||
change in between releases on labs channel, the old versions are guaranteed to be backward compatible.
|
||||
|
||||
|
||||
## Linked copies `COPY --link`, `ADD --link`
|
||||
|
||||
To use this flag set Dockerfile version to at least `1.4`.
|
||||
|
||||
```dockerfile
|
||||
# syntax=docker/dockerfile:1.4
|
||||
```
|
||||
|
||||
Enabling this flag in `COPY` or `ADD` commands allows you to copy files with enhanced semantics where your files remain independent on their own layer and don't get invalidated when commands on previous layers are changed.
|
||||
|
||||
When `--link` is used your source files are copied into an empty destination directory. That directory is turned into a layer that is linked on top of your previous state.
|
||||
|
||||
```dockerfile
|
||||
# syntax=docker/dockerfile:1.4
|
||||
FROM alpine
|
||||
COPY --link /foo /bar
|
||||
```
|
||||
|
||||
Is equivalent of doing two builds:
|
||||
|
||||
```dockerfile
|
||||
FROM alpine
|
||||
```
|
||||
|
||||
and
|
||||
|
||||
```dockerfile
|
||||
FROM scratch
|
||||
COPY /foo /bar
|
||||
```
|
||||
|
||||
and merging all the layers of both images together.
|
||||
|
||||
#### Benefits of using `--link`
|
||||
|
||||
Using `--link` allows to reuse already built layers in subsequent builds with `--cache-from` even if the previous layers have changed. This is especially important for multi-stage builds where a `COPY --from` statement would previously get invalidated if any previous commands in the same stage changed, causing the need to rebuild the intermediate stages again. With `--link` the layer the previous build generated is reused and merged on top of the new layers. This also means you can easily rebase your images when the base images receive updates, without having to execute the whole build again. In backends that support it, BuildKit can do this rebase action without the need to push or pull any layers between the client and the registry. BuildKit will detect this case and only create new image manifest that contains the new layers and old layers in correct order.
|
||||
|
||||
The same behavior where BuildKit can avoid pulling down the base image can also happen when using `--link` and no other commands that would require access to the files in the base image. In that case BuildKit will only build the layers for the `COPY` commands and push them to the registry directly on top of the layers of the base image.
|
||||
|
||||
#### Incompatibilities with `--link=false`
|
||||
|
||||
When using `--link` the `COPY/ADD` commands are not allowed to read any files from the previous state. This means that if in previous state the destination directory was a path that contained a symlink, `COPY/ADD` can not follow it. In the final image the destination path created with `--link` will always be a path containing only directories.
|
||||
|
||||
If you don't rely on the behavior of following symlinks in the destination path, using `--link` is always recommended. The performance of `--link` is equivalent or better than the default behavior and it creates much better conditions for cache reuse.
|
||||
|
||||
|
||||
## Build Mounts `RUN --mount=...`
|
||||
|
||||
To use this flag set Dockerfile version to at least `1.2`
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ type AddCommand struct {
|
|||
SourcesAndDest
|
||||
Chown string
|
||||
Chmod string
|
||||
Link bool
|
||||
}
|
||||
|
||||
// Expand variables
|
||||
|
|
@ -249,6 +250,7 @@ type CopyCommand struct {
|
|||
From string
|
||||
Chown string
|
||||
Chmod string
|
||||
Link bool
|
||||
}
|
||||
|
||||
// Expand variables
|
||||
|
|
|
|||
|
|
@ -280,6 +280,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
|
|||
}
|
||||
flChown := req.flags.AddString("chown", "")
|
||||
flChmod := req.flags.AddString("chmod", "")
|
||||
flLink := req.flags.AddBool("link", false)
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -294,6 +295,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
|
|||
SourcesAndDest: *sourcesAndDest,
|
||||
Chown: flChown.Value,
|
||||
Chmod: flChmod.Value,
|
||||
Link: flLink.Value == "true",
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -304,6 +306,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
|
|||
flChown := req.flags.AddString("chown", "")
|
||||
flFrom := req.flags.AddString("from", "")
|
||||
flChmod := req.flags.AddString("chmod", "")
|
||||
flLink := req.flags.AddBool("link", false)
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -319,6 +322,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
|
|||
From: flFrom.Value,
|
||||
Chown: flChown.Value,
|
||||
Chmod: flChmod.Value,
|
||||
Link: flLink.Value == "true",
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue