JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2132 from jedevc/dockerfile-heredocs 

Dockerfile heredocs
v0.9
Tõnis Tiigi 2021-06-10 09:23:04 -07:00 committed by GitHub
parent 0f9f55ff7c fa632c5b21
commit 4518627f4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 1551 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.golangci.yml
View File

@ -8,6 +8,7 @@ run:
build-tags:
- dfrunsecurity
- dfrunnetwork
- dfheredoc
linters:
enable:

93
frontend/dockerfile/dockerfile2llb/convert.go
View File

@ -507,7 +507,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
case *instructions.AddCommand:
err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, c.Chown, c.Chmod, c.Location(), opt)
if err == nil {
for _, src := range c.Sources() {
for _, src := range c.SourcePaths {
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
}
@ -542,7 +542,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
}
err = dispatchCopy(d, c.SourcesAndDest, l, false, c, c.Chown, c.Chmod, c.Location(), opt)
if err == nil && len(cmd.sources) == 0 {
for _, src := range c.Sources() {
for _, src := range c.SourcePaths {
d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
}
}
@ -647,15 +647,63 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand) error {
}
func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv, sources []*dispatchState, dopt dispatchOpt) error {
var opt []llb.RunOption
var args []string = c.CmdLine
if len(c.Files) > 0 {
if len(args) != 1 {
return fmt.Errorf("parsing produced an invalid run command: %v", args)
}
if heredoc := parser.MustParseHeredoc(args[0]); heredoc != nil {
if d.image.OS != "windows" && strings.HasPrefix(c.Files[0].Data, "#!") {
// This is a single heredoc with a shebang, so create a file
// and run it.
// NOTE: choosing to expand doesn't really make sense here, so
// we silently ignore that option if it was provided.
sourcePath := "/"
destPath := "/dev/pipes/"
f := c.Files[0].Name
data := c.Files[0].Data
if c.Files[0].Chomp {
data = parser.ChompHeredocContent(data)
}
st := llb.Scratch().Dir(sourcePath).File(llb.Mkfile(f, 0755, []byte(data)))
mount := llb.AddMount(destPath, st, llb.SourcePath(sourcePath), llb.Readonly)
opt = append(opt, mount)
args[0] = path.Join(destPath, f)
} else {
// Just a simple heredoc, so just run the contents in the
// shell: this creates the effect of a "fake"-heredoc, so that
// the syntax can still be used for shells that don't support
// heredocs directly.
// NOTE: like above, we ignore the expand option.
data := c.Files[0].Data
if c.Files[0].Chomp {
data = parser.ChompHeredocContent(data)
}
args[0] = data
}
} else {
// More complex heredoc, so reconstitute it, and pass it to the
// shell to handle.
for _, file := range c.Files {
args[0] += "\n" + file.Data + file.Name
}
}
}
if c.PrependShell {
args = withShell(d.image, args)
}
env, err := d.state.Env(context.TODO())
if err != nil {
return err
}
opt := []llb.RunOption{llb.Args(args), dfCmd(c), location(dopt.sourceMap, c.Location())}
opt = append(opt, llb.Args(args), dfCmd(c), location(dopt.sourceMap, c.Location()))
if d.ignoreCache {
opt = append(opt, llb.IgnoreCache)
}
@ -735,12 +783,12 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
}
func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, chmod string, loc []parser.Range, opt dispatchOpt) error {
pp, err := pathRelativeToWorkingDir(d.state, c.Dest())
pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
if err != nil {
return err
}
dest := path.Join("/", pp)
if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
dest += string(filepath.Separator)
}
@ -768,7 +816,7 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
var a *llb.FileAction
for _, src := range c.Sources() {
for _, src := range c.SourcePaths {
commitMessage.WriteString(" " + src)
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !isAddCommand {
@ -818,7 +866,24 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
}
}
commitMessage.WriteString(" " + c.Dest())
for _, src := range c.SourceContents {
data := src.Data
f := src.Path
st := llb.Scratch().Dir("/").File(llb.Mkfile(f, 0664, []byte(data)))
opts := append([]llb.CopyOption{&llb.CopyInfo{
Mode: mode,
CreateDestPath: true,
}}, copyOpt...)
if a == nil {
a = llb.Copy(st, f, dest, opts...)
} else {
a = a.Copy(st, f, dest, opts...)
}
}
commitMessage.WriteString(" " + c.DestPath)
platform := opt.targetPlatform
if d.platform != nil {
@ -847,6 +912,10 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
return dispatchCopyFileOp(d, c, sourceState, isAddCommand, cmdToPrint, chown, chmod, loc, opt)
}
if len(c.SourceContents) > 0 {
return errors.New("inline content copy is not supported")
}
if chmod != "" {
if opt.llbCaps != nil && opt.llbCaps.Supports(pb.CapFileBase) != nil {
return errors.Wrap(opt.llbCaps.Supports(pb.CapFileBase), "chmod is not supported")
@ -855,18 +924,18 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
}
img := llb.Image(opt.copyImage, llb.MarkImageInternal, llb.Platform(opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
pp, err := pathRelativeToWorkingDir(d.state, c.Dest())
pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
if err != nil {
return err
}
dest := path.Join(".", pp)
if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
dest += string(filepath.Separator)
}
args := []string{"copy"}
unpack := isAddCommand
mounts := make([]llb.RunOption, 0, len(c.Sources()))
mounts := make([]llb.RunOption, 0, len(c.SourcePaths))
if chown != "" {
args = append(args, fmt.Sprintf("--chown=%s", chown))
_, _, err := parseUser(chown)
@ -883,7 +952,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
commitMessage.WriteString("COPY")
}
for i, src := range c.Sources() {
for i, src := range c.SourcePaths {
commitMessage.WriteString(" " + src)
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !isAddCommand {
@ -920,7 +989,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
}
}
commitMessage.WriteString(" " + c.Dest())
commitMessage.WriteString(" " + c.DestPath)
args = append(args, dest)
if unpack {

503
frontend/dockerfile/dockerfile_heredoc_test.go Normal file
View File

@ -0,0 +1,503 @@
// +build dfheredoc
package dockerfile
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/containerd/continuity/fs/fstest"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/frontend/dockerfile/builder"
"github.com/moby/buildkit/util/testutil/integration"
"github.com/stretchr/testify/require"
)
var hdTests = []integration.Test{
testCopyHeredoc,
testRunBasicHeredoc,
testRunFakeHeredoc,
testRunShebangHeredoc,
testRunComplexHeredoc,
testHeredocIndent,
testHeredocVarSubstitution,
}
func init() {
heredocTests = append(heredocTests, hdTests...)
}
func testCopyHeredoc(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox AS build
RUN adduser -D user
WORKDIR /dest
COPY <<EOF single
single file
EOF
COPY <<EOF <<EOF2 double/
first file
EOF
second file
EOF2
RUN mkdir -p /permfiles
COPY --chmod=777 <<EOF /permfiles/all
dummy content
EOF
COPY --chmod=0644 <<EOF /permfiles/rw
dummy content
EOF
COPY --chown=user:user <<EOF /permfiles/owned
dummy content
EOF
RUN stat -c "%04a" /permfiles/all >> perms && \
stat -c "%04a" /permfiles/rw >> perms && \
stat -c "%U:%G" /permfiles/owned >> perms
FROM scratch
COPY --from=build /dest /
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
contents := map[string]string{
"single": "single file\n",
"double/EOF": "first file\n",
"double/EOF2": "second file\n",
"perms": "0777\n0644\nuser:user\n",
}
for name, content := range contents {
dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
require.NoError(t, err)
require.Equal(t, content, string(dt))
}
}
func testRunBasicHeredoc(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox AS build
RUN <<EOF
echo "i am" >> /dest
whoami >> /dest
EOF
FROM scratch
COPY --from=build /dest /dest
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest"))
require.NoError(t, err)
require.Equal(t, "i am\nroot\n", string(dt))
}
func testRunFakeHeredoc(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox AS build
SHELL ["/bin/awk"]
RUN <<EOF
BEGIN {
print "foo" > "/dest"
}
EOF
FROM scratch
COPY --from=build /dest /dest
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest"))
require.NoError(t, err)
require.Equal(t, "foo\n", string(dt))
}
func testRunShebangHeredoc(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox AS build
RUN <<EOF
#!/bin/awk -f
BEGIN {
print "hello" >> "/dest"
print "world" >> "/dest"
}
EOF
FROM scratch
COPY --from=build /dest /dest
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest"))
require.NoError(t, err)
require.Equal(t, "hello\nworld\n", string(dt))
}
func testRunComplexHeredoc(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox AS build
WORKDIR /dest
RUN cat <<EOF1 | tr '[:upper:]' '[:lower:]' > ./out1; \
cat <<EOF2 | tr '[:lower:]' '[:upper:]' > ./out2
hello WORLD
EOF1
HELLO world
EOF2
RUN <<EOF 3<<IN1 4<<IN2 awk -f -
BEGIN {
while ((getline line < "/proc/self/fd/3") > 0)
print tolower(line) > "./fd3"
while ((getline line < "/proc/self/fd/4") > 0)
print toupper(line) > "./fd4"
}
EOF
hello WORLD
IN1
HELLO world
IN2
FROM scratch
COPY --from=build /dest /
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
contents := map[string]string{
"out1": "hello world\n",
"out2": "HELLO WORLD\n",
"fd3": "hello world\n",
"fd4": "HELLO WORLD\n",
}
for name, content := range contents {
dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
require.NoError(t, err)
require.Equal(t, content, string(dt))
}
}
func testHeredocIndent(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox AS build
COPY <<EOF /dest/foo-copy
foo
EOF
COPY <<-EOF /dest/bar-copy
bar
EOF
RUN <<EOF
echo "
foo" > /dest/foo-run
EOF
RUN <<-EOF
echo "
bar" > /dest/bar-run
EOF
RUN <<EOF
#!/bin/sh
echo "
foo" > /dest/foo2-run
EOF
RUN <<-EOF
#!/bin/sh
echo "
bar" > /dest/bar2-run
EOF
RUN <<EOF sh > /dest/foo3-run
echo "
foo"
EOF
RUN <<-EOF sh > /dest/bar3-run
echo "
bar"
EOF
FROM scratch
COPY --from=build /dest /
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
contents := map[string]string{
"foo-copy": "\tfoo\n",
"foo-run": "\n\tfoo\n",
"foo2-run": "\n\tfoo\n",
"foo3-run": "\n\tfoo\n",
"bar-copy": "bar\n",
"bar-run": "\nbar\n",
"bar2-run": "\nbar\n",
"bar3-run": "\nbar\n",
}
for name, content := range contents {
dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
require.NoError(t, err)
require.Equal(t, content, string(dt))
}
}
func testHeredocVarSubstitution(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM busybox as build
ARG name=world
COPY <<EOF /dest/c1
Hello ${name}!
EOF
COPY <<'EOF' /dest/c2
Hello ${name}!
EOF
COPY <<"EOF" /dest/c3
Hello ${name}!
EOF
RUN <<EOF
greeting="Hello"
echo "${greeting} ${name}!" > /dest/r1
EOF
RUN <<EOF
name="new world"
echo "Hello ${name}!" > /dest/r2
EOF
FROM scratch
COPY --from=build /dest /
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
contents := map[string]string{
"c1": "Hello world!\n",
"c2": "Hello ${name}!\n",
"c3": "Hello ${name}!\n",
"r1": "Hello world!\n",
"r2": "Hello new world!\n",
}
for name, content := range contents {
dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
require.NoError(t, err)
require.Equal(t, content, string(dt))
}
}

4
frontend/dockerfile/dockerfile_test.go
View File

@ -142,6 +142,9 @@ var securityTests = []integration.Test{}
// Tests that depend on the `network.*` entitlements
var networkTests = []integration.Test{}
// Tests that depend on heredoc support
var heredocTests = []integration.Test{}
var opts []integration.TestOpt
var securityOpts []integration.TestOpt
@ -194,6 +197,7 @@ func TestIntegration(t *testing.T) {
"granted": networkHostGranted,
"denied": networkHostDenied,
}))...)
integration.Run(t, heredocTests, opts...)
}
func testDefaultEnvWithArgs(t *testing.T, sb integration.Sandbox) {

92
frontend/dockerfile/docs/syntax.md
View File

@ -175,7 +175,7 @@ However, pem files with passphrases are not supported.
## Security context `RUN --security=insecure|sandbox`
To use this flag set Dockerfile version to `labs` channel.
To use this flag, set Dockerfile version to `labs` channel.
```
# syntax=docker/dockerfile:1.2-labs
@ -204,7 +204,7 @@ RUN --security=insecure cat /proc/self/status | grep CapEff
## Network modes `RUN --network=none|host|default`
To use this flag set Dockerfile version to `labs` channel.
To use this flag, set Dockerfile version to `labs` channel.
```
# syntax=docker/dockerfile:1.2-labs
@ -237,3 +237,91 @@ RUN --network=none pip install --find-links wheels mypackage
`pip` will only be able to install the packages provided in the tarfile, which
can be controlled by an earlier build stage.
## Here-Documents
To use this flag, set Dockerfile version to `labs` channel. Currently this feature is only available
in `docker/dockerfile-upstream:master-labs` image.
```
# syntax=docker/dockerfile-upstream:master-labs
```
Here-documents allow redirection of subsequent Dockerfile lines to the input of `RUN` or `COPY` commands.
If such command contains a [here-document](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_07_04)
Dockerfile will consider the next lines until the line only containing a here-doc delimiter as part of the same command.
#### Example: running a multi-line script
```dockerfile
# syntax = docker/dockerfile-upstream:master-labs
FROM debian
RUN <<eot bash
apt-get update
apt-get install -y vim
eot
```
If the command only contains a here-document, its contents is evaluated with the default shell.
```dockerfile
# syntax = docker/dockerfile-upstream:master-labs
FROM debian
RUN <<eot
mkdir -p foo/bar
eot
```
Alternatively, shebang header can be used to define an interpreter.
```dockerfile
# syntax = docker/dockerfile-upstream:master-labs
FROM python:3.6
RUN <<eot
#!/usr/bin/env python
print("hello world")
eot
```
More complex examples may use multiple here-documents.
```dockerfile
# syntax = docker/dockerfile-upstream:master-labs
FROM alpine
RUN <<FILE1 cat > file1 && <<FILE2 cat > file2
I am
first
FILE1
I am
second
FILE2
```
#### Example: creating inline files
In `COPY` commands source parameters can be replaced with here-doc indicators.
Regular here-doc [variable expansion and tab stripping rules](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_07_04) apply.
```dockerfile
# syntax = docker/dockerfile-upstream:master-labs
FROM alpine
ARG FOO=bar
COPY <<-eot /app/foo
hello ${FOO}
eot
```
```dockerfile
# syntax = docker/dockerfile-upstream:master-labs
FROM alpine
COPY <<-"eot" /app/script.sh
echo hello ${FOO}
eot
RUN FOO=abc ash /app/script.sh
```

62
frontend/dockerfile/instructions/commands.go
View File

@ -165,19 +165,45 @@ func (c *LabelCommand) Expand(expander SingleWordExpander) error {
return expandKvpsInPlace(c.Labels, expander)
}
// SourcesAndDest represent a list of source files and a destination
type SourcesAndDest []string
// Sources list the source paths
func (s SourcesAndDest) Sources() []string {
res := make([]string, len(s)-1)
copy(res, s[:len(s)-1])
return res
// SourceContent represents an anonymous file object
type SourceContent struct {
Path string
Data string
Expand bool
}
// Dest path of the operation
func (s SourcesAndDest) Dest() string {
return s[len(s)-1]
// SourcesAndDest represent a collection of sources and a destination
type SourcesAndDest struct {
DestPath string
SourcePaths []string
SourceContents []SourceContent
}
func (s *SourcesAndDest) Expand(expander SingleWordExpander) error {
for i, content := range s.SourceContents {
if !content.Expand {
continue
}
expandedData, err := expander(content.Data)
if err != nil {
return err
}
s.SourceContents[i].Data = expandedData
}
err := expandSliceInPlace(s.SourcePaths, expander)
if err != nil {
return err
}
expandedDestPath, err := expander(s.DestPath)
if err != nil {
return err
}
s.DestPath = expandedDestPath
return nil
}
// AddCommand : ADD foo /path
@ -199,7 +225,8 @@ func (c *AddCommand) Expand(expander SingleWordExpander) error {
return err
}
c.Chown = expandedChown
return expandSliceInPlace(c.SourcesAndDest, expander)
return c.SourcesAndDest.Expand(expander)
}
// CopyCommand : COPY foo /path
@ -221,7 +248,8 @@ func (c *CopyCommand) Expand(expander SingleWordExpander) error {
return err
}
c.Chown = expandedChown
return expandSliceInPlace(c.SourcesAndDest, expander)
return c.SourcesAndDest.Expand(expander)
}
// OnbuildCommand : ONBUILD <some other command>
@ -249,9 +277,17 @@ func (c *WorkdirCommand) Expand(expander SingleWordExpander) error {
return nil
}
// ShellInlineFile represents an inline file created for a shell command
type ShellInlineFile struct {
Name string
Data string
Chomp bool
}
// ShellDependantCmdLine represents a cmdline optionally prepended with the shell
type ShellDependantCmdLine struct {
CmdLine strslice.StrSlice
Files []ShellInlineFile
PrependShell bool
}

106
frontend/dockerfile/instructions/parse.go
View File

@ -18,6 +18,7 @@ import (
type parseRequest struct {
command string
args []string
heredocs []parser.Heredoc
attributes map[string]bool
flags *BFlags
original string
@ -47,6 +48,7 @@ func newParseRequestFromNode(node *parser.Node) parseRequest {
return parseRequest{
command: node.Value,
args: nodeArgs(node),
heredocs: node.Heredocs,
attributes: node.Attributes,
original: node.Original,
flags: NewBFlagsWithArgs(node.Flags),
@ -236,6 +238,45 @@ func parseLabel(req parseRequest) (*LabelCommand, error) {
}, nil
}
func parseSourcesAndDest(req parseRequest, command string) (*SourcesAndDest, error) {
srcs := req.args[:len(req.args)-1]
dest := req.args[len(req.args)-1]
if heredoc := parser.MustParseHeredoc(dest); heredoc != nil {
return nil, errBadHeredoc(command, "a destination")
}
heredocLookup := make(map[string]parser.Heredoc)
for _, heredoc := range req.heredocs {
heredocLookup[heredoc.Name] = heredoc
}
var sourcePaths []string
var sourceContents []SourceContent
for _, src := range srcs {
if heredoc := parser.MustParseHeredoc(src); heredoc != nil {
content := heredocLookup[heredoc.Name].Content
if heredoc.Chomp {
content = parser.ChompHeredocContent(content)
}
sourceContents = append(sourceContents,
SourceContent{
Data: content,
Path: heredoc.Name,
Expand: heredoc.Expand,
},
)
} else {
sourcePaths = append(sourcePaths, src)
}
}
return &SourcesAndDest{
DestPath: dest,
SourcePaths: sourcePaths,
SourceContents: sourceContents,
}, nil
}
func parseAdd(req parseRequest) (*AddCommand, error) {
if len(req.args) < 2 {
return nil, errNoDestinationArgument("ADD")
@ -245,9 +286,15 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
sourcesAndDest, err := parseSourcesAndDest(req, "ADD")
if err != nil {
return nil, err
}
return &AddCommand{
SourcesAndDest: SourcesAndDest(req.args),
withNameAndCode: newWithNameAndCode(req),
SourcesAndDest: *sourcesAndDest,
Chown: flChown.Value,
Chmod: flChmod.Value,
}, nil
@ -263,10 +310,16 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
sourcesAndDest, err := parseSourcesAndDest(req, "COPY")
if err != nil {
return nil, err
}
return &CopyCommand{
SourcesAndDest: SourcesAndDest(req.args),
From: flFrom.Value,
withNameAndCode: newWithNameAndCode(req),
SourcesAndDest: *sourcesAndDest,
From: flFrom.Value,
Chown: flChown.Value,
Chmod: flChmod.Value,
}, nil
@ -351,7 +404,17 @@ func parseWorkdir(req parseRequest) (*WorkdirCommand, error) {
}
func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependantCmdLine {
func parseShellDependentCommand(req parseRequest, command string, emptyAsNil bool) (ShellDependantCmdLine, error) {
var files []ShellInlineFile
for _, heredoc := range req.heredocs {
file := ShellInlineFile{
Name: heredoc.Name,
Data: heredoc.Content,
Chomp: heredoc.Chomp,
}
files = append(files, file)
}
args := handleJSONArgs(req.args, req.attributes)
cmd := strslice.StrSlice(args)
if emptyAsNil && len(cmd) == 0 {
@ -359,8 +422,9 @@ func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependan
}
return ShellDependantCmdLine{
CmdLine: cmd,
Files: files,
PrependShell: !req.attributes["json"],
}
}, nil
}
func parseRun(req parseRequest) (*RunCommand, error) {
@ -376,7 +440,13 @@ func parseRun(req parseRequest) (*RunCommand, error) {
return nil, err
}
cmd.FlagsUsed = req.flags.Used()
cmd.ShellDependantCmdLine = parseShellDependentCommand(req, false)
cmdline, err := parseShellDependentCommand(req, "RUN", false)
if err != nil {
return nil, err
}
cmd.ShellDependantCmdLine = cmdline
cmd.withNameAndCode = newWithNameAndCode(req)
for _, fn := range parseRunPostHooks {
@ -392,11 +462,16 @@ func parseCmd(req parseRequest) (*CmdCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
cmdline, err := parseShellDependentCommand(req, "CMD", false)
if err != nil {
return nil, err
}
return &CmdCommand{
ShellDependantCmdLine: parseShellDependentCommand(req, false),
ShellDependantCmdLine: cmdline,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
@ -404,12 +479,15 @@ func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
return nil, err
}
cmd := &EntrypointCommand{
ShellDependantCmdLine: parseShellDependentCommand(req, true),
withNameAndCode: newWithNameAndCode(req),
cmdline, err := parseShellDependentCommand(req, "ENTRYPOINT", true)
if err != nil {
return nil, err
}
return cmd, nil
return &EntrypointCommand{
ShellDependantCmdLine: cmdline,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
// parseOptInterval(flag) is the duration of flag.Value, or 0 if
@ -651,6 +729,10 @@ func errNoDestinationArgument(command string) error {
return errors.Errorf("%s requires at least two arguments, but only one was provided. Destination could not be determined.", command)
}
func errBadHeredoc(command string, option string) error {
return errors.Errorf("%s cannot accept a heredoc as %s", command, option)
}
func errBlankCommandNames(command string) error {
return errors.Errorf("%s names can not be blank", command)
}

245
frontend/dockerfile/instructions/parse_heredoc_test.go Normal file
View File

@ -0,0 +1,245 @@
// +build dfheredoc
package instructions
import (
"strings"
"testing"
"github.com/docker/docker/api/types/strslice"
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/stretchr/testify/require"
)
func TestErrorCasesHeredoc(t *testing.T) {
cases := []struct {
name string
dockerfile string
expectedError string
}{
{
name: "COPY heredoc destination",
dockerfile: "COPY /foo <<EOF\nEOF",
expectedError: "COPY cannot accept a heredoc as a destination",
},
}
for _, c := range cases {
r := strings.NewReader(c.dockerfile)
ast, err := parser.Parse(r)
if err != nil {
t.Fatalf("Error when parsing Dockerfile: %s", err)
}
n := ast.AST.Children[0]
_, err = ParseInstruction(n)
require.Error(t, err)
require.Contains(t, err.Error(), c.expectedError)
}
}
func TestCopyHeredoc(t *testing.T) {
cases := []struct {
dockerfile string
sourcesAndDest SourcesAndDest
}{
{
dockerfile: "COPY /foo /bar",
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourcePaths: []string{"/foo"},
},
},
{
dockerfile: `COPY <<EOF /bar
EOF`,
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourceContents: []SourceContent{
{
Path: "EOF",
Data: "",
Expand: true,
},
},
},
},
{
dockerfile: `COPY <<EOF /bar
TESTING
EOF`,
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourceContents: []SourceContent{
{
Path: "EOF",
Data: "TESTING\n",
Expand: true,
},
},
},
},
{
dockerfile: `COPY <<-EOF /bar
TESTING
EOF`,
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourceContents: []SourceContent{
{
Path: "EOF",
Data: "TESTING\n",
Expand: true,
},
},
},
},
{
dockerfile: `COPY <<'EOF' /bar
TESTING
EOF`,
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourceContents: []SourceContent{
{
Path: "EOF",
Data: "TESTING\n",
Expand: false,
},
},
},
},
{
dockerfile: `COPY <<EOF1 <<EOF2 /bar
this is the first file
EOF1
this is the second file
EOF2`,
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourceContents: []SourceContent{
{
Path: "EOF1",
Data: "this is the first file\n",
Expand: true,
},
{
Path: "EOF2",
Data: "this is the second file\n",
Expand: true,
},
},
},
},
{
dockerfile: `COPY <<EOF foo.txt /bar
this is inline
EOF`,
sourcesAndDest: SourcesAndDest{
DestPath: "/bar",
SourcePaths: []string{"foo.txt"},
SourceContents: []SourceContent{
{
Path: "EOF",
Data: "this is inline\n",
Expand: true,
},
},
},
},
}
for _, c := range cases {
r := strings.NewReader(c.dockerfile)
ast, err := parser.Parse(r)
require.NoError(t, err)
n := ast.AST.Children[0]
comm, err := ParseInstruction(n)
require.NoError(t, err)
sd := comm.(*CopyCommand).SourcesAndDest
require.Equal(t, c.sourcesAndDest, sd)
}
}
func TestRunHeredoc(t *testing.T) {
cases := []struct {
dockerfile string
shell bool
command strslice.StrSlice
files []ShellInlineFile
}{
{
dockerfile: `RUN ["ls", "/"]`,
command: strslice.StrSlice{"ls", "/"},
shell: false,
},
{
dockerfile: `RUN ["<<EOF"]`,
command: strslice.StrSlice{"<<EOF"},
shell: false,
},
{
dockerfile: "RUN ls /",
command: strslice.StrSlice{"ls /"},
shell: true,
},
{
dockerfile: `RUN <<EOF
ls /
whoami
EOF`,
command: strslice.StrSlice{"<<EOF"},
files: []ShellInlineFile{
{
Name: "EOF",
Data: "ls /\nwhoami\n",
},
},
shell: true,
},
{
dockerfile: `RUN <<'EOF' | python
print("hello")
print("world")
EOF`,
command: strslice.StrSlice{"<<'EOF' | python"},
files: []ShellInlineFile{
{
Name: "EOF",
Data: `print("hello")
print("world")
`,
},
},
shell: true,
},
{
dockerfile: `RUN <<-EOF
echo test
EOF`,
command: strslice.StrSlice{"<<-EOF"},
files: []ShellInlineFile{
{
Name: "EOF",
Data: "\techo test\n",
Chomp: true,
},
},
shell: true,
},
}
for _, c := range cases {
r := strings.NewReader(c.dockerfile)
ast, err := parser.Parse(r)
require.NoError(t, err)
n := ast.AST.Children[0]
comm, err := ParseInstruction(n)
require.NoError(t, err)
require.Equal(t, c.shell, comm.(*RunCommand).PrependShell)
require.Equal(t, c.command, comm.(*RunCommand).CmdLine)
require.Equal(t, c.files, comm.(*RunCommand).Files)
}
}

10
frontend/dockerfile/instructions/parse_test.go
View File

@ -192,11 +192,6 @@ func TestErrorCases(t *testing.T) {
dockerfile: "ONBUILD FROM scratch",
expectedError: "FROM isn't allowed as an ONBUILD trigger",
},
{
name: "ONBUILD forbidden MAINTAINER",
dockerfile: "ONBUILD MAINTAINER docker.io",
expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger",
},
{
name: "MAINTAINER unknown flag",
dockerfile: "MAINTAINER --boo joe@example.com",
@ -212,6 +207,11 @@ func TestErrorCases(t *testing.T) {
dockerfile: `foo bar`,
expectedError: "unknown instruction: FOO",
},
{
name: "Invalid instruction",
dockerfile: `foo bar`,
expectedError: "unknown instruction: FOO",
},
}
for _, c := range cases {
r := strings.NewReader(c.dockerfile)

145
frontend/dockerfile/parser/parser.go
View File

@ -12,6 +12,7 @@ import (
"unicode"
"github.com/moby/buildkit/frontend/dockerfile/command"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/pkg/errors"
)
@ -31,6 +32,7 @@ type Node struct {
Value string // actual content
Next *Node // the next item in the current sexp
Children []*Node // the children of this sexp
Heredocs []Heredoc // extra heredoc content attachments
Attributes map[string]bool // special attributes for this node
Original string // original line used before parsing
Flags []string // only top Node should have this set
@ -74,6 +76,17 @@ func (node *Node) lines(start, end int) {
node.EndLine = end
}
func (node *Node) canContainHeredoc() bool {
if _, allowedDirective := heredocDirectives[node.Value]; !allowedDirective {
return false
}
if _, isJSON := node.Attributes["json"]; isJSON {
return false
}
return true
}
// AddChild adds a new child node, and updates line information
func (node *Node) AddChild(child *Node, startLine, endLine int) {
child.lines(startLine, endLine)
@ -84,11 +97,22 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
node.Children = append(node.Children, child)
}
type Heredoc struct {
Name string
FileDescriptor uint
Expand bool
Chomp bool
Content string
}
var (
dispatch map[string]func(string, *directives) (*Node, map[string]bool, error)
heredocDirectives map[string]bool
reWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
reDirectives = regexp.MustCompile(`^#\s*([a-zA-Z][a-zA-Z0-9]*)\s*=\s*(.+?)\s*$`)
reComment = regexp.MustCompile(`^#.*$`)
reHeredoc = regexp.MustCompile(`^(\d*)<<(-?)(['"]?)([a-zA-Z][a-zA-Z0-9]*)(['"]?)$`)
reLeadingTabs = regexp.MustCompile(`(?m)^\t+`)
)
// DefaultEscapeToken is the default escape token
@ -252,6 +276,7 @@ func Parse(rwc io.Reader) (*Result, error) {
currentLine := 0
root := &Node{StartLine: -1}
scanner := bufio.NewScanner(rwc)
scanner.Split(scanLines)
warnings := []string{}
var comments []string
@ -312,8 +337,40 @@ func Parse(rwc io.Reader) (*Result, error) {
if err != nil {
return nil, withLocation(err, startLine, currentLine)
}
comments = nil
if child.canContainHeredoc() {
heredocs, err := heredocsFromLine(line)
if err != nil {
return nil, withLocation(err, startLine, currentLine)
}
for _, heredoc := range heredocs {
terminator := []byte(heredoc.Name)
terminated := false
for scanner.Scan() {
bytesRead := scanner.Bytes()
currentLine++
possibleTerminator := trimNewline(bytesRead)
if heredoc.Chomp {
possibleTerminator = trimLeadingTabs(possibleTerminator)
}
if bytes.Equal(possibleTerminator, terminator) {
terminated = true
break
}
heredoc.Content += string(bytesRead)
}
if !terminated {
return nil, withLocation(errors.New("unterminated heredoc"), startLine, currentLine)
}
child.Heredocs = append(child.Heredocs, heredoc)
}
}
root.AddChild(child, startLine, currentLine)
comments = nil
}
if len(warnings) > 0 {
@ -331,20 +388,83 @@ func Parse(rwc io.Reader) (*Result, error) {
}, withLocation(handleScannerError(scanner.Err()), currentLine, 0)
}
func heredocFromMatch(match []string) (*Heredoc, error) {
if len(match) == 0 {
return nil, nil
}
fileDescriptor, _ := strconv.ParseUint(match[1], 10, 0)
chomp := match[2] == "-"
quoteOpen := match[3]
name := match[4]
quoteClose := match[5]
expand := true
if quoteOpen != "" || quoteClose != "" {
if quoteOpen != quoteClose {
return nil, errors.New("quoted heredoc quotes do not match")
}
expand = false
}
return &Heredoc{
Name: name,
Expand: expand,
Chomp: chomp,
FileDescriptor: uint(fileDescriptor),
}, nil
}
func ParseHeredoc(src string) (*Heredoc, error) {
return heredocFromMatch(reHeredoc.FindStringSubmatch(src))
}
func MustParseHeredoc(src string) *Heredoc {
heredoc, _ := ParseHeredoc(src)
return heredoc
}
func heredocsFromLine(line string) ([]Heredoc, error) {
shlex := shell.NewLex('\\')
shlex.RawQuotes = true
words, _ := shlex.ProcessWords(line, []string{})
var docs []Heredoc
for _, word := range words {
heredoc, err := ParseHeredoc(word)
if err != nil {
return nil, err
}
if heredoc != nil {
docs = append(docs, *heredoc)
}
}
return docs, nil
}
func ChompHeredocContent(src string) string {
return reLeadingTabs.ReplaceAllString(src, "")
}
func trimComments(src []byte) []byte {
return reComment.ReplaceAll(src, []byte{})
}
func trimWhitespace(src []byte) []byte {
func trimLeadingWhitespace(src []byte) []byte {
return bytes.TrimLeftFunc(src, unicode.IsSpace)
}
func trimLeadingTabs(src []byte) []byte {
return bytes.TrimLeft(src, "\t")
}
func trimNewline(src []byte) []byte {
return bytes.TrimRight(src, "\r\n")
}
func isComment(line []byte) bool {
return reComment.Match(trimWhitespace(line))
return reComment.Match(trimLeadingWhitespace(trimNewline(line)))
}
func isEmptyContinuationLine(line []byte) bool {
return len(trimWhitespace(line)) == 0
return len(trimLeadingWhitespace(trimNewline(line))) == 0
}
var utf8bom = []byte{0xEF, 0xBB, 0xBF}
@ -360,12 +480,27 @@ func trimContinuationCharacter(line string, d *directives) (string, bool) {
// TODO: remove stripLeftWhitespace after deprecation period. It seems silly
// to preserve whitespace on continuation lines. Why is that done?
func processLine(d *directives, token []byte, stripLeftWhitespace bool) ([]byte, error) {
token = trimNewline(token)
if stripLeftWhitespace {
token = trimWhitespace(token)
token = trimLeadingWhitespace(token)
}
return trimComments(token), d.possibleParserDirective(string(token))
}
// Variation of bufio.ScanLines that preserves the line endings
func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
return i + 1, data[0 : i+1], nil
}
if atEOF {
return len(data), data, nil
}
return 0, nil, nil
}
func handleScannerError(err error) error {
switch err {
case bufio.ErrTooLong:

13
frontend/dockerfile/parser/parser_heredoc.go Normal file
View File

@ -0,0 +1,13 @@
// +build dfheredoc
package parser
import "github.com/moby/buildkit/frontend/dockerfile/command"
func init() {
heredocDirectives = map[string]bool{
command.Add: true,
command.Copy: true,
command.Run: true,
}
}

314
frontend/dockerfile/parser/parser_heredoc_test.go Normal file
View File

@ -0,0 +1,314 @@
// +build dfheredoc
package parser
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
)
func TestParseExtractsHeredoc(t *testing.T) {
dockerfile := bytes.NewBufferString(`
FROM alpine:3.6
ENV NAME=me
RUN ls
USER <<INVALID
INVALID
RUN <<EMPTY
EMPTY
RUN 3<<EMPTY2
EMPTY2
RUN "<<NOHEREDOC"
RUN <<INDENT
foo
bar
INDENT
RUN <<-UNINDENT
baz
quux
UNINDENT
RUN <<-UNINDENT2
baz
quux
UNINDENT2
RUN <<-EXPAND
expand $NAME
EXPAND
RUN <<-'NOEXPAND'
don't expand $NAME
NOEXPAND
RUN <<COPY
echo hello world
echo foo bar
COPY
RUN <<COMMENT
# internal comment
echo hello world
echo foo bar # trailing comment
COMMENT
RUN --mount=type=cache,target=/foo <<MOUNT
echo hello
MOUNT
COPY <<FILE1 <<FILE2 /dest
content 1
FILE1
content 2
FILE2
COPY <<X <<Y /dest
Y
X
X
Y
`)
tests := [][]Heredoc{
nil, // ENV EXAMPLE=bla
nil, // RUN ls
nil, // USER <<INVALID
nil, // INVALID
{
// RUN <<EMPTY
{
Name: "EMPTY",
Content: "",
Expand: true,
},
},
{
// RUN <<EMPTY2
{
Name: "EMPTY2",
Content: "",
Expand: true,
FileDescriptor: 3,
},
},
nil, // RUN "<<NOHEREDOC"
{
// RUN <<INDENT
{
Name: "INDENT",
Content: "\tfoo\n\tbar\n",
Expand: true,
},
},
{
// RUN <<-UNINDENT
{
Name: "UNINDENT",
Content: "\tbaz\n\tquux\n",
Expand: true,
Chomp: true,
},
},
{
// RUN <<-UNINDENT2
{
Name: "UNINDENT2",
Content: "\tbaz\n\tquux\n",
Expand: true,
Chomp: true,
},
},
{
// RUN <<-EXPAND
{
Name: "EXPAND",
Content: "\texpand $NAME\n",
Expand: true,
Chomp: true,
},
},
{
// RUN <<-'NOEXPAND'
{
Name: "NOEXPAND",
Content: "\tdon't expand $NAME\n",
Expand: false,
Chomp: true,
},
},
{
// RUN <<COPY
{
Name: "COPY",
Content: "echo hello world\necho foo bar\n",
Expand: true,
},
},
{
// RUN <<COMMENT
{
Name: "COMMENT",
Content: "# internal comment\necho hello world\necho foo bar # trailing comment\n",
Expand: true,
},
},
{
// RUN <<MOUNT
{
Name: "MOUNT",
Content: "echo hello\n",
Expand: true,
},
},
{
// COPY <<FILE1 <<FILE2 /dest
{
Name: "FILE1",
Content: "content 1\n",
Expand: true,
},
{
Name: "FILE2",
Content: "content 2\n",
Expand: true,
},
},
{
// COPY <<X <<Y /dest
{
Name: "X",
Content: "Y\n",
Expand: true,
},
{
Name: "Y",
Content: "X\n",
Expand: true,
},
},
}
result, err := Parse(dockerfile)
require.NoError(t, err)
for i, test := range tests {
child := result.AST.Children[i+1]
require.Equal(t, test, child.Heredocs)
}
}
func TestParseJSONHeredoc(t *testing.T) {
dockerfile := bytes.NewBufferString(`
FROM alpine:3.6
RUN ["whoami"]
RUN ["<<EOF"]
RUN ["<<'EOF'"]
`)
result, err := Parse(dockerfile)
require.NoError(t, err)
for i := 1; i <= 3; i++ {
child := result.AST.Children[i]
require.Nil(t, child.Heredocs)
}
}
func TestHeredocChomp(t *testing.T) {
content := "\thello\n\tworld\n"
require.Equal(t, "hello\nworld\n", ChompHeredocContent(content))
}
func TestParseHeredocHelpers(t *testing.T) {
validHeredocs := []string{
"<<EOF",
"<<'EOF'",
`<<"EOF"`,
"<<-EOF",
"<<-'EOF'",
`<<-"EOF"`,
}
invalidHeredocs := []string{
"<<'EOF",
"<<\"EOF",
"<<EOF'",
"<<EOF\"",
}
notHeredocs := []string{
"",
"EOF",
"<<",
"<<-",
"<EOF",
"<<<EOF",
}
for _, src := range notHeredocs {
heredoc, err := ParseHeredoc(src)
require.NoError(t, err)
require.Nil(t, heredoc)
}
for _, src := range validHeredocs {
heredoc, err := ParseHeredoc(src)
require.NoError(t, err)
require.Equal(t, heredoc.Name, "EOF")
}
for _, src := range invalidHeredocs {
_, err := ParseHeredoc(src)
require.Error(t, err)
}
}
func TestHeredocsFromLine(t *testing.T) {
srcs := []struct {
line string
heredocNames []string
}{
{
line: "RUN <<EOF",
heredocNames: []string{"EOF"},
},
{
line: "RUN <<-EOF",
heredocNames: []string{"EOF"},
},
{
line: "RUN <<'EOF'",
heredocNames: []string{"EOF"},
},
{
line: "RUN 4<<EOF",
heredocNames: []string{"EOF"},
},
{
line: "RUN <<EOF <<EOF2",
heredocNames: []string{"EOF", "EOF2"},
},
{
line: "RUN '<<EOF'",
heredocNames: nil,
},
{
line: `RUN "<<EOF"`,
heredocNames: nil,
},
}
for _, src := range srcs {
heredocs, err := heredocsFromLine(src.line)
require.NoError(t, err)
for i, heredoc := range heredocs {
require.Equal(t, heredoc.Name, src.heredocNames[i])
}
}
}

4
frontend/dockerfile/parser/parser_test.go
View File

@ -137,6 +137,9 @@ func TestParseWarnsOnEmptyContinutationLine(t *testing.T) {
dockerfile := bytes.NewBufferString(`
FROM alpine:3.6
RUN valid \
continuation
RUN something \
following \
@ -146,6 +149,7 @@ RUN something \
RUN another \
thing
RUN non-indented \
# this is a comment
after-comment

2
frontend/dockerfile/release/labs/tags
View File

@ -1 +1 @@
dfrunsecurity dfrunnetwork
dfrunsecurity dfrunnetwork dfheredoc