Merge pull request #2132 from jedevc/dockerfile-heredocs

Dockerfile heredocs

.golangci.yml

@@ -8,6 +8,7 @@ run:
   build-tags:
     - dfrunsecurity
     - dfrunnetwork
+    - dfheredoc
 
 linters:
   enable:

frontend/dockerfile/dockerfile2llb/convert.go

@@ -507,7 +507,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 	case *instructions.AddCommand:
 		err = dispatchCopy(d, c.SourcesAndDest, opt.buildContext, true, c, c.Chown, c.Chmod, c.Location(), opt)
 		if err == nil {
-			for _, src := range c.Sources() {
+			for _, src := range c.SourcePaths {
 				if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
 					d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
 				}
@@ -542,7 +542,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 		}
 		err = dispatchCopy(d, c.SourcesAndDest, l, false, c, c.Chown, c.Chmod, c.Location(), opt)
 		if err == nil && len(cmd.sources) == 0 {
-			for _, src := range c.Sources() {
+			for _, src := range c.SourcePaths {
 				d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
 			}
 		}
@@ -647,15 +647,63 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand) error {
 }
 
 func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv, sources []*dispatchState, dopt dispatchOpt) error {
+	var opt []llb.RunOption
+
 	var args []string = c.CmdLine
+	if len(c.Files) > 0 {
+		if len(args) != 1 {
+			return fmt.Errorf("parsing produced an invalid run command: %v", args)
+		}
+
+		if heredoc := parser.MustParseHeredoc(args[0]); heredoc != nil {
+			if d.image.OS != "windows" && strings.HasPrefix(c.Files[0].Data, "#!") {
+				// This is a single heredoc with a shebang, so create a file
+				// and run it.
+				// NOTE: choosing to expand doesn't really make sense here, so
+				// we silently ignore that option if it was provided.
+				sourcePath := "/"
+				destPath := "/dev/pipes/"
+
+				f := c.Files[0].Name
+				data := c.Files[0].Data
+				if c.Files[0].Chomp {
+					data = parser.ChompHeredocContent(data)
+				}
+				st := llb.Scratch().Dir(sourcePath).File(llb.Mkfile(f, 0755, []byte(data)))
+
+				mount := llb.AddMount(destPath, st, llb.SourcePath(sourcePath), llb.Readonly)
+				opt = append(opt, mount)
+
+				args[0] = path.Join(destPath, f)
+			} else {
+				// Just a simple heredoc, so just run the contents in the
+				// shell: this creates the effect of a "fake"-heredoc, so that
+				// the syntax can still be used for shells that don't support
+				// heredocs directly.
+				// NOTE: like above, we ignore the expand option.
+				data := c.Files[0].Data
+				if c.Files[0].Chomp {
+					data = parser.ChompHeredocContent(data)
+				}
+				args[0] = data
+			}
+		} else {
+			// More complex heredoc, so reconstitute it, and pass it to the
+			// shell to handle.
+			for _, file := range c.Files {
+				args[0] += "\n" + file.Data + file.Name
+			}
+		}
+	}
 	if c.PrependShell {
 		args = withShell(d.image, args)
 	}
+
 	env, err := d.state.Env(context.TODO())
 	if err != nil {
 		return err
 	}
-	opt := []llb.RunOption{llb.Args(args), dfCmd(c), location(dopt.sourceMap, c.Location())}
+	opt = append(opt, llb.Args(args), dfCmd(c), location(dopt.sourceMap, c.Location()))
 	if d.ignoreCache {
 		opt = append(opt, llb.IgnoreCache)
 	}
@@ -735,12 +783,12 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
 }
 
 func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, chmod string, loc []parser.Range, opt dispatchOpt) error {
-	pp, err := pathRelativeToWorkingDir(d.state, c.Dest())
+	pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
 	if err != nil {
 		return err
 	}
 	dest := path.Join("/", pp)
-	if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
+	if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
 		dest += string(filepath.Separator)
 	}
 
@@ -768,7 +816,7 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
 
 	var a *llb.FileAction
 
-	for _, src := range c.Sources() {
+	for _, src := range c.SourcePaths {
 		commitMessage.WriteString(" " + src)
 		if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
 			if !isAddCommand {
@@ -818,7 +866,24 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
 		}
 	}
 
-	commitMessage.WriteString(" " + c.Dest())
+	for _, src := range c.SourceContents {
+		data := src.Data
+		f := src.Path
+		st := llb.Scratch().Dir("/").File(llb.Mkfile(f, 0664, []byte(data)))
+
+		opts := append([]llb.CopyOption{&llb.CopyInfo{
+			Mode:           mode,
+			CreateDestPath: true,
+		}}, copyOpt...)
+
+		if a == nil {
+			a = llb.Copy(st, f, dest, opts...)
+		} else {
+			a = a.Copy(st, f, dest, opts...)
+		}
+	}
+
+	commitMessage.WriteString(" " + c.DestPath)
 
 	platform := opt.targetPlatform
 	if d.platform != nil {
@@ -847,6 +912,10 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
 		return dispatchCopyFileOp(d, c, sourceState, isAddCommand, cmdToPrint, chown, chmod, loc, opt)
 	}
 
+	if len(c.SourceContents) > 0 {
+		return errors.New("inline content copy is not supported")
+	}
+
 	if chmod != "" {
 		if opt.llbCaps != nil && opt.llbCaps.Supports(pb.CapFileBase) != nil {
 			return errors.Wrap(opt.llbCaps.Supports(pb.CapFileBase), "chmod is not supported")
@@ -855,18 +924,18 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
 	}
 
 	img := llb.Image(opt.copyImage, llb.MarkImageInternal, llb.Platform(opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
-	pp, err := pathRelativeToWorkingDir(d.state, c.Dest())
+	pp, err := pathRelativeToWorkingDir(d.state, c.DestPath)
 	if err != nil {
 		return err
 	}
 	dest := path.Join(".", pp)
-	if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
+	if c.DestPath == "." || c.DestPath == "" || c.DestPath[len(c.DestPath)-1] == filepath.Separator {
 		dest += string(filepath.Separator)
 	}
 	args := []string{"copy"}
 	unpack := isAddCommand
 
-	mounts := make([]llb.RunOption, 0, len(c.Sources()))
+	mounts := make([]llb.RunOption, 0, len(c.SourcePaths))
 	if chown != "" {
 		args = append(args, fmt.Sprintf("--chown=%s", chown))
 		_, _, err := parseUser(chown)
@@ -883,7 +952,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
 		commitMessage.WriteString("COPY")
 	}
 
-	for i, src := range c.Sources() {
+	for i, src := range c.SourcePaths {
 		commitMessage.WriteString(" " + src)
 		if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
 			if !isAddCommand {
@@ -920,7 +989,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
 		}
 	}
 
-	commitMessage.WriteString(" " + c.Dest())
+	commitMessage.WriteString(" " + c.DestPath)
 
 	args = append(args, dest)
 	if unpack {

frontend/dockerfile/dockerfile_heredoc_test.go

@@ -0,0 +1,503 @@
+// +build dfheredoc
+
+package dockerfile
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/frontend/dockerfile/builder"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/stretchr/testify/require"
+)
+
+var hdTests = []integration.Test{
+	testCopyHeredoc,
+	testRunBasicHeredoc,
+	testRunFakeHeredoc,
+	testRunShebangHeredoc,
+	testRunComplexHeredoc,
+	testHeredocIndent,
+	testHeredocVarSubstitution,
+}
+
+func init() {
+	heredocTests = append(heredocTests, hdTests...)
+}
+
+func testCopyHeredoc(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS build
+
+RUN adduser -D user
+WORKDIR /dest
+
+COPY <<EOF single
+single file
+EOF
+
+COPY <<EOF <<EOF2 double/
+first file
+EOF
+second file
+EOF2
+
+RUN mkdir -p /permfiles
+COPY --chmod=777 <<EOF /permfiles/all
+dummy content
+EOF
+COPY --chmod=0644 <<EOF /permfiles/rw
+dummy content
+EOF
+COPY --chown=user:user <<EOF /permfiles/owned
+dummy content
+EOF
+RUN stat -c "%04a" /permfiles/all >> perms && \
+	stat -c "%04a" /permfiles/rw >> perms && \
+	stat -c "%U:%G" /permfiles/owned >> perms
+
+FROM scratch
+COPY --from=build /dest /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	contents := map[string]string{
+		"single":      "single file\n",
+		"double/EOF":  "first file\n",
+		"double/EOF2": "second file\n",
+		"perms":       "0777\n0644\nuser:user\n",
+	}
+
+	for name, content := range contents {
+		dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
+		require.NoError(t, err)
+		require.Equal(t, content, string(dt))
+	}
+}
+
+func testRunBasicHeredoc(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS build
+
+RUN <<EOF
+echo "i am" >> /dest
+whoami >> /dest
+EOF
+
+FROM scratch
+COPY --from=build /dest /dest
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest"))
+	require.NoError(t, err)
+	require.Equal(t, "i am\nroot\n", string(dt))
+}
+
+func testRunFakeHeredoc(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS build
+
+SHELL ["/bin/awk"]
+RUN <<EOF
+BEGIN {
+	print "foo" > "/dest"
+}
+EOF
+
+FROM scratch
+COPY --from=build /dest /dest
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest"))
+	require.NoError(t, err)
+	require.Equal(t, "foo\n", string(dt))
+}
+
+func testRunShebangHeredoc(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS build
+
+RUN <<EOF
+#!/bin/awk -f
+BEGIN {
+	print "hello" >> "/dest"
+	print "world" >> "/dest"
+}
+EOF
+
+FROM scratch
+COPY --from=build /dest /dest
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "dest"))
+	require.NoError(t, err)
+	require.Equal(t, "hello\nworld\n", string(dt))
+}
+
+func testRunComplexHeredoc(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS build
+
+WORKDIR /dest
+
+RUN cat <<EOF1 | tr '[:upper:]' '[:lower:]' > ./out1; \
+	cat <<EOF2 | tr '[:lower:]' '[:upper:]' > ./out2
+hello WORLD
+EOF1
+HELLO world
+EOF2
+
+RUN <<EOF 3<<IN1 4<<IN2 awk -f -
+BEGIN {
+	while ((getline line < "/proc/self/fd/3") > 0)
+		print tolower(line) > "./fd3"
+	while ((getline line < "/proc/self/fd/4") > 0)
+		print toupper(line) > "./fd4"
+}
+EOF
+hello WORLD
+IN1
+HELLO world
+IN2
+
+FROM scratch
+COPY --from=build /dest /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	contents := map[string]string{
+		"out1": "hello world\n",
+		"out2": "HELLO WORLD\n",
+		"fd3":  "hello world\n",
+		"fd4":  "HELLO WORLD\n",
+	}
+
+	for name, content := range contents {
+		dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
+		require.NoError(t, err)
+		require.Equal(t, content, string(dt))
+	}
+}
+
+func testHeredocIndent(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS build
+
+COPY <<EOF /dest/foo-copy
+	foo
+EOF
+
+COPY <<-EOF /dest/bar-copy
+	bar
+EOF
+
+RUN <<EOF
+echo "
+	foo" > /dest/foo-run
+EOF
+
+RUN <<-EOF
+echo "
+	bar" > /dest/bar-run
+EOF
+
+RUN <<EOF
+#!/bin/sh
+echo "
+	foo" > /dest/foo2-run
+EOF
+
+RUN <<-EOF
+#!/bin/sh
+echo "
+	bar" > /dest/bar2-run
+EOF
+
+RUN <<EOF sh > /dest/foo3-run
+echo "
+	foo"
+EOF
+
+RUN <<-EOF sh > /dest/bar3-run
+echo "
+	bar"
+EOF
+
+FROM scratch
+COPY --from=build /dest /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	contents := map[string]string{
+		"foo-copy": "\tfoo\n",
+		"foo-run":  "\n\tfoo\n",
+		"foo2-run": "\n\tfoo\n",
+		"foo3-run": "\n\tfoo\n",
+		"bar-copy": "bar\n",
+		"bar-run":  "\nbar\n",
+		"bar2-run": "\nbar\n",
+		"bar3-run": "\nbar\n",
+	}
+
+	for name, content := range contents {
+		dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
+		require.NoError(t, err)
+		require.Equal(t, content, string(dt))
+	}
+}
+
+func testHeredocVarSubstitution(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox as build
+
+ARG name=world
+
+COPY <<EOF /dest/c1
+Hello ${name}!
+EOF
+COPY <<'EOF' /dest/c2
+Hello ${name}!
+EOF
+COPY <<"EOF" /dest/c3
+Hello ${name}!
+EOF
+
+RUN <<EOF
+greeting="Hello"
+echo "${greeting} ${name}!" > /dest/r1
+EOF
+RUN <<EOF
+name="new world"
+echo "Hello ${name}!" > /dest/r2
+EOF
+
+FROM scratch
+COPY --from=build /dest /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	contents := map[string]string{
+		"c1": "Hello world!\n",
+		"c2": "Hello ${name}!\n",
+		"c3": "Hello ${name}!\n",
+		"r1": "Hello world!\n",
+		"r2": "Hello new world!\n",
+	}
+
+	for name, content := range contents {
+		dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
+		require.NoError(t, err)
+		require.Equal(t, content, string(dt))
+	}
+}

frontend/dockerfile/dockerfile_test.go

@@ -142,6 +142,9 @@ var securityTests = []integration.Test{}
 // Tests that depend on the `network.*` entitlements
 var networkTests = []integration.Test{}
 
+// Tests that depend on heredoc support
+var heredocTests = []integration.Test{}
+
 var opts []integration.TestOpt
 var securityOpts []integration.TestOpt
 
@@ -194,6 +197,7 @@ func TestIntegration(t *testing.T) {
 			"granted": networkHostGranted,
 			"denied":  networkHostDenied,
 		}))...)
+	integration.Run(t, heredocTests, opts...)
 }
 
 func testDefaultEnvWithArgs(t *testing.T, sb integration.Sandbox) {

frontend/dockerfile/docs/syntax.md

@@ -30,7 +30,7 @@ change in between releases on labs channel, the old versions are guaranteed to b
 To use this flag set Dockerfile version to at least `1.2`
 
 ```
-#syntax=docker/dockerfile:1.2
+# syntax=docker/dockerfile:1.2
 ```
 
 `RUN --mount` allows you to create mounts that process running as part of the build can access. This can be used to bind
@@ -175,10 +175,10 @@ However, pem files with passphrases are not supported.
 
 ## Security context `RUN --security=insecure|sandbox`
 
-To use this flag set Dockerfile version to `labs` channel.
+To use this flag, set Dockerfile version to `labs` channel.
 
 ```
-#syntax=docker/dockerfile:1.2-labs
+# syntax=docker/dockerfile:1.2-labs
 ```
 
 With `--security=insecure`, builder runs the command without sandbox in insecure mode,
@@ -204,10 +204,10 @@ RUN --security=insecure cat /proc/self/status | grep CapEff
 
 ## Network modes `RUN --network=none|host|default`
 
-To use this flag set Dockerfile version to `labs` channel.
+To use this flag, set Dockerfile version to `labs` channel.
 
 ```
-#syntax=docker/dockerfile:1.2-labs
+# syntax=docker/dockerfile:1.2-labs
 ```
 
 `RUN --network` allows control over which networking environment the command is run in.
@@ -237,3 +237,91 @@ RUN --network=none pip install --find-links wheels mypackage
 
 `pip` will only be able to install the packages provided in the tarfile, which
 can be controlled by an earlier build stage.
+
+
+## Here-Documents
+
+To use this flag, set Dockerfile version to `labs` channel. Currently this feature is only available
+in `docker/dockerfile-upstream:master-labs` image.
+
+```
+# syntax=docker/dockerfile-upstream:master-labs
+```
+
+Here-documents allow redirection of subsequent Dockerfile lines to the input of `RUN` or `COPY` commands.
+If such command contains a [here-document](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_07_04)
+Dockerfile will consider the next lines until the line only containing a here-doc delimiter as part of the same command.
+
+#### Example: running a multi-line script
+
+```dockerfile
+# syntax = docker/dockerfile-upstream:master-labs
+FROM debian
+RUN <<eot bash
+  apt-get update
+  apt-get install -y vim
+eot
+```
+
+If the command only contains a here-document, its contents is evaluated with the default shell.
+
+```dockerfile
+# syntax = docker/dockerfile-upstream:master-labs
+FROM debian
+RUN <<eot
+  mkdir -p foo/bar
+eot
+```
+
+Alternatively, shebang header can be used to define an interpreter.
+
+```dockerfile
+# syntax = docker/dockerfile-upstream:master-labs
+FROM python:3.6
+RUN <<eot
+#!/usr/bin/env python
+print("hello world")
+eot
+```
+
+More complex examples may use multiple here-documents.
+
+```dockerfile
+# syntax = docker/dockerfile-upstream:master-labs
+FROM alpine
+RUN <<FILE1 cat > file1 && <<FILE2 cat > file2
+I am
+first
+FILE1
+I am
+second
+FILE2
+```
+
+#### Example: creating inline files
+
+In `COPY` commands source parameters can be replaced with here-doc indicators.
+Regular here-doc [variable expansion and tab stripping rules](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_07_04) apply.
+
+```dockerfile
+# syntax = docker/dockerfile-upstream:master-labs
+FROM alpine
+ARG FOO=bar
+COPY <<-eot /app/foo
+	hello ${FOO}
+eot
+```
+
+```dockerfile
+# syntax = docker/dockerfile-upstream:master-labs
+FROM alpine
+COPY <<-"eot" /app/script.sh
+	echo hello ${FOO}
+eot
+RUN FOO=abc ash /app/script.sh
+```
+
+
+
+
+

frontend/dockerfile/instructions/commands.go

@@ -165,19 +165,45 @@ func (c *LabelCommand) Expand(expander SingleWordExpander) error {
 	return expandKvpsInPlace(c.Labels, expander)
 }
 
-// SourcesAndDest represent a list of source files and a destination
-type SourcesAndDest []string
-
-// Sources list the source paths
-func (s SourcesAndDest) Sources() []string {
-	res := make([]string, len(s)-1)
-	copy(res, s[:len(s)-1])
-	return res
+// SourceContent represents an anonymous file object
+type SourceContent struct {
+	Path   string
+	Data   string
+	Expand bool
 }
 
-// Dest path of the operation
-func (s SourcesAndDest) Dest() string {
-	return s[len(s)-1]
+// SourcesAndDest represent a collection of sources and a destination
+type SourcesAndDest struct {
+	DestPath       string
+	SourcePaths    []string
+	SourceContents []SourceContent
+}
+
+func (s *SourcesAndDest) Expand(expander SingleWordExpander) error {
+	for i, content := range s.SourceContents {
+		if !content.Expand {
+			continue
+		}
+
+		expandedData, err := expander(content.Data)
+		if err != nil {
+			return err
+		}
+		s.SourceContents[i].Data = expandedData
+	}
+
+	err := expandSliceInPlace(s.SourcePaths, expander)
+	if err != nil {
+		return err
+	}
+
+	expandedDestPath, err := expander(s.DestPath)
+	if err != nil {
+		return err
+	}
+	s.DestPath = expandedDestPath
+
+	return nil
 }
 
 // AddCommand : ADD foo /path
@@ -199,7 +225,8 @@ func (c *AddCommand) Expand(expander SingleWordExpander) error {
 		return err
 	}
 	c.Chown = expandedChown
-	return expandSliceInPlace(c.SourcesAndDest, expander)
+
+	return c.SourcesAndDest.Expand(expander)
 }
 
 // CopyCommand : COPY foo /path
@@ -221,7 +248,8 @@ func (c *CopyCommand) Expand(expander SingleWordExpander) error {
 		return err
 	}
 	c.Chown = expandedChown
-	return expandSliceInPlace(c.SourcesAndDest, expander)
+
+	return c.SourcesAndDest.Expand(expander)
 }
 
 // OnbuildCommand : ONBUILD <some other command>
@@ -249,9 +277,17 @@ func (c *WorkdirCommand) Expand(expander SingleWordExpander) error {
 	return nil
 }
 
+// ShellInlineFile represents an inline file created for a shell command
+type ShellInlineFile struct {
+	Name  string
+	Data  string
+	Chomp bool
+}
+
 // ShellDependantCmdLine represents a cmdline optionally prepended with the shell
 type ShellDependantCmdLine struct {
 	CmdLine      strslice.StrSlice
+	Files        []ShellInlineFile
 	PrependShell bool
 }
 

frontend/dockerfile/instructions/parse.go

@@ -18,6 +18,7 @@ import (
 type parseRequest struct {
 	command    string
 	args       []string
+	heredocs   []parser.Heredoc
 	attributes map[string]bool
 	flags      *BFlags
 	original   string
@@ -47,6 +48,7 @@ func newParseRequestFromNode(node *parser.Node) parseRequest {
 	return parseRequest{
 		command:    node.Value,
 		args:       nodeArgs(node),
+		heredocs:   node.Heredocs,
 		attributes: node.Attributes,
 		original:   node.Original,
 		flags:      NewBFlagsWithArgs(node.Flags),
@@ -236,6 +238,45 @@ func parseLabel(req parseRequest) (*LabelCommand, error) {
 	}, nil
 }
 
+func parseSourcesAndDest(req parseRequest, command string) (*SourcesAndDest, error) {
+	srcs := req.args[:len(req.args)-1]
+	dest := req.args[len(req.args)-1]
+	if heredoc := parser.MustParseHeredoc(dest); heredoc != nil {
+		return nil, errBadHeredoc(command, "a destination")
+	}
+
+	heredocLookup := make(map[string]parser.Heredoc)
+	for _, heredoc := range req.heredocs {
+		heredocLookup[heredoc.Name] = heredoc
+	}
+
+	var sourcePaths []string
+	var sourceContents []SourceContent
+	for _, src := range srcs {
+		if heredoc := parser.MustParseHeredoc(src); heredoc != nil {
+			content := heredocLookup[heredoc.Name].Content
+			if heredoc.Chomp {
+				content = parser.ChompHeredocContent(content)
+			}
+			sourceContents = append(sourceContents,
+				SourceContent{
+					Data:   content,
+					Path:   heredoc.Name,
+					Expand: heredoc.Expand,
+				},
+			)
+		} else {
+			sourcePaths = append(sourcePaths, src)
+		}
+	}
+
+	return &SourcesAndDest{
+		DestPath:       dest,
+		SourcePaths:    sourcePaths,
+		SourceContents: sourceContents,
+	}, nil
+}
+
 func parseAdd(req parseRequest) (*AddCommand, error) {
 	if len(req.args) < 2 {
 		return nil, errNoDestinationArgument("ADD")
@@ -245,9 +286,15 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 	if err := req.flags.Parse(); err != nil {
 		return nil, err
 	}
+
+	sourcesAndDest, err := parseSourcesAndDest(req, "ADD")
+	if err != nil {
+		return nil, err
+	}
+
 	return &AddCommand{
-		SourcesAndDest:  SourcesAndDest(req.args),
 		withNameAndCode: newWithNameAndCode(req),
+		SourcesAndDest:  *sourcesAndDest,
 		Chown:           flChown.Value,
 		Chmod:           flChmod.Value,
 	}, nil
@@ -263,10 +310,16 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
 	if err := req.flags.Parse(); err != nil {
 		return nil, err
 	}
+
+	sourcesAndDest, err := parseSourcesAndDest(req, "COPY")
+	if err != nil {
+		return nil, err
+	}
+
 	return &CopyCommand{
-		SourcesAndDest:  SourcesAndDest(req.args),
-		From:            flFrom.Value,
 		withNameAndCode: newWithNameAndCode(req),
+		SourcesAndDest:  *sourcesAndDest,
+		From:            flFrom.Value,
 		Chown:           flChown.Value,
 		Chmod:           flChmod.Value,
 	}, nil
@@ -351,7 +404,17 @@ func parseWorkdir(req parseRequest) (*WorkdirCommand, error) {
 
 }
 
-func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependantCmdLine {
+func parseShellDependentCommand(req parseRequest, command string, emptyAsNil bool) (ShellDependantCmdLine, error) {
+	var files []ShellInlineFile
+	for _, heredoc := range req.heredocs {
+		file := ShellInlineFile{
+			Name:  heredoc.Name,
+			Data:  heredoc.Content,
+			Chomp: heredoc.Chomp,
+		}
+		files = append(files, file)
+	}
+
 	args := handleJSONArgs(req.args, req.attributes)
 	cmd := strslice.StrSlice(args)
 	if emptyAsNil && len(cmd) == 0 {
@@ -359,8 +422,9 @@ func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependan
 	}
 	return ShellDependantCmdLine{
 		CmdLine:      cmd,
+		Files:        files,
 		PrependShell: !req.attributes["json"],
-	}
+	}, nil
 }
 
 func parseRun(req parseRequest) (*RunCommand, error) {
@@ -376,7 +440,13 @@ func parseRun(req parseRequest) (*RunCommand, error) {
 		return nil, err
 	}
 	cmd.FlagsUsed = req.flags.Used()
-	cmd.ShellDependantCmdLine = parseShellDependentCommand(req, false)
+
+	cmdline, err := parseShellDependentCommand(req, "RUN", false)
+	if err != nil {
+		return nil, err
+	}
+	cmd.ShellDependantCmdLine = cmdline
+
 	cmd.withNameAndCode = newWithNameAndCode(req)
 
 	for _, fn := range parseRunPostHooks {
@@ -392,11 +462,16 @@ func parseCmd(req parseRequest) (*CmdCommand, error) {
 	if err := req.flags.Parse(); err != nil {
 		return nil, err
 	}
+
+	cmdline, err := parseShellDependentCommand(req, "CMD", false)
+	if err != nil {
+		return nil, err
+	}
+
 	return &CmdCommand{
-		ShellDependantCmdLine: parseShellDependentCommand(req, false),
+		ShellDependantCmdLine: cmdline,
 		withNameAndCode:       newWithNameAndCode(req),
 	}, nil
-
 }
 
 func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
@@ -404,12 +479,15 @@ func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
 		return nil, err
 	}
 
-	cmd := &EntrypointCommand{
-		ShellDependantCmdLine: parseShellDependentCommand(req, true),
-		withNameAndCode:       newWithNameAndCode(req),
+	cmdline, err := parseShellDependentCommand(req, "ENTRYPOINT", true)
+	if err != nil {
+		return nil, err
 	}
 
-	return cmd, nil
+	return &EntrypointCommand{
+		ShellDependantCmdLine: cmdline,
+		withNameAndCode:       newWithNameAndCode(req),
+	}, nil
 }
 
 // parseOptInterval(flag) is the duration of flag.Value, or 0 if
@@ -651,6 +729,10 @@ func errNoDestinationArgument(command string) error {
 	return errors.Errorf("%s requires at least two arguments, but only one was provided. Destination could not be determined.", command)
 }
 
+func errBadHeredoc(command string, option string) error {
+	return errors.Errorf("%s cannot accept a heredoc as %s", command, option)
+}
+
 func errBlankCommandNames(command string) error {
 	return errors.Errorf("%s names can not be blank", command)
 }

frontend/dockerfile/instructions/parse_heredoc_test.go

@@ -0,0 +1,245 @@
+// +build dfheredoc
+
+package instructions
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/stretchr/testify/require"
+)
+
+func TestErrorCasesHeredoc(t *testing.T) {
+	cases := []struct {
+		name          string
+		dockerfile    string
+		expectedError string
+	}{
+		{
+			name:          "COPY heredoc destination",
+			dockerfile:    "COPY /foo <<EOF\nEOF",
+			expectedError: "COPY cannot accept a heredoc as a destination",
+		},
+	}
+	for _, c := range cases {
+		r := strings.NewReader(c.dockerfile)
+		ast, err := parser.Parse(r)
+
+		if err != nil {
+			t.Fatalf("Error when parsing Dockerfile: %s", err)
+		}
+		n := ast.AST.Children[0]
+		_, err = ParseInstruction(n)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), c.expectedError)
+	}
+}
+
+func TestCopyHeredoc(t *testing.T) {
+	cases := []struct {
+		dockerfile     string
+		sourcesAndDest SourcesAndDest
+	}{
+		{
+			dockerfile: "COPY /foo /bar",
+			sourcesAndDest: SourcesAndDest{
+				DestPath:    "/bar",
+				SourcePaths: []string{"/foo"},
+			},
+		},
+		{
+			dockerfile: `COPY <<EOF /bar
+EOF`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath: "/bar",
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF",
+						Data:   "",
+						Expand: true,
+					},
+				},
+			},
+		},
+		{
+			dockerfile: `COPY <<EOF /bar
+TESTING
+EOF`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath: "/bar",
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF",
+						Data:   "TESTING\n",
+						Expand: true,
+					},
+				},
+			},
+		},
+		{
+			dockerfile: `COPY <<-EOF /bar
+	TESTING
+EOF`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath: "/bar",
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF",
+						Data:   "TESTING\n",
+						Expand: true,
+					},
+				},
+			},
+		},
+		{
+			dockerfile: `COPY <<'EOF' /bar
+TESTING
+EOF`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath: "/bar",
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF",
+						Data:   "TESTING\n",
+						Expand: false,
+					},
+				},
+			},
+		},
+		{
+			dockerfile: `COPY <<EOF1 <<EOF2 /bar
+this is the first file
+EOF1
+this is the second file
+EOF2`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath: "/bar",
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF1",
+						Data:   "this is the first file\n",
+						Expand: true,
+					},
+					{
+						Path:   "EOF2",
+						Data:   "this is the second file\n",
+						Expand: true,
+					},
+				},
+			},
+		},
+		{
+			dockerfile: `COPY <<EOF foo.txt /bar
+this is inline
+EOF`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath:    "/bar",
+				SourcePaths: []string{"foo.txt"},
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF",
+						Data:   "this is inline\n",
+						Expand: true,
+					},
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		r := strings.NewReader(c.dockerfile)
+		ast, err := parser.Parse(r)
+		require.NoError(t, err)
+
+		n := ast.AST.Children[0]
+		comm, err := ParseInstruction(n)
+		require.NoError(t, err)
+
+		sd := comm.(*CopyCommand).SourcesAndDest
+		require.Equal(t, c.sourcesAndDest, sd)
+	}
+}
+
+func TestRunHeredoc(t *testing.T) {
+	cases := []struct {
+		dockerfile string
+		shell      bool
+		command    strslice.StrSlice
+		files      []ShellInlineFile
+	}{
+		{
+			dockerfile: `RUN ["ls", "/"]`,
+			command:    strslice.StrSlice{"ls", "/"},
+			shell:      false,
+		},
+		{
+			dockerfile: `RUN ["<<EOF"]`,
+			command:    strslice.StrSlice{"<<EOF"},
+			shell:      false,
+		},
+		{
+			dockerfile: "RUN ls /",
+			command:    strslice.StrSlice{"ls /"},
+			shell:      true,
+		},
+		{
+			dockerfile: `RUN <<EOF
+ls /
+whoami
+EOF`,
+			command: strslice.StrSlice{"<<EOF"},
+			files: []ShellInlineFile{
+				{
+					Name: "EOF",
+					Data: "ls /\nwhoami\n",
+				},
+			},
+			shell: true,
+		},
+		{
+			dockerfile: `RUN <<'EOF' | python
+print("hello")
+print("world")
+EOF`,
+			command: strslice.StrSlice{"<<'EOF' | python"},
+			files: []ShellInlineFile{
+				{
+					Name: "EOF",
+					Data: `print("hello")
+print("world")
+`,
+				},
+			},
+			shell: true,
+		},
+		{
+			dockerfile: `RUN <<-EOF
+	echo test
+EOF`,
+			command: strslice.StrSlice{"<<-EOF"},
+			files: []ShellInlineFile{
+				{
+					Name:  "EOF",
+					Data:  "\techo test\n",
+					Chomp: true,
+				},
+			},
+			shell: true,
+		},
+	}
+
+	for _, c := range cases {
+		r := strings.NewReader(c.dockerfile)
+		ast, err := parser.Parse(r)
+		require.NoError(t, err)
+
+		n := ast.AST.Children[0]
+		comm, err := ParseInstruction(n)
+		require.NoError(t, err)
+		require.Equal(t, c.shell, comm.(*RunCommand).PrependShell)
+		require.Equal(t, c.command, comm.(*RunCommand).CmdLine)
+		require.Equal(t, c.files, comm.(*RunCommand).Files)
+	}
+}

frontend/dockerfile/instructions/parse_test.go

@@ -192,11 +192,6 @@ func TestErrorCases(t *testing.T) {
 			dockerfile:    "ONBUILD FROM scratch",
 			expectedError: "FROM isn't allowed as an ONBUILD trigger",
 		},
-		{
-			name:          "ONBUILD forbidden MAINTAINER",
-			dockerfile:    "ONBUILD MAINTAINER docker.io",
-			expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger",
-		},
 		{
 			name:          "MAINTAINER unknown flag",
 			dockerfile:    "MAINTAINER --boo joe@example.com",
@@ -212,6 +207,11 @@ func TestErrorCases(t *testing.T) {
 			dockerfile:    `foo bar`,
 			expectedError: "unknown instruction: FOO",
 		},
+		{
+			name:          "Invalid instruction",
+			dockerfile:    `foo bar`,
+			expectedError: "unknown instruction: FOO",
+		},
 	}
 	for _, c := range cases {
 		r := strings.NewReader(c.dockerfile)

frontend/dockerfile/parser/parser.go

@@ -12,6 +12,7 @@ import (
 	"unicode"
 
 	"github.com/moby/buildkit/frontend/dockerfile/command"
+	"github.com/moby/buildkit/frontend/dockerfile/shell"
 	"github.com/pkg/errors"
 )
 
@@ -31,6 +32,7 @@ type Node struct {
 	Value       string          // actual content
 	Next        *Node           // the next item in the current sexp
 	Children    []*Node         // the children of this sexp
+	Heredocs    []Heredoc       // extra heredoc content attachments
 	Attributes  map[string]bool // special attributes for this node
 	Original    string          // original line used before parsing
 	Flags       []string        // only top Node should have this set
@@ -74,6 +76,17 @@ func (node *Node) lines(start, end int) {
 	node.EndLine = end
 }
 
+func (node *Node) canContainHeredoc() bool {
+	if _, allowedDirective := heredocDirectives[node.Value]; !allowedDirective {
+		return false
+	}
+	if _, isJSON := node.Attributes["json"]; isJSON {
+		return false
+	}
+
+	return true
+}
+
 // AddChild adds a new child node, and updates line information
 func (node *Node) AddChild(child *Node, startLine, endLine int) {
 	child.lines(startLine, endLine)
@@ -84,11 +97,22 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
 	node.Children = append(node.Children, child)
 }
 
+type Heredoc struct {
+	Name           string
+	FileDescriptor uint
+	Expand         bool
+	Chomp          bool
+	Content        string
+}
+
 var (
-	dispatch     map[string]func(string, *directives) (*Node, map[string]bool, error)
-	reWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
-	reDirectives = regexp.MustCompile(`^#\s*([a-zA-Z][a-zA-Z0-9]*)\s*=\s*(.+?)\s*$`)
-	reComment    = regexp.MustCompile(`^#.*$`)
+	dispatch          map[string]func(string, *directives) (*Node, map[string]bool, error)
+	heredocDirectives map[string]bool
+	reWhitespace      = regexp.MustCompile(`[\t\v\f\r ]+`)
+	reDirectives      = regexp.MustCompile(`^#\s*([a-zA-Z][a-zA-Z0-9]*)\s*=\s*(.+?)\s*$`)
+	reComment         = regexp.MustCompile(`^#.*$`)
+	reHeredoc         = regexp.MustCompile(`^(\d*)<<(-?)(['"]?)([a-zA-Z][a-zA-Z0-9]*)(['"]?)$`)
+	reLeadingTabs     = regexp.MustCompile(`(?m)^\t+`)
 )
 
 // DefaultEscapeToken is the default escape token
@@ -252,6 +276,7 @@ func Parse(rwc io.Reader) (*Result, error) {
 	currentLine := 0
 	root := &Node{StartLine: -1}
 	scanner := bufio.NewScanner(rwc)
+	scanner.Split(scanLines)
 	warnings := []string{}
 	var comments []string
 
@@ -312,8 +337,40 @@ func Parse(rwc io.Reader) (*Result, error) {
 		if err != nil {
 			return nil, withLocation(err, startLine, currentLine)
 		}
-		comments = nil
+
+		if child.canContainHeredoc() {
+			heredocs, err := heredocsFromLine(line)
+			if err != nil {
+				return nil, withLocation(err, startLine, currentLine)
+			}
+
+			for _, heredoc := range heredocs {
+				terminator := []byte(heredoc.Name)
+				terminated := false
+				for scanner.Scan() {
+					bytesRead := scanner.Bytes()
+					currentLine++
+
+					possibleTerminator := trimNewline(bytesRead)
+					if heredoc.Chomp {
+						possibleTerminator = trimLeadingTabs(possibleTerminator)
+					}
+					if bytes.Equal(possibleTerminator, terminator) {
+						terminated = true
+						break
+					}
+					heredoc.Content += string(bytesRead)
+				}
+				if !terminated {
+					return nil, withLocation(errors.New("unterminated heredoc"), startLine, currentLine)
+				}
+
+				child.Heredocs = append(child.Heredocs, heredoc)
+			}
+		}
+
 		root.AddChild(child, startLine, currentLine)
+		comments = nil
 	}
 
 	if len(warnings) > 0 {
@@ -331,20 +388,83 @@ func Parse(rwc io.Reader) (*Result, error) {
 	}, withLocation(handleScannerError(scanner.Err()), currentLine, 0)
 }
 
+func heredocFromMatch(match []string) (*Heredoc, error) {
+	if len(match) == 0 {
+		return nil, nil
+	}
+
+	fileDescriptor, _ := strconv.ParseUint(match[1], 10, 0)
+	chomp := match[2] == "-"
+	quoteOpen := match[3]
+	name := match[4]
+	quoteClose := match[5]
+
+	expand := true
+	if quoteOpen != "" || quoteClose != "" {
+		if quoteOpen != quoteClose {
+			return nil, errors.New("quoted heredoc quotes do not match")
+		}
+		expand = false
+	}
+
+	return &Heredoc{
+		Name:           name,
+		Expand:         expand,
+		Chomp:          chomp,
+		FileDescriptor: uint(fileDescriptor),
+	}, nil
+}
+
+func ParseHeredoc(src string) (*Heredoc, error) {
+	return heredocFromMatch(reHeredoc.FindStringSubmatch(src))
+}
+func MustParseHeredoc(src string) *Heredoc {
+	heredoc, _ := ParseHeredoc(src)
+	return heredoc
+}
+
+func heredocsFromLine(line string) ([]Heredoc, error) {
+	shlex := shell.NewLex('\\')
+	shlex.RawQuotes = true
+	words, _ := shlex.ProcessWords(line, []string{})
+
+	var docs []Heredoc
+	for _, word := range words {
+		heredoc, err := ParseHeredoc(word)
+		if err != nil {
+			return nil, err
+		}
+		if heredoc != nil {
+			docs = append(docs, *heredoc)
+		}
+	}
+	return docs, nil
+}
+
+func ChompHeredocContent(src string) string {
+	return reLeadingTabs.ReplaceAllString(src, "")
+}
+
 func trimComments(src []byte) []byte {
 	return reComment.ReplaceAll(src, []byte{})
 }
 
-func trimWhitespace(src []byte) []byte {
+func trimLeadingWhitespace(src []byte) []byte {
 	return bytes.TrimLeftFunc(src, unicode.IsSpace)
 }
+func trimLeadingTabs(src []byte) []byte {
+	return bytes.TrimLeft(src, "\t")
+}
+func trimNewline(src []byte) []byte {
+	return bytes.TrimRight(src, "\r\n")
+}
 
 func isComment(line []byte) bool {
-	return reComment.Match(trimWhitespace(line))
+	return reComment.Match(trimLeadingWhitespace(trimNewline(line)))
 }
 
 func isEmptyContinuationLine(line []byte) bool {
-	return len(trimWhitespace(line)) == 0
+	return len(trimLeadingWhitespace(trimNewline(line))) == 0
 }
 
 var utf8bom = []byte{0xEF, 0xBB, 0xBF}
@@ -360,12 +480,27 @@ func trimContinuationCharacter(line string, d *directives) (string, bool) {
 // TODO: remove stripLeftWhitespace after deprecation period. It seems silly
 // to preserve whitespace on continuation lines. Why is that done?
 func processLine(d *directives, token []byte, stripLeftWhitespace bool) ([]byte, error) {
+	token = trimNewline(token)
 	if stripLeftWhitespace {
-		token = trimWhitespace(token)
+		token = trimLeadingWhitespace(token)
 	}
 	return trimComments(token), d.possibleParserDirective(string(token))
 }
 
+// Variation of bufio.ScanLines that preserves the line endings
+func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+	if i := bytes.IndexByte(data, '\n'); i >= 0 {
+		return i + 1, data[0 : i+1], nil
+	}
+	if atEOF {
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
 func handleScannerError(err error) error {
 	switch err {
 	case bufio.ErrTooLong:

frontend/dockerfile/parser/parser_heredoc.go

@@ -0,0 +1,13 @@
+// +build dfheredoc
+
+package parser
+
+import "github.com/moby/buildkit/frontend/dockerfile/command"
+
+func init() {
+	heredocDirectives = map[string]bool{
+		command.Add:  true,
+		command.Copy: true,
+		command.Run:  true,
+	}
+}

frontend/dockerfile/parser/parser_heredoc_test.go

@@ -0,0 +1,314 @@
+// +build dfheredoc
+
+package parser
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseExtractsHeredoc(t *testing.T) {
+	dockerfile := bytes.NewBufferString(`
+FROM alpine:3.6
+
+ENV NAME=me
+
+RUN ls
+
+USER <<INVALID
+INVALID
+
+RUN <<EMPTY
+EMPTY
+
+RUN 3<<EMPTY2
+EMPTY2
+
+RUN "<<NOHEREDOC"
+
+RUN <<INDENT
+	foo
+	bar
+INDENT
+
+RUN <<-UNINDENT
+	baz
+	quux
+UNINDENT
+
+RUN <<-UNINDENT2
+	baz
+	quux
+	UNINDENT2
+
+RUN <<-EXPAND
+	expand $NAME
+EXPAND
+
+RUN <<-'NOEXPAND'
+	don't expand $NAME
+NOEXPAND
+
+RUN <<COPY
+echo hello world
+echo foo bar
+COPY
+
+RUN <<COMMENT
+# internal comment
+echo hello world
+echo foo bar # trailing comment
+COMMENT
+
+RUN --mount=type=cache,target=/foo <<MOUNT
+echo hello
+MOUNT
+
+COPY <<FILE1 <<FILE2 /dest
+content 1
+FILE1
+content 2
+FILE2
+
+COPY <<X <<Y /dest
+Y
+X
+X
+Y
+	`)
+
+	tests := [][]Heredoc{
+		nil, // ENV EXAMPLE=bla
+		nil, // RUN ls
+		nil, // USER <<INVALID
+		nil, // INVALID
+		{
+			// RUN <<EMPTY
+			{
+				Name:    "EMPTY",
+				Content: "",
+				Expand:  true,
+			},
+		},
+		{
+			// RUN <<EMPTY2
+			{
+				Name:           "EMPTY2",
+				Content:        "",
+				Expand:         true,
+				FileDescriptor: 3,
+			},
+		},
+		nil, // RUN "<<NOHEREDOC"
+		{
+			// RUN <<INDENT
+			{
+				Name:    "INDENT",
+				Content: "\tfoo\n\tbar\n",
+				Expand:  true,
+			},
+		},
+		{
+			// RUN <<-UNINDENT
+			{
+				Name:    "UNINDENT",
+				Content: "\tbaz\n\tquux\n",
+				Expand:  true,
+				Chomp:   true,
+			},
+		},
+		{
+			// RUN <<-UNINDENT2
+			{
+				Name:    "UNINDENT2",
+				Content: "\tbaz\n\tquux\n",
+				Expand:  true,
+				Chomp:   true,
+			},
+		},
+		{
+			// RUN <<-EXPAND
+			{
+				Name:    "EXPAND",
+				Content: "\texpand $NAME\n",
+				Expand:  true,
+				Chomp:   true,
+			},
+		},
+		{
+			// RUN <<-'NOEXPAND'
+			{
+				Name:    "NOEXPAND",
+				Content: "\tdon't expand $NAME\n",
+				Expand:  false,
+				Chomp:   true,
+			},
+		},
+		{
+			// RUN <<COPY
+			{
+				Name:    "COPY",
+				Content: "echo hello world\necho foo bar\n",
+				Expand:  true,
+			},
+		},
+		{
+			// RUN <<COMMENT
+			{
+				Name:    "COMMENT",
+				Content: "# internal comment\necho hello world\necho foo bar # trailing comment\n",
+				Expand:  true,
+			},
+		},
+		{
+			// RUN <<MOUNT
+			{
+				Name:    "MOUNT",
+				Content: "echo hello\n",
+				Expand:  true,
+			},
+		},
+		{
+			// COPY <<FILE1 <<FILE2 /dest
+			{
+				Name:    "FILE1",
+				Content: "content 1\n",
+				Expand:  true,
+			},
+			{
+				Name:    "FILE2",
+				Content: "content 2\n",
+				Expand:  true,
+			},
+		},
+		{
+			// COPY <<X <<Y /dest
+			{
+				Name:    "X",
+				Content: "Y\n",
+				Expand:  true,
+			},
+			{
+				Name:    "Y",
+				Content: "X\n",
+				Expand:  true,
+			},
+		},
+	}
+
+	result, err := Parse(dockerfile)
+	require.NoError(t, err)
+
+	for i, test := range tests {
+		child := result.AST.Children[i+1]
+		require.Equal(t, test, child.Heredocs)
+	}
+}
+
+func TestParseJSONHeredoc(t *testing.T) {
+	dockerfile := bytes.NewBufferString(`
+FROM alpine:3.6
+
+RUN ["whoami"]
+RUN ["<<EOF"]
+RUN ["<<'EOF'"]
+	`)
+
+	result, err := Parse(dockerfile)
+	require.NoError(t, err)
+
+	for i := 1; i <= 3; i++ {
+		child := result.AST.Children[i]
+		require.Nil(t, child.Heredocs)
+	}
+}
+
+func TestHeredocChomp(t *testing.T) {
+	content := "\thello\n\tworld\n"
+	require.Equal(t, "hello\nworld\n", ChompHeredocContent(content))
+}
+
+func TestParseHeredocHelpers(t *testing.T) {
+	validHeredocs := []string{
+		"<<EOF",
+		"<<'EOF'",
+		`<<"EOF"`,
+		"<<-EOF",
+		"<<-'EOF'",
+		`<<-"EOF"`,
+	}
+	invalidHeredocs := []string{
+		"<<'EOF",
+		"<<\"EOF",
+		"<<EOF'",
+		"<<EOF\"",
+	}
+	notHeredocs := []string{
+		"",
+		"EOF",
+		"<<",
+		"<<-",
+		"<EOF",
+		"<<<EOF",
+	}
+	for _, src := range notHeredocs {
+		heredoc, err := ParseHeredoc(src)
+		require.NoError(t, err)
+		require.Nil(t, heredoc)
+	}
+	for _, src := range validHeredocs {
+		heredoc, err := ParseHeredoc(src)
+		require.NoError(t, err)
+		require.Equal(t, heredoc.Name, "EOF")
+	}
+	for _, src := range invalidHeredocs {
+		_, err := ParseHeredoc(src)
+		require.Error(t, err)
+	}
+}
+
+func TestHeredocsFromLine(t *testing.T) {
+	srcs := []struct {
+		line         string
+		heredocNames []string
+	}{
+		{
+			line:         "RUN <<EOF",
+			heredocNames: []string{"EOF"},
+		},
+		{
+			line:         "RUN <<-EOF",
+			heredocNames: []string{"EOF"},
+		},
+		{
+			line:         "RUN <<'EOF'",
+			heredocNames: []string{"EOF"},
+		},
+		{
+			line:         "RUN 4<<EOF",
+			heredocNames: []string{"EOF"},
+		},
+		{
+			line:         "RUN <<EOF <<EOF2",
+			heredocNames: []string{"EOF", "EOF2"},
+		},
+		{
+			line:         "RUN '<<EOF'",
+			heredocNames: nil,
+		},
+		{
+			line:         `RUN "<<EOF"`,
+			heredocNames: nil,
+		},
+	}
+
+	for _, src := range srcs {
+		heredocs, err := heredocsFromLine(src.line)
+		require.NoError(t, err)
+		for i, heredoc := range heredocs {
+			require.Equal(t, heredoc.Name, src.heredocNames[i])
+		}
+	}
+}

frontend/dockerfile/parser/parser_test.go

@@ -137,6 +137,9 @@ func TestParseWarnsOnEmptyContinutationLine(t *testing.T) {
 	dockerfile := bytes.NewBufferString(`
 FROM alpine:3.6
 
+RUN valid \
+    continuation
+
 RUN something \
 
     following \
@@ -146,6 +149,7 @@ RUN something \
 RUN another \
 
     thing
+
 RUN non-indented \
 # this is a comment
    after-comment

frontend/dockerfile/release/labs/tags

@@ -1 +1 @@
-dfrunsecurity dfrunnetwork
+dfrunsecurity dfrunnetwork dfheredoc