Merge pull request #2442 from jedevc/heredoc-copy-symbols

Fix #2439 (`COPY` with Heredocs eats quotes)

frontend/dockerfile/dockerfile2llb/convert.go

@@ -594,6 +594,21 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 			return err
 		}
 	}
+	if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansionRaw); ok {
+		err := ex.ExpandRaw(func(word string) (string, error) {
+			env, err := d.state.Env(context.TODO())
+			if err != nil {
+				return "", err
+			}
+
+			lex := shell.NewLex('\\')
+			lex.SkipProcessQuotes = true
+			return lex.ProcessWord(word, env)
+		})
+		if err != nil {
+			return err
+		}
+	}
 
 	var err error
 	switch c := cmd.Command.(type) {

frontend/dockerfile/dockerfile_heredoc_test.go

@@ -20,6 +20,7 @@ import (
 
 var hdTests = integration.TestFuncs(
 	testCopyHeredoc,
+	testCopyHeredocSpecialSymbols,
 	testRunBasicHeredoc,
 	testRunFakeHeredoc,
 	testRunShebangHeredoc,
@@ -112,6 +113,94 @@ COPY --from=build /dest /
 	}
 }
 
+func testCopyHeredocSpecialSymbols(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM scratch
+
+COPY <<EOF quotefile
+"quotes in file"
+EOF
+
+COPY <<EOF slashfile1
+\
+EOF
+COPY <<EOF slashfile2
+\\
+EOF
+COPY <<EOF slashfile3
+\$
+EOF
+
+COPY <<"EOF" rawslashfile1
+\
+EOF
+COPY <<"EOF" rawslashfile2
+\\
+EOF
+COPY <<"EOF" rawslashfile3
+\$
+EOF
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "quotefile"))
+	require.NoError(t, err)
+	require.Equal(t, "\"quotes in file\"\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "slashfile1"))
+	require.NoError(t, err)
+	require.Equal(t, "\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "slashfile2"))
+	require.NoError(t, err)
+	require.Equal(t, "\\\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "slashfile3"))
+	require.NoError(t, err)
+	require.Equal(t, "$\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "rawslashfile1"))
+	require.NoError(t, err)
+	require.Equal(t, "\\\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "rawslashfile2"))
+	require.NoError(t, err)
+	require.Equal(t, "\\\\\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "rawslashfile3"))
+	require.NoError(t, err)
+	require.Equal(t, "\\$\n", string(dt))
+}
+
 func testRunBasicHeredoc(t *testing.T, sb integration.Sandbox) {
 	f := getFrontend(t, sb)
 
@@ -449,6 +538,25 @@ COPY <<"EOF" /dest/c3
 Hello ${name}!
 EOF
 
+COPY <<EOF /dest/q1
+Hello '${name}'!
+EOF
+COPY <<EOF /dest/q2
+Hello "${name}"!
+EOF
+COPY <<'EOF' /dest/qsingle1
+Hello '${name}'!
+EOF
+COPY <<'EOF' /dest/qsingle2
+Hello "${name}"!
+EOF
+COPY <<"EOF" /dest/qdouble1
+Hello '${name}'!
+EOF
+COPY <<"EOF" /dest/qdouble2
+Hello "${name}"!
+EOF
+
 RUN <<EOF
 greeting="Hello"
 echo "${greeting} ${name}!" > /dest/r1
@@ -491,11 +599,17 @@ COPY --from=build /dest /
 	require.NoError(t, err)
 
 	contents := map[string]string{
-		"c1": "Hello world!\n",
+		"c1":       "Hello world!\n",
-		"c2": "Hello ${name}!\n",
+		"c2":       "Hello ${name}!\n",
-		"c3": "Hello ${name}!\n",
+		"c3":       "Hello ${name}!\n",
-		"r1": "Hello world!\n",
+		"q1":       "Hello 'world'!\n",
-		"r2": "Hello new world!\n",
+		"q2":       "Hello \"world\"!\n",
+		"qsingle1": "Hello '${name}'!\n",
+		"qsingle2": "Hello \"${name}\"!\n",
+		"qdouble1": "Hello '${name}'!\n",
+		"qdouble2": "Hello \"${name}\"!\n",
+		"r1":       "Hello world!\n",
+		"r2":       "Hello new world!\n",
 	}
 
 	for name, content := range contents {

frontend/dockerfile/instructions/commands.go

@@ -71,11 +71,18 @@ func newWithNameAndCode(req parseRequest) withNameAndCode {
 // SingleWordExpander is a provider for variable expansion where 1 word => 1 output
 type SingleWordExpander func(word string) (string, error)
 
-// SupportsSingleWordExpansion interface marks a command as supporting variable expansion
+// SupportsSingleWordExpansion interface marks a command as supporting variable
+// expansion
 type SupportsSingleWordExpansion interface {
 	Expand(expander SingleWordExpander) error
 }
 
+// SupportsSingleWordExpansionRaw interface marks a command as supporting
+// variable expansion, while ensuring that quotes are preserved
+type SupportsSingleWordExpansionRaw interface {
+	ExpandRaw(expander SingleWordExpander) error
+}
+
 // PlatformSpecific adds platform checks to a command
 type PlatformSpecific interface {
 	CheckPlatform(platform string) error
@@ -180,18 +187,6 @@ type SourcesAndDest struct {
 }
 
 func (s *SourcesAndDest) Expand(expander SingleWordExpander) error {
-	for i, content := range s.SourceContents {
-		if !content.Expand {
-			continue
-		}
-
-		expandedData, err := expander(content.Data)
-		if err != nil {
-			return err
-		}
-		s.SourceContents[i].Data = expandedData
-	}
-
 	err := expandSliceInPlace(s.SourcePaths, expander)
 	if err != nil {
 		return err
@@ -206,6 +201,21 @@ func (s *SourcesAndDest) Expand(expander SingleWordExpander) error {
 	return nil
 }
 
+func (s *SourcesAndDest) ExpandRaw(expander SingleWordExpander) error {
+	for i, content := range s.SourceContents {
+		if !content.Expand {
+			continue
+		}
+
+		expandedData, err := expander(content.Data)
+		if err != nil {
+			return err
+		}
+		s.SourceContents[i].Data = expandedData
+	}
+	return nil
+}
+
 // AddCommand : ADD foo /path
 //
 // Add the file 'foo' to '/path'. Tarball and Remote URL (http, https) handling

frontend/dockerfile/instructions/parse_heredoc_test.go

@@ -147,6 +147,21 @@ EOF`,
 				},
 			},
 		},
+		{
+			dockerfile: `COPY <<EOF /quotes
+"quotes"
+EOF`,
+			sourcesAndDest: SourcesAndDest{
+				DestPath: "/quotes",
+				SourceContents: []SourceContent{
+					{
+						Path:   "EOF",
+						Data:   "\"quotes\"\n",
+						Expand: true,
+					},
+				},
+			},
+		},
 	}
 
 	for _, c := range cases {

frontend/dockerfile/parser/parser_heredoc_test.go

@@ -73,6 +73,11 @@ FILE1
 content 2
 FILE2
 
+COPY <<EOF /quotes
+"foo"
+'bar'
+EOF
+
 COPY <<X <<Y /dest
 Y
 X
@@ -211,6 +216,14 @@ $EOF
 				Expand:  true,
 			},
 		},
+		{
+			// COPY <<EOF /quotes
+			{
+				Name:    "EOF",
+				Content: "\"foo\"\n'bar'\n",
+				Expand:  true,
+			},
+		},
 		{
 			// COPY <<X <<Y /dest
 			{

frontend/dockerfile/shell/lex.go

@@ -18,10 +18,11 @@ import (
 // It doesn't support all flavors of ${xx:...} formats but new ones can
 // be added by adding code to the "special ${} format processing" section
 type Lex struct {
-	escapeToken  rune
+	escapeToken       rune
-	RawQuotes    bool
+	RawQuotes         bool
-	RawEscapes   bool
+	RawEscapes        bool
-	SkipUnsetEnv bool
+	SkipProcessQuotes bool
+	SkipUnsetEnv      bool
 }
 
 // NewLex creates a new Lex which uses escapeToken to escape quotes.
@@ -62,23 +63,25 @@ func (s *Lex) ProcessWordsWithMap(word string, env map[string]string) ([]string,
 
 func (s *Lex) process(word string, env map[string]string) (string, []string, error) {
 	sw := &shellWord{
-		envs:         env,
+		envs:              env,
-		escapeToken:  s.escapeToken,
+		escapeToken:       s.escapeToken,
-		skipUnsetEnv: s.SkipUnsetEnv,
+		skipUnsetEnv:      s.SkipUnsetEnv,
-		rawQuotes:    s.RawQuotes,
+		skipProcessQuotes: s.SkipProcessQuotes,
-		rawEscapes:   s.RawEscapes,
+		rawQuotes:         s.RawQuotes,
+		rawEscapes:        s.RawEscapes,
 	}
 	sw.scanner.Init(strings.NewReader(word))
 	return sw.process(word)
 }
 
 type shellWord struct {
-	scanner      scanner.Scanner
+	scanner           scanner.Scanner
-	envs         map[string]string
+	envs              map[string]string
-	escapeToken  rune
+	escapeToken       rune
-	rawQuotes    bool
+	rawQuotes         bool
-	rawEscapes   bool
+	rawEscapes        bool
-	skipUnsetEnv bool
+	skipUnsetEnv      bool
+	skipProcessQuotes bool
 }
 
 func (sw *shellWord) process(source string) (string, []string, error) {
@@ -141,9 +144,11 @@ func (sw *shellWord) processStopOn(stopChar rune) (string, []string, error) {
 	var words wordsStruct
 
 	var charFuncMapping = map[rune]func() (string, error){
-		'\'': sw.processSingleQuote,
+		'$': sw.processDollar,
-		'"':  sw.processDoubleQuote,
+	}
-		'$':  sw.processDollar,
+	if !sw.skipProcessQuotes {
+		charFuncMapping['\''] = sw.processSingleQuote
+		charFuncMapping['"'] = sw.processDoubleQuote
 	}
 
 	for sw.scanner.Peek() != scanner.EOF {
@@ -173,6 +178,7 @@ func (sw *shellWord) processStopOn(stopChar rune) (string, []string, error) {
 			if ch == sw.escapeToken {
 				if sw.rawEscapes {
 					words.addRawChar(ch)
+					result.WriteRune(ch)
 				}
 
 				// '\' (default escape token, but ` allowed) escapes, except end of line