Merge pull request #2442 from jedevc/heredoc-copy-symbols

Fix #2439 (`COPY` with Heredocs eats quotes)
master
Tõnis Tiigi 2022-01-31 10:37:41 -08:00 committed by GitHub
commit 2633c96bac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 209 additions and 36 deletions

View File

@ -594,6 +594,21 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
return err
}
}
if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansionRaw); ok {
err := ex.ExpandRaw(func(word string) (string, error) {
env, err := d.state.Env(context.TODO())
if err != nil {
return "", err
}
lex := shell.NewLex('\\')
lex.SkipProcessQuotes = true
return lex.ProcessWord(word, env)
})
if err != nil {
return err
}
}
var err error
switch c := cmd.Command.(type) {

View File

@ -20,6 +20,7 @@ import (
var hdTests = integration.TestFuncs(
testCopyHeredoc,
testCopyHeredocSpecialSymbols,
testRunBasicHeredoc,
testRunFakeHeredoc,
testRunShebangHeredoc,
@ -112,6 +113,94 @@ COPY --from=build /dest /
}
}
func testCopyHeredocSpecialSymbols(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
dockerfile := []byte(`
FROM scratch
COPY <<EOF quotefile
"quotes in file"
EOF
COPY <<EOF slashfile1
\
EOF
COPY <<EOF slashfile2
\\
EOF
COPY <<EOF slashfile3
\$
EOF
COPY <<"EOF" rawslashfile1
\
EOF
COPY <<"EOF" rawslashfile2
\\
EOF
COPY <<"EOF" rawslashfile3
\$
EOF
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)
c, err := client.New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()
destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
dt, err := ioutil.ReadFile(filepath.Join(destDir, "quotefile"))
require.NoError(t, err)
require.Equal(t, "\"quotes in file\"\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "slashfile1"))
require.NoError(t, err)
require.Equal(t, "\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "slashfile2"))
require.NoError(t, err)
require.Equal(t, "\\\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "slashfile3"))
require.NoError(t, err)
require.Equal(t, "$\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "rawslashfile1"))
require.NoError(t, err)
require.Equal(t, "\\\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "rawslashfile2"))
require.NoError(t, err)
require.Equal(t, "\\\\\n", string(dt))
dt, err = ioutil.ReadFile(filepath.Join(destDir, "rawslashfile3"))
require.NoError(t, err)
require.Equal(t, "\\$\n", string(dt))
}
func testRunBasicHeredoc(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)
@ -449,6 +538,25 @@ COPY <<"EOF" /dest/c3
Hello ${name}!
EOF
COPY <<EOF /dest/q1
Hello '${name}'!
EOF
COPY <<EOF /dest/q2
Hello "${name}"!
EOF
COPY <<'EOF' /dest/qsingle1
Hello '${name}'!
EOF
COPY <<'EOF' /dest/qsingle2
Hello "${name}"!
EOF
COPY <<"EOF" /dest/qdouble1
Hello '${name}'!
EOF
COPY <<"EOF" /dest/qdouble2
Hello "${name}"!
EOF
RUN <<EOF
greeting="Hello"
echo "${greeting} ${name}!" > /dest/r1
@ -491,11 +599,17 @@ COPY --from=build /dest /
require.NoError(t, err)
contents := map[string]string{
"c1": "Hello world!\n",
"c2": "Hello ${name}!\n",
"c3": "Hello ${name}!\n",
"r1": "Hello world!\n",
"r2": "Hello new world!\n",
"c1": "Hello world!\n",
"c2": "Hello ${name}!\n",
"c3": "Hello ${name}!\n",
"q1": "Hello 'world'!\n",
"q2": "Hello \"world\"!\n",
"qsingle1": "Hello '${name}'!\n",
"qsingle2": "Hello \"${name}\"!\n",
"qdouble1": "Hello '${name}'!\n",
"qdouble2": "Hello \"${name}\"!\n",
"r1": "Hello world!\n",
"r2": "Hello new world!\n",
}
for name, content := range contents {

View File

@ -71,11 +71,18 @@ func newWithNameAndCode(req parseRequest) withNameAndCode {
// SingleWordExpander is a provider for variable expansion where 1 word => 1 output
type SingleWordExpander func(word string) (string, error)
// SupportsSingleWordExpansion interface marks a command as supporting variable expansion
// SupportsSingleWordExpansion interface marks a command as supporting variable
// expansion
type SupportsSingleWordExpansion interface {
Expand(expander SingleWordExpander) error
}
// SupportsSingleWordExpansionRaw interface marks a command as supporting
// variable expansion, while ensuring that quotes are preserved
type SupportsSingleWordExpansionRaw interface {
ExpandRaw(expander SingleWordExpander) error
}
// PlatformSpecific adds platform checks to a command
type PlatformSpecific interface {
CheckPlatform(platform string) error
@ -180,18 +187,6 @@ type SourcesAndDest struct {
}
func (s *SourcesAndDest) Expand(expander SingleWordExpander) error {
for i, content := range s.SourceContents {
if !content.Expand {
continue
}
expandedData, err := expander(content.Data)
if err != nil {
return err
}
s.SourceContents[i].Data = expandedData
}
err := expandSliceInPlace(s.SourcePaths, expander)
if err != nil {
return err
@ -206,6 +201,21 @@ func (s *SourcesAndDest) Expand(expander SingleWordExpander) error {
return nil
}
func (s *SourcesAndDest) ExpandRaw(expander SingleWordExpander) error {
for i, content := range s.SourceContents {
if !content.Expand {
continue
}
expandedData, err := expander(content.Data)
if err != nil {
return err
}
s.SourceContents[i].Data = expandedData
}
return nil
}
// AddCommand : ADD foo /path
//
// Add the file 'foo' to '/path'. Tarball and Remote URL (http, https) handling

View File

@ -147,6 +147,21 @@ EOF`,
},
},
},
{
dockerfile: `COPY <<EOF /quotes
"quotes"
EOF`,
sourcesAndDest: SourcesAndDest{
DestPath: "/quotes",
SourceContents: []SourceContent{
{
Path: "EOF",
Data: "\"quotes\"\n",
Expand: true,
},
},
},
},
}
for _, c := range cases {

View File

@ -73,6 +73,11 @@ FILE1
content 2
FILE2
COPY <<EOF /quotes
"foo"
'bar'
EOF
COPY <<X <<Y /dest
Y
X
@ -211,6 +216,14 @@ $EOF
Expand: true,
},
},
{
// COPY <<EOF /quotes
{
Name: "EOF",
Content: "\"foo\"\n'bar'\n",
Expand: true,
},
},
{
// COPY <<X <<Y /dest
{

View File

@ -18,10 +18,11 @@ import (
// It doesn't support all flavors of ${xx:...} formats but new ones can
// be added by adding code to the "special ${} format processing" section
type Lex struct {
escapeToken rune
RawQuotes bool
RawEscapes bool
SkipUnsetEnv bool
escapeToken rune
RawQuotes bool
RawEscapes bool
SkipProcessQuotes bool
SkipUnsetEnv bool
}
// NewLex creates a new Lex which uses escapeToken to escape quotes.
@ -62,23 +63,25 @@ func (s *Lex) ProcessWordsWithMap(word string, env map[string]string) ([]string,
func (s *Lex) process(word string, env map[string]string) (string, []string, error) {
sw := &shellWord{
envs: env,
escapeToken: s.escapeToken,
skipUnsetEnv: s.SkipUnsetEnv,
rawQuotes: s.RawQuotes,
rawEscapes: s.RawEscapes,
envs: env,
escapeToken: s.escapeToken,
skipUnsetEnv: s.SkipUnsetEnv,
skipProcessQuotes: s.SkipProcessQuotes,
rawQuotes: s.RawQuotes,
rawEscapes: s.RawEscapes,
}
sw.scanner.Init(strings.NewReader(word))
return sw.process(word)
}
type shellWord struct {
scanner scanner.Scanner
envs map[string]string
escapeToken rune
rawQuotes bool
rawEscapes bool
skipUnsetEnv bool
scanner scanner.Scanner
envs map[string]string
escapeToken rune
rawQuotes bool
rawEscapes bool
skipUnsetEnv bool
skipProcessQuotes bool
}
func (sw *shellWord) process(source string) (string, []string, error) {
@ -141,9 +144,11 @@ func (sw *shellWord) processStopOn(stopChar rune) (string, []string, error) {
var words wordsStruct
var charFuncMapping = map[rune]func() (string, error){
'\'': sw.processSingleQuote,
'"': sw.processDoubleQuote,
'$': sw.processDollar,
'$': sw.processDollar,
}
if !sw.skipProcessQuotes {
charFuncMapping['\''] = sw.processSingleQuote
charFuncMapping['"'] = sw.processDoubleQuote
}
for sw.scanner.Peek() != scanner.EOF {
@ -173,6 +178,7 @@ func (sw *shellWord) processStopOn(stopChar rune) (string, []string, error) {
if ch == sw.escapeToken {
if sw.rawEscapes {
words.addRawChar(ch)
result.WriteRune(ch)
}
// '\' (default escape token, but ` allowed) escapes, except end of line