Introduce a typed command system and 2 phase parse/dispatch build

This is a work base to introduce more features like build time
dockerfile optimisations, dependency analysis and parallel build, as
well as a first step to go from a dispatch-inline process to a
frontend+backend process.

Signed-off-by: Simon Ferquel <simon.ferquel@docker.com>

rewritten from github.com/moby/moby 669c0677980b04bcbf871bb7c2d9f07caccfd42b
docker-18.09
Simon Ferquel 2017-05-22 17:21:17 +02:00 committed by Tonis Tiigi
parent 52c2623316
commit 8cde317b4c
9 changed files with 1725 additions and 0 deletions

View File

@ -0,0 +1,183 @@
package instructions
import (
"fmt"
"strings"
)
// FlagType is the type of the build flag
type FlagType int
const (
boolType FlagType = iota
stringType
)
// BFlags contains all flags information for the builder
type BFlags struct {
Args []string // actual flags/args from cmd line
flags map[string]*Flag
used map[string]*Flag
Err error
}
// Flag contains all information for a flag
type Flag struct {
bf *BFlags
name string
flagType FlagType
Value string
}
// NewBFlags returns the new BFlags struct
func NewBFlags() *BFlags {
return &BFlags{
flags: make(map[string]*Flag),
used: make(map[string]*Flag),
}
}
// NewBFlagsWithArgs returns the new BFlags struct with Args set to args
func NewBFlagsWithArgs(args []string) *BFlags {
flags := NewBFlags()
flags.Args = args
return flags
}
// AddBool adds a bool flag to BFlags
// Note, any error will be generated when Parse() is called (see Parse).
func (bf *BFlags) AddBool(name string, def bool) *Flag {
flag := bf.addFlag(name, boolType)
if flag == nil {
return nil
}
if def {
flag.Value = "true"
} else {
flag.Value = "false"
}
return flag
}
// AddString adds a string flag to BFlags
// Note, any error will be generated when Parse() is called (see Parse).
func (bf *BFlags) AddString(name string, def string) *Flag {
flag := bf.addFlag(name, stringType)
if flag == nil {
return nil
}
flag.Value = def
return flag
}
// addFlag is a generic func used by the other AddXXX() func
// to add a new flag to the BFlags struct.
// Note, any error will be generated when Parse() is called (see Parse).
func (bf *BFlags) addFlag(name string, flagType FlagType) *Flag {
if _, ok := bf.flags[name]; ok {
bf.Err = fmt.Errorf("Duplicate flag defined: %s", name)
return nil
}
newFlag := &Flag{
bf: bf,
name: name,
flagType: flagType,
}
bf.flags[name] = newFlag
return newFlag
}
// IsUsed checks if the flag is used
func (fl *Flag) IsUsed() bool {
if _, ok := fl.bf.used[fl.name]; ok {
return true
}
return false
}
// IsTrue checks if a bool flag is true
func (fl *Flag) IsTrue() bool {
if fl.flagType != boolType {
// Should never get here
panic(fmt.Errorf("Trying to use IsTrue on a non-boolean: %s", fl.name))
}
return fl.Value == "true"
}
// Parse parses and checks if the BFlags is valid.
// Any error noticed during the AddXXX() funcs will be generated/returned
// here. We do this because an error during AddXXX() is more like a
// compile time error so it doesn't matter too much when we stop our
// processing as long as we do stop it, so this allows the code
// around AddXXX() to be just:
// defFlag := AddString("description", "")
// w/o needing to add an if-statement around each one.
func (bf *BFlags) Parse() error {
// If there was an error while defining the possible flags
// go ahead and bubble it back up here since we didn't do it
// earlier in the processing
if bf.Err != nil {
return fmt.Errorf("Error setting up flags: %s", bf.Err)
}
for _, arg := range bf.Args {
if !strings.HasPrefix(arg, "--") {
return fmt.Errorf("Arg should start with -- : %s", arg)
}
if arg == "--" {
return nil
}
arg = arg[2:]
value := ""
index := strings.Index(arg, "=")
if index >= 0 {
value = arg[index+1:]
arg = arg[:index]
}
flag, ok := bf.flags[arg]
if !ok {
return fmt.Errorf("Unknown flag: %s", arg)
}
if _, ok = bf.used[arg]; ok {
return fmt.Errorf("Duplicate flag specified: %s", arg)
}
bf.used[arg] = flag
switch flag.flagType {
case boolType:
// value == "" is only ok if no "=" was specified
if index >= 0 && value == "" {
return fmt.Errorf("Missing a value on flag: %s", arg)
}
lower := strings.ToLower(value)
if lower == "" {
flag.Value = "true"
} else if lower == "true" || lower == "false" {
flag.Value = lower
} else {
return fmt.Errorf("Expecting boolean value for flag %s, not: %s", arg, value)
}
case stringType:
if index < 0 {
return fmt.Errorf("Missing a value on flag: %s", arg)
}
flag.Value = value
default:
panic("No idea what kind of flag we have! Should never get here!")
}
}
return nil
}

View File

@ -0,0 +1,187 @@
package instructions
import (
"testing"
)
func TestBuilderFlags(t *testing.T) {
var expected string
var err error
// ---
bf := NewBFlags()
bf.Args = []string{}
if err := bf.Parse(); err != nil {
t.Fatalf("Test1 of %q was supposed to work: %s", bf.Args, err)
}
// ---
bf = NewBFlags()
bf.Args = []string{"--"}
if err := bf.Parse(); err != nil {
t.Fatalf("Test2 of %q was supposed to work: %s", bf.Args, err)
}
// ---
bf = NewBFlags()
flStr1 := bf.AddString("str1", "")
flBool1 := bf.AddBool("bool1", false)
bf.Args = []string{}
if err = bf.Parse(); err != nil {
t.Fatalf("Test3 of %q was supposed to work: %s", bf.Args, err)
}
if flStr1.IsUsed() {
t.Fatal("Test3 - str1 was not used!")
}
if flBool1.IsUsed() {
t.Fatal("Test3 - bool1 was not used!")
}
// ---
bf = NewBFlags()
flStr1 = bf.AddString("str1", "HI")
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{}
if err = bf.Parse(); err != nil {
t.Fatalf("Test4 of %q was supposed to work: %s", bf.Args, err)
}
if flStr1.Value != "HI" {
t.Fatal("Str1 was supposed to default to: HI")
}
if flBool1.IsTrue() {
t.Fatal("Bool1 was supposed to default to: false")
}
if flStr1.IsUsed() {
t.Fatal("Str1 was not used!")
}
if flBool1.IsUsed() {
t.Fatal("Bool1 was not used!")
}
// ---
bf = NewBFlags()
flStr1 = bf.AddString("str1", "HI")
bf.Args = []string{"--str1"}
if err = bf.Parse(); err == nil {
t.Fatalf("Test %q was supposed to fail", bf.Args)
}
// ---
bf = NewBFlags()
flStr1 = bf.AddString("str1", "HI")
bf.Args = []string{"--str1="}
if err = bf.Parse(); err != nil {
t.Fatalf("Test %q was supposed to work: %s", bf.Args, err)
}
expected = ""
if flStr1.Value != expected {
t.Fatalf("Str1 (%q) should be: %q", flStr1.Value, expected)
}
// ---
bf = NewBFlags()
flStr1 = bf.AddString("str1", "HI")
bf.Args = []string{"--str1=BYE"}
if err = bf.Parse(); err != nil {
t.Fatalf("Test %q was supposed to work: %s", bf.Args, err)
}
expected = "BYE"
if flStr1.Value != expected {
t.Fatalf("Str1 (%q) should be: %q", flStr1.Value, expected)
}
// ---
bf = NewBFlags()
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{"--bool1"}
if err = bf.Parse(); err != nil {
t.Fatalf("Test %q was supposed to work: %s", bf.Args, err)
}
if !flBool1.IsTrue() {
t.Fatal("Test-b1 Bool1 was supposed to be true")
}
// ---
bf = NewBFlags()
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{"--bool1=true"}
if err = bf.Parse(); err != nil {
t.Fatalf("Test %q was supposed to work: %s", bf.Args, err)
}
if !flBool1.IsTrue() {
t.Fatal("Test-b2 Bool1 was supposed to be true")
}
// ---
bf = NewBFlags()
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{"--bool1=false"}
if err = bf.Parse(); err != nil {
t.Fatalf("Test %q was supposed to work: %s", bf.Args, err)
}
if flBool1.IsTrue() {
t.Fatal("Test-b3 Bool1 was supposed to be false")
}
// ---
bf = NewBFlags()
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{"--bool1=false1"}
if err = bf.Parse(); err == nil {
t.Fatalf("Test %q was supposed to fail", bf.Args)
}
// ---
bf = NewBFlags()
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{"--bool2"}
if err = bf.Parse(); err == nil {
t.Fatalf("Test %q was supposed to fail", bf.Args)
}
// ---
bf = NewBFlags()
flStr1 = bf.AddString("str1", "HI")
flBool1 = bf.AddBool("bool1", false)
bf.Args = []string{"--bool1", "--str1=BYE"}
if err = bf.Parse(); err != nil {
t.Fatalf("Test %q was supposed to work: %s", bf.Args, err)
}
if flStr1.Value != "BYE" {
t.Fatalf("Test %s, str1 should be BYE", bf.Args)
}
if !flBool1.IsTrue() {
t.Fatalf("Test %s, bool1 should be true", bf.Args)
}
}

View File

@ -0,0 +1,396 @@
package instructions
import (
"errors"
"strings"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/strslice"
)
// KeyValuePair represent an arbitrary named value (usefull in slice insted of map[string] string to preserve ordering)
type KeyValuePair struct {
Key string
Value string
}
func (kvp *KeyValuePair) String() string {
return kvp.Key + "=" + kvp.Value
}
// Command is implemented by every command present in a dockerfile
type Command interface {
Name() string
}
// KeyValuePairs is a slice of KeyValuePair
type KeyValuePairs []KeyValuePair
// withNameAndCode is the base of every command in a Dockerfile (String() returns its source code)
type withNameAndCode struct {
code string
name string
}
func (c *withNameAndCode) String() string {
return c.code
}
// Name of the command
func (c *withNameAndCode) Name() string {
return c.name
}
func newWithNameAndCode(req parseRequest) withNameAndCode {
return withNameAndCode{code: strings.TrimSpace(req.original), name: req.command}
}
// SingleWordExpander is a provider for variable expansion where 1 word => 1 output
type SingleWordExpander func(word string) (string, error)
// SupportsSingleWordExpansion interface marks a command as supporting variable expansion
type SupportsSingleWordExpansion interface {
Expand(expander SingleWordExpander) error
}
// PlatformSpecific adds platform checks to a command
type PlatformSpecific interface {
CheckPlatform(platform string) error
}
func expandKvp(kvp KeyValuePair, expander SingleWordExpander) (KeyValuePair, error) {
key, err := expander(kvp.Key)
if err != nil {
return KeyValuePair{}, err
}
value, err := expander(kvp.Value)
if err != nil {
return KeyValuePair{}, err
}
return KeyValuePair{Key: key, Value: value}, nil
}
func expandKvpsInPlace(kvps KeyValuePairs, expander SingleWordExpander) error {
for i, kvp := range kvps {
newKvp, err := expandKvp(kvp, expander)
if err != nil {
return err
}
kvps[i] = newKvp
}
return nil
}
func expandSliceInPlace(values []string, expander SingleWordExpander) error {
for i, v := range values {
newValue, err := expander(v)
if err != nil {
return err
}
values[i] = newValue
}
return nil
}
// EnvCommand : ENV key1 value1 [keyN valueN...]
type EnvCommand struct {
withNameAndCode
Env KeyValuePairs // kvp slice instead of map to preserve ordering
}
// Expand variables
func (c *EnvCommand) Expand(expander SingleWordExpander) error {
return expandKvpsInPlace(c.Env, expander)
}
// MaintainerCommand : MAINTAINER maintainer_name
type MaintainerCommand struct {
withNameAndCode
Maintainer string
}
// LabelCommand : LABEL some json data describing the image
//
// Sets the Label variable foo to bar,
//
type LabelCommand struct {
withNameAndCode
Labels KeyValuePairs // kvp slice instead of map to preserve ordering
}
// Expand variables
func (c *LabelCommand) Expand(expander SingleWordExpander) error {
return expandKvpsInPlace(c.Labels, expander)
}
// SourcesAndDest represent a list of source files and a destination
type SourcesAndDest []string
// Sources list the source paths
func (s SourcesAndDest) Sources() []string {
res := make([]string, len(s)-1)
copy(res, s[:len(s)-1])
return res
}
// Dest path of the operation
func (s SourcesAndDest) Dest() string {
return s[len(s)-1]
}
// AddCommand : ADD foo /path
//
// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling
// exist here. If you do not wish to have this automatic handling, use COPY.
//
type AddCommand struct {
withNameAndCode
SourcesAndDest
Chown string
}
// Expand variables
func (c *AddCommand) Expand(expander SingleWordExpander) error {
return expandSliceInPlace(c.SourcesAndDest, expander)
}
// CopyCommand : COPY foo /path
//
// Same as 'ADD' but without the tar and remote url handling.
//
type CopyCommand struct {
withNameAndCode
SourcesAndDest
From string
Chown string
}
// Expand variables
func (c *CopyCommand) Expand(expander SingleWordExpander) error {
return expandSliceInPlace(c.SourcesAndDest, expander)
}
// OnbuildCommand : ONBUILD <some other command>
type OnbuildCommand struct {
withNameAndCode
Expression string
}
// WorkdirCommand : WORKDIR /tmp
//
// Set the working directory for future RUN/CMD/etc statements.
//
type WorkdirCommand struct {
withNameAndCode
Path string
}
// Expand variables
func (c *WorkdirCommand) Expand(expander SingleWordExpander) error {
p, err := expander(c.Path)
if err != nil {
return err
}
c.Path = p
return nil
}
// ShellDependantCmdLine represents a cmdline optionaly prepended with the shell
type ShellDependantCmdLine struct {
CmdLine strslice.StrSlice
PrependShell bool
}
// RunCommand : RUN some command yo
//
// run a command and commit the image. Args are automatically prepended with
// the current SHELL which defaults to 'sh -c' under linux or 'cmd /S /C' under
// Windows, in the event there is only one argument The difference in processing:
//
// RUN echo hi # sh -c echo hi (Linux)
// RUN echo hi # cmd /S /C echo hi (Windows)
// RUN [ "echo", "hi" ] # echo hi
//
type RunCommand struct {
withNameAndCode
ShellDependantCmdLine
}
// CmdCommand : CMD foo
//
// Set the default command to run in the container (which may be empty).
// Argument handling is the same as RUN.
//
type CmdCommand struct {
withNameAndCode
ShellDependantCmdLine
}
// HealthCheckCommand : HEALTHCHECK foo
//
// Set the default healthcheck command to run in the container (which may be empty).
// Argument handling is the same as RUN.
//
type HealthCheckCommand struct {
withNameAndCode
Health *container.HealthConfig
}
// EntrypointCommand : ENTRYPOINT /usr/sbin/nginx
//
// Set the entrypoint to /usr/sbin/nginx. Will accept the CMD as the arguments
// to /usr/sbin/nginx. Uses the default shell if not in JSON format.
//
// Handles command processing similar to CMD and RUN, only req.runConfig.Entrypoint
// is initialized at newBuilder time instead of through argument parsing.
//
type EntrypointCommand struct {
withNameAndCode
ShellDependantCmdLine
}
// ExposeCommand : EXPOSE 6667/tcp 7000/tcp
//
// Expose ports for links and port mappings. This all ends up in
// req.runConfig.ExposedPorts for runconfig.
//
type ExposeCommand struct {
withNameAndCode
Ports []string
}
// UserCommand : USER foo
//
// Set the user to 'foo' for future commands and when running the
// ENTRYPOINT/CMD at container run time.
//
type UserCommand struct {
withNameAndCode
User string
}
// Expand variables
func (c *UserCommand) Expand(expander SingleWordExpander) error {
p, err := expander(c.User)
if err != nil {
return err
}
c.User = p
return nil
}
// VolumeCommand : VOLUME /foo
//
// Expose the volume /foo for use. Will also accept the JSON array form.
//
type VolumeCommand struct {
withNameAndCode
Volumes []string
}
// Expand variables
func (c *VolumeCommand) Expand(expander SingleWordExpander) error {
return expandSliceInPlace(c.Volumes, expander)
}
// StopSignalCommand : STOPSIGNAL signal
//
// Set the signal that will be used to kill the container.
type StopSignalCommand struct {
withNameAndCode
Signal string
}
// Expand variables
func (c *StopSignalCommand) Expand(expander SingleWordExpander) error {
p, err := expander(c.Signal)
if err != nil {
return err
}
c.Signal = p
return nil
}
// CheckPlatform checks that the command is supported in the target platform
func (c *StopSignalCommand) CheckPlatform(platform string) error {
if platform == "windows" {
return errors.New("The daemon on this platform does not support the command stopsignal")
}
return nil
}
// ArgCommand : ARG name[=value]
//
// Adds the variable foo to the trusted list of variables that can be passed
// to builder using the --build-arg flag for expansion/substitution or passing to 'run'.
// Dockerfile author may optionally set a default value of this variable.
type ArgCommand struct {
withNameAndCode
Key string
Value *string
}
// Expand variables
func (c *ArgCommand) Expand(expander SingleWordExpander) error {
p, err := expander(c.Key)
if err != nil {
return err
}
c.Key = p
if c.Value != nil {
p, err = expander(*c.Value)
if err != nil {
return err
}
c.Value = &p
}
return nil
}
// ShellCommand : SHELL powershell -command
//
// Set the non-default shell to use.
type ShellCommand struct {
withNameAndCode
Shell strslice.StrSlice
}
// Stage represents a single stage in a multi-stage build
type Stage struct {
Name string
Commands []Command
BaseName string
SourceCode string
}
// AddCommand to the stage
func (s *Stage) AddCommand(cmd Command) {
// todo: validate cmd type
s.Commands = append(s.Commands, cmd)
}
// IsCurrentStage check if the stage name is the current stage
func IsCurrentStage(s []Stage, name string) bool {
if len(s) == 0 {
return false
}
return s[len(s)-1].Name == name
}
// CurrentStage return the last stage in a slice
func CurrentStage(s []Stage) (*Stage, error) {
if len(s) == 0 {
return nil, errors.New("No build stage in current context")
}
return &s[len(s)-1], nil
}
// HasStage looks for the presence of a given stage name
func HasStage(s []Stage, name string) (int, bool) {
for i, stage := range s {
if stage.Name == name {
return i, true
}
}
return -1, false
}

View File

@ -0,0 +1,9 @@
// +build !windows
package instructions
import "fmt"
func errNotJSON(command, _ string) error {
return fmt.Errorf("%s requires the arguments to be in JSON form", command)
}

View File

@ -0,0 +1,27 @@
package instructions
import (
"fmt"
"path/filepath"
"regexp"
"strings"
)
func errNotJSON(command, original string) error {
// For Windows users, give a hint if it looks like it might contain
// a path which hasn't been escaped such as ["c:\windows\system32\prog.exe", "-param"],
// as JSON must be escaped. Unfortunate...
//
// Specifically looking for quote-driveletter-colon-backslash, there's no
// double backslash and a [] pair. No, this is not perfect, but it doesn't
// have to be. It's simply a hint to make life a little easier.
extra := ""
original = filepath.FromSlash(strings.ToLower(strings.Replace(strings.ToLower(original), strings.ToLower(command)+" ", "", -1)))
if len(regexp.MustCompile(`"[a-z]:\\.*`).FindStringSubmatch(original)) > 0 &&
!strings.Contains(original, `\\`) &&
strings.Contains(original, "[") &&
strings.Contains(original, "]") {
extra = fmt.Sprintf(`. It looks like '%s' includes a file path without an escaped back-slash. JSON requires back-slashes to be escaped such as ["c:\\path\\to\\file.exe", "/parameter"]`, original)
}
return fmt.Errorf("%s requires the arguments to be in JSON form%s", command, extra)
}

View File

@ -0,0 +1,635 @@
package instructions
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/strslice"
"github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/pkg/errors"
)
type parseRequest struct {
command string
args []string
attributes map[string]bool
flags *BFlags
original string
}
func nodeArgs(node *parser.Node) []string {
result := []string{}
for ; node.Next != nil; node = node.Next {
arg := node.Next
if len(arg.Children) == 0 {
result = append(result, arg.Value)
} else if len(arg.Children) == 1 {
//sub command
result = append(result, arg.Children[0].Value)
result = append(result, nodeArgs(arg.Children[0])...)
}
}
return result
}
func newParseRequestFromNode(node *parser.Node) parseRequest {
return parseRequest{
command: node.Value,
args: nodeArgs(node),
attributes: node.Attributes,
original: node.Original,
flags: NewBFlagsWithArgs(node.Flags),
}
}
// ParseInstruction converts an AST to a typed instruction (either a command or a build stage beginning when encountering a `FROM` statement)
func ParseInstruction(node *parser.Node) (interface{}, error) {
req := newParseRequestFromNode(node)
switch node.Value {
case command.Env:
return parseEnv(req)
case command.Maintainer:
return parseMaintainer(req)
case command.Label:
return parseLabel(req)
case command.Add:
return parseAdd(req)
case command.Copy:
return parseCopy(req)
case command.From:
return parseFrom(req)
case command.Onbuild:
return parseOnBuild(req)
case command.Workdir:
return parseWorkdir(req)
case command.Run:
return parseRun(req)
case command.Cmd:
return parseCmd(req)
case command.Healthcheck:
return parseHealthcheck(req)
case command.Entrypoint:
return parseEntrypoint(req)
case command.Expose:
return parseExpose(req)
case command.User:
return parseUser(req)
case command.Volume:
return parseVolume(req)
case command.StopSignal:
return parseStopSignal(req)
case command.Arg:
return parseArg(req)
case command.Shell:
return parseShell(req)
}
return nil, &UnknownInstruction{Instruction: node.Value, Line: node.StartLine}
}
// ParseCommand converts an AST to a typed Command
func ParseCommand(node *parser.Node) (Command, error) {
s, err := ParseInstruction(node)
if err != nil {
return nil, err
}
if c, ok := s.(Command); ok {
return c, nil
}
return nil, errors.Errorf("%T is not a command type", s)
}
// UnknownInstruction represents an error occuring when a command is unresolvable
type UnknownInstruction struct {
Line int
Instruction string
}
func (e *UnknownInstruction) Error() string {
return fmt.Sprintf("unknown instruction: %s", strings.ToUpper(e.Instruction))
}
// IsUnknownInstruction checks if the error is an UnknownInstruction or a parseError containing an UnknownInstruction
func IsUnknownInstruction(err error) bool {
_, ok := err.(*UnknownInstruction)
if !ok {
var pe *parseError
if pe, ok = err.(*parseError); ok {
_, ok = pe.inner.(*UnknownInstruction)
}
}
return ok
}
type parseError struct {
inner error
node *parser.Node
}
func (e *parseError) Error() string {
return fmt.Sprintf("Dockerfile parse error line %d: %v", e.node.StartLine, e.inner.Error())
}
// Parse a docker file into a collection of buildable stages
func Parse(ast *parser.Node) (stages []Stage, metaArgs []ArgCommand, err error) {
for _, n := range ast.Children {
cmd, err := ParseInstruction(n)
if err != nil {
return nil, nil, &parseError{inner: err, node: n}
}
if len(stages) == 0 {
// meta arg case
if a, isArg := cmd.(*ArgCommand); isArg {
metaArgs = append(metaArgs, *a)
continue
}
}
switch c := cmd.(type) {
case *Stage:
stages = append(stages, *c)
case Command:
stage, err := CurrentStage(stages)
if err != nil {
return nil, nil, err
}
stage.AddCommand(c)
default:
return nil, nil, errors.Errorf("%T is not a command type", cmd)
}
}
return stages, metaArgs, nil
}
func parseKvps(args []string, cmdName string) (KeyValuePairs, error) {
if len(args) == 0 {
return nil, errAtLeastOneArgument(cmdName)
}
if len(args)%2 != 0 {
// should never get here, but just in case
return nil, errTooManyArguments(cmdName)
}
var res KeyValuePairs
for j := 0; j < len(args); j += 2 {
if len(args[j]) == 0 {
return nil, errBlankCommandNames(cmdName)
}
name := args[j]
value := args[j+1]
res = append(res, KeyValuePair{Key: name, Value: value})
}
return res, nil
}
func parseEnv(req parseRequest) (*EnvCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
envs, err := parseKvps(req.args, "ENV")
if err != nil {
return nil, err
}
return &EnvCommand{
Env: envs,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseMaintainer(req parseRequest) (*MaintainerCommand, error) {
if len(req.args) != 1 {
return nil, errExactlyOneArgument("MAINTAINER")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
return &MaintainerCommand{
Maintainer: req.args[0],
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseLabel(req parseRequest) (*LabelCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
labels, err := parseKvps(req.args, "LABEL")
if err != nil {
return nil, err
}
return &LabelCommand{
Labels: labels,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseAdd(req parseRequest) (*AddCommand, error) {
if len(req.args) < 2 {
return nil, errAtLeastTwoArguments("ADD")
}
flChown := req.flags.AddString("chown", "")
if err := req.flags.Parse(); err != nil {
return nil, err
}
return &AddCommand{
SourcesAndDest: SourcesAndDest(req.args),
withNameAndCode: newWithNameAndCode(req),
Chown: flChown.Value,
}, nil
}
func parseCopy(req parseRequest) (*CopyCommand, error) {
if len(req.args) < 2 {
return nil, errAtLeastTwoArguments("COPY")
}
flChown := req.flags.AddString("chown", "")
flFrom := req.flags.AddString("from", "")
if err := req.flags.Parse(); err != nil {
return nil, err
}
return &CopyCommand{
SourcesAndDest: SourcesAndDest(req.args),
From: flFrom.Value,
withNameAndCode: newWithNameAndCode(req),
Chown: flChown.Value,
}, nil
}
func parseFrom(req parseRequest) (*Stage, error) {
stageName, err := parseBuildStageName(req.args)
if err != nil {
return nil, err
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
code := strings.TrimSpace(req.original)
return &Stage{
BaseName: req.args[0],
Name: stageName,
SourceCode: code,
Commands: []Command{},
}, nil
}
func parseBuildStageName(args []string) (string, error) {
stageName := ""
switch {
case len(args) == 3 && strings.EqualFold(args[1], "as"):
stageName = strings.ToLower(args[2])
if ok, _ := regexp.MatchString("^[a-z][a-z0-9-_\\.]*$", stageName); !ok {
return "", errors.Errorf("invalid name for build stage: %q, name can't start with a number or contain symbols", stageName)
}
case len(args) != 1:
return "", errors.New("FROM requires either one or three arguments")
}
return stageName, nil
}
func parseOnBuild(req parseRequest) (*OnbuildCommand, error) {
if len(req.args) == 0 {
return nil, errAtLeastOneArgument("ONBUILD")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
triggerInstruction := strings.ToUpper(strings.TrimSpace(req.args[0]))
switch strings.ToUpper(triggerInstruction) {
case "ONBUILD":
return nil, errors.New("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed")
case "MAINTAINER", "FROM":
return nil, fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction)
}
original := regexp.MustCompile(`(?i)^\s*ONBUILD\s*`).ReplaceAllString(req.original, "")
return &OnbuildCommand{
Expression: original,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseWorkdir(req parseRequest) (*WorkdirCommand, error) {
if len(req.args) != 1 {
return nil, errExactlyOneArgument("WORKDIR")
}
err := req.flags.Parse()
if err != nil {
return nil, err
}
return &WorkdirCommand{
Path: req.args[0],
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependantCmdLine {
args := handleJSONArgs(req.args, req.attributes)
cmd := strslice.StrSlice(args)
if emptyAsNil && len(cmd) == 0 {
cmd = nil
}
return ShellDependantCmdLine{
CmdLine: cmd,
PrependShell: !req.attributes["json"],
}
}
func parseRun(req parseRequest) (*RunCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
return &RunCommand{
ShellDependantCmdLine: parseShellDependentCommand(req, false),
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseCmd(req parseRequest) (*CmdCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
return &CmdCommand{
ShellDependantCmdLine: parseShellDependentCommand(req, false),
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
cmd := &EntrypointCommand{
ShellDependantCmdLine: parseShellDependentCommand(req, true),
withNameAndCode: newWithNameAndCode(req),
}
return cmd, nil
}
// parseOptInterval(flag) is the duration of flag.Value, or 0 if
// empty. An error is reported if the value is given and less than minimum duration.
func parseOptInterval(f *Flag) (time.Duration, error) {
s := f.Value
if s == "" {
return 0, nil
}
d, err := time.ParseDuration(s)
if err != nil {
return 0, err
}
if d < container.MinimumDuration {
return 0, fmt.Errorf("Interval %#v cannot be less than %s", f.name, container.MinimumDuration)
}
return d, nil
}
func parseHealthcheck(req parseRequest) (*HealthCheckCommand, error) {
if len(req.args) == 0 {
return nil, errAtLeastOneArgument("HEALTHCHECK")
}
cmd := &HealthCheckCommand{
withNameAndCode: newWithNameAndCode(req),
}
typ := strings.ToUpper(req.args[0])
args := req.args[1:]
if typ == "NONE" {
if len(args) != 0 {
return nil, errors.New("HEALTHCHECK NONE takes no arguments")
}
test := strslice.StrSlice{typ}
cmd.Health = &container.HealthConfig{
Test: test,
}
} else {
healthcheck := container.HealthConfig{}
flInterval := req.flags.AddString("interval", "")
flTimeout := req.flags.AddString("timeout", "")
flStartPeriod := req.flags.AddString("start-period", "")
flRetries := req.flags.AddString("retries", "")
if err := req.flags.Parse(); err != nil {
return nil, err
}
switch typ {
case "CMD":
cmdSlice := handleJSONArgs(args, req.attributes)
if len(cmdSlice) == 0 {
return nil, errors.New("Missing command after HEALTHCHECK CMD")
}
if !req.attributes["json"] {
typ = "CMD-SHELL"
}
healthcheck.Test = strslice.StrSlice(append([]string{typ}, cmdSlice...))
default:
return nil, fmt.Errorf("Unknown type %#v in HEALTHCHECK (try CMD)", typ)
}
interval, err := parseOptInterval(flInterval)
if err != nil {
return nil, err
}
healthcheck.Interval = interval
timeout, err := parseOptInterval(flTimeout)
if err != nil {
return nil, err
}
healthcheck.Timeout = timeout
startPeriod, err := parseOptInterval(flStartPeriod)
if err != nil {
return nil, err
}
healthcheck.StartPeriod = startPeriod
if flRetries.Value != "" {
retries, err := strconv.ParseInt(flRetries.Value, 10, 32)
if err != nil {
return nil, err
}
if retries < 1 {
return nil, fmt.Errorf("--retries must be at least 1 (not %d)", retries)
}
healthcheck.Retries = int(retries)
} else {
healthcheck.Retries = 0
}
cmd.Health = &healthcheck
}
return cmd, nil
}
func parseExpose(req parseRequest) (*ExposeCommand, error) {
portsTab := req.args
if len(req.args) == 0 {
return nil, errAtLeastOneArgument("EXPOSE")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
sort.Strings(portsTab)
return &ExposeCommand{
Ports: portsTab,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseUser(req parseRequest) (*UserCommand, error) {
if len(req.args) != 1 {
return nil, errExactlyOneArgument("USER")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
return &UserCommand{
User: req.args[0],
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseVolume(req parseRequest) (*VolumeCommand, error) {
if len(req.args) == 0 {
return nil, errAtLeastOneArgument("VOLUME")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
cmd := &VolumeCommand{
withNameAndCode: newWithNameAndCode(req),
}
for _, v := range req.args {
v = strings.TrimSpace(v)
if v == "" {
return nil, errors.New("VOLUME specified can not be an empty string")
}
cmd.Volumes = append(cmd.Volumes, v)
}
return cmd, nil
}
func parseStopSignal(req parseRequest) (*StopSignalCommand, error) {
if len(req.args) != 1 {
return nil, errExactlyOneArgument("STOPSIGNAL")
}
sig := req.args[0]
cmd := &StopSignalCommand{
Signal: sig,
withNameAndCode: newWithNameAndCode(req),
}
return cmd, nil
}
func parseArg(req parseRequest) (*ArgCommand, error) {
if len(req.args) != 1 {
return nil, errExactlyOneArgument("ARG")
}
var (
name string
newValue *string
)
arg := req.args[0]
// 'arg' can just be a name or name-value pair. Note that this is different
// from 'env' that handles the split of name and value at the parser level.
// The reason for doing it differently for 'arg' is that we support just
// defining an arg and not assign it a value (while 'env' always expects a
// name-value pair). If possible, it will be good to harmonize the two.
if strings.Contains(arg, "=") {
parts := strings.SplitN(arg, "=", 2)
if len(parts[0]) == 0 {
return nil, errBlankCommandNames("ARG")
}
name = parts[0]
newValue = &parts[1]
} else {
name = arg
}
return &ArgCommand{
Key: name,
Value: newValue,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
func parseShell(req parseRequest) (*ShellCommand, error) {
if err := req.flags.Parse(); err != nil {
return nil, err
}
shellSlice := handleJSONArgs(req.args, req.attributes)
switch {
case len(shellSlice) == 0:
// SHELL []
return nil, errAtLeastOneArgument("SHELL")
case req.attributes["json"]:
// SHELL ["powershell", "-command"]
return &ShellCommand{
Shell: strslice.StrSlice(shellSlice),
withNameAndCode: newWithNameAndCode(req),
}, nil
default:
// SHELL powershell -command - not JSON
return nil, errNotJSON("SHELL", req.original)
}
}
func errAtLeastOneArgument(command string) error {
return errors.Errorf("%s requires at least one argument", command)
}
func errExactlyOneArgument(command string) error {
return errors.Errorf("%s requires exactly one argument", command)
}
func errAtLeastTwoArguments(command string) error {
return errors.Errorf("%s requires at least two arguments", command)
}
func errBlankCommandNames(command string) error {
return errors.Errorf("%s names can not be blank", command)
}
func errTooManyArguments(command string) error {
return errors.Errorf("Bad input to %s, too many arguments", command)
}

View File

@ -0,0 +1,204 @@
package instructions
import (
"strings"
"testing"
"github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/internal/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCommandsExactlyOneArgument(t *testing.T) {
commands := []string{
"MAINTAINER",
"WORKDIR",
"USER",
"STOPSIGNAL",
}
for _, command := range commands {
ast, err := parser.Parse(strings.NewReader(command))
require.NoError(t, err)
_, err = ParseInstruction(ast.AST.Children[0])
assert.EqualError(t, err, errExactlyOneArgument(command).Error())
}
}
func TestCommandsAtLeastOneArgument(t *testing.T) {
commands := []string{
"ENV",
"LABEL",
"ONBUILD",
"HEALTHCHECK",
"EXPOSE",
"VOLUME",
}
for _, command := range commands {
ast, err := parser.Parse(strings.NewReader(command))
require.NoError(t, err)
_, err = ParseInstruction(ast.AST.Children[0])
assert.EqualError(t, err, errAtLeastOneArgument(command).Error())
}
}
func TestCommandsAtLeastTwoArgument(t *testing.T) {
commands := []string{
"ADD",
"COPY",
}
for _, command := range commands {
ast, err := parser.Parse(strings.NewReader(command + " arg1"))
require.NoError(t, err)
_, err = ParseInstruction(ast.AST.Children[0])
assert.EqualError(t, err, errAtLeastTwoArguments(command).Error())
}
}
func TestCommandsTooManyArguments(t *testing.T) {
commands := []string{
"ENV",
"LABEL",
}
for _, command := range commands {
node := &parser.Node{
Original: command + "arg1 arg2 arg3",
Value: strings.ToLower(command),
Next: &parser.Node{
Value: "arg1",
Next: &parser.Node{
Value: "arg2",
Next: &parser.Node{
Value: "arg3",
},
},
},
}
_, err := ParseInstruction(node)
assert.EqualError(t, err, errTooManyArguments(command).Error())
}
}
func TestCommandsBlankNames(t *testing.T) {
commands := []string{
"ENV",
"LABEL",
}
for _, command := range commands {
node := &parser.Node{
Original: command + " =arg2",
Value: strings.ToLower(command),
Next: &parser.Node{
Value: "",
Next: &parser.Node{
Value: "arg2",
},
},
}
_, err := ParseInstruction(node)
assert.EqualError(t, err, errBlankCommandNames(command).Error())
}
}
func TestHealthCheckCmd(t *testing.T) {
node := &parser.Node{
Value: command.Healthcheck,
Next: &parser.Node{
Value: "CMD",
Next: &parser.Node{
Value: "hello",
Next: &parser.Node{
Value: "world",
},
},
},
}
cmd, err := ParseInstruction(node)
assert.NoError(t, err)
hc, ok := cmd.(*HealthCheckCommand)
assert.True(t, ok)
expected := []string{"CMD-SHELL", "hello world"}
assert.Equal(t, expected, hc.Health.Test)
}
func TestParseOptInterval(t *testing.T) {
flInterval := &Flag{
name: "interval",
flagType: stringType,
Value: "50ns",
}
_, err := parseOptInterval(flInterval)
testutil.ErrorContains(t, err, "cannot be less than 1ms")
flInterval.Value = "1ms"
_, err = parseOptInterval(flInterval)
require.NoError(t, err)
}
func TestErrorCases(t *testing.T) {
cases := []struct {
name string
dockerfile string
expectedError string
}{
{
name: "copyEmptyWhitespace",
dockerfile: `COPY
quux \
bar`,
expectedError: "COPY requires at least two arguments",
},
{
name: "ONBUILD forbidden FROM",
dockerfile: "ONBUILD FROM scratch",
expectedError: "FROM isn't allowed as an ONBUILD trigger",
},
{
name: "ONBUILD forbidden MAINTAINER",
dockerfile: "ONBUILD MAINTAINER docker.io",
expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger",
},
{
name: "ARG two arguments",
dockerfile: "ARG foo bar",
expectedError: "ARG requires exactly one argument",
},
{
name: "MAINTAINER unknown flag",
dockerfile: "MAINTAINER --boo joe@example.com",
expectedError: "Unknown flag: boo",
},
{
name: "Chaining ONBUILD",
dockerfile: `ONBUILD ONBUILD RUN touch foobar`,
expectedError: "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed",
},
{
name: "Invalid instruction",
dockerfile: `foo bar`,
expectedError: "unknown instruction: FOO",
},
}
for _, c := range cases {
r := strings.NewReader(c.dockerfile)
ast, err := parser.Parse(r)
if err != nil {
t.Fatalf("Error when parsing Dockerfile: %s", err)
}
n := ast.AST.Children[0]
_, err = ParseInstruction(n)
if err != nil {
testutil.ErrorContains(t, err, c.expectedError)
return
}
t.Fatalf("No error when executing test %s", c.name)
}
}

View File

@ -0,0 +1,19 @@
package instructions
import "strings"
// handleJSONArgs parses command passed to CMD, ENTRYPOINT, RUN and SHELL instruction in Dockerfile
// for exec form it returns untouched args slice
// for shell form it returns concatenated args as the first element of a slice
func handleJSONArgs(args []string, attributes map[string]bool) []string {
if len(args) == 0 {
return []string{}
}
if attributes != nil && attributes["json"] {
return args
}
// literal string command, not an exec array
return []string{strings.Join(args, " ")}
}

View File

@ -0,0 +1,65 @@
package instructions
import "testing"
type testCase struct {
name string
args []string
attributes map[string]bool
expected []string
}
func initTestCases() []testCase {
testCases := []testCase{}
testCases = append(testCases, testCase{
name: "empty args",
args: []string{},
attributes: make(map[string]bool),
expected: []string{},
})
jsonAttributes := make(map[string]bool)
jsonAttributes["json"] = true
testCases = append(testCases, testCase{
name: "json attribute with one element",
args: []string{"foo"},
attributes: jsonAttributes,
expected: []string{"foo"},
})
testCases = append(testCases, testCase{
name: "json attribute with two elements",
args: []string{"foo", "bar"},
attributes: jsonAttributes,
expected: []string{"foo", "bar"},
})
testCases = append(testCases, testCase{
name: "no attributes",
args: []string{"foo", "bar"},
attributes: nil,
expected: []string{"foo bar"},
})
return testCases
}
func TestHandleJSONArgs(t *testing.T) {
testCases := initTestCases()
for _, test := range testCases {
arguments := handleJSONArgs(test.args, test.attributes)
if len(arguments) != len(test.expected) {
t.Fatalf("In test \"%s\": length of returned slice is incorrect. Expected: %d, got: %d", test.name, len(test.expected), len(arguments))
}
for i := range test.expected {
if arguments[i] != test.expected[i] {
t.Fatalf("In test \"%s\": element as position %d is incorrect. Expected: %s, got: %s", test.name, i, test.expected[i], arguments[i])
}
}
}
}