Move builder files to builder/dockerfile

Signed-off-by: Tibor Vass <tibor@docker.com>

rewritten from github.com/moby/moby f41230b93a77ca7f4d84718f39a4847b6117f694

frontend/dockerfile/command/command.go

@@ -0,0 +1,42 @@
+// Package command contains the set of Dockerfile commands.
+package command
+
+// Define constants for the command strings
+const (
+	Env        = "env"
+	Label      = "label"
+	Maintainer = "maintainer"
+	Add        = "add"
+	Copy       = "copy"
+	From       = "from"
+	Onbuild    = "onbuild"
+	Workdir    = "workdir"
+	Run        = "run"
+	Cmd        = "cmd"
+	Entrypoint = "entrypoint"
+	Expose     = "expose"
+	Volume     = "volume"
+	User       = "user"
+	StopSignal = "stopsignal"
+	Arg        = "arg"
+)
+
+// Commands is list of all Dockerfile commands
+var Commands = map[string]struct{}{
+	Env:        {},
+	Label:      {},
+	Maintainer: {},
+	Add:        {},
+	Copy:       {},
+	From:       {},
+	Onbuild:    {},
+	Workdir:    {},
+	Run:        {},
+	Cmd:        {},
+	Entrypoint: {},
+	Expose:     {},
+	Volume:     {},
+	User:       {},
+	StopSignal: {},
+	Arg:        {},
+}

frontend/dockerfile/parser/dumper/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/docker/docker/builder/parser"
+)
+
+func main() {
+	var f *os.File
+	var err error
+
+	if len(os.Args) < 2 {
+		fmt.Println("please supply filename(s)")
+		os.Exit(1)
+	}
+
+	for _, fn := range os.Args[1:] {
+		f, err = os.Open(fn)
+		if err != nil {
+			panic(err)
+		}
+
+		ast, err := parser.Parse(f)
+		if err != nil {
+			panic(err)
+		} else {
+			fmt.Println(ast.Dump())
+		}
+	}
+}

frontend/dockerfile/parser/json_test.go

@@ -0,0 +1,55 @@
+package parser
+
+import (
+	"testing"
+)
+
+var invalidJSONArraysOfStrings = []string{
+	`["a",42,"b"]`,
+	`["a",123.456,"b"]`,
+	`["a",{},"b"]`,
+	`["a",{"c": "d"},"b"]`,
+	`["a",["c"],"b"]`,
+	`["a",true,"b"]`,
+	`["a",false,"b"]`,
+	`["a",null,"b"]`,
+}
+
+var validJSONArraysOfStrings = map[string][]string{
+	`[]`:           {},
+	`[""]`:         {""},
+	`["a"]`:        {"a"},
+	`["a","b"]`:    {"a", "b"},
+	`[ "a", "b" ]`: {"a", "b"},
+	`[	"a",	"b"	]`: {"a", "b"},
+	`	[	"a",	"b"	]	`: {"a", "b"},
+	`["abc 123", "♥", "☃", "\" \\ \/ \b \f \n \r \t \u0000"]`: {"abc 123", "♥", "☃", "\" \\ / \b \f \n \r \t \u0000"},
+}
+
+func TestJSONArraysOfStrings(t *testing.T) {
+	for json, expected := range validJSONArraysOfStrings {
+		if node, _, err := parseJSON(json); err != nil {
+			t.Fatalf("%q should be a valid JSON array of strings, but wasn't! (err: %q)", json, err)
+		} else {
+			i := 0
+			for node != nil {
+				if i >= len(expected) {
+					t.Fatalf("expected result is shorter than parsed result (%d vs %d+) in %q", len(expected), i+1, json)
+				}
+				if node.Value != expected[i] {
+					t.Fatalf("expected %q (not %q) in %q at pos %d", expected[i], node.Value, json, i)
+				}
+				node = node.Next
+				i++
+			}
+			if i != len(expected) {
+				t.Fatalf("expected result is longer than parsed result (%d vs %d) in %q", len(expected), i+1, json)
+			}
+		}
+	}
+	for _, json := range invalidJSONArraysOfStrings {
+		if _, _, err := parseJSON(json); err != errDockerfileNotStringArray {
+			t.Fatalf("%q should be an invalid JSON array of strings, but wasn't!", json)
+		}
+	}
+}

frontend/dockerfile/parser/line_parsers.go

@@ -0,0 +1,331 @@
+package parser
+
+// line parsers are dispatch calls that parse a single unit of text into a
+// Node object which contains the whole statement. Dockerfiles have varied
+// (but not usually unique, see ONBUILD for a unique example) parsing rules
+// per-command, and these unify the processing in a way that makes it
+// manageable.
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"unicode"
+)
+
+var (
+	errDockerfileNotStringArray = errors.New("When using JSON array syntax, arrays must be comprised of strings only.")
+)
+
+// ignore the current argument. This will still leave a command parsed, but
+// will not incorporate the arguments into the ast.
+func parseIgnore(rest string) (*Node, map[string]bool, error) {
+	return &Node{}, nil, nil
+}
+
+// used for onbuild. Could potentially be used for anything that represents a
+// statement with sub-statements.
+//
+// ONBUILD RUN foo bar -> (onbuild (run foo bar))
+//
+func parseSubCommand(rest string) (*Node, map[string]bool, error) {
+	if rest == "" {
+		return nil, nil, nil
+	}
+
+	_, child, err := parseLine(rest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &Node{Children: []*Node{child}}, nil, nil
+}
+
+// helper to parse words (i.e space delimited or quoted strings) in a statement.
+// The quotes are preserved as part of this function and they are stripped later
+// as part of processWords().
+func parseWords(rest string) []string {
+	const (
+		inSpaces = iota // looking for start of a word
+		inWord
+		inQuote
+	)
+
+	words := []string{}
+	phase := inSpaces
+	word := ""
+	quote := '\000'
+	blankOK := false
+	var ch rune
+
+	for pos := 0; pos <= len(rest); pos++ {
+		if pos != len(rest) {
+			ch = rune(rest[pos])
+		}
+
+		if phase == inSpaces { // Looking for start of word
+			if pos == len(rest) { // end of input
+				break
+			}
+			if unicode.IsSpace(ch) { // skip spaces
+				continue
+			}
+			phase = inWord // found it, fall thru
+		}
+		if (phase == inWord || phase == inQuote) && (pos == len(rest)) {
+			if blankOK || len(word) > 0 {
+				words = append(words, word)
+			}
+			break
+		}
+		if phase == inWord {
+			if unicode.IsSpace(ch) {
+				phase = inSpaces
+				if blankOK || len(word) > 0 {
+					words = append(words, word)
+				}
+				word = ""
+				blankOK = false
+				continue
+			}
+			if ch == '\'' || ch == '"' {
+				quote = ch
+				blankOK = true
+				phase = inQuote
+			}
+			if ch == '\\' {
+				if pos+1 == len(rest) {
+					continue // just skip \ at end
+				}
+				// If we're not quoted and we see a \, then always just
+				// add \ plus the char to the word, even if the char
+				// is a quote.
+				word += string(ch)
+				pos++
+				ch = rune(rest[pos])
+			}
+			word += string(ch)
+			continue
+		}
+		if phase == inQuote {
+			if ch == quote {
+				phase = inWord
+			}
+			// \ is special except for ' quotes - can't escape anything for '
+			if ch == '\\' && quote != '\'' {
+				if pos+1 == len(rest) {
+					phase = inWord
+					continue // just skip \ at end
+				}
+				pos++
+				nextCh := rune(rest[pos])
+				word += string(ch)
+				ch = nextCh
+			}
+			word += string(ch)
+		}
+	}
+
+	return words
+}
+
+// parse environment like statements. Note that this does *not* handle
+// variable interpolation, which will be handled in the evaluator.
+func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
+	// This is kind of tricky because we need to support the old
+	// variant:   KEY name value
+	// as well as the new one:    KEY name=value ...
+	// The trigger to know which one is being used will be whether we hit
+	// a space or = first.  space ==> old, "=" ==> new
+
+	words := parseWords(rest)
+	if len(words) == 0 {
+		return nil, nil, nil
+	}
+
+	var rootnode *Node
+
+	// Old format (KEY name value)
+	if !strings.Contains(words[0], "=") {
+		node := &Node{}
+		rootnode = node
+		strs := tokenWhitespace.Split(rest, 2)
+
+		if len(strs) < 2 {
+			return nil, nil, fmt.Errorf(key + " must have two arguments")
+		}
+
+		node.Value = strs[0]
+		node.Next = &Node{}
+		node.Next.Value = strs[1]
+	} else {
+		var prevNode *Node
+		for i, word := range words {
+			if !strings.Contains(word, "=") {
+				return nil, nil, fmt.Errorf("Syntax error - can't find = in %q. Must be of the form: name=value", word)
+			}
+			parts := strings.SplitN(word, "=", 2)
+
+			name := &Node{}
+			value := &Node{}
+
+			name.Next = value
+			name.Value = parts[0]
+			value.Value = parts[1]
+
+			if i == 0 {
+				rootnode = name
+			} else {
+				prevNode.Next = name
+			}
+			prevNode = value
+		}
+	}
+
+	return rootnode, nil, nil
+}
+
+func parseEnv(rest string) (*Node, map[string]bool, error) {
+	return parseNameVal(rest, "ENV")
+}
+
+func parseLabel(rest string) (*Node, map[string]bool, error) {
+	return parseNameVal(rest, "LABEL")
+}
+
+// parses a statement containing one or more keyword definition(s) and/or
+// value assignments, like `name1 name2= name3="" name4=value`.
+// Note that this is a stricter format than the old format of assignment,
+// allowed by parseNameVal(), in a way that this only allows assignment of the
+// form `keyword=[<value>]` like  `name2=`, `name3=""`, and `name4=value` above.
+// In addition, a keyword definition alone is of the form `keyword` like `name1`
+// above. And the assignments `name2=` and `name3=""` are equivalent and
+// assign an empty value to the respective keywords.
+func parseNameOrNameVal(rest string) (*Node, map[string]bool, error) {
+	words := parseWords(rest)
+	if len(words) == 0 {
+		return nil, nil, nil
+	}
+
+	var (
+		rootnode *Node
+		prevNode *Node
+	)
+	for i, word := range words {
+		node := &Node{}
+		node.Value = word
+		if i == 0 {
+			rootnode = node
+		} else {
+			prevNode.Next = node
+		}
+		prevNode = node
+	}
+
+	return rootnode, nil, nil
+}
+
+// parses a whitespace-delimited set of arguments. The result is effectively a
+// linked list of string arguments.
+func parseStringsWhitespaceDelimited(rest string) (*Node, map[string]bool, error) {
+	if rest == "" {
+		return nil, nil, nil
+	}
+
+	node := &Node{}
+	rootnode := node
+	prevnode := node
+	for _, str := range tokenWhitespace.Split(rest, -1) { // use regexp
+		prevnode = node
+		node.Value = str
+		node.Next = &Node{}
+		node = node.Next
+	}
+
+	// XXX to get around regexp.Split *always* providing an empty string at the
+	// end due to how our loop is constructed, nil out the last node in the
+	// chain.
+	prevnode.Next = nil
+
+	return rootnode, nil, nil
+}
+
+// parsestring just wraps the string in quotes and returns a working node.
+func parseString(rest string) (*Node, map[string]bool, error) {
+	if rest == "" {
+		return nil, nil, nil
+	}
+	n := &Node{}
+	n.Value = rest
+	return n, nil, nil
+}
+
+// parseJSON converts JSON arrays to an AST.
+func parseJSON(rest string) (*Node, map[string]bool, error) {
+	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
+	if !strings.HasPrefix(rest, "[") {
+		return nil, nil, fmt.Errorf(`Error parsing "%s" as a JSON array`, rest)
+	}
+
+	var myJSON []interface{}
+	if err := json.NewDecoder(strings.NewReader(rest)).Decode(&myJSON); err != nil {
+		return nil, nil, err
+	}
+
+	var top, prev *Node
+	for _, str := range myJSON {
+		s, ok := str.(string)
+		if !ok {
+			return nil, nil, errDockerfileNotStringArray
+		}
+
+		node := &Node{Value: s}
+		if prev == nil {
+			top = node
+		} else {
+			prev.Next = node
+		}
+		prev = node
+	}
+
+	return top, map[string]bool{"json": true}, nil
+}
+
+// parseMaybeJSON determines if the argument appears to be a JSON array. If
+// so, passes to parseJSON; if not, quotes the result and returns a single
+// node.
+func parseMaybeJSON(rest string) (*Node, map[string]bool, error) {
+	if rest == "" {
+		return nil, nil, nil
+	}
+
+	node, attrs, err := parseJSON(rest)
+
+	if err == nil {
+		return node, attrs, nil
+	}
+	if err == errDockerfileNotStringArray {
+		return nil, nil, err
+	}
+
+	node = &Node{}
+	node.Value = rest
+	return node, nil, nil
+}
+
+// parseMaybeJSONToList determines if the argument appears to be a JSON array. If
+// so, passes to parseJSON; if not, attempts to parse it as a whitespace
+// delimited string.
+func parseMaybeJSONToList(rest string) (*Node, map[string]bool, error) {
+	node, attrs, err := parseJSON(rest)
+
+	if err == nil {
+		return node, attrs, nil
+	}
+	if err == errDockerfileNotStringArray {
+		return nil, nil, err
+	}
+
+	return parseStringsWhitespaceDelimited(rest)
+}

frontend/dockerfile/parser/parser.go

@@ -0,0 +1,145 @@
+// Package parser implements a parser and parse tree dumper for Dockerfiles.
+package parser
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"github.com/docker/docker/builder/dockerfile/command"
+)
+
+// Node is a structure used to represent a parse tree.
+//
+// In the node there are three fields, Value, Next, and Children. Value is the
+// current token's string value. Next is always the next non-child token, and
+// children contains all the children. Here's an example:
+//
+// (value next (child child-next child-next-next) next-next)
+//
+// This data structure is frankly pretty lousy for handling complex languages,
+// but lucky for us the Dockerfile isn't very complicated. This structure
+// works a little more effectively than a "proper" parse tree for our needs.
+//
+type Node struct {
+	Value      string          // actual content
+	Next       *Node           // the next item in the current sexp
+	Children   []*Node         // the children of this sexp
+	Attributes map[string]bool // special attributes for this node
+	Original   string          // original line used before parsing
+	Flags      []string        // only top Node should have this set
+}
+
+var (
+	dispatch              map[string]func(string) (*Node, map[string]bool, error)
+	tokenWhitespace       = regexp.MustCompile(`[\t\v\f\r ]+`)
+	tokenLineContinuation = regexp.MustCompile(`\\[ \t]*$`)
+	tokenComment          = regexp.MustCompile(`^#.*$`)
+)
+
+func init() {
+	// Dispatch Table. see line_parsers.go for the parse functions.
+	// The command is parsed and mapped to the line parser. The line parser
+	// receives the arguments but not the command, and returns an AST after
+	// reformulating the arguments according to the rules in the parser
+	// functions. Errors are propagated up by Parse() and the resulting AST can
+	// be incorporated directly into the existing AST as a next.
+	dispatch = map[string]func(string) (*Node, map[string]bool, error){
+		command.User:       parseString,
+		command.Onbuild:    parseSubCommand,
+		command.Workdir:    parseString,
+		command.Env:        parseEnv,
+		command.Label:      parseLabel,
+		command.Maintainer: parseString,
+		command.From:       parseString,
+		command.Add:        parseMaybeJSONToList,
+		command.Copy:       parseMaybeJSONToList,
+		command.Run:        parseMaybeJSON,
+		command.Cmd:        parseMaybeJSON,
+		command.Entrypoint: parseMaybeJSON,
+		command.Expose:     parseStringsWhitespaceDelimited,
+		command.Volume:     parseMaybeJSONToList,
+		command.StopSignal: parseString,
+		command.Arg:        parseNameOrNameVal,
+	}
+}
+
+// parse a line and return the remainder.
+func parseLine(line string) (string, *Node, error) {
+	if line = stripComments(line); line == "" {
+		return "", nil, nil
+	}
+
+	if tokenLineContinuation.MatchString(line) {
+		line = tokenLineContinuation.ReplaceAllString(line, "")
+		return line, nil, nil
+	}
+
+	cmd, flags, args, err := splitCommand(line)
+	if err != nil {
+		return "", nil, err
+	}
+
+	node := &Node{}
+	node.Value = cmd
+
+	sexp, attrs, err := fullDispatch(cmd, args)
+	if err != nil {
+		return "", nil, err
+	}
+
+	node.Next = sexp
+	node.Attributes = attrs
+	node.Original = line
+	node.Flags = flags
+
+	return "", node, nil
+}
+
+// Parse is the main parse routine.
+// It handles an io.ReadWriteCloser and returns the root of the AST.
+func Parse(rwc io.Reader) (*Node, error) {
+	root := &Node{}
+	scanner := bufio.NewScanner(rwc)
+
+	for scanner.Scan() {
+		scannedLine := strings.TrimLeftFunc(scanner.Text(), unicode.IsSpace)
+		line, child, err := parseLine(scannedLine)
+		if err != nil {
+			return nil, err
+		}
+
+		if line != "" && child == nil {
+			for scanner.Scan() {
+				newline := scanner.Text()
+
+				if stripComments(strings.TrimSpace(newline)) == "" {
+					continue
+				}
+
+				line, child, err = parseLine(line + newline)
+				if err != nil {
+					return nil, err
+				}
+
+				if child != nil {
+					break
+				}
+			}
+			if child == nil && line != "" {
+				line, child, err = parseLine(line)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+
+		if child != nil {
+			root.Children = append(root.Children, child)
+		}
+	}
+
+	return root, nil
+}

frontend/dockerfile/parser/parser_test.go

@@ -0,0 +1,112 @@
+package parser
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const testDir = "testfiles"
+const negativeTestDir = "testfiles-negative"
+
+func getDirs(t *testing.T, dir string) []string {
+	f, err := os.Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer f.Close()
+
+	dirs, err := f.Readdirnames(0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return dirs
+}
+
+func TestTestNegative(t *testing.T) {
+	for _, dir := range getDirs(t, negativeTestDir) {
+		dockerfile := filepath.Join(negativeTestDir, dir, "Dockerfile")
+
+		df, err := os.Open(dockerfile)
+		if err != nil {
+			t.Fatalf("Dockerfile missing for %s: %v", dir, err)
+		}
+
+		_, err = Parse(df)
+		if err == nil {
+			t.Fatalf("No error parsing broken dockerfile for %s", dir)
+		}
+
+		df.Close()
+	}
+}
+
+func TestTestData(t *testing.T) {
+	for _, dir := range getDirs(t, testDir) {
+		dockerfile := filepath.Join(testDir, dir, "Dockerfile")
+		resultfile := filepath.Join(testDir, dir, "result")
+
+		df, err := os.Open(dockerfile)
+		if err != nil {
+			t.Fatalf("Dockerfile missing for %s: %v", dir, err)
+		}
+		defer df.Close()
+
+		ast, err := Parse(df)
+		if err != nil {
+			t.Fatalf("Error parsing %s's dockerfile: %v", dir, err)
+		}
+
+		content, err := ioutil.ReadFile(resultfile)
+		if err != nil {
+			t.Fatalf("Error reading %s's result file: %v", dir, err)
+		}
+
+		if ast.Dump()+"\n" != string(content) {
+			fmt.Fprintln(os.Stderr, "Result:\n"+ast.Dump())
+			fmt.Fprintln(os.Stderr, "Expected:\n"+string(content))
+			t.Fatalf("%s: AST dump of dockerfile does not match result", dir)
+		}
+	}
+}
+
+func TestParseWords(t *testing.T) {
+	tests := []map[string][]string{
+		{
+			"input":  {"foo"},
+			"expect": {"foo"},
+		},
+		{
+			"input":  {"foo bar"},
+			"expect": {"foo", "bar"},
+		},
+		{
+			"input":  {"foo=bar"},
+			"expect": {"foo=bar"},
+		},
+		{
+			"input":  {"foo bar 'abc xyz'"},
+			"expect": {"foo", "bar", "'abc xyz'"},
+		},
+		{
+			"input":  {`foo bar "abc xyz"`},
+			"expect": {"foo", "bar", `"abc xyz"`},
+		},
+	}
+
+	for _, test := range tests {
+		words := parseWords(test["input"][0])
+		if len(words) != len(test["expect"]) {
+			t.Fatalf("length check failed. input: %v, expect: %v, output: %v", test["input"][0], test["expect"], words)
+		}
+		for i, word := range words {
+			if word != test["expect"][i] {
+				t.Fatalf("word check failed for word: %q. input: %v, expect: %v, output: %v", word, test["input"][0], test["expect"], words)
+			}
+		}
+	}
+}

frontend/dockerfile/parser/testfiles-negative/env_no_value/Dockerfile

@@ -0,0 +1,3 @@
+FROM busybox
+
+ENV PATH

frontend/dockerfile/parser/testfiles-negative/shykes-nested-json/Dockerfile

@@ -0,0 +1 @@
+CMD [ "echo", [ "nested json" ] ]

frontend/dockerfile/parser/testfiles/ADD-COPY-with-JSON/Dockerfile

@@ -0,0 +1,11 @@
+FROM	ubuntu:14.04
+MAINTAINER	Seongyeol Lim <seongyeol37@gmail.com>
+
+COPY	.	/go/src/github.com/docker/docker
+ADD		.	/
+ADD		null /
+COPY	nullfile /tmp
+ADD		[ "vimrc", "/tmp" ]
+COPY	[ "bashrc", "/tmp" ]
+COPY	[ "test file", "/tmp" ]
+ADD		[ "test file", "/tmp/test file" ]

frontend/dockerfile/parser/testfiles/ADD-COPY-with-JSON/result

@@ -0,0 +1,10 @@
+(from "ubuntu:14.04")
+(maintainer "Seongyeol Lim <seongyeol37@gmail.com>")
+(copy "." "/go/src/github.com/docker/docker")
+(add "." "/")
+(add "null" "/")
+(copy "nullfile" "/tmp")
+(add "vimrc" "/tmp")
+(copy "bashrc" "/tmp")
+(copy "test file" "/tmp")
+(add "test file" "/tmp/test file")

frontend/dockerfile/parser/testfiles/brimstone-consuldock/Dockerfile

@@ -0,0 +1,25 @@
+FROM brimstone/ubuntu:14.04
+
+MAINTAINER brimstone@the.narro.ws
+
+# TORUN -v /var/run/docker.sock:/var/run/docker.sock
+
+ENV GOPATH /go
+
+# Set our command
+ENTRYPOINT ["/usr/local/bin/consuldock"]
+
+# Install the packages we need, clean up after them and us
+RUN apt-get update \
+	&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.clean \
+    && apt-get install -y --no-install-recommends git golang ca-certificates \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists \
+
+	&& go get -v github.com/brimstone/consuldock \
+    && mv $GOPATH/bin/consuldock /usr/local/bin/consuldock \
+
+	&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.dirty \
+	&& apt-get remove --purge -y $(diff /tmp/dpkg.clean /tmp/dpkg.dirty | awk '/^>/ {print $2}') \
+	&& rm /tmp/dpkg.* \
+	&& rm -rf $GOPATH

frontend/dockerfile/parser/testfiles/brimstone-consuldock/result

@@ -0,0 +1,5 @@
+(from "brimstone/ubuntu:14.04")
+(maintainer "brimstone@the.narro.ws")
+(env "GOPATH" "/go")
+(entrypoint "/usr/local/bin/consuldock")
+(run "apt-get update \t&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.clean     && apt-get install -y --no-install-recommends git golang ca-certificates     && apt-get clean     && rm -rf /var/lib/apt/lists \t&& go get -v github.com/brimstone/consuldock     && mv $GOPATH/bin/consuldock /usr/local/bin/consuldock \t&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.dirty \t&& apt-get remove --purge -y $(diff /tmp/dpkg.clean /tmp/dpkg.dirty | awk '/^>/ {print $2}') \t&& rm /tmp/dpkg.* \t&& rm -rf $GOPATH")

frontend/dockerfile/parser/testfiles/brimstone-docker-consul/Dockerfile

@@ -0,0 +1,52 @@
+FROM brimstone/ubuntu:14.04
+
+CMD []
+
+ENTRYPOINT ["/usr/bin/consul", "agent", "-server", "-data-dir=/consul", "-client=0.0.0.0", "-ui-dir=/webui"]
+
+EXPOSE 8500 8600 8400 8301 8302
+
+RUN apt-get update \
+    && apt-get install -y unzip wget \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists
+
+RUN cd /tmp \
+    && wget https://dl.bintray.com/mitchellh/consul/0.3.1_web_ui.zip \
+       -O web_ui.zip \
+    && unzip web_ui.zip \
+    && mv dist /webui \
+    && rm web_ui.zip
+
+RUN apt-get update \
+	&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.clean \
+    && apt-get install -y --no-install-recommends unzip wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists \
+
+    && cd /tmp \
+    && wget https://dl.bintray.com/mitchellh/consul/0.3.1_web_ui.zip \
+       -O web_ui.zip \
+    && unzip web_ui.zip \
+    && mv dist /webui \
+    && rm web_ui.zip \
+
+	&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.dirty \
+	&& apt-get remove --purge -y $(diff /tmp/dpkg.clean /tmp/dpkg.dirty | awk '/^>/ {print $2}') \
+	&& rm /tmp/dpkg.*
+
+ENV GOPATH /go
+
+RUN apt-get update \
+	&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.clean \
+    && apt-get install -y --no-install-recommends git golang ca-certificates build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists \
+
+	&& go get -v github.com/hashicorp/consul \
+	&& mv $GOPATH/bin/consul /usr/bin/consul \
+
+	&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.dirty \
+	&& apt-get remove --purge -y $(diff /tmp/dpkg.clean /tmp/dpkg.dirty | awk '/^>/ {print $2}') \
+	&& rm /tmp/dpkg.* \
+	&& rm -rf $GOPATH

frontend/dockerfile/parser/testfiles/brimstone-docker-consul/result

@@ -0,0 +1,9 @@
+(from "brimstone/ubuntu:14.04")
+(cmd)
+(entrypoint "/usr/bin/consul" "agent" "-server" "-data-dir=/consul" "-client=0.0.0.0" "-ui-dir=/webui")
+(expose "8500" "8600" "8400" "8301" "8302")
+(run "apt-get update     && apt-get install -y unzip wget \t&& apt-get clean \t&& rm -rf /var/lib/apt/lists")
+(run "cd /tmp     && wget https://dl.bintray.com/mitchellh/consul/0.3.1_web_ui.zip        -O web_ui.zip     && unzip web_ui.zip     && mv dist /webui     && rm web_ui.zip")
+(run "apt-get update \t&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.clean     && apt-get install -y --no-install-recommends unzip wget     && apt-get clean     && rm -rf /var/lib/apt/lists     && cd /tmp     && wget https://dl.bintray.com/mitchellh/consul/0.3.1_web_ui.zip        -O web_ui.zip     && unzip web_ui.zip     && mv dist /webui     && rm web_ui.zip \t&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.dirty \t&& apt-get remove --purge -y $(diff /tmp/dpkg.clean /tmp/dpkg.dirty | awk '/^>/ {print $2}') \t&& rm /tmp/dpkg.*")
+(env "GOPATH" "/go")
+(run "apt-get update \t&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.clean     && apt-get install -y --no-install-recommends git golang ca-certificates build-essential     && apt-get clean     && rm -rf /var/lib/apt/lists \t&& go get -v github.com/hashicorp/consul \t&& mv $GOPATH/bin/consul /usr/bin/consul \t&& dpkg -l | awk '/^ii/ {print $2}' > /tmp/dpkg.dirty \t&& apt-get remove --purge -y $(diff /tmp/dpkg.clean /tmp/dpkg.dirty | awk '/^>/ {print $2}') \t&& rm /tmp/dpkg.* \t&& rm -rf $GOPATH")

frontend/dockerfile/parser/testfiles/continueIndent/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:14.04
+
+RUN echo hello\
+  world\
+  goodnight  \
+  moon\
+  light\
+ning
+RUN echo hello  \
+  world
+RUN echo hello  \
+world
+RUN echo hello \
+goodbye\
+frog
+RUN echo hello  \  
+world
+RUN echo hi \
+ \
+ world \
+\
+ good\
+\
+night
+RUN echo goodbye\
+frog
+RUN echo good\
+bye\
+frog
+
+RUN echo hello \
+# this is a comment
+
+# this is a comment with a blank line surrounding it
+
+this is some more useful stuff

frontend/dockerfile/parser/testfiles/continueIndent/result

@@ -0,0 +1,10 @@
+(from "ubuntu:14.04")
+(run "echo hello  world  goodnight    moon  lightning")
+(run "echo hello    world")
+(run "echo hello  world")
+(run "echo hello goodbyefrog")
+(run "echo hello  world")
+(run "echo hi   world  goodnight")
+(run "echo goodbyefrog")
+(run "echo goodbyefrog")
+(run "echo hello this is some more useful stuff")

frontend/dockerfile/parser/testfiles/cpuguy83-nagios/Dockerfile

@@ -0,0 +1,54 @@
+FROM cpuguy83/ubuntu
+ENV NAGIOS_HOME /opt/nagios
+ENV NAGIOS_USER nagios
+ENV NAGIOS_GROUP nagios
+ENV NAGIOS_CMDUSER nagios
+ENV NAGIOS_CMDGROUP nagios
+ENV NAGIOSADMIN_USER nagiosadmin
+ENV NAGIOSADMIN_PASS nagios
+ENV APACHE_RUN_USER nagios
+ENV APACHE_RUN_GROUP nagios
+ENV NAGIOS_TIMEZONE UTC
+
+RUN sed -i 's/universe/universe multiverse/' /etc/apt/sources.list
+RUN apt-get update && apt-get install -y iputils-ping netcat build-essential snmp snmpd snmp-mibs-downloader php5-cli apache2 libapache2-mod-php5 runit bc postfix bsd-mailx
+RUN ( egrep -i  "^${NAGIOS_GROUP}" /etc/group || groupadd $NAGIOS_GROUP ) && ( egrep -i "^${NAGIOS_CMDGROUP}" /etc/group || groupadd $NAGIOS_CMDGROUP )
+RUN ( id -u $NAGIOS_USER || useradd --system $NAGIOS_USER -g $NAGIOS_GROUP -d $NAGIOS_HOME ) && ( id -u $NAGIOS_CMDUSER || useradd --system -d $NAGIOS_HOME -g $NAGIOS_CMDGROUP $NAGIOS_CMDUSER )
+
+ADD http://downloads.sourceforge.net/project/nagios/nagios-3.x/nagios-3.5.1/nagios-3.5.1.tar.gz?r=http%3A%2F%2Fwww.nagios.org%2Fdownload%2Fcore%2Fthanks%2F%3Ft%3D1398863696&ts=1398863718&use_mirror=superb-dca3 /tmp/nagios.tar.gz
+RUN cd /tmp && tar -zxvf nagios.tar.gz && cd nagios  && ./configure --prefix=${NAGIOS_HOME} --exec-prefix=${NAGIOS_HOME} --enable-event-broker --with-nagios-command-user=${NAGIOS_CMDUSER} --with-command-group=${NAGIOS_CMDGROUP} --with-nagios-user=${NAGIOS_USER} --with-nagios-group=${NAGIOS_GROUP} && make all && make install && make install-config && make install-commandmode && cp sample-config/httpd.conf /etc/apache2/conf.d/nagios.conf
+ADD http://www.nagios-plugins.org/download/nagios-plugins-1.5.tar.gz /tmp/
+RUN cd /tmp && tar -zxvf nagios-plugins-1.5.tar.gz && cd nagios-plugins-1.5 && ./configure --prefix=${NAGIOS_HOME} && make && make install
+
+RUN sed -i.bak 's/.*\=www\-data//g' /etc/apache2/envvars
+RUN export DOC_ROOT="DocumentRoot $(echo $NAGIOS_HOME/share)"; sed -i "s,DocumentRoot.*,$DOC_ROOT," /etc/apache2/sites-enabled/000-default
+
+RUN ln -s ${NAGIOS_HOME}/bin/nagios /usr/local/bin/nagios && mkdir -p /usr/share/snmp/mibs && chmod 0755 /usr/share/snmp/mibs && touch /usr/share/snmp/mibs/.foo
+
+RUN echo "use_timezone=$NAGIOS_TIMEZONE" >> ${NAGIOS_HOME}/etc/nagios.cfg && echo "SetEnv TZ \"${NAGIOS_TIMEZONE}\"" >> /etc/apache2/conf.d/nagios.conf
+
+RUN mkdir -p ${NAGIOS_HOME}/etc/conf.d && mkdir -p ${NAGIOS_HOME}/etc/monitor && ln -s /usr/share/snmp/mibs ${NAGIOS_HOME}/libexec/mibs
+RUN echo "cfg_dir=${NAGIOS_HOME}/etc/conf.d" >> ${NAGIOS_HOME}/etc/nagios.cfg
+RUN echo "cfg_dir=${NAGIOS_HOME}/etc/monitor" >> ${NAGIOS_HOME}/etc/nagios.cfg
+RUN download-mibs && echo "mibs +ALL" > /etc/snmp/snmp.conf
+
+RUN sed -i 's,/bin/mail,/usr/bin/mail,' /opt/nagios/etc/objects/commands.cfg && \
+  sed -i 's,/usr/usr,/usr,' /opt/nagios/etc/objects/commands.cfg
+RUN cp /etc/services /var/spool/postfix/etc/
+
+RUN mkdir -p /etc/sv/nagios && mkdir -p /etc/sv/apache && rm -rf /etc/sv/getty-5 && mkdir -p /etc/sv/postfix
+ADD nagios.init /etc/sv/nagios/run
+ADD apache.init /etc/sv/apache/run
+ADD postfix.init /etc/sv/postfix/run
+ADD postfix.stop /etc/sv/postfix/finish
+
+ADD start.sh /usr/local/bin/start_nagios
+
+ENV APACHE_LOCK_DIR /var/run
+ENV APACHE_LOG_DIR /var/log/apache2
+
+EXPOSE 80
+
+VOLUME ["/opt/nagios/var", "/opt/nagios/etc", "/opt/nagios/libexec", "/var/log/apache2", "/usr/share/snmp/mibs"]
+
+CMD ["/usr/local/bin/start_nagios"]

frontend/dockerfile/parser/testfiles/cpuguy83-nagios/result

@@ -0,0 +1,40 @@
+(from "cpuguy83/ubuntu")
+(env "NAGIOS_HOME" "/opt/nagios")
+(env "NAGIOS_USER" "nagios")
+(env "NAGIOS_GROUP" "nagios")
+(env "NAGIOS_CMDUSER" "nagios")
+(env "NAGIOS_CMDGROUP" "nagios")
+(env "NAGIOSADMIN_USER" "nagiosadmin")
+(env "NAGIOSADMIN_PASS" "nagios")
+(env "APACHE_RUN_USER" "nagios")
+(env "APACHE_RUN_GROUP" "nagios")
+(env "NAGIOS_TIMEZONE" "UTC")
+(run "sed -i 's/universe/universe multiverse/' /etc/apt/sources.list")
+(run "apt-get update && apt-get install -y iputils-ping netcat build-essential snmp snmpd snmp-mibs-downloader php5-cli apache2 libapache2-mod-php5 runit bc postfix bsd-mailx")
+(run "( egrep -i  \"^${NAGIOS_GROUP}\" /etc/group || groupadd $NAGIOS_GROUP ) && ( egrep -i \"^${NAGIOS_CMDGROUP}\" /etc/group || groupadd $NAGIOS_CMDGROUP )")
+(run "( id -u $NAGIOS_USER || useradd --system $NAGIOS_USER -g $NAGIOS_GROUP -d $NAGIOS_HOME ) && ( id -u $NAGIOS_CMDUSER || useradd --system -d $NAGIOS_HOME -g $NAGIOS_CMDGROUP $NAGIOS_CMDUSER )")
+(add "http://downloads.sourceforge.net/project/nagios/nagios-3.x/nagios-3.5.1/nagios-3.5.1.tar.gz?r=http%3A%2F%2Fwww.nagios.org%2Fdownload%2Fcore%2Fthanks%2F%3Ft%3D1398863696&ts=1398863718&use_mirror=superb-dca3" "/tmp/nagios.tar.gz")
+(run "cd /tmp && tar -zxvf nagios.tar.gz && cd nagios  && ./configure --prefix=${NAGIOS_HOME} --exec-prefix=${NAGIOS_HOME} --enable-event-broker --with-nagios-command-user=${NAGIOS_CMDUSER} --with-command-group=${NAGIOS_CMDGROUP} --with-nagios-user=${NAGIOS_USER} --with-nagios-group=${NAGIOS_GROUP} && make all && make install && make install-config && make install-commandmode && cp sample-config/httpd.conf /etc/apache2/conf.d/nagios.conf")
+(add "http://www.nagios-plugins.org/download/nagios-plugins-1.5.tar.gz" "/tmp/")
+(run "cd /tmp && tar -zxvf nagios-plugins-1.5.tar.gz && cd nagios-plugins-1.5 && ./configure --prefix=${NAGIOS_HOME} && make && make install")
+(run "sed -i.bak 's/.*\\=www\\-data//g' /etc/apache2/envvars")
+(run "export DOC_ROOT=\"DocumentRoot $(echo $NAGIOS_HOME/share)\"; sed -i \"s,DocumentRoot.*,$DOC_ROOT,\" /etc/apache2/sites-enabled/000-default")
+(run "ln -s ${NAGIOS_HOME}/bin/nagios /usr/local/bin/nagios && mkdir -p /usr/share/snmp/mibs && chmod 0755 /usr/share/snmp/mibs && touch /usr/share/snmp/mibs/.foo")
+(run "echo \"use_timezone=$NAGIOS_TIMEZONE\" >> ${NAGIOS_HOME}/etc/nagios.cfg && echo \"SetEnv TZ \\\"${NAGIOS_TIMEZONE}\\\"\" >> /etc/apache2/conf.d/nagios.conf")
+(run "mkdir -p ${NAGIOS_HOME}/etc/conf.d && mkdir -p ${NAGIOS_HOME}/etc/monitor && ln -s /usr/share/snmp/mibs ${NAGIOS_HOME}/libexec/mibs")
+(run "echo \"cfg_dir=${NAGIOS_HOME}/etc/conf.d\" >> ${NAGIOS_HOME}/etc/nagios.cfg")
+(run "echo \"cfg_dir=${NAGIOS_HOME}/etc/monitor\" >> ${NAGIOS_HOME}/etc/nagios.cfg")
+(run "download-mibs && echo \"mibs +ALL\" > /etc/snmp/snmp.conf")
+(run "sed -i 's,/bin/mail,/usr/bin/mail,' /opt/nagios/etc/objects/commands.cfg &&   sed -i 's,/usr/usr,/usr,' /opt/nagios/etc/objects/commands.cfg")
+(run "cp /etc/services /var/spool/postfix/etc/")
+(run "mkdir -p /etc/sv/nagios && mkdir -p /etc/sv/apache && rm -rf /etc/sv/getty-5 && mkdir -p /etc/sv/postfix")
+(add "nagios.init" "/etc/sv/nagios/run")
+(add "apache.init" "/etc/sv/apache/run")
+(add "postfix.init" "/etc/sv/postfix/run")
+(add "postfix.stop" "/etc/sv/postfix/finish")
+(add "start.sh" "/usr/local/bin/start_nagios")
+(env "APACHE_LOCK_DIR" "/var/run")
+(env "APACHE_LOG_DIR" "/var/log/apache2")
+(expose "80")
+(volume "/opt/nagios/var" "/opt/nagios/etc" "/opt/nagios/libexec" "/var/log/apache2" "/usr/share/snmp/mibs")
+(cmd "/usr/local/bin/start_nagios")

frontend/dockerfile/parser/testfiles/docker/Dockerfile

@@ -0,0 +1,104 @@
+# This file describes the standard way to build Docker, using docker
+#
+# Usage:
+#
+# # Assemble the full dev environment. This is slow the first time.
+# docker build -t docker .
+#
+# # Mount your source in an interactive container for quick testing:
+# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
+#
+# # Run the test suite:
+# docker run --privileged docker hack/make.sh test
+#
+# # Publish a release:
+# docker run --privileged \
+#  -e AWS_S3_BUCKET=baz \
+#  -e AWS_ACCESS_KEY=foo \
+#  -e AWS_SECRET_KEY=bar \
+#  -e GPG_PASSPHRASE=gloubiboulga \
+#  docker hack/release.sh
+#
+# Note: AppArmor used to mess with privileged mode, but this is no longer
+# the case. Therefore, you don't have to disable it anymore.
+#
+
+FROM	ubuntu:14.04
+MAINTAINER	Tianon Gravi <admwiggin@gmail.com> (@tianon)
+
+# Packaged dependencies
+RUN	apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+	apt-utils \
+	aufs-tools \
+	automake \
+	btrfs-tools \
+	build-essential \
+	curl \
+	dpkg-sig \
+	git \
+	iptables \
+	libapparmor-dev \
+	libcap-dev \
+	libsqlite3-dev \
+	lxc=1.0* \
+	mercurial \
+	pandoc \
+	parallel \
+	reprepro \
+	ruby1.9.1 \
+	ruby1.9.1-dev \
+	s3cmd=1.1.0* \
+	--no-install-recommends
+
+# Get lvm2 source for compiling statically
+RUN	git clone --no-checkout https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout -q v2_02_103
+# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
+# note: we don't use "git clone -b" above because it then spews big nasty warnings about 'detached HEAD' state that we can't silence as easily as we can silence them using "git checkout" directly
+
+# Compile and install lvm2
+RUN	cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper
+# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
+
+# Install Go
+RUN	curl -sSL https://golang.org/dl/go1.3.src.tar.gz | tar -v -C /usr/local -xz
+ENV	PATH	/usr/local/go/bin:$PATH
+ENV	GOPATH	/go:/go/src/github.com/docker/docker/vendor
+RUN	cd /usr/local/go/src && ./make.bash --no-clean 2>&1
+
+# Compile Go for cross compilation
+ENV	DOCKER_CROSSPLATFORMS	\
+	linux/386 linux/arm \
+	darwin/amd64 darwin/386 \
+	freebsd/amd64 freebsd/386 freebsd/arm
+# (set an explicit GOARM of 5 for maximum compatibility)
+ENV	GOARM	5
+RUN	cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done'
+
+# Grab Go's cover tool for dead-simple code coverage testing
+RUN	go get golang.org/x/tools/cmd/cover
+
+# TODO replace FPM with some very minimal debhelper stuff
+RUN	gem install --no-rdoc --no-ri fpm --version 1.0.2
+
+# Get the "busybox" image source so we can build locally instead of pulling
+RUN	git clone -b buildroot-2014.02 https://github.com/jpetazzo/docker-busybox.git /docker-busybox
+
+# Setup s3cmd config
+RUN	/bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY' > /.s3cfg
+
+# Set user.email so crosbymichael's in-container merge commits go smoothly
+RUN	git config --global user.email 'docker-dummy@example.com'
+
+# Add an unprivileged user to be used for tests which need it
+RUN groupadd -r docker
+RUN useradd --create-home --gid docker unprivilegeduser
+
+VOLUME	/var/lib/docker
+WORKDIR	/go/src/github.com/docker/docker
+ENV	DOCKER_BUILDTAGS	apparmor selinux
+
+# Wrap all commands in the "docker-in-docker" script to allow nested containers
+ENTRYPOINT	["hack/dind"]
+
+# Upload docker source
+COPY	.	/go/src/github.com/docker/docker

frontend/dockerfile/parser/testfiles/docker/result

@@ -0,0 +1,24 @@
+(from "ubuntu:14.04")
+(maintainer "Tianon Gravi <admwiggin@gmail.com> (@tianon)")
+(run "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \tapt-utils \taufs-tools \tautomake \tbtrfs-tools \tbuild-essential \tcurl \tdpkg-sig \tgit \tiptables \tlibapparmor-dev \tlibcap-dev \tlibsqlite3-dev \tlxc=1.0* \tmercurial \tpandoc \tparallel \treprepro \truby1.9.1 \truby1.9.1-dev \ts3cmd=1.1.0* \t--no-install-recommends")
+(run "git clone --no-checkout https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout -q v2_02_103")
+(run "cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper")
+(run "curl -sSL https://golang.org/dl/go1.3.src.tar.gz | tar -v -C /usr/local -xz")
+(env "PATH" "/usr/local/go/bin:$PATH")
+(env "GOPATH" "/go:/go/src/github.com/docker/docker/vendor")
+(run "cd /usr/local/go/src && ./make.bash --no-clean 2>&1")
+(env "DOCKER_CROSSPLATFORMS" "linux/386 linux/arm \tdarwin/amd64 darwin/386 \tfreebsd/amd64 freebsd/386 freebsd/arm")
+(env "GOARM" "5")
+(run "cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done'")
+(run "go get golang.org/x/tools/cmd/cover")
+(run "gem install --no-rdoc --no-ri fpm --version 1.0.2")
+(run "git clone -b buildroot-2014.02 https://github.com/jpetazzo/docker-busybox.git /docker-busybox")
+(run "/bin/echo -e '[default]\\naccess_key=$AWS_ACCESS_KEY\\nsecret_key=$AWS_SECRET_KEY' > /.s3cfg")
+(run "git config --global user.email 'docker-dummy@example.com'")
+(run "groupadd -r docker")
+(run "useradd --create-home --gid docker unprivilegeduser")
+(volume "/var/lib/docker")
+(workdir "/go/src/github.com/docker/docker")
+(env "DOCKER_BUILDTAGS" "apparmor selinux")
+(entrypoint "hack/dind")
+(copy "." "/go/src/github.com/docker/docker")

frontend/dockerfile/parser/testfiles/env/Dockerfile

@@ -0,0 +1,23 @@
+FROM ubuntu
+ENV name value
+ENV name=value
+ENV name=value name2=value2
+ENV name="value value1"
+ENV name=value\ value2
+ENV name="value'quote space'value2"
+ENV name='value"double quote"value2'
+ENV name=value\ value2 name2=value2\ value3
+ENV name="a\"b"
+ENV name="a\'b"
+ENV name='a\'b'
+ENV name='a\'b''
+ENV name='a\"b'
+ENV name="''"
+# don't put anything after the next line - it must be the last line of the
+# Dockerfile and it must end with \
+ENV name=value \
+    name1=value1 \
+    name2="value2a \
+           value2b" \
+    name3="value3a\n\"value3b\"" \
+	name4="value4a\\nvalue4b" \

frontend/dockerfile/parser/testfiles/env/result

@@ -0,0 +1,16 @@
+(from "ubuntu")
+(env "name" "value")
+(env "name" "value")
+(env "name" "value" "name2" "value2")
+(env "name" "\"value value1\"")
+(env "name" "value\\ value2")
+(env "name" "\"value'quote space'value2\"")
+(env "name" "'value\"double quote\"value2'")
+(env "name" "value\\ value2" "name2" "value2\\ value3")
+(env "name" "\"a\\\"b\"")
+(env "name" "\"a\\'b\"")
+(env "name" "'a\\'b'")
+(env "name" "'a\\'b''")
+(env "name" "'a\\\"b'")
+(env "name" "\"''\"")
+(env "name" "value" "name1" "value1" "name2" "\"value2a            value2b\"" "name3" "\"value3a\\n\\\"value3b\\\"\"" "name4" "\"value4a\\\\nvalue4b\"")

frontend/dockerfile/parser/testfiles/escapes/Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:14.04
+MAINTAINER Erik \\Hollensbe <erik@hollensbe.org>\"
+
+RUN apt-get \update && \
+  apt-get \"install znc -y
+ADD \conf\\" /.znc
+
+RUN foo \
+
+bar \
+
+baz
+
+CMD [ "\/usr\\\"/bin/znc", "-f", "-r" ]

frontend/dockerfile/parser/testfiles/escapes/result

@@ -0,0 +1,6 @@
+(from "ubuntu:14.04")
+(maintainer "Erik \\\\Hollensbe <erik@hollensbe.org>\\\"")
+(run "apt-get \\update &&   apt-get \\\"install znc -y")
+(add "\\conf\\\\\"" "/.znc")
+(run "foo bar baz")
+(cmd "/usr\\\"/bin/znc" "-f" "-r")

frontend/dockerfile/parser/testfiles/flags/Dockerfile

@@ -0,0 +1,10 @@
+FROM scratch
+COPY foo /tmp/
+COPY --user=me foo /tmp/
+COPY --doit=true foo /tmp/
+COPY --user=me --doit=true foo /tmp/
+COPY --doit=true -- foo /tmp/
+COPY -- foo /tmp/
+CMD --doit [ "a", "b" ]
+CMD --doit=true -- [ "a", "b" ]
+CMD --doit -- [ ]

frontend/dockerfile/parser/testfiles/flags/result

@@ -0,0 +1,10 @@
+(from "scratch")
+(copy "foo" "/tmp/")
+(copy ["--user=me"] "foo" "/tmp/")
+(copy ["--doit=true"] "foo" "/tmp/")
+(copy ["--user=me" "--doit=true"] "foo" "/tmp/")
+(copy ["--doit=true"] "foo" "/tmp/")
+(copy "foo" "/tmp/")
+(cmd ["--doit"] "a" "b")
+(cmd ["--doit=true"] "a" "b")
+(cmd ["--doit"])

frontend/dockerfile/parser/testfiles/influxdb/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:14.04
+
+RUN apt-get update && apt-get install wget -y
+RUN wget http://s3.amazonaws.com/influxdb/influxdb_latest_amd64.deb
+RUN dpkg -i influxdb_latest_amd64.deb
+RUN rm -r /opt/influxdb/shared
+
+VOLUME /opt/influxdb/shared
+
+CMD /usr/bin/influxdb --pidfile /var/run/influxdb.pid -config /opt/influxdb/shared/config.toml
+
+EXPOSE 8083
+EXPOSE 8086
+EXPOSE 8090
+EXPOSE 8099

frontend/dockerfile/parser/testfiles/influxdb/result

@@ -0,0 +1,11 @@
+(from "ubuntu:14.04")
+(run "apt-get update && apt-get install wget -y")
+(run "wget http://s3.amazonaws.com/influxdb/influxdb_latest_amd64.deb")
+(run "dpkg -i influxdb_latest_amd64.deb")
+(run "rm -r /opt/influxdb/shared")
+(volume "/opt/influxdb/shared")
+(cmd "/usr/bin/influxdb --pidfile /var/run/influxdb.pid -config /opt/influxdb/shared/config.toml")
+(expose "8083")
+(expose "8086")
+(expose "8090")
+(expose "8099")

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-json-inside-string-double/Dockerfile

@@ -0,0 +1 @@
+CMD "[\"echo\", \"Phew, I just managed to escaped those double quotes\"]"

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-json-inside-string-double/result

@@ -0,0 +1 @@
+(cmd "\"[\\\"echo\\\", \\\"Phew, I just managed to escaped those double quotes\\\"]\"")

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-json-inside-string/Dockerfile

@@ -0,0 +1 @@
+CMD '["echo", "Well, JSON in a string is JSON too?"]'

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-json-inside-string/result

@@ -0,0 +1 @@
+(cmd "'[\"echo\", \"Well, JSON in a string is JSON too?\"]'")

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-single-quotes/Dockerfile

@@ -0,0 +1 @@
+CMD ['echo','single quotes are invalid JSON']

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-single-quotes/result

@@ -0,0 +1 @@
+(cmd "['echo','single quotes are invalid JSON']")

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-unterminated-bracket/Dockerfile

@@ -0,0 +1 @@
+CMD ["echo", "Please, close the brackets when you're done"

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-unterminated-bracket/result

@@ -0,0 +1 @@
+(cmd "[\"echo\", \"Please, close the brackets when you're done\"")

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-unterminated-string/Dockerfile

@@ -0,0 +1 @@
+CMD ["echo", "look ma, no quote!]

frontend/dockerfile/parser/testfiles/jeztah-invalid-json-unterminated-string/result

@@ -0,0 +1 @@
+(cmd "[\"echo\", \"look ma, no quote!]")

frontend/dockerfile/parser/testfiles/json/Dockerfile

@@ -0,0 +1,8 @@
+CMD []
+CMD [""]
+CMD ["a"]
+CMD ["a","b"]
+CMD [ "a", "b" ]
+CMD [	"a",	"b"	]
+CMD	[	"a",	"b"	]	
+CMD ["abc 123", "♥", "☃", "\" \\ \/ \b \f \n \r \t \u0000"]

frontend/dockerfile/parser/testfiles/json/result

@@ -0,0 +1,8 @@
+(cmd)
+(cmd "")
+(cmd "a")
+(cmd "a" "b")
+(cmd "a" "b")
+(cmd "a" "b")
+(cmd "a" "b")
+(cmd "abc 123" "♥" "☃" "\" \\ / \b \f \n \r \t \x00")

frontend/dockerfile/parser/testfiles/kartar-entrypoint-oddities/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:14.04
+MAINTAINER James Turnbull "james@example.com"
+ENV REFRESHED_AT 2014-06-01
+RUN apt-get update
+RUN apt-get -y install redis-server redis-tools
+EXPOSE 6379
+ENTRYPOINT [ "/usr/bin/redis-server" ]

frontend/dockerfile/parser/testfiles/kartar-entrypoint-oddities/result

@@ -0,0 +1,7 @@
+(from "ubuntu:14.04")
+(maintainer "James Turnbull \"james@example.com\"")
+(env "REFRESHED_AT" "2014-06-01")
+(run "apt-get update")
+(run "apt-get -y install redis-server redis-tools")
+(expose "6379")
+(entrypoint "/usr/bin/redis-server")

frontend/dockerfile/parser/testfiles/lk4d4-the-edge-case-generator/Dockerfile

@@ -0,0 +1,48 @@
+FROM busybox:buildroot-2014.02
+
+MAINTAINER docker <docker@docker.io>
+
+ONBUILD RUN ["echo", "test"]
+ONBUILD RUN echo test
+ONBUILD COPY . /
+
+
+# RUN Commands \
+# linebreak in comment \
+RUN ["ls", "-la"]
+RUN ["echo", "'1234'"]
+RUN echo "1234"
+RUN echo 1234
+RUN echo '1234' && \
+    echo "456" && \
+    echo 789
+RUN    sh -c 'echo root:testpass \
+        > /tmp/passwd'
+RUN mkdir -p /test /test2 /test3/test
+
+# ENV \
+ENV SCUBA 1 DUBA 3
+ENV SCUBA "1 DUBA 3"
+
+# CMD \
+CMD ["echo", "test"]
+CMD echo test
+CMD echo "test"
+CMD echo 'test'
+CMD echo 'test' | wc -
+
+#EXPOSE\
+EXPOSE 3000
+EXPOSE 9000 5000 6000
+
+USER docker
+USER docker:root
+
+VOLUME ["/test"]
+VOLUME ["/test", "/test2"]
+VOLUME /test3
+
+WORKDIR /test
+
+ADD . /
+COPY . copy

frontend/dockerfile/parser/testfiles/lk4d4-the-edge-case-generator/result

@@ -0,0 +1,29 @@
+(from "busybox:buildroot-2014.02")
+(maintainer "docker <docker@docker.io>")
+(onbuild (run "echo" "test"))
+(onbuild (run "echo test"))
+(onbuild (copy "." "/"))
+(run "ls" "-la")
+(run "echo" "'1234'")
+(run "echo \"1234\"")
+(run "echo 1234")
+(run "echo '1234' &&     echo \"456\" &&     echo 789")
+(run "sh -c 'echo root:testpass         > /tmp/passwd'")
+(run "mkdir -p /test /test2 /test3/test")
+(env "SCUBA" "1 DUBA 3")
+(env "SCUBA" "\"1 DUBA 3\"")
+(cmd "echo" "test")
+(cmd "echo test")
+(cmd "echo \"test\"")
+(cmd "echo 'test'")
+(cmd "echo 'test' | wc -")
+(expose "3000")
+(expose "9000" "5000" "6000")
+(user "docker")
+(user "docker:root")
+(volume "/test")
+(volume "/test" "/test2")
+(volume "/test3")
+(workdir "/test")
+(add "." "/")
+(copy "." "copy")

frontend/dockerfile/parser/testfiles/mail/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:14.04
+
+RUN apt-get update -qy && apt-get install mutt offlineimap vim-nox abook elinks curl tmux cron zsh -y
+ADD .muttrc /
+ADD .offlineimaprc /
+ADD .tmux.conf /
+ADD mutt /.mutt
+ADD vim /.vim
+ADD vimrc /.vimrc
+ADD crontab /etc/crontab
+RUN chmod 644 /etc/crontab
+RUN mkdir /Mail
+RUN mkdir /.offlineimap
+RUN echo "export TERM=screen-256color" >/.zshenv
+
+CMD setsid cron; tmux -2

frontend/dockerfile/parser/testfiles/mail/result

@@ -0,0 +1,14 @@
+(from "ubuntu:14.04")
+(run "apt-get update -qy && apt-get install mutt offlineimap vim-nox abook elinks curl tmux cron zsh -y")
+(add ".muttrc" "/")
+(add ".offlineimaprc" "/")
+(add ".tmux.conf" "/")
+(add "mutt" "/.mutt")
+(add "vim" "/.vim")
+(add "vimrc" "/.vimrc")
+(add "crontab" "/etc/crontab")
+(run "chmod 644 /etc/crontab")
+(run "mkdir /Mail")
+(run "mkdir /.offlineimap")
+(run "echo \"export TERM=screen-256color\" >/.zshenv")
+(cmd "setsid cron; tmux -2")

frontend/dockerfile/parser/testfiles/multiple-volumes/Dockerfile

@@ -0,0 +1,3 @@
+FROM foo
+
+VOLUME /opt/nagios/var /opt/nagios/etc /opt/nagios/libexec /var/log/apache2 /usr/share/snmp/mibs

frontend/dockerfile/parser/testfiles/multiple-volumes/result

@@ -0,0 +1,2 @@
+(from "foo")
+(volume "/opt/nagios/var" "/opt/nagios/etc" "/opt/nagios/libexec" "/var/log/apache2" "/usr/share/snmp/mibs")

frontend/dockerfile/parser/testfiles/mumble/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:14.04
+
+RUN apt-get update && apt-get install libcap2-bin mumble-server -y
+
+ADD ./mumble-server.ini /etc/mumble-server.ini
+
+CMD /usr/sbin/murmurd

frontend/dockerfile/parser/testfiles/mumble/result

@@ -0,0 +1,4 @@
+(from "ubuntu:14.04")
+(run "apt-get update && apt-get install libcap2-bin mumble-server -y")
+(add "./mumble-server.ini" "/etc/mumble-server.ini")
+(cmd "/usr/sbin/murmurd")

frontend/dockerfile/parser/testfiles/nginx/Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:14.04
+MAINTAINER Erik Hollensbe <erik@hollensbe.org>
+
+RUN apt-get update && apt-get install nginx-full -y
+RUN rm -rf /etc/nginx
+ADD etc /etc/nginx
+RUN chown -R root:root /etc/nginx
+RUN /usr/sbin/nginx -qt
+RUN mkdir /www
+
+CMD ["/usr/sbin/nginx"]
+
+VOLUME /www
+EXPOSE 80

frontend/dockerfile/parser/testfiles/nginx/result

@@ -0,0 +1,11 @@
+(from "ubuntu:14.04")
+(maintainer "Erik Hollensbe <erik@hollensbe.org>")
+(run "apt-get update && apt-get install nginx-full -y")
+(run "rm -rf /etc/nginx")
+(add "etc" "/etc/nginx")
+(run "chown -R root:root /etc/nginx")
+(run "/usr/sbin/nginx -qt")
+(run "mkdir /www")
+(cmd "/usr/sbin/nginx")
+(volume "/www")
+(expose "80")

frontend/dockerfile/parser/testfiles/tf2/Dockerfile

@@ -0,0 +1,23 @@
+FROM ubuntu:12.04
+
+EXPOSE 27015
+EXPOSE 27005
+EXPOSE 26901
+EXPOSE 27020
+
+RUN apt-get update && apt-get install libc6-dev-i386 curl unzip -y
+RUN mkdir -p /steam
+RUN curl http://media.steampowered.com/client/steamcmd_linux.tar.gz | tar vxz -C /steam
+ADD ./script /steam/script
+RUN /steam/steamcmd.sh +runscript /steam/script
+RUN curl http://mirror.pointysoftware.net/alliedmodders/mmsource-1.10.0-linux.tar.gz | tar vxz -C /steam/tf2/tf
+RUN curl http://mirror.pointysoftware.net/alliedmodders/sourcemod-1.5.3-linux.tar.gz | tar vxz -C /steam/tf2/tf
+ADD ./server.cfg /steam/tf2/tf/cfg/server.cfg
+ADD ./ctf_2fort.cfg /steam/tf2/tf/cfg/ctf_2fort.cfg
+ADD ./sourcemod.cfg /steam/tf2/tf/cfg/sourcemod/sourcemod.cfg
+RUN rm -r /steam/tf2/tf/addons/sourcemod/configs
+ADD ./configs /steam/tf2/tf/addons/sourcemod/configs
+RUN mkdir -p /steam/tf2/tf/addons/sourcemod/translations/en
+RUN cp /steam/tf2/tf/addons/sourcemod/translations/*.txt /steam/tf2/tf/addons/sourcemod/translations/en
+
+CMD cd /steam/tf2 && ./srcds_run -port 27015 +ip 0.0.0.0 +map ctf_2fort -autoupdate -steam_dir /steam -steamcmd_script /steam/script +tf_bot_quota 12 +tf_bot_quota_mode fill

frontend/dockerfile/parser/testfiles/tf2/result

@@ -0,0 +1,20 @@
+(from "ubuntu:12.04")
+(expose "27015")
+(expose "27005")
+(expose "26901")
+(expose "27020")
+(run "apt-get update && apt-get install libc6-dev-i386 curl unzip -y")
+(run "mkdir -p /steam")
+(run "curl http://media.steampowered.com/client/steamcmd_linux.tar.gz | tar vxz -C /steam")
+(add "./script" "/steam/script")
+(run "/steam/steamcmd.sh +runscript /steam/script")
+(run "curl http://mirror.pointysoftware.net/alliedmodders/mmsource-1.10.0-linux.tar.gz | tar vxz -C /steam/tf2/tf")
+(run "curl http://mirror.pointysoftware.net/alliedmodders/sourcemod-1.5.3-linux.tar.gz | tar vxz -C /steam/tf2/tf")
+(add "./server.cfg" "/steam/tf2/tf/cfg/server.cfg")
+(add "./ctf_2fort.cfg" "/steam/tf2/tf/cfg/ctf_2fort.cfg")
+(add "./sourcemod.cfg" "/steam/tf2/tf/cfg/sourcemod/sourcemod.cfg")
+(run "rm -r /steam/tf2/tf/addons/sourcemod/configs")
+(add "./configs" "/steam/tf2/tf/addons/sourcemod/configs")
+(run "mkdir -p /steam/tf2/tf/addons/sourcemod/translations/en")
+(run "cp /steam/tf2/tf/addons/sourcemod/translations/*.txt /steam/tf2/tf/addons/sourcemod/translations/en")
+(cmd "cd /steam/tf2 && ./srcds_run -port 27015 +ip 0.0.0.0 +map ctf_2fort -autoupdate -steam_dir /steam -steamcmd_script /steam/script +tf_bot_quota 12 +tf_bot_quota_mode fill")

frontend/dockerfile/parser/testfiles/weechat/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:14.04
+
+RUN apt-get update -qy && apt-get install tmux zsh weechat-curses -y
+
+ADD .weechat /.weechat
+ADD .tmux.conf /
+RUN echo "export TERM=screen-256color" >/.zshenv
+
+CMD zsh -c weechat

frontend/dockerfile/parser/testfiles/weechat/result

@@ -0,0 +1,6 @@
+(from "ubuntu:14.04")
+(run "apt-get update -qy && apt-get install tmux zsh weechat-curses -y")
+(add ".weechat" "/.weechat")
+(add ".tmux.conf" "/")
+(run "echo \"export TERM=screen-256color\" >/.zshenv")
+(cmd "zsh -c weechat")

frontend/dockerfile/parser/testfiles/znc/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:14.04
+MAINTAINER Erik Hollensbe <erik@hollensbe.org>
+
+RUN apt-get update && apt-get install znc -y
+ADD conf /.znc
+
+CMD [ "/usr/bin/znc", "-f", "-r" ]

frontend/dockerfile/parser/testfiles/znc/result

@@ -0,0 +1,5 @@
+(from "ubuntu:14.04")
+(maintainer "Erik Hollensbe <erik@hollensbe.org>")
+(run "apt-get update && apt-get install znc -y")
+(add "conf" "/.znc")
+(cmd "/usr/bin/znc" "-f" "-r")

frontend/dockerfile/parser/utils.go

@@ -0,0 +1,176 @@
+package parser
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+// Dump dumps the AST defined by `node` as a list of sexps.
+// Returns a string suitable for printing.
+func (node *Node) Dump() string {
+	str := ""
+	str += node.Value
+
+	if len(node.Flags) > 0 {
+		str += fmt.Sprintf(" %q", node.Flags)
+	}
+
+	for _, n := range node.Children {
+		str += "(" + n.Dump() + ")\n"
+	}
+
+	if node.Next != nil {
+		for n := node.Next; n != nil; n = n.Next {
+			if len(n.Children) > 0 {
+				str += " " + n.Dump()
+			} else {
+				str += " " + strconv.Quote(n.Value)
+			}
+		}
+	}
+
+	return strings.TrimSpace(str)
+}
+
+// performs the dispatch based on the two primal strings, cmd and args. Please
+// look at the dispatch table in parser.go to see how these dispatchers work.
+func fullDispatch(cmd, args string) (*Node, map[string]bool, error) {
+	fn := dispatch[cmd]
+
+	// Ignore invalid Dockerfile instructions
+	if fn == nil {
+		fn = parseIgnore
+	}
+
+	sexp, attrs, err := fn(args)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return sexp, attrs, nil
+}
+
+// splitCommand takes a single line of text and parses out the cmd and args,
+// which are used for dispatching to more exact parsing functions.
+func splitCommand(line string) (string, []string, string, error) {
+	var args string
+	var flags []string
+
+	// Make sure we get the same results irrespective of leading/trailing spaces
+	cmdline := tokenWhitespace.Split(strings.TrimSpace(line), 2)
+	cmd := strings.ToLower(cmdline[0])
+
+	if len(cmdline) == 2 {
+		var err error
+		args, flags, err = extractBuilderFlags(cmdline[1])
+		if err != nil {
+			return "", nil, "", err
+		}
+	}
+
+	return cmd, flags, strings.TrimSpace(args), nil
+}
+
+// covers comments and empty lines. Lines should be trimmed before passing to
+// this function.
+func stripComments(line string) string {
+	// string is already trimmed at this point
+	if tokenComment.MatchString(line) {
+		return tokenComment.ReplaceAllString(line, "")
+	}
+
+	return line
+}
+
+func extractBuilderFlags(line string) (string, []string, error) {
+	// Parses the BuilderFlags and returns the remaining part of the line
+
+	const (
+		inSpaces = iota // looking for start of a word
+		inWord
+		inQuote
+	)
+
+	words := []string{}
+	phase := inSpaces
+	word := ""
+	quote := '\000'
+	blankOK := false
+	var ch rune
+
+	for pos := 0; pos <= len(line); pos++ {
+		if pos != len(line) {
+			ch = rune(line[pos])
+		}
+
+		if phase == inSpaces { // Looking for start of word
+			if pos == len(line) { // end of input
+				break
+			}
+			if unicode.IsSpace(ch) { // skip spaces
+				continue
+			}
+
+			// Only keep going if the next word starts with --
+			if ch != '-' || pos+1 == len(line) || rune(line[pos+1]) != '-' {
+				return line[pos:], words, nil
+			}
+
+			phase = inWord // found someting with "--", fall thru
+		}
+		if (phase == inWord || phase == inQuote) && (pos == len(line)) {
+			if word != "--" && (blankOK || len(word) > 0) {
+				words = append(words, word)
+			}
+			break
+		}
+		if phase == inWord {
+			if unicode.IsSpace(ch) {
+				phase = inSpaces
+				if word == "--" {
+					return line[pos:], words, nil
+				}
+				if blankOK || len(word) > 0 {
+					words = append(words, word)
+				}
+				word = ""
+				blankOK = false
+				continue
+			}
+			if ch == '\'' || ch == '"' {
+				quote = ch
+				blankOK = true
+				phase = inQuote
+				continue
+			}
+			if ch == '\\' {
+				if pos+1 == len(line) {
+					continue // just skip \ at end
+				}
+				pos++
+				ch = rune(line[pos])
+			}
+			word += string(ch)
+			continue
+		}
+		if phase == inQuote {
+			if ch == quote {
+				phase = inWord
+				continue
+			}
+			if ch == '\\' {
+				if pos+1 == len(line) {
+					phase = inWord
+					continue // just skip \ at end
+				}
+				pos++
+				ch = rune(line[pos])
+			}
+			word += string(ch)
+		}
+	}
+
+	return "", words, nil
+}