Merge pull request #1693 from tonistiigi/dockerfile-comments

dockerfile: parse comments associated with args and stages

frontend/dockerfile/instructions/commands.go

@@ -21,8 +21,9 @@ func (kvp *KeyValuePair) String() string {
 
 // KeyValuePairOptional is the same as KeyValuePair but Value is optional
 type KeyValuePairOptional struct {
-	Key   string
-	Value *string
+	Key     string
+	Value   *string
+	Comment string
 }
 
 func (kvpo *KeyValuePairOptional) ValueString() string {
@@ -419,6 +420,7 @@ type Stage struct {
 	SourceCode string
 	Platform   string
 	Location   []parser.Range
+	Comment    string
 }
 
 // AddCommand to the stage

frontend/dockerfile/instructions/parse.go

@@ -22,6 +22,7 @@ type parseRequest struct {
 	flags      *BFlags
 	original   string
 	location   []parser.Range
+	comments   []string
 }
 
 var parseRunPreHooks []func(*RunCommand, parseRequest) error
@@ -50,6 +51,7 @@ func newParseRequestFromNode(node *parser.Node) parseRequest {
 		original:   node.Original,
 		flags:      NewBFlagsWithArgs(node.Flags),
 		location:   node.Location(),
+		comments:   node.PrevComment,
 	}
 }
 
@@ -289,6 +291,7 @@ func parseFrom(req parseRequest) (*Stage, error) {
 		Commands:   []Command{},
 		Platform:   flPlatform.Value,
 		Location:   req.location,
+		Comment:    getComment(req.comments, stageName),
 	}, nil
 
 }
@@ -604,6 +607,7 @@ func parseArg(req parseRequest) (*ArgCommand, error) {
 		} else {
 			kvpo.Key = arg
 		}
+		kvpo.Comment = getComment(req.comments, kvpo.Key)
 		pairs[i] = kvpo
 	}
 
@@ -654,3 +658,15 @@ func errBlankCommandNames(command string) error {
 func errTooManyArguments(command string) error {
 	return errors.Errorf("Bad input to %s, too many arguments", command)
 }
+
+func getComment(comments []string, name string) string {
+	if name == "" {
+		return ""
+	}
+	for _, line := range comments {
+		if strings.HasPrefix(line, name+" ") {
+			return strings.TrimPrefix(line, name+" ")
+		}
+	}
+	return ""
+}

frontend/dockerfile/instructions/parse_test.go

@@ -1,6 +1,7 @@
 package instructions
 
 import (
+	"bytes"
 	"strings"
 	"testing"
 
@@ -140,6 +141,39 @@ func TestParseOptInterval(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestCommentsDetection(t *testing.T) {
+	dt := `# foo sets foo
+ARG foo=bar
+
+# base defines first stage
+FROM busybox AS base
+# this is irrelevant
+ARG foo
+# bar defines bar
+# baz is something else
+ARG bar baz=123
+`
+
+	ast, err := parser.Parse(bytes.NewBuffer([]byte(dt)))
+	require.NoError(t, err)
+
+	stages, meta, err := Parse(ast.AST)
+	require.NoError(t, err)
+
+	require.Equal(t, "defines first stage", stages[0].Comment)
+	require.Equal(t, "foo", meta[0].Args[0].Key)
+	require.Equal(t, "sets foo", meta[0].Args[0].Comment)
+
+	st := stages[0]
+
+	require.Equal(t, "foo", st.Commands[0].(*ArgCommand).Args[0].Key)
+	require.Equal(t, "", st.Commands[0].(*ArgCommand).Args[0].Comment)
+	require.Equal(t, "bar", st.Commands[1].(*ArgCommand).Args[0].Key)
+	require.Equal(t, "defines bar", st.Commands[1].(*ArgCommand).Args[0].Comment)
+	require.Equal(t, "baz", st.Commands[1].(*ArgCommand).Args[1].Key)
+	require.Equal(t, "is something else", st.Commands[1].(*ArgCommand).Args[1].Comment)
+}
+
 func TestErrorCases(t *testing.T) {
 	cases := []struct {
 		name          string

frontend/dockerfile/parser/line_parsers.go

@@ -40,7 +40,7 @@ func parseSubCommand(rest string, d *directives) (*Node, map[string]bool, error)
 		return nil, nil, nil
 	}
 
-	child, err := newNodeFromLine(rest, d)
+	child, err := newNodeFromLine(rest, d, nil)
 	if err != nil {
 		return nil, nil, err
 	}

frontend/dockerfile/parser/parser.go

@@ -28,14 +28,15 @@ import (
 // works a little more effectively than a "proper" parse tree for our needs.
 //
 type Node struct {
-	Value      string          // actual content
-	Next       *Node           // the next item in the current sexp
-	Children   []*Node         // the children of this sexp
-	Attributes map[string]bool // special attributes for this node
-	Original   string          // original line used before parsing
-	Flags      []string        // only top Node should have this set
-	StartLine  int             // the line in the original dockerfile where the node begins
-	EndLine    int             // the line in the original dockerfile where the node ends
+	Value       string          // actual content
+	Next        *Node           // the next item in the current sexp
+	Children    []*Node         // the children of this sexp
+	Attributes  map[string]bool // special attributes for this node
+	Original    string          // original line used before parsing
+	Flags       []string        // only top Node should have this set
+	StartLine   int             // the line in the original dockerfile where the node begins
+	EndLine     int             // the line in the original dockerfile where the node ends
+	PrevComment []string
 }
 
 // Location return the location of node in source code
@@ -191,7 +192,7 @@ func init() {
 // newNodeFromLine splits the line into parts, and dispatches to a function
 // based on the command and command arguments. A Node is created from the
 // result of the dispatch.
-func newNodeFromLine(line string, d *directives) (*Node, error) {
+func newNodeFromLine(line string, d *directives, comments []string) (*Node, error) {
 	cmd, flags, args, err := splitCommand(line)
 	if err != nil {
 		return nil, err
@@ -208,11 +209,12 @@ func newNodeFromLine(line string, d *directives) (*Node, error) {
 	}
 
 	return &Node{
-		Value:      cmd,
-		Original:   line,
-		Flags:      flags,
-		Next:       next,
-		Attributes: attrs,
+		Value:       cmd,
+		Original:    line,
+		Flags:       flags,
+		Next:        next,
+		Attributes:  attrs,
+		PrevComment: comments,
 	}, nil
 }
 
@@ -239,6 +241,7 @@ func Parse(rwc io.Reader) (*Result, error) {
 	root := &Node{StartLine: -1}
 	scanner := bufio.NewScanner(rwc)
 	warnings := []string{}
+	var comments []string
 
 	var err error
 	for scanner.Scan() {
@@ -247,6 +250,14 @@ func Parse(rwc io.Reader) (*Result, error) {
 			// First line, strip the byte-order-marker if present
 			bytesRead = bytes.TrimPrefix(bytesRead, utf8bom)
 		}
+		if isComment(bytesRead) {
+			comment := strings.TrimSpace(string(bytesRead[1:]))
+			if comment == "" {
+				comments = nil
+			} else {
+				comments = append(comments, comment)
+			}
+		}
 		bytesRead, err = processLine(d, bytesRead, true)
 		if err != nil {
 			return nil, withLocation(err, currentLine, 0)
@@ -285,10 +296,11 @@ func Parse(rwc io.Reader) (*Result, error) {
 			warnings = append(warnings, "[WARNING]: Empty continuation line found in:\n    "+line)
 		}
 
-		child, err := newNodeFromLine(line, d)
+		child, err := newNodeFromLine(line, d, comments)
 		if err != nil {
 			return nil, withLocation(err, startLine, currentLine)
 		}
+		comments = nil
 		root.AddChild(child, startLine, currentLine)
 	}