Merge pull request #52 from tonistiigi/debug-handlers

buildd: enable debug handlers

cmd/buildd/debug.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"expvar"
+	"net"
+	"net/http"
+	"net/http/pprof"
+
+	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/trace"
+)
+
+func setupDebugHandlers(addr string) error {
+	m := http.NewServeMux()
+	m.Handle("/debug/vars", expvar.Handler())
+	m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+	m.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+	m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+	m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+	m.Handle("/debug/requests", http.HandlerFunc(trace.Traces))
+	m.Handle("/debug/events", http.HandlerFunc(trace.Events))
+
+	// setting debugaddr is opt-in. permission is defined by listener address
+	trace.AuthRequest = func(_ *http.Request) (bool, bool) {
+		return true, true
+	}
+
+	l, err := net.Listen("tcp", addr)
+	if err != nil {
+		return err
+	}
+	logrus.Debugf("debug handlers listening at %s", addr)
+	go http.Serve(l, m)
+	return nil
+}

cmd/buildd/main.go

@@ -35,6 +35,11 @@ func main() {
 			Usage: "listening socket",
 			Value: "/run/buildkit/buildd.sock",
 		},
+		cli.StringFlag{
+			Name:  "debugaddr",
+			Usage: "Debugging address (eg. 0.0.0.0:6060)",
+			Value: "",
+		},
 	}
 
 	app.Flags = appendFlags(app.Flags)
@@ -43,6 +48,12 @@ func main() {
 		signals := make(chan os.Signal, 2048)
 		signal.Notify(signals, unix.SIGTERM, unix.SIGINT)
 
+		if debugAddr := c.GlobalString("debugaddr"); debugAddr != "" {
+			if err := setupDebugHandlers(debugAddr); err != nil {
+				return err
+			}
+		}
+
 		server := grpc.NewServer(debugGrpcErrors())
 
 		// relative path does not work with nightlyone/lockfile

vendor.conf

@@ -11,7 +11,7 @@ golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
 github.com/Sirupsen/logrus v0.11.0
 google.golang.org/grpc v1.3.0
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448
-golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6
+golang.org/x/net 1f9224279e98554b6a6432d4dd998a739f8b2b7c
 github.com/gogo/protobuf d2e1ade2d719b78fe5b061b4c18a9f7111b5bdc8
 github.com/golang/protobuf 5a0f697c9ed9d68fef0116532c6e05cfeae00e55
 github.com/containerd/continuity 86cec1535a968310e7532819f699ff2830ed7463

vendor/golang.org/x/net/http2/configure_transport.go

@@ -56,7 +56,7 @@ func configureTransport(t1 *http.Transport) (*Transport, error) {
 }
 
 // registerHTTPSProtocol calls Transport.RegisterProtocol but
-// convering panics into errors.
+// converting panics into errors.
 func registerHTTPSProtocol(t *http.Transport, rt http.RoundTripper) (err error) {
 	defer func() {
 		if e := recover(); e != nil {

vendor/golang.org/x/net/http2/errors.go

@@ -87,13 +87,16 @@ type goAwayFlowError struct{}
 
 func (goAwayFlowError) Error() string { return "connection exceeded flow control window size" }
 
-// connErrorReason wraps a ConnectionError with an informative error about why it occurs.
-
+// connError represents an HTTP/2 ConnectionError error code, along
+// with a string (for debugging) explaining why.
+//
 // Errors of this type are only returned by the frame parser functions
-// and converted into ConnectionError(ErrCodeProtocol).
+// and converted into ConnectionError(Code), after stashing away
+// the Reason into the Framer's errDetail field, accessible via
+// the (*Framer).ErrorDetail method.
 type connError struct {
-	Code   ErrCode
-	Reason string
+	Code   ErrCode // the ConnectionError error code
+	Reason string  // additional reason
 }
 
 func (e connError) Error() string {

vendor/golang.org/x/net/http2/go18.go

@@ -52,3 +52,5 @@ func reqGetBody(req *http.Request) func() (io.ReadCloser, error) {
 func reqBodyIsNoBody(body io.ReadCloser) bool {
 	return body == http.NoBody
 }
+
+func go18httpNoBody() io.ReadCloser { return http.NoBody } // for tests only

vendor/golang.org/x/net/http2/http2.go

@@ -376,12 +376,16 @@ func (s *sorter) SortStrings(ss []string) {
 // validPseudoPath reports whether v is a valid :path pseudo-header
 // value. It must be either:
 //
-//     *) a non-empty string starting with '/', but not with with "//",
+//     *) a non-empty string starting with '/'
 //     *) the string '*', for OPTIONS requests.
 //
 // For now this is only used a quick check for deciding when to clean
 // up Opaque URLs before sending requests from the Transport.
 // See golang.org/issue/16847
+//
+// We used to enforce that the path also didn't start with "//", but
+// Google's GFE accepts such paths and Chrome sends them, so ignore
+// that part of the spec. See golang.org/issue/19103.
 func validPseudoPath(v string) bool {
-	return (len(v) > 0 && v[0] == '/' && (len(v) == 1 || v[1] != '/')) || v == "*"
+	return (len(v) > 0 && v[0] == '/') || v == "*"
 }

vendor/golang.org/x/net/http2/not_go18.go

@@ -25,3 +25,5 @@ func reqGetBody(req *http.Request) func() (io.ReadCloser, error) {
 }
 
 func reqBodyIsNoBody(io.ReadCloser) bool { return false }
+
+func go18httpNoBody() io.ReadCloser { return nil } // for tests only

vendor/golang.org/x/net/http2/pipe.go

@@ -50,7 +50,7 @@ func (p *pipe) Read(d []byte) (n int, err error) {
 		if p.breakErr != nil {
 			return 0, p.breakErr
 		}
-		if p.b.Len() > 0 {
+		if p.b != nil && p.b.Len() > 0 {
 			return p.b.Read(d)
 		}
 		if p.err != nil {

vendor/golang.org/x/net/http2/server.go

@@ -2252,6 +2252,7 @@ type responseWriterState struct {
 	wroteHeader   bool        // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet.
 	sentHeader    bool        // have we sent the header frame?
 	handlerDone   bool        // handler has finished
+	dirty         bool        // a Write failed; don't reuse this responseWriterState
 
 	sentContentLen int64 // non-zero if handler set a Content-Length header
 	wroteBytes     int64
@@ -2333,6 +2334,7 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) {
 			date:          date,
 		})
 		if err != nil {
+			rws.dirty = true
 			return 0, err
 		}
 		if endStream {
@@ -2354,6 +2356,7 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) {
 	if len(p) > 0 || endStream {
 		// only send a 0 byte DATA frame if we're ending the stream.
 		if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil {
+			rws.dirty = true
 			return 0, err
 		}
 	}
@@ -2365,6 +2368,9 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) {
 			trailers:  rws.trailers,
 			endStream: true,
 		})
+		if err != nil {
+			rws.dirty = true
+		}
 		return len(p), err
 	}
 	return len(p), nil
@@ -2504,7 +2510,7 @@ func cloneHeader(h http.Header) http.Header {
 //
 // * Handler calls w.Write or w.WriteString ->
 // * -> rws.bw (*bufio.Writer) ->
-// * (Handler migth call Flush)
+// * (Handler might call Flush)
 // * -> chunkWriter{rws}
 // * -> responseWriterState.writeChunk(p []byte)
 // * -> responseWriterState.writeChunk (most of the magic; see comment there)
@@ -2543,10 +2549,19 @@ func (w *responseWriter) write(lenData int, dataB []byte, dataS string) (n int,
 
 func (w *responseWriter) handlerDone() {
 	rws := w.rws
+	dirty := rws.dirty
 	rws.handlerDone = true
 	w.Flush()
 	w.rws = nil
-	responseWriterStatePool.Put(rws)
+	if !dirty {
+		// Only recycle the pool if all prior Write calls to
+		// the serverConn goroutine completed successfully. If
+		// they returned earlier due to resets from the peer
+		// there might still be write goroutines outstanding
+		// from the serverConn referencing the rws memory. See
+		// issue 20704.
+		responseWriterStatePool.Put(rws)
+	}
 }
 
 // Push errors.

vendor/golang.org/x/net/http2/transport.go

@@ -694,7 +694,7 @@ func checkConnHeaders(req *http.Request) error {
 // req.ContentLength, where 0 actually means zero (not unknown) and -1
 // means unknown.
 func actualContentLength(req *http.Request) int64 {
-	if req.Body == nil {
+	if req.Body == nil || reqBodyIsNoBody(req.Body) {
 		return 0
 	}
 	if req.ContentLength != 0 {
@@ -725,8 +725,8 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) {
 	}
 
 	body := req.Body
-	hasBody := body != nil
 	contentLen := actualContentLength(req)
+	hasBody := contentLen != 0
 
 	// TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere?
 	var requestedGzip bool
@@ -1713,16 +1713,27 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error {
 		}
 		// Return any padded flow control now, since we won't
 		// refund it later on body reads.
-		if pad := int32(f.Length) - int32(len(data)); pad > 0 {
-			cs.inflow.add(pad)
-			cc.inflow.add(pad)
+		var refund int
+		if pad := int(f.Length) - len(data); pad > 0 {
+			refund += pad
+		}
+		// Return len(data) now if the stream is already closed,
+		// since data will never be read.
+		didReset := cs.didReset
+		if didReset {
+			refund += len(data)
+		}
+		if refund > 0 {
+			cc.inflow.add(int32(refund))
 			cc.wmu.Lock()
-			cc.fr.WriteWindowUpdate(0, uint32(pad))
-			cc.fr.WriteWindowUpdate(cs.ID, uint32(pad))
+			cc.fr.WriteWindowUpdate(0, uint32(refund))
+			if !didReset {
+				cs.inflow.add(int32(refund))
+				cc.fr.WriteWindowUpdate(cs.ID, uint32(refund))
+			}
 			cc.bw.Flush()
 			cc.wmu.Unlock()
 		}
-		didReset := cs.didReset
 		cc.mu.Unlock()
 
 		if len(data) > 0 && !didReset {

vendor/golang.org/x/net/idna/idna.go

@@ -67,6 +67,15 @@ func VerifyDNSLength(verify bool) Option {
 	return func(o *options) { o.verifyDNSLength = verify }
 }
 
+// RemoveLeadingDots removes leading label separators. Leading runes that map to
+// dots, such as U+3002, are removed as well.
+//
+// This is the behavior suggested by the UTS #46 and is adopted by some
+// browsers.
+func RemoveLeadingDots(remove bool) Option {
+	return func(o *options) { o.removeLeadingDots = remove }
+}
+
 // ValidateLabels sets whether to check the mandatory label validation criteria
 // as defined in Section 5.4 of RFC 5891. This includes testing for correct use
 // of hyphens ('-'), normalization, validity of runes, and the context rules.
@@ -133,14 +142,16 @@ func MapForLookup() Option {
 		o.mapping = validateAndMap
 		StrictDomainName(true)(o)
 		ValidateLabels(true)(o)
+		RemoveLeadingDots(true)(o)
 	}
 }
 
 type options struct {
-	transitional    bool
-	useSTD3Rules    bool
-	validateLabels  bool
-	verifyDNSLength bool
+	transitional      bool
+	useSTD3Rules      bool
+	validateLabels    bool
+	verifyDNSLength   bool
+	removeLeadingDots bool
 
 	trie *idnaTrie
 
@@ -240,21 +251,23 @@ var (
 
 	punycode = &Profile{}
 	lookup   = &Profile{options{
-		transitional:   true,
-		useSTD3Rules:   true,
-		validateLabels: true,
-		trie:           trie,
-		fromPuny:       validateFromPunycode,
-		mapping:        validateAndMap,
-		bidirule:       bidirule.ValidString,
+		transitional:      true,
+		useSTD3Rules:      true,
+		validateLabels:    true,
+		removeLeadingDots: true,
+		trie:              trie,
+		fromPuny:          validateFromPunycode,
+		mapping:           validateAndMap,
+		bidirule:          bidirule.ValidString,
 	}}
 	display = &Profile{options{
-		useSTD3Rules:   true,
-		validateLabels: true,
-		trie:           trie,
-		fromPuny:       validateFromPunycode,
-		mapping:        validateAndMap,
-		bidirule:       bidirule.ValidString,
+		useSTD3Rules:      true,
+		validateLabels:    true,
+		removeLeadingDots: true,
+		trie:              trie,
+		fromPuny:          validateFromPunycode,
+		mapping:           validateAndMap,
+		bidirule:          bidirule.ValidString,
 	}}
 	registration = &Profile{options{
 		useSTD3Rules:    true,
@@ -293,7 +306,9 @@ func (p *Profile) process(s string, toASCII bool) (string, error) {
 		s, err = p.mapping(p, s)
 	}
 	// Remove leading empty labels.
-	for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
+	if p.removeLeadingDots {
+		for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
+		}
 	}
 	// It seems like we should only create this error on ToASCII, but the
 	// UTS 46 conformance tests suggests we should always check this.
@@ -373,23 +388,20 @@ func validateRegistration(p *Profile, s string) (string, error) {
 	if !norm.NFC.IsNormalString(s) {
 		return s, &labelError{s, "V1"}
 	}
-	var err error
 	for i := 0; i < len(s); {
 		v, sz := trie.lookupString(s[i:])
-		i += sz
 		// Copy bytes not copied so far.
 		switch p.simplify(info(v).category()) {
 		// TODO: handle the NV8 defined in the Unicode idna data set to allow
 		// for strict conformance to IDNA2008.
 		case valid, deviation:
 		case disallowed, mapped, unknown, ignored:
-			if err == nil {
-				r, _ := utf8.DecodeRuneInString(s[i:])
-				err = runeError(r)
-			}
+			r, _ := utf8.DecodeRuneInString(s[i:])
+			return s, runeError(r)
 		}
+		i += sz
 	}
-	return s, err
+	return s, nil
 }
 
 func validateAndMap(p *Profile, s string) (string, error) {
@@ -408,7 +420,7 @@ func validateAndMap(p *Profile, s string) (string, error) {
 			continue
 		case disallowed:
 			if err == nil {
-				r, _ := utf8.DecodeRuneInString(s[i:])
+				r, _ := utf8.DecodeRuneInString(s[start:])
 				err = runeError(r)
 			}
 			continue

vendor/golang.org/x/net/trace/events.go

@@ -39,9 +39,9 @@ var buckets = []bucket{
 }
 
 // RenderEvents renders the HTML page typically served at /debug/events.
-// It does not do any auth checking; see AuthRequest for the default auth check
-// used by the handler registered on http.DefaultServeMux.
-// req may be nil.
+// It does not do any auth checking. The request may be nil.
+//
+// Most users will use the Events handler.
 func RenderEvents(w http.ResponseWriter, req *http.Request, sensitive bool) {
 	now := time.Now()
 	data := &struct {

vendor/golang.org/x/net/trace/trace.go

@@ -110,30 +110,46 @@ var AuthRequest = func(req *http.Request) (any, sensitive bool) {
 }
 
 func init() {
-	http.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
-		any, sensitive := AuthRequest(req)
-		if !any {
-			http.Error(w, "not allowed", http.StatusUnauthorized)
-			return
-		}
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		Render(w, req, sensitive)
-	})
-	http.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
-		any, sensitive := AuthRequest(req)
-		if !any {
-			http.Error(w, "not allowed", http.StatusUnauthorized)
-			return
-		}
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		RenderEvents(w, req, sensitive)
-	})
+	// TODO(jbd): Serve Traces from /debug/traces in the future?
+	// There is no requirement for a request to be present to have traces.
+	http.HandleFunc("/debug/requests", Traces)
+	http.HandleFunc("/debug/events", Events)
+}
+
+// Traces responds with traces from the program.
+// The package initialization registers it in http.DefaultServeMux
+// at /debug/requests.
+//
+// It performs authorization by running AuthRequest.
+func Traces(w http.ResponseWriter, req *http.Request) {
+	any, sensitive := AuthRequest(req)
+	if !any {
+		http.Error(w, "not allowed", http.StatusUnauthorized)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	Render(w, req, sensitive)
+}
+
+// Events responds with a page of events collected by EventLogs.
+// The package initialization registers it in http.DefaultServeMux
+// at /debug/events.
+//
+// It performs authorization by running AuthRequest.
+func Events(w http.ResponseWriter, req *http.Request) {
+	any, sensitive := AuthRequest(req)
+	if !any {
+		http.Error(w, "not allowed", http.StatusUnauthorized)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	RenderEvents(w, req, sensitive)
 }
 
 // Render renders the HTML page typically served at /debug/requests.
-// It does not do any auth checking; see AuthRequest for the default auth check
-// used by the handler registered on http.DefaultServeMux.
-// req may be nil.
+// It does not do any auth checking. The request may be nil.
+//
+// Most users will use the Traces handler.
 func Render(w io.Writer, req *http.Request, sensitive bool) {
 	data := &struct {
 		Families         []string