Merge pull request #1432 from projectdiscovery/issue-1312-headless-matchers

Adding support for navigation history to matchers

v2/pkg/protocols/headless/engine/page.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"net/url"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -10,10 +12,18 @@ import (
 
 // Page is a single page in an isolated browser instance
 type Page struct {
-	page     *rod.Page
-	rules    []requestRule
-	instance *Instance
-	router   *rod.HijackRouter
+	page         *rod.Page
+	rules        []requestRule
+	instance     *Instance
+	router       *rod.HijackRouter
+	historyMutex *sync.RWMutex
+	History      []HistoryData
+}
+
+// HistoryData contains the page request/response pairs
+type HistoryData struct {
+	RawRequest  string
+	RawResponse string
 }
 
 // Run runs a list of actions by creating a new page in the browser.
@@ -30,7 +40,7 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio
 		}
 	}
 
-	createdPage := &Page{page: page, instance: i}
+	createdPage := &Page{page: page, instance: i, historyMutex: &sync.RWMutex{}}
 	router := page.HijackRequests()
 	if routerErr := router.Add("*", "", createdPage.routingRuleHandler); routerErr != nil {
 		return nil, nil, routerErr
@@ -81,3 +91,24 @@ func (p *Page) URL() string {
 	}
 	return info.URL
 }
+
+// DumpHistory returns the full page navigation history
+func (p *Page) DumpHistory() string {
+	p.historyMutex.RLock()
+	defer p.historyMutex.RUnlock()
+
+	var historyDump strings.Builder
+	for _, historyData := range p.History {
+		historyDump.WriteString(historyData.RawRequest)
+		historyDump.WriteString(historyData.RawResponse)
+	}
+	return historyDump.String()
+}
+
+// addToHistory adds a request/response pair to the page history
+func (p *Page) addToHistory(historyData HistoryData) {
+	p.historyMutex.Lock()
+	defer p.historyMutex.Unlock()
+
+	p.History = append(p.History, historyData)
+}

v2/pkg/protocols/headless/engine/rules.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"fmt"
+	"net/http/httputil"
+	"strings"
 
 	"github.com/go-rod/rod"
 )
@@ -10,7 +12,6 @@ import (
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 	// usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless
 	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
-
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue
@@ -51,4 +52,32 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 			ctx.Response.SetBody(rule.Args["body"])
 		}
 	}
+
+	// store history
+	req := ctx.Request.Req()
+	var rawReq string
+	if raw, err := httputil.DumpRequestOut(req, true); err == nil {
+		rawReq = string(raw)
+	}
+
+	// attempts to rebuild the response
+	var rawResp strings.Builder
+	respPayloads := ctx.Response.Payload()
+	if respPayloads != nil {
+		rawResp.WriteString("HTTP/1.1 ")
+		rawResp.WriteString(fmt.Sprint(respPayloads.ResponseCode))
+		rawResp.WriteString(" " + respPayloads.ResponsePhrase + "+\n")
+		for _, header := range respPayloads.ResponseHeaders {
+			rawResp.WriteString(header.Name + ": " + header.Value + "\n")
+		}
+		rawResp.WriteString("\n")
+		rawResp.WriteString(ctx.Response.Body())
+	}
+
+	// dump request
+	historyData := HistoryData{
+		RawRequest:  rawReq,
+		RawResponse: rawResp.String(),
+	}
+	p.addToHistory(historyData)
 }

v2/pkg/protocols/headless/operators.go

@@ -54,6 +54,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 	switch part {
 	case "body", "resp", "":
 		part = "data"
+	case "history":
+		part = "history"
 	}
 
 	item, ok := data[part]
@@ -66,12 +68,13 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 }
 
 // responseToDSLMap converts a headless response to a map for use in DSL matching
-func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent {
+func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent {
 	return output.InternalEvent{
 		"host":          host,
 		"matched":       matched,
 		"req":           req,
 		"data":          resp,
+		"history":       history,
 		"type":          request.Type().String(),
 		"template-id":   request.options.TemplateID,
 		"template-info": request.options.TemplateInfo,

v2/pkg/protocols/headless/request.go

@@ -66,7 +66,7 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o
 	if err == nil {
 		responseBody, _ = html.HTML()
 	}
-	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL)
+	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
 	for k, v := range out {
 		outputEvent[k] = v
 	}