Adding support for full navigation history to headless matchers

v2/pkg/protocols/headless/engine/page.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -14,6 +15,12 @@ type Page struct {
 	rules    []requestRule
 	instance *Instance
 	router   *rod.HijackRouter
+	History  []HistoryData
+}
+
+type HistoryData struct {
+	RawRequest  string
+	RawResponse string
 }
 
 // Run runs a list of actions by creating a new page in the browser.
@@ -81,3 +88,12 @@ func (p *Page) URL() string {
 	}
 	return info.URL
 }
+
+func (p *Page) DumpHistory() string {
+	var historyDump strings.Builder
+	for _, historyData := range p.History {
+		historyDump.WriteString(historyData.RawRequest)
+		historyDump.WriteString(historyData.RawResponse)
+	}
+	return historyDump.String()
+}

v2/pkg/protocols/headless/engine/rules.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"fmt"
+	"net/http/httputil"
+	"strings"
 
 	"github.com/go-rod/rod"
 )
@@ -10,7 +12,6 @@ import (
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 	// usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless
 	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
-
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue
@@ -51,4 +52,30 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 			ctx.Response.SetBody(rule.Args["body"])
 		}
 	}
+
+	// store history
+	req := ctx.Request.Req()
+	var rawReq string
+	if raw, err := httputil.DumpRequestOut(req, true); err == nil {
+		rawReq = string(raw)
+	}
+
+	// attempts to rebuild the response
+	var rawResp strings.Builder
+	respPayloads := ctx.Response.Payload()
+	if respPayloads != nil {
+		rawResp.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\n", respPayloads.ResponseCode, respPayloads.ResponsePhrase))
+		for _, header := range respPayloads.ResponseHeaders {
+			rawResp.WriteString(fmt.Sprintf("%s: %s\n", header.Name, header.Value))
+		}
+		rawResp.WriteString("\n")
+		rawResp.WriteString(ctx.Response.Body())
+	}
+
+	// dump request
+	historyData := HistoryData{
+		RawRequest:  rawReq,
+		RawResponse: rawResp.String(),
+	}
+	p.History = append(p.History, historyData)
 }

v2/pkg/protocols/headless/operators.go

@@ -54,6 +54,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 	switch part {
 	case "body", "resp", "":
 		part = "data"
+	case "history":
+		part = "history"
 	}
 
 	item, ok := data[part]
@@ -66,12 +68,13 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 }
 
 // responseToDSLMap converts a headless response to a map for use in DSL matching
-func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent {
+func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent {
 	return output.InternalEvent{
 		"host":          host,
 		"matched":       matched,
 		"req":           req,
 		"data":          resp,
+		"history":       history,
 		"type":          request.Type().String(),
 		"template-id":   request.options.TemplateID,
 		"template-info": request.options.TemplateInfo,

v2/pkg/protocols/headless/request.go

@@ -66,7 +66,7 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o
 	if err == nil {
 		responseBody, _ = html.HTML()
 	}
-	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL)
+	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
 	for k, v := range out {
 		outputEvent[k] = v
 	}