Merge branch 'dev' into feature-ldap

integration_tests/http/interactsh-stop-at-first-match.yaml

@@ -0,0 +1,23 @@
+id: interactsh-stop-at-first-match-integration-test
+
+info:
+  name: Interactsh StopAtFirstMatch Integration Test
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+      - "{{BaseURL}}"
+      - "{{BaseURL}}"
+    headers:
+      url: 'http://{{interactsh-url}}'
+
+    stop-at-first-match: true
+
+    matchers:
+      - type: word
+        part: interactsh_protocol # Confirms the HTTP Interaction
+        words:
+          - "http"

v2/cmd/integration-test/http.go

@@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/request-condition.yaml":                  &httpRequestCondition{},
 	"http/request-condition-new.yaml":              &httpRequestCondition{},
 	"http/interactsh.yaml":                         &httpInteractshRequest{},
+	"http/interactsh-stop-at-first-match.yaml":     &httpInteractshStopAtFirstMatchRequest{},
 	"http/self-contained.yaml":                     &httpRequestSelContained{},
 	"http/get-case-insensitive.yaml":               &httpGetCaseInsensitive{},
 	"http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
@@ -67,6 +68,29 @@ func (h *httpInteractshRequest) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type httpInteractshStopAtFirstMatchRequest struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		value := r.Header.Get("url")
+		if value != "" {
+			if resp, _ := http.DefaultClient.Get(value); resp != nil {
+				resp.Body.Close()
+			}
+		}
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
 type httpGetHeaders struct{}
 
 // Execute executes a test case and returns an error if occurred

v2/internal/runner/runner.go

@@ -159,7 +159,7 @@ func New(options *types.Options) (*Runner, error) {
 	opts.ColldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second
 	opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
 	opts.NoInteractsh = runner.options.NoInteractsh
-
+	opts.StopAtFirstMatch = runner.options.StopAtFirstMatch
 	interactshClient, err := interactsh.New(opts)
 	if err != nil {
 		gologger.Error().Msgf("Could not create interactsh client: %s", err)

v2/pkg/protocols/common/interactsh/interactsh.go

@@ -2,6 +2,8 @@ package interactsh
 
 import (
 	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
 	"net/url"
 	"os"
@@ -32,6 +34,8 @@ type Client struct {
 	requests *ccache.Cache
 	// interactions is a stored cache for interactsh-interaction->interactsh-url data
 	interactions *ccache.Cache
+	// matchedTemplates is a stored cache to track matched templates
+	matchedTemplates *ccache.Cache
 
 	options          *Options
 	eviction         time.Duration
@@ -73,8 +77,12 @@ type Options struct {
 	Progress progress.Progress
 	// Debug specifies whether debugging output should be shown for interactsh-client
 	Debug bool
-
+	// HttpFallback controls http retry in case of https failure for server url
+	HttpFallback bool
+	// NoInteractsh disables the engine
 	NoInteractsh bool
+
+	StopAtFirstMatch bool
 }
 
 const defaultMaxInteractionsCount = 5000
@@ -94,9 +102,12 @@ func New(options *Options) (*Client, error) {
 	interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount)
 	interactionsCache := ccache.New(interactionsCfg)
 
+	matchedTemplateCache := ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))
+
 	interactClient := &Client{
 		eviction:         options.Eviction,
 		interactions:     interactionsCache,
+		matchedTemplates: matchedTemplateCache,
 		dotHostname:      "." + parsed.Host,
 		options:          options,
 		requests:         cache,
@@ -117,6 +128,7 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre
 		Output:         output,
 		IssuesClient:   reporting,
 		Progress:       progress,
+		HttpFallback:   true,
 	}
 }
 
@@ -128,6 +140,7 @@ func (c *Client) firstTimeInitializeClient() error {
 		ServerURL:         c.options.ServerURL,
 		Token:             c.options.Authorization,
 		PersistentSession: false,
+		HTTPFallback:      c.options.HttpFallback,
 	})
 	if err != nil {
 		return errors.Wrap(err, "could not create client")
@@ -135,6 +148,9 @@ func (c *Client) firstTimeInitializeClient() error {
 	c.interactsh = interactsh
 
 	interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
+		if c.options.StopAtFirstMatch && c.matched {
+			return
+		}
 		if c.options.Debug {
 			debugPrintInteraction(interaction)
 		}
@@ -155,6 +171,14 @@ func (c *Client) firstTimeInitializeClient() error {
 		if !ok {
 			return
 		}
+
+		if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok {
+			gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent["template-id"].(string), request.Event.InternalEvent["host"].(string)))
+			if gotItem != nil {
+				return
+			}
+		}
+
 		_ = c.processInteractionForRequest(interaction, request)
 	})
 	return nil
@@ -184,6 +208,9 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
 
 	if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) {
 		c.matched = true
+		if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok {
+			c.matchedTemplates.Set(hash(data.Event.InternalEvent["template-id"].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration)
+		}
 	}
 	return true
 }
@@ -228,6 +255,11 @@ func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, [
 	return data, interactshURLs
 }
 
+// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options
+func (c *Client) SetStopAtFirstMatch() {
+	c.options.StopAtFirstMatch = true
+}
+
 // MakeResultEventFunc is a result making function for nuclei
 type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
 
@@ -243,6 +275,9 @@ type RequestData struct {
 // RequestEvent is the event for a network request sent by nuclei.
 func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) {
 	for _, interactshURL := range interactshURLs {
+		if c.options.StopAtFirstMatch && c.matched {
+			break
+		}
 		id := strings.TrimSuffix(interactshURL, c.dotHostname)
 
 		interaction := c.interactions.Get(id)
@@ -313,3 +348,10 @@ func debugPrintInteraction(interaction *server.Interaction) {
 	}
 	fmt.Fprint(os.Stderr, builder.String())
 }
+
+func hash(templateID, host string) string {
+	h := sha1.New()
+	h.Write([]byte(templateID))
+	h.Write([]byte(host))
+	return hex.EncodeToString(h.Sum(nil))
+}

v2/pkg/protocols/headless/engine/page.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"net/url"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -10,10 +12,18 @@ import (
 
 // Page is a single page in an isolated browser instance
 type Page struct {
-	page     *rod.Page
-	rules    []requestRule
-	instance *Instance
-	router   *rod.HijackRouter
+	page         *rod.Page
+	rules        []requestRule
+	instance     *Instance
+	router       *rod.HijackRouter
+	historyMutex *sync.RWMutex
+	History      []HistoryData
+}
+
+// HistoryData contains the page request/response pairs
+type HistoryData struct {
+	RawRequest  string
+	RawResponse string
 }
 
 // Run runs a list of actions by creating a new page in the browser.
@@ -30,7 +40,7 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio
 		}
 	}
 
-	createdPage := &Page{page: page, instance: i}
+	createdPage := &Page{page: page, instance: i, historyMutex: &sync.RWMutex{}}
 	router := page.HijackRequests()
 	if routerErr := router.Add("*", "", createdPage.routingRuleHandler); routerErr != nil {
 		return nil, nil, routerErr
@@ -81,3 +91,24 @@ func (p *Page) URL() string {
 	}
 	return info.URL
 }
+
+// DumpHistory returns the full page navigation history
+func (p *Page) DumpHistory() string {
+	p.historyMutex.RLock()
+	defer p.historyMutex.RUnlock()
+
+	var historyDump strings.Builder
+	for _, historyData := range p.History {
+		historyDump.WriteString(historyData.RawRequest)
+		historyDump.WriteString(historyData.RawResponse)
+	}
+	return historyDump.String()
+}
+
+// addToHistory adds a request/response pair to the page history
+func (p *Page) addToHistory(historyData HistoryData) {
+	p.historyMutex.Lock()
+	defer p.historyMutex.Unlock()
+
+	p.History = append(p.History, historyData)
+}

v2/pkg/protocols/headless/engine/rules.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"fmt"
+	"net/http/httputil"
+	"strings"
 
 	"github.com/go-rod/rod"
 )
@@ -10,7 +12,6 @@ import (
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 	// usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless
 	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
-
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue
@@ -51,4 +52,32 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 			ctx.Response.SetBody(rule.Args["body"])
 		}
 	}
+
+	// store history
+	req := ctx.Request.Req()
+	var rawReq string
+	if raw, err := httputil.DumpRequestOut(req, true); err == nil {
+		rawReq = string(raw)
+	}
+
+	// attempts to rebuild the response
+	var rawResp strings.Builder
+	respPayloads := ctx.Response.Payload()
+	if respPayloads != nil {
+		rawResp.WriteString("HTTP/1.1 ")
+		rawResp.WriteString(fmt.Sprint(respPayloads.ResponseCode))
+		rawResp.WriteString(" " + respPayloads.ResponsePhrase + "+\n")
+		for _, header := range respPayloads.ResponseHeaders {
+			rawResp.WriteString(header.Name + ": " + header.Value + "\n")
+		}
+		rawResp.WriteString("\n")
+		rawResp.WriteString(ctx.Response.Body())
+	}
+
+	// dump request
+	historyData := HistoryData{
+		RawRequest:  rawReq,
+		RawResponse: rawResp.String(),
+	}
+	p.addToHistory(historyData)
 }

v2/pkg/protocols/headless/operators.go

@@ -54,6 +54,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 	switch part {
 	case "body", "resp", "":
 		part = "data"
+	case "history":
+		part = "history"
 	}
 
 	item, ok := data[part]
@@ -66,12 +68,13 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 }
 
 // responseToDSLMap converts a headless response to a map for use in DSL matching
-func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent {
+func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent {
 	return output.InternalEvent{
 		"host":          host,
 		"matched":       matched,
 		"req":           req,
 		"data":          resp,
+		"history":       history,
 		"type":          request.Type().String(),
 		"template-id":   request.options.TemplateID,
 		"template-info": request.options.TemplateInfo,

v2/pkg/protocols/headless/request.go

@@ -66,7 +66,7 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o
 	if err == nil {
 		responseBody, _ = html.HTML()
 	}
-	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL)
+	outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
 	for k, v := range out {
 		outputEvent[k] = v
 	}

v2/pkg/protocols/http/build_request.go

@@ -61,7 +61,8 @@ func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues ma
 	ctx := context.Background()
 
 	if r.options.Interactsh != nil {
-		data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
+
+		data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, []string{})
 		for payloadName, payloadValue := range payloads {
 			payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs)
 		}

v2/pkg/protocols/http/build_request_test.go

@@ -236,6 +236,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 		Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
 		ColldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second,
 		PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+		HttpFallback:   true,
 	})
 	require.Nil(t, err, "could not create interactsh client")
 

v2/pkg/protocols/http/operators.go

@@ -125,6 +125,10 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
 	data["template-id"] = request.options.TemplateID
 	data["template-info"] = request.options.TemplateInfo
 	data["template-path"] = request.options.TemplatePath
+
+	if request.StopAtFirstMatch || request.options.StopAtFirstMatch {
+		data["stop-at-first-match"] = true
+	}
 	return data
 }
 

v2/pkg/protocols/network/request.go

@@ -247,6 +247,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 	response := responseBuilder.String()
 	outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)
 	outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
+	if request.options.StopAtFirstMatch {
+		outputEvent["stop-at-first-match"] = true
+	}
 	for k, v := range previous {
 		outputEvent[k] = v
 	}