diff --git a/integration_tests/http/interactsh-stop-at-first-match.yaml b/integration_tests/http/interactsh-stop-at-first-match.yaml new file mode 100644 index 00000000..415a634e --- /dev/null +++ b/integration_tests/http/interactsh-stop-at-first-match.yaml @@ -0,0 +1,23 @@ +id: interactsh-stop-at-first-match-integration-test + +info: + name: Interactsh StopAtFirstMatch Integration Test + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + - "{{BaseURL}}" + - "{{BaseURL}}" + headers: + url: 'http://{{interactsh-url}}' + + stop-at-first-match: true + + matchers: + - type: word + part: interactsh_protocol # Confirms the HTTP Interaction + words: + - "http" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index be640ec3..a95cea55 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, "http/interactsh.yaml": &httpInteractshRequest{}, + "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, "http/self-contained.yaml": &httpRequestSelContained{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, @@ -67,6 +68,29 @@ func (h *httpInteractshRequest) Execute(filePath string) error { return expectResultsCount(results, 1) } +type httpInteractshStopAtFirstMatchRequest struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + value := r.Header.Get("url") + if value != "" { + if resp, _ := http.DefaultClient.Get(value); resp != nil { + resp.Body.Close() + } + } + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 56eb0536..2d9f2870 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -159,7 +159,7 @@ func New(options *types.Options) (*Runner, error) { opts.ColldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second opts.NoInteractsh = runner.options.NoInteractsh - + opts.StopAtFirstMatch = runner.options.StopAtFirstMatch interactshClient, err := interactsh.New(opts) if err != nil { gologger.Error().Msgf("Could not create interactsh client: %s", err) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 175ad0ee..87dde8a8 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -2,6 +2,8 @@ package interactsh import ( "bytes" + "crypto/sha1" + "encoding/hex" "fmt" "net/url" "os" @@ -32,6 +34,8 @@ type Client struct { requests *ccache.Cache // interactions is a stored cache for interactsh-interaction->interactsh-url data interactions *ccache.Cache + // matchedTemplates is a stored cache to track matched templates + matchedTemplates *ccache.Cache options *Options eviction time.Duration @@ -73,8 +77,12 @@ type Options struct { Progress progress.Progress // Debug specifies whether debugging output should be shown for interactsh-client Debug bool - + // HttpFallback controls http retry in case of https failure for server url + HttpFallback bool + // NoInteractsh disables the engine NoInteractsh bool + + StopAtFirstMatch bool } const defaultMaxInteractionsCount = 5000 @@ -94,9 +102,12 @@ func New(options *Options) (*Client, error) { interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount) interactionsCache := ccache.New(interactionsCfg) + matchedTemplateCache := ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)) + interactClient := &Client{ eviction: options.Eviction, interactions: interactionsCache, + matchedTemplates: matchedTemplateCache, dotHostname: "." + parsed.Host, options: options, requests: cache, @@ -117,6 +128,7 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre Output: output, IssuesClient: reporting, Progress: progress, + HttpFallback: true, } } @@ -128,6 +140,7 @@ func (c *Client) firstTimeInitializeClient() error { ServerURL: c.options.ServerURL, Token: c.options.Authorization, PersistentSession: false, + HTTPFallback: c.options.HttpFallback, }) if err != nil { return errors.Wrap(err, "could not create client") @@ -135,6 +148,9 @@ func (c *Client) firstTimeInitializeClient() error { c.interactsh = interactsh interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) { + if c.options.StopAtFirstMatch && c.matched { + return + } if c.options.Debug { debugPrintInteraction(interaction) } @@ -155,6 +171,14 @@ func (c *Client) firstTimeInitializeClient() error { if !ok { return } + + if _, ok := request.Event.InternalEvent["stop-at-first-match"]; ok { + gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent["template-id"].(string), request.Event.InternalEvent["host"].(string))) + if gotItem != nil { + return + } + } + _ = c.processInteractionForRequest(interaction, request) }) return nil @@ -184,6 +208,9 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched = true + if _, ok := data.Event.InternalEvent["stop-at-first-match"]; ok { + c.matchedTemplates.Set(hash(data.Event.InternalEvent["template-id"].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) + } } return true } @@ -228,6 +255,11 @@ func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, [ return data, interactshURLs } +// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options +func (c *Client) SetStopAtFirstMatch() { + c.options.StopAtFirstMatch = true +} + // MakeResultEventFunc is a result making function for nuclei type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent @@ -243,6 +275,9 @@ type RequestData struct { // RequestEvent is the event for a network request sent by nuclei. func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { for _, interactshURL := range interactshURLs { + if c.options.StopAtFirstMatch && c.matched { + break + } id := strings.TrimSuffix(interactshURL, c.dotHostname) interaction := c.interactions.Get(id) @@ -313,3 +348,10 @@ func debugPrintInteraction(interaction *server.Interaction) { } fmt.Fprint(os.Stderr, builder.String()) } + +func hash(templateID, host string) string { + h := sha1.New() + h.Write([]byte(templateID)) + h.Write([]byte(host)) + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index afd59fd2..6543a8fb 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -2,6 +2,8 @@ package engine import ( "net/url" + "strings" + "sync" "time" "github.com/go-rod/rod" @@ -10,10 +12,18 @@ import ( // Page is a single page in an isolated browser instance type Page struct { - page *rod.Page - rules []requestRule - instance *Instance - router *rod.HijackRouter + page *rod.Page + rules []requestRule + instance *Instance + router *rod.HijackRouter + historyMutex *sync.RWMutex + History []HistoryData +} + +// HistoryData contains the page request/response pairs +type HistoryData struct { + RawRequest string + RawResponse string } // Run runs a list of actions by creating a new page in the browser. @@ -30,7 +40,7 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio } } - createdPage := &Page{page: page, instance: i} + createdPage := &Page{page: page, instance: i, historyMutex: &sync.RWMutex{}} router := page.HijackRequests() if routerErr := router.Add("*", "", createdPage.routingRuleHandler); routerErr != nil { return nil, nil, routerErr @@ -81,3 +91,24 @@ func (p *Page) URL() string { } return info.URL } + +// DumpHistory returns the full page navigation history +func (p *Page) DumpHistory() string { + p.historyMutex.RLock() + defer p.historyMutex.RUnlock() + + var historyDump strings.Builder + for _, historyData := range p.History { + historyDump.WriteString(historyData.RawRequest) + historyDump.WriteString(historyData.RawResponse) + } + return historyDump.String() +} + +// addToHistory adds a request/response pair to the page history +func (p *Page) addToHistory(historyData HistoryData) { + p.historyMutex.Lock() + defer p.historyMutex.Unlock() + + p.History = append(p.History, historyData) +} diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 8dc20687..5aa0c91d 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -2,6 +2,8 @@ package engine import ( "fmt" + "net/http/httputil" + "strings" "github.com/go-rod/rod" ) @@ -10,7 +12,6 @@ import ( func (p *Page) routingRuleHandler(ctx *rod.Hijack) { // usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) - for _, rule := range p.rules { if rule.Part != "request" { continue @@ -51,4 +52,32 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { ctx.Response.SetBody(rule.Args["body"]) } } + + // store history + req := ctx.Request.Req() + var rawReq string + if raw, err := httputil.DumpRequestOut(req, true); err == nil { + rawReq = string(raw) + } + + // attempts to rebuild the response + var rawResp strings.Builder + respPayloads := ctx.Response.Payload() + if respPayloads != nil { + rawResp.WriteString("HTTP/1.1 ") + rawResp.WriteString(fmt.Sprint(respPayloads.ResponseCode)) + rawResp.WriteString(" " + respPayloads.ResponsePhrase + "+\n") + for _, header := range respPayloads.ResponseHeaders { + rawResp.WriteString(header.Name + ": " + header.Value + "\n") + } + rawResp.WriteString("\n") + rawResp.WriteString(ctx.Response.Body()) + } + + // dump request + historyData := HistoryData{ + RawRequest: rawReq, + RawResponse: rawResp.String(), + } + p.addToHistory(historyData) } diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index b9f9b4cc..5d48371c 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -54,6 +54,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st switch part { case "body", "resp", "": part = "data" + case "history": + part = "history" } item, ok := data[part] @@ -66,12 +68,13 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st } // responseToDSLMap converts a headless response to a map for use in DSL matching -func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, "req": req, "data": resp, + "history": history, "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 639909c4..41c26884 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -66,7 +66,7 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o if err == nil { responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory()) for k, v := range out { outputEvent[k] = v } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 70e212c7..ce6a49c4 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -61,7 +61,8 @@ func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues ma ctx := context.Background() if r.options.Interactsh != nil { - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, []string{}) for payloadName, payloadValue := range payloads { payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) } diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 8b8e3f61..4e4890db 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -236,6 +236,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { Eviction: time.Duration(options.InteractionsEviction) * time.Second, ColldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, + HttpFallback: true, }) require.Nil(t, err, "could not create interactsh client") diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 70fd4d4c..0f4726d8 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -125,6 +125,10 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo data["template-path"] = request.options.TemplatePath + + if request.StopAtFirstMatch || request.options.StopAtFirstMatch { + data["stop-at-first-match"] = true + } return data } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index b3cb5237..37bb71b8 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -247,6 +247,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac response := responseBuilder.String() outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) outputEvent["ip"] = request.dialer.GetDialedIP(hostname) + if request.options.StopAtFirstMatch { + outputEvent["stop-at-first-match"] = true + } for k, v := range previous { outputEvent[k] = v }