Merge from dev

v2/cmd/nuclei/main.go

@@ -109,7 +109,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
 		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
 		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
-		flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
+		flagSet.IntVar(&options.InteractionsCooldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
 		flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"),
 	)
 

v2/internal/runner/runner.go

@@ -151,7 +151,7 @@ func New(options *types.Options) (*Runner, error) {
 		opts.Authorization = options.InteractshToken
 		opts.CacheSize = int64(options.InteractionsCacheSize)
 		opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second
-		opts.ColldownPeriod = time.Duration(options.InteractionsColldownPeriod) * time.Second
+		opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second
 		opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
 
 		interactshClient, err := interactsh.New(opts)

v2/pkg/protocols/common/interactsh/interactsh.go

@@ -220,12 +220,14 @@ func (c *Client) Close() bool {
 //
 // It accepts data to replace as well as the URL to replace placeholders
 // with generated uniquely for each request.
-func (c *Client) ReplaceMarkers(data, interactshURL string) string {
+func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) {
-	if !strings.Contains(data, interactshURLMarker) {
+
-		return data
+	for strings.Contains(data, interactshURLMarker) {
+		url := c.URL()
+		interactshURLs = append(interactshURLs, url)
+		data = strings.Replace(data, interactshURLMarker, url, 1)
 	}
-	replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data)
+	return data, interactshURLs
-	return replaced
 }
 
 // MakeResultEventFunc is a result making function for nuclei
@@ -241,7 +243,8 @@ type RequestData struct {
 }
 
 // RequestEvent is the event for a network request sent by nuclei.
-func (c *Client) RequestEvent(interactshURL string, data *RequestData) {
+func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) {
+	for _, interactshURL := range interactshURLs {
 		id := strings.TrimSuffix(interactshURL, c.dotHostname)
 
 		interaction := c.interactions.Get(id)
@@ -252,21 +255,19 @@ func (c *Client) RequestEvent(interactshURL string, data *RequestData) {
 				c.requests.Set(id, data, c.eviction)
 				return
 			}
-		matched := false
 			for _, interaction := range interactions {
 				if c.processInteractionForRequest(interaction, data) {
-				matched = true
+					c.interactions.Delete(id)
 					break
 				}
 			}
-		if matched {
-			c.interactions.Delete(id)
-		}
 		} else {
 			c.requests.Set(id, data, c.eviction)
 		}
 	}
 
+}
+
 // HasMatchers returns true if an operator has interactsh part
 // matchers or extractors.
 //

v2/pkg/protocols/http/build_request.go

@@ -38,6 +38,7 @@ type generatedRequest struct {
 	pipelinedClient *rawhttp.PipelineClient
 	request         *retryablehttp.Request
 	dynamicValues   map[string]interface{}
+	interactshURLs  []string
 }
 
 func (g *generatedRequest) URL() string {
@@ -52,9 +53,9 @@ func (g *generatedRequest) URL() string {
 
 // Make creates a http request for the provided input.
 // It returns io.EOF as error when all the requests have been exhausted.
-func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
+func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	if r.request.SelfContained {
-		return r.makeSelfContainedRequest(dynamicValues, interactURL)
+		return r.makeSelfContainedRequest(dynamicValues)
 	}
 	// We get the next payload for the request.
 	data, payloads, ok := r.nextValue()
@@ -63,12 +64,10 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	}
 	ctx := context.Background()
 
-	if interactURL != "" {
+	data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
-		data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
 
 	for payloadName, payloadValue := range payloads {
-			payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL)
+		payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs)
-		}
 	}
 
 	parsed, err := url.Parse(baseURL)
@@ -98,12 +97,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	// If data contains \n it's a raw request, process it like raw. Else
 	// continue with the template based request flow.
 	if isRawRequest {
-		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
+		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
 	}
-	return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
+	return r.makeHTTPRequestFromModel(ctx, data, values, payloads)
 }
 
-func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
+func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	// We get the next payload for the request.
 	data, payloads, ok := r.nextValue()
 	if !ok {
@@ -136,13 +135,13 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int
 			generators.BuildPayloadFromOptions(r.request.options.Options),
 		)
 
-		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
+		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
 	}
 	values := generators.MergeMaps(
 		dynamicValues,
 		generators.BuildPayloadFromOptions(r.request.options.Options),
 	)
-	return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
+	return r.makeHTTPRequestFromModel(ctx, data, values, payloads)
 }
 
 // Total returns the total number of requests for the generator
@@ -171,10 +170,8 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
 }
 
 // MakeHTTPRequestFromModel creates a *http.Request from a request template
-func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
+func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
-	if interactURL != "" {
+	data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
-		data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
-	}
 
 	// Combine the template payloads along with base
 	// request values.
@@ -198,18 +195,16 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
 		return nil, err
 	}
 
-	request, err := r.fillRequest(req, finalValues, interactURL)
+	request, err := r.fillRequest(req, finalValues)
 	if err != nil {
 		return nil, err
 	}
-	return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil
+	return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil
 }
 
 // makeHTTPRequestFromRaw creates a *http.Request from a raw request
-func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}, interactURL string) (*generatedRequest, error) {
+func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) {
-	if interactURL != "" {
+	data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
-		data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
-	}
 	return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads)
 }
 
@@ -258,21 +253,19 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
 			req.Host = value
 		}
 	}
-	request, err := r.fillRequest(req, finalValues, "")
+	request, err := r.fillRequest(req, finalValues)
 	if err != nil {
 		return nil, err
 	}
 
-	return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil
+	return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil
 }
 
 // fillRequest fills various headers in the request with values
-func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}, interactURL string) (*retryablehttp.Request, error) {
+func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
 	// Set the header values requested
 	for header, value := range r.request.Headers {
-		if interactURL != "" {
+		value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs)
-			value = r.options.Interactsh.ReplaceMarkers(value, interactURL)
-		}
 		value, err := expressions.Evaluate(value, values)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not evaluate helper expressions")
@@ -290,10 +283,8 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
 
 	// Check if the user requested a request body
 	if r.request.Body != "" {
-		body := r.request.Body
+		var body string
-		if interactURL != "" {
+		body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs)
-			body = r.options.Interactsh.ReplaceMarkers(body, interactURL)
-		}
 		body, err := expressions.Evaluate(body, values)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not evaluate helper expressions")

v2/pkg/protocols/http/build_request_test.go

@@ -3,13 +3,14 @@ package http
 import (
 	"net/url"
 	"testing"
-
+	"time"
-	"github.com/stretchr/testify/require"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBaseURLWithTemplatePrefs(t *testing.T) {
@@ -85,7 +86,7 @@ func TestMakeRequestFromModal(t *testing.T) {
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
+	req, err := generator.Make("https://example.com", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	bodyBytes, _ := req.request.BodyBytes()
@@ -112,12 +113,12 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}, "")
+	req, err := generator.Make("https://example.com/test.php", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator()
-	req, err = generator.Make("https://example.com/test/", map[string]interface{}{}, "")
+	req, err = generator.Make("https://example.com/test/", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -150,12 +151,12 @@ Accept-Encoding: gzip`},
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
+	req, err := generator.Make("https://example.com", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
-	req, err = generator.Make("https://example.com", map[string]interface{}{}, "")
+	req, err = generator.Make("https://example.com", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -189,13 +190,66 @@ Accept-Encoding: gzip`},
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
+	req, err := generator.Make("https://example.com", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
-	req, err = generator.Make("https://example.com", map[string]interface{}{}, "")
+	req, err = generator.Make("https://example.com", map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
 }
+
+func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
+
+	options := testutils.DefaultOptions
+
+	testutils.Init(options)
+	templateID := "testing-unique-interactsh"
+	request := &Request{
+		ID:     templateID,
+		Name:   "testing",
+		Path:   []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"},
+		Method: "GET",
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	err := request.Compile(executerOpts)
+	require.Nil(t, err, "could not compile http request")
+
+	generator := request.newGenerator()
+
+	generator.options.Interactsh, err = interactsh.New(&interactsh.Options{
+		ServerURL:      options.InteractshURL,
+		CacheSize:      int64(options.InteractionsCacheSize),
+		Eviction:       time.Duration(options.InteractionsEviction) * time.Second,
+		ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second,
+		PollDuration:   time.Duration(options.InteractionsPollDuration) * time.Second,
+	})
+	require.Nil(t, err, "could not create interactsh client")
+
+	got, err := generator.Make("https://example.com", map[string]interface{}{})
+	require.Nil(t, err, "could not make http request")
+
+	// check if all the interactsh markers are replaced with unique urls
+	require.NotContains(t, got.request.URL.String(), "{{interactsh-url}}", "could not get correct interactsh url")
+	// check the length of returned urls
+	require.Equal(t, len(got.interactshURLs), 4, "could not get correct interactsh url")
+	// check if the interactsh urls are unique
+	require.True(t, areUnique(got.interactshURLs), "interactsh urls are not unique")
+}
+
+// areUnique checks if the elements of string slice are unique
+func areUnique(elements []string) bool {
+	encountered := map[string]bool{}
+	for v := range elements {
+		if encountered[elements[v]] {
+			return false
+		}
+		encountered[elements[v]] = true
+	}
+	return true
+}

v2/pkg/protocols/http/request.go

@@ -47,7 +47,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
 	// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
 	// This will introduce a delay and will populate in hacky way the field "request" of outputEvent
 	generator := request.newGenerator()
-	requestForDump, err := generator.Make(reqURL, nil, "")
+	requestForDump, err := generator.Make(reqURL, nil)
 	if err != nil {
 		return err
 	}
@@ -65,7 +65,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
 	// Pre-Generate requests
 	for i := 0; i < request.RaceNumberRequests; i++ {
 		generator := request.newGenerator()
-		generatedRequest, err := generator.Make(reqURL, nil, "")
+		generatedRequest, err := generator.Make(reqURL, nil)
 		if err != nil {
 			return err
 		}
@@ -104,7 +104,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
 	var requestErr error
 	mutex := &sync.Mutex{}
 	for {
-		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "")
+		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
 		if err == io.EOF {
 			break
 		}
@@ -167,7 +167,7 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
 	var requestErr error
 	mutex := &sync.Mutex{}
 	for {
-		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "")
+		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
 		if err == io.EOF {
 			break
 		}
@@ -221,11 +221,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 	for {
 		hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
 
-		var interactURL string
+		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
-		if request.options.Interactsh != nil && hasInteractMarkers {
-			interactURL = request.options.Interactsh.URL()
-		}
-		generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL)
 		if err == io.EOF {
 			break
 		}
@@ -251,7 +247,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 				dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues)
 			}
 			if hasInteractMarkers && request.options.Interactsh != nil {
-				request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
+				request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{
 					MakeResultFunc: request.MakeResultEvent,
 					Event:          event,
 					Operators:      request.CompiledOperators,

v2/pkg/protocols/http/request_generator.go

@@ -16,6 +16,7 @@ type requestGenerator struct {
 	request         *Request
 	options         *protocols.ExecuterOptions
 	payloadIterator *generators.Iterator
+	interactshURLs  []string
 }
 
 // newGenerator creates a new request generator instance

v2/pkg/protocols/network/request.go

@@ -126,11 +126,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	defer conn.Close()
 	_ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second))
 
-	hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
+	var interactshURLs []string
-	var interactURL string
-	if request.options.Interactsh != nil && hasInteractMarkers {
-		interactURL = request.options.Interactsh.URL()
-	}
 
 	responseBuilder := &strings.Builder{}
 	reqBuilder := &strings.Builder{}
@@ -143,9 +139,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 		case "hex":
 			data, err = hex.DecodeString(input.Data)
 		default:
-			if interactURL != "" {
+			input.Data, interactshURLs = request.options.Interactsh.ReplaceMarkers(input.Data, []string{})
-				input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL)
-			}
 			data = []byte(input.Data)
 		}
 		if err != nil {
@@ -267,14 +261,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	}
 
 	var event *output.InternalWrappedEvent
-	if interactURL == "" {
+	if len(interactshURLs) == 0 {
 		event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
 			wrappedEvent.OperatorsResult.PayloadValues = payloads
 		})
 		callback(event)
 	} else if request.options.Interactsh != nil {
 		event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
-		request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
+		request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{
 			MakeResultFunc: request.MakeResultEvent,
 			Event:          event,
 			Operators:      request.CompiledOperators,

v2/pkg/testutils/testutils.go

@@ -58,6 +58,11 @@ var DefaultOptions = &types.Options{
 	Templates:                  []string{},
 	ExcludedTemplates:          []string{},
 	CustomHeaders:              []string{},
+	InteractshURL:              "https://interactsh.com",
+	InteractionsCacheSize:      5000,
+	InteractionsEviction:       60,
+	InteractionsCooldownPeriod: 5,
+	InteractionsPollDuration:   5,
 }
 
 // TemplateInfo contains info for a mock executed template.

v2/pkg/types/types.go

@@ -109,9 +109,9 @@ type Options struct {
 	// Eviction is the number of seconds after which to automatically discard
 	// interaction requests.
 	InteractionsEviction int
-	// InteractionsColldownPeriod is additional seconds to wait for interactions after closing
+	// InteractionsCooldownPeriod is additional seconds to wait for interactions after closing
 	// of the poller.
-	InteractionsColldownPeriod int
+	InteractionsCooldownPeriod int
 	// OfflineHTTP is a flag that specific offline processing of http response
 	// using same matchers/extractors from http protocol without the need
 	// to send a new request, reading responses from a file.