Add template option to disable merging target url path with raw request path (#3799)

* add template option to disable merging target url path with raw request path

* rename disable-merge-path -> disable-path-automerge
add integration test

integration_tests/http/disable-path-automerge.yaml

@@ -0,0 +1,18 @@
+id: test
+
+info:
+  name: test
+  author: pdteam
+  severity: info
+
+http:
+  - raw:
+      - |
+        GET /api/v1/test?id=123 HTTP/1.1
+        Host: {{Hostname}}
+    disable-path-automerge: true
+    matchers:
+      - type: status
+        status:
+          - 200
+    

v2/cmd/integration-test/http.go

@@ -78,6 +78,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/cl-body-with-header.yaml":                 &httpCLBodyWithHeader{},
 	"http/save-extractor-values-to-file.yaml":       &httpSaveExtractorValuesToFile{},
 	"http/cli-with-constants.yaml":                  &ConstantWithCliVar{},
+	"http/disable-path-automerge.yaml":              &httpDisablePathAutomerge{},
 }
 
 type httpInteractshRequest struct{}
@@ -1423,3 +1424,21 @@ func (h *ConstantWithCliVar) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 1)
 }
+
+// disable path automerge in raw request
+type httpDisablePathAutomerge struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpDisablePathAutomerge) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/api/v1/test", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprint(w, r.URL.Query().Get("id"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/api/v1/user", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(got, 1)
+}

v2/pkg/protocols/http/build_request.go

@@ -264,7 +264,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		// in self contained requests baseURL is extracted from raw request itself
 		rawRequestData, err = raw.ParseRawRequest(rawRequest, r.request.Unsafe)
 	} else {
-		rawRequestData, err = raw.Parse(rawRequest, baseURL, r.request.Unsafe)
+		rawRequestData, err = raw.Parse(rawRequest, baseURL, r.request.Unsafe, r.request.DisablePathAutomerge)
 	}
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("failed to parse raw request")

v2/pkg/protocols/http/http.go

@@ -197,6 +197,9 @@ type Request struct {
 	// description: |
 	//   DigestAuthPassword specifies the password for digest authentication
 	DigestAuthPassword string `yaml:"digest-password,omitempty" json:"digest-password,omitempty" jsonschema:"title=specifies the password for digest authentication,description=Optional parameter which specifies the password for digest auth"`
+	// description: |
+	//  DisablePathAutomerge disables merging target url path with raw request path
+	DisablePathAutomerge bool `yaml:"disable-path-automerge,omitempty" json:"disable-path-automerge,omitempty" jsonschema:"title=disable auto merging of path,description=Disable merging target url path with raw request path"`
 }
 
 // Options returns executer options for http request

v2/pkg/protocols/http/raw/raw.go

@@ -26,7 +26,7 @@ type Request struct {
 }
 
 // Parse parses the raw request as supplied by the user
-func Parse(request string, inputURL *urlutil.URL, unsafe bool) (*Request, error) {
+func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge bool) (*Request, error) {
 	rawrequest, err := readRawRequest(request, unsafe)
 	if err != nil {
 		return nil, err
@@ -45,6 +45,9 @@ func Parse(request string, inputURL *urlutil.URL, unsafe bool) (*Request, error)
 			return nil, errorutil.NewWithErr(err).WithTag("raw").Msgf("failed to parse url %v from template", rawrequest.Path)
 		}
 		cloned := inputURL.Clone()
+		if disablePathAutomerge {
+			cloned.Path = ""
+		}
 		parseErr := cloned.MergePath(urlx.GetRelativePath(), true)
 		if parseErr != nil {
 			return nil, errorutil.NewWithTag("raw", "could not automergepath for template path %v", urlx.GetRelativePath()).Wrap(parseErr)
@@ -71,6 +74,9 @@ func Parse(request string, inputURL *urlutil.URL, unsafe bool) (*Request, error)
 				}
 			}
 		} else {
+			if disablePathAutomerge {
+				cloned.Path = ""
+			}
 			err = cloned.MergePath(rawrequest.Path, true)
 			if err != nil {
 				return nil, errorutil.NewWithErr(err).WithTag("raw").Msgf("failed to automerge %v from unsafe template", rawrequest.Path)
@@ -81,6 +87,9 @@ func Parse(request string, inputURL *urlutil.URL, unsafe bool) (*Request, error)
 
 	default:
 		cloned := inputURL.Clone()
+		if disablePathAutomerge {
+			cloned.Path = ""
+		}
 		parseErr := cloned.MergePath(rawrequest.Path, true)
 		if parseErr != nil {
 			return nil, errorutil.NewWithTag("raw", "could not automergepath for template path %v", rawrequest.Path).Wrap(parseErr)

v2/pkg/protocols/http/raw/raw_test.go

@@ -14,32 +14,32 @@ Origin: {{BaseURL}}
 Connection: close
 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
-Accept-Language: en-US,en;q=0.9`, parseURL(t, "https://example.com:8080"), false)
+Accept-Language: en-US,en;q=0.9`, parseURL(t, "https://example.com:8080"), false, false)
 	require.Nil(t, err, "could not parse GET request")
 	require.Equal(t, "https://example.com:8080/gg/phpinfo.php", request.FullURL, "Could not parse request url correctly")
 	require.Equal(t, "/gg/phpinfo.php", request.Path, "Could not parse request path correctly")
 
 	t.Run("path-suffix", func(t *testing.T) {
 		request, err := Parse(`GET /hello HTTP/1.1
-Host: {{Hostname}}`, parseURL(t, "https://example.com:8080/test"), false)
+Host: {{Hostname}}`, parseURL(t, "https://example.com:8080/test"), false, false)
 		require.Nil(t, err, "could not parse GET request")
 		require.Equal(t, "https://example.com:8080/test/hello", request.FullURL, "Could not parse request url correctly")
 	})
 
 	t.Run("query-values", func(t *testing.T) {
 		request, err := Parse(`GET ?username=test&password=test HTTP/1.1
-Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test"), false)
+Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test"), false, false)
 		require.Nil(t, err, "could not parse GET request")
 		// url.values are sorted to avoid randomness of using maps
 		require.Equal(t, "https://example.com:8080/test?password=test&username=test", request.FullURL, "Could not parse request url correctly")
 
 		request, err = Parse(`GET ?username=test&password=test HTTP/1.1
-Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false)
+Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false)
 		require.Nil(t, err, "could not parse GET request")
 		require.Equal(t, "https://example.com:8080/test/?password=test&username=test", request.FullURL, "Could not parse request url correctly")
 
 		request, err = Parse(`GET /?username=test&password=test HTTP/1.1
-		Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false)
+		Host: {{Hostname}}:123`, parseURL(t, "https://example.com:8080/test/"), false, false)
 		require.Nil(t, err, "could not parse GET request")
 		require.Equal(t, "https://example.com:8080/test/?password=test&username=test", request.FullURL, "Could not parse request url correctly")
 	})
@@ -51,7 +51,7 @@ Host: {{Hostname}}
 Authorization: Basic {{base64('username:password')}}
 User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0
 Accept-Language: en-US,en;q=0.9
-Connection: close`, parseURL(t, "https://test.com"), false)
+Connection: close`, parseURL(t, "https://test.com"), false, false)
 	require.Nil(t, err, "could not parse GET request")
 	require.Equal(t, "GET", request.Method, "Could not parse GET method request correctly")
 	require.Equal(t, "/manager/html", request.Path, "Could not parse request path correctly")
@@ -61,7 +61,7 @@ Host: {{Hostname}}
 Content-Type: application/x-www-form-urlencoded
 Connection: close
 
-username=admin&password=login`, parseURL(t, "https://test.com"), false)
+username=admin&password=login`, parseURL(t, "https://test.com"), false, false)
 	require.Nil(t, err, "could not parse POST request")
 	require.Equal(t, "POST", request.Method, "Could not parse POST method request correctly")
 	require.Equal(t, "username=admin&password=login", request.Data, "Could not parse request data correctly")
@@ -73,13 +73,13 @@ Host: {{Hostname}}
 Authorization: Basic {{base64('username:password')}}
 User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0
 Accept-Language: en-US,en;q=0.9
-Connection: close`, parseURL(t, "https://test.com/test/"), true)
+Connection: close`, parseURL(t, "https://test.com/test/"), true, false)
 	require.Nil(t, err, "could not parse unsafe request")
 	require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly")
 
 	request, err = Parse(`GET ?a=b HTTP/1.1
 	Host: {{Hostname}}
-	Origin: {{BaseURL}}`, parseURL(t, "https://test.com/test.js"), true)
+	Origin: {{BaseURL}}`, parseURL(t, "https://test.com/test.js"), true, false)
 	require.Nil(t, err, "could not parse unsafe request")
 	require.Contains(t, string(request.UnsafeRawBytes), "GET /test.js?a=b", "Could not parse unsafe method request path correctly")
 }
@@ -87,13 +87,26 @@ Connection: close`, parseURL(t, "https://test.com/test/"), true)
 func TestTryFillCustomHeaders(t *testing.T) {
 	testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n"
 	expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n"
-	request, err := Parse(testValue, parseURL(t, "https://test.com/test/"), true)
+	request, err := Parse(testValue, parseURL(t, "https://test.com/test/"), true, false)
 	require.Nil(t, err, "could not parse unsafe request")
 	err = request.TryFillCustomHeaders([]string{"test: test"})
 	require.Nil(t, err, "could not add custom headers")
 	require.Equal(t, expected, string(request.UnsafeRawBytes), "actual value and expected value are different")
 }
 
+func TestDisableMergePath(t *testing.T) {
+	request, err := Parse(` GET /api/v1/id=123 HTTP/1.1
+	Host: {{Hostname}}`, parseURL(t, "https://example.com/api/v1/user"), false, true)
+	require.Nil(t, err, "could not parse GET request with disable merge path")
+	require.Equal(t, "https://example.com/api/v1/id=123", request.FullURL, "Could not parse request url with disable merge path correctly")
+
+	request, err = Parse(` GET /api/v1/id=123 HTTP/1.1
+	Host: {{Hostname}}`, parseURL(t, "https://example.com/api/v1/user"), false, false)
+	require.Nil(t, err, "could not parse GET request with merge path")
+	require.Equal(t, "https://example.com/api/v1/user/api/v1/id=123", request.FullURL, "Could not parse request url with merge path correctly")
+
+}
+
 func parseURL(t *testing.T, inputurl string) *urlutil.URL {
 	urlx, err := urlutil.Parse(inputurl)
 	if err != nil {