From e9ab5f498a9882825ea68e18597ca1e4c263cf5c Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Fri, 20 Oct 2023 17:54:10 +0530
Subject: [PATCH] template preprocessor + multi request variables indexing bug
 fix (#4262)

* add randstr preprocessor to defaults

* fix indexing in http + preprocessor integration test

* add multi-request integration test

* skip test if asnmap is down
---
 cmd/integration-test/http.go                  | 56 +++++++++++++++++++
 .../protocols/http/http-preprocessor.yaml     | 17 ++++++
 .../protocols/http/multi-request.yaml         | 26 +++++++++
 pkg/core/inputs/hybrid/hmap_test.go           |  4 ++
 pkg/protocols/http/request.go                 |  2 +-
 pkg/templates/compile.go                      |  2 +-
 pkg/templates/preprocessors.go                |  4 +-
 7 files changed, 108 insertions(+), 3 deletions(-)
 create mode 100644 integration_tests/protocols/http/http-preprocessor.yaml
 create mode 100644 integration_tests/protocols/http/multi-request.yaml

diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go
index 79d718fa..de3f9c15 100644
--- a/cmd/integration-test/http.go
+++ b/cmd/integration-test/http.go
@@ -10,6 +10,7 @@ import (
 	"net/http/httputil"
 	"os"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -80,6 +81,8 @@ var httpTestcases = []TestCaseInfo{
 	{Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}},
 	{Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}},
 	{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
+	{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
+	{Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}},
 }
 
 type httpInteractshRequest struct{}
@@ -1475,3 +1478,56 @@ func (h *httpInteractshRequestsWithMCAnd) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 1)
 }
+
+// integration test to check if preprocessor i.e {{randstr}}
+// is working correctly
+type httpPreprocessor struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpPreprocessor) Execute(filePath string) error {
+	router := httprouter.New()
+	re := regexp.MustCompile(`[A-Za-z0-9]{25,}`)
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		value := r.URL.RequestURI()
+		if re.MatchString(value) {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprint(w, "ok")
+		} else {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, "not ok")
+		}
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type httpMultiRequest struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpMultiRequest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/ping", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "ping")
+	})
+	router.GET("/pong", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "pong")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
diff --git a/integration_tests/protocols/http/http-preprocessor.yaml b/integration_tests/protocols/http/http-preprocessor.yaml
new file mode 100644
index 00000000..b856f3cc
--- /dev/null
+++ b/integration_tests/protocols/http/http-preprocessor.yaml
@@ -0,0 +1,17 @@
+id: http-preprocessor
+
+info:
+  name: Test Http Preprocessor
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET /?test={{randstr}} HTTP/1.1
+        Host: {{Hostname}}
+
+    matchers:
+      - type: status
+        status:
+          - 200
\ No newline at end of file
diff --git a/integration_tests/protocols/http/multi-request.yaml b/integration_tests/protocols/http/multi-request.yaml
new file mode 100644
index 00000000..4ede5e37
--- /dev/null
+++ b/integration_tests/protocols/http/multi-request.yaml
@@ -0,0 +1,26 @@
+id: http-multi-request
+
+info:
+  name: http multi request template
+  author: pdteam
+  severity: info
+  description: template with multiple http request with combined logic
+  reference: https://example-reference-link
+
+# requestURI is reflected back as response body here
+http:
+  - raw:
+      - |
+        GET /ping HTTP/1.1
+        Host: {{Hostname}}
+     
+      - |
+        GET /pong HTTP/1.1
+        Host: {{Hostname}}
+    
+    matchers:
+        - type: dsl
+          dsl:
+            - 'body_1 == "ping"'
+            - 'body_2 == "pong"'
+          condition: and
\ No newline at end of file
diff --git a/pkg/core/inputs/hybrid/hmap_test.go b/pkg/core/inputs/hybrid/hmap_test.go
index fee92f46..3dbbdb73 100644
--- a/pkg/core/inputs/hybrid/hmap_test.go
+++ b/pkg/core/inputs/hybrid/hmap_test.go
@@ -180,6 +180,10 @@ func Test_expandASNInputValue(t *testing.T) {
 			got = append(got, metainput.Input)
 			return nil
 		})
+		if len(got) == 0 {
+			// asnmap server is down
+			t.SkipNow()
+		}
 		// read the expected IPs from the file
 		fileContent, err := os.ReadFile(tt.expectedOutputFile)
 		require.Nil(t, err, "could not read the expectedOutputFile file")
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 9131ba4a..aac2bad7 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -793,7 +793,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if request.NeedsRequestCondition() {
 			for k, v := range outputEvent {
 				key := fmt.Sprintf("%s_%d", k, requestCount)
-				if previousEvent[key] != nil {
+				if previousEvent != nil {
 					previousEvent[key] = v
 				}
 				finalEvent[key] = v
diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go
index 3c6a9694..5f5e6345 100644
--- a/pkg/templates/compile.go
+++ b/pkg/templates/compile.go
@@ -272,7 +272,7 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
 	// persist verified status value and then
 	// expand all preprocessor and reparse template
 
-	// === signature verification befoer preprocessors ===
+	// === signature verification before preprocessors ===
 	template, err := parseTemplate(data, options)
 	if err != nil {
 		return nil, err
diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go
index f730ec09..d7e5864d 100644
--- a/pkg/templates/preprocessors.go
+++ b/pkg/templates/preprocessors.go
@@ -17,7 +17,9 @@ type Preprocessor interface {
 
 var (
 	preprocessorRegex    = regexp.MustCompile(`{{([a-z0-9_]+)}}`)
-	defaultPreprocessors = []Preprocessor{}
+	defaultPreprocessors = []Preprocessor{
+		&randStrPreprocessor{},
+	}
 )
 
 func getPreprocessors(preprocessor Preprocessor) []Preprocessor {