Merge branch 'dev' into issue-1231-zcrypto

2022-02-03 09:02:09 +01:00 · 2022-02-03 09:02:09 +01:00 · 62fc7a8110
parent 672d21cce7 ae45519551
commit 62fc7a8110
30 changed files with 610 additions and 180 deletions
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@ -2506,6 +2506,8 @@ Enum Values:
  - <code>TXT</code>

  - <code>AAAA</code>
+
+  - <code>CAA</code>
 </div>

 <hr />
--- a/integration_tests/dns/caa.yaml
+++ b/integration_tests/dns/caa.yaml
@ -0,0 +1,22 @@
+id: caa-fingerprinting
+
+info:
+  name: CAA Fingerprint
+  author: pdteam
+  severity: info
+  tags: dns,caa
+
+dns:
+  - name: "{{FQDN}}"
+    type: CAA
+
+    matchers:
+      - type: word
+        words:
+          - "IN\tCAA"
+
+    extractors:
+      - type: regex
+        group: 1
+        regex:
+          - "IN\tCAA\t(.+)"
--- a/integration_tests/http/stop-at-first-match-with-extractors.yaml
+++ b/integration_tests/http/stop-at-first-match-with-extractors.yaml
@ -0,0 +1,18 @@
+id: stop-at-first-match-with-extractors
+
+info:
+  name: Stop at first match Request with extractors
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?a=1"
+      - "{{BaseURL}}?a=2"
+    stop-at-first-match: true
+    extractors:
+      - type: kval
+        part: header
+        kval:
+          - "date"
--- a/integration_tests/http/stop-at-first-match.yaml
+++ b/integration_tests/http/stop-at-first-match.yaml
@ -0,0 +1,17 @@
+id: stop-at-first-match
+
+info:
+  name: Stop at first match Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}?a=1"
+      - "{{BaseURL}}?a=2"
+    matchers:
+      - type: word
+        words:
+          - "This is test"
+    stop-at-first-match: true
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@ -338,7 +338,8 @@
        "PTR",
        "MX",
        "TXT",
-        "AAAA"
+        "AAAA",
+        "CAA"
      ],
      "type": "string",
      "title": "type of DNS request to make",
--- a/v2/cmd/integration-test/dns.go
+++ b/v2/cmd/integration-test/dns.go
@ -7,6 +7,7 @@ import (
 var dnsTestCases = map[string]testutils.TestCase{
 	"dns/basic.yaml": &dnsBasic{},
 	"dns/ptr.yaml":   &dnsPtr{},
+	"dns/caa.yaml":   &dnsCAA{},
 }

 type dnsBasic struct{}
@ -40,3 +41,19 @@ func (h *dnsPtr) Execute(filePath string) error {
 	}
 	return expectResultsCount(results, 1)
 }
+
+type dnsCAA struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *dnsCAA) Execute(filePath string) error {
+	var routerErr error
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	return expectResultsCount(results, 1)
+}
--- a/v2/cmd/integration-test/http.go
+++ b/v2/cmd/integration-test/http.go
@ -18,32 +18,34 @@ import (
 )

 var httpTestcases = map[string]testutils.TestCase{
-	"http/get-headers.yaml":                        &httpGetHeaders{},
-	"http/get-query-string.yaml":                   &httpGetQueryString{},
-	"http/get-redirects.yaml":                      &httpGetRedirects{},
-	"http/get.yaml":                                &httpGet{},
-	"http/post-body.yaml":                          &httpPostBody{},
-	"http/post-json-body.yaml":                     &httpPostJSONBody{},
-	"http/post-multipart-body.yaml":                &httpPostMultipartBody{},
-	"http/raw-cookie-reuse.yaml":                   &httpRawCookieReuse{},
-	"http/raw-dynamic-extractor.yaml":              &httpRawDynamicExtractor{},
-	"http/raw-get-query.yaml":                      &httpRawGetQuery{},
-	"http/raw-get.yaml":                            &httpRawGet{},
-	"http/raw-payload.yaml":                        &httpRawPayload{},
-	"http/raw-post-body.yaml":                      &httpRawPostBody{},
-	"http/raw-unsafe-request.yaml":                 &httpRawUnsafeRequest{},
-	"http/request-condition.yaml":                  &httpRequestCondition{},
-	"http/request-condition-new.yaml":              &httpRequestCondition{},
-	"http/interactsh.yaml":                         &httpInteractshRequest{},
-	"http/interactsh-stop-at-first-match.yaml":     &httpInteractshStopAtFirstMatchRequest{},
-	"http/self-contained.yaml":                     &httpRequestSelContained{},
-	"http/get-case-insensitive.yaml":               &httpGetCaseInsensitive{},
-	"http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{},
-	"http/get-redirects-chain-headers.yaml":        &httpGetRedirectsChainHeaders{},
-	"http/dsl-matcher-variable.yaml":               &httpDSLVariable{},
-	"http/dsl-functions.yaml":                      &httpDSLFunctions{},
-	"http/race-simple.yaml":                        &httpRaceSimple{},
-	"http/race-multiple.yaml":                      &httpRaceMultiple{},
+	"http/get-headers.yaml":                         &httpGetHeaders{},
+	"http/get-query-string.yaml":                    &httpGetQueryString{},
+	"http/get-redirects.yaml":                       &httpGetRedirects{},
+	"http/get.yaml":                                 &httpGet{},
+	"http/post-body.yaml":                           &httpPostBody{},
+	"http/post-json-body.yaml":                      &httpPostJSONBody{},
+	"http/post-multipart-body.yaml":                 &httpPostMultipartBody{},
+	"http/raw-cookie-reuse.yaml":                    &httpRawCookieReuse{},
+	"http/raw-dynamic-extractor.yaml":               &httpRawDynamicExtractor{},
+	"http/raw-get-query.yaml":                       &httpRawGetQuery{},
+	"http/raw-get.yaml":                             &httpRawGet{},
+	"http/raw-payload.yaml":                         &httpRawPayload{},
+	"http/raw-post-body.yaml":                       &httpRawPostBody{},
+	"http/raw-unsafe-request.yaml":                  &httpRawUnsafeRequest{},
+	"http/request-condition.yaml":                   &httpRequestCondition{},
+	"http/request-condition-new.yaml":               &httpRequestCondition{},
+	"http/interactsh.yaml":                          &httpInteractshRequest{},
+	"http/interactsh-stop-at-first-match.yaml":      &httpInteractshStopAtFirstMatchRequest{},
+	"http/self-contained.yaml":                      &httpRequestSelContained{},
+	"http/get-case-insensitive.yaml":                &httpGetCaseInsensitive{},
+	"http/get.yaml,http/get-case-insensitive.yaml":  &httpGetCaseInsensitiveCluster{},
+	"http/get-redirects-chain-headers.yaml":         &httpGetRedirectsChainHeaders{},
+	"http/dsl-matcher-variable.yaml":                &httpDSLVariable{},
+	"http/dsl-functions.yaml":                       &httpDSLFunctions{},
+	"http/race-simple.yaml":                         &httpRaceSimple{},
+	"http/race-multiple.yaml":                       &httpRaceMultiple{},
+	"http/stop-at-first-match.yaml":                 &httpStopAtFirstMatch{},
+	"http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{},
 }

 type httpInteractshRequest struct{}
@ -727,3 +729,41 @@ func (h *httpRaceMultiple) Execute(filePath string) error {
 	}
 	return expectResultsCount(results, 5)
 }
+
+type httpStopAtFirstMatch struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpStopAtFirstMatch) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type httpStopAtFirstMatchWithExtractors struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpStopAtFirstMatchWithExtractors) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 2)
+}
--- a/v2/cmd/integration-test/loader.go
+++ b/v2/cmd/integration-test/loader.go
@ -2,6 +2,7 @@ package main

 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
@ -17,6 +18,7 @@ var loaderTestcases = map[string]testutils.TestCase{
 	"loader/workflow-list.yaml":             &remoteWorkflowList{},
 	"loader/nonexistent-template-list.yaml": &nonExistentTemplateList{},
 	"loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{},
+	"loader/template-list-not-allowed.yaml": &remoteTemplateListNotAllowed{},
 }

 type remoteTemplateList struct{}
@ -45,7 +47,14 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()

-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+	err := ioutil.WriteFile("test-config.yaml", []byte(configFileData), os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer os.Remove("test-config.yaml")
+
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@ -53,6 +62,41 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	return expectResultsCount(results, 2)
 }

+type remoteTemplateListNotAllowed struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
+	router := httprouter.New()
+
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	})
+
+	router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, err := os.ReadFile(templateList)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+		_, err = w.Write(file)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	if err == nil {
+		return fmt.Errorf("expected error for not allowed remote template list url")
+	}
+
+	return nil
+
+}
+
 type remoteWorkflowList struct{}

 // Execute executes a test case and returns an error if occurred
@ -79,7 +123,14 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()

-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list")
+	configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+	err := ioutil.WriteFile("test-config.yaml", []byte(configFileData), os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer os.Remove("test-config.yaml")
+
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@ -91,6 +91,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", []string{}, "URL containing list of workflows to run"),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"),
 	)

 	createGroup(flagSet, "filters", "Filtering",
--- a/v2/go.mod
+++ b/v2/go.mod
@ -30,7 +30,7 @@ require (
 	github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47
 	github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
 	github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5
-	github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a
+	github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
 	github.com/projectdiscovery/interactsh v0.0.8-0.20220112083504-b0b3b2f359a5
@ -38,12 +38,12 @@ require (
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660
 	github.com/projectdiscovery/retryablehttp-go v1.0.2
-	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
+	github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700
 	github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0 // indirect
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil/v3 v3.21.12
+	github.com/shirou/gopsutil/v3 v3.22.1
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.4.1
 	github.com/syndtr/goleveldb v1.0.0
@ -63,7 +63,7 @@ require (
 	moul.io/http2curl v1.0.0
 )

-require github.com/aws/aws-sdk-go v1.42.37
+require github.com/aws/aws-sdk-go v1.42.45

 require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e

@ -148,7 +148,7 @@ require (
 	goftp.io/server/v2 v2.0.0 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+	golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
--- a/v2/go.sum
+++ b/v2/go.sum
@ -82,8 +82,8 @@ github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3st
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.42.37 h1:EIziSq3REaoi1LgUBgxoQr29DQS7GYHnBbZPajtJmXM=
-github.com/aws/aws-sdk-go v1.42.37/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
+github.com/aws/aws-sdk-go v1.42.45 h1:rzYlmOX2EqdsYKvo0WBBffuff3BuckL1UB2KyzWhXyQ=
+github.com/aws/aws-sdk-go v1.42.45/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@ -221,8 +221,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@ -421,8 +422,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h
 github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e h1:RJJuYyuwskYtzZi2gziy6SE/b7saWEzyskaA252E0VY=
 github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI=
 github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
-github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk=
-github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
+github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0 h1:KtCp/dCsxXNdT8m0yyWc/4ou4YaKWVakAr3G03TjQCk=
+github.com/projectdiscovery/goflags v0.0.8-0.20220121110825-48035ad3ffe0/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
@ -460,8 +461,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB
 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
 github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
-github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk=
+github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4=
@ -484,8 +486,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
-github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA=
-github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
+github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
+github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@ -790,10 +792,10 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
+golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
--- a/v2/internal/runner/options.go
+++ b/v2/internal/runner/options.go
@ -2,6 +2,8 @@ package runner

 import (
 	"bufio"
+	"io"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
@ -152,7 +154,7 @@ func configureOutput(options *types.Options) {
 	if options.Verbose || options.Validate {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
-	if options.Debug {
+	if options.Debug || options.DebugRequests || options.DebugResponse {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
 	}
 	if options.NoColor {
@ -161,6 +163,10 @@ func configureOutput(options *types.Options) {
 	if options.Silent {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
 	}
+
+	// disable standard logger (ref: https://github.com/golang/go/issues/19895)
+	log.SetFlags(0)
+	log.SetOutput(io.Discard)
 }

 // loadResolvers loads resolvers from both user provided flag and file
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@ -183,6 +183,9 @@ func New(options *types.Options) (*Runner, error) {
 	opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
 	opts.NoInteractsh = runner.options.NoInteractsh
 	opts.StopAtFirstMatch = runner.options.StopAtFirstMatch
+	opts.Debug = runner.options.Debug
+	opts.DebugRequest = runner.options.DebugRequests
+	opts.DebugResponse = runner.options.DebugResponse
 	interactshClient, err := interactsh.New(opts)
 	if err != nil {
 		gologger.Error().Msgf("Could not create interactsh client: %s", err)
--- a/v2/pkg/catalog/find.go
+++ b/v2/pkg/catalog/find.go
@ -18,14 +18,21 @@ func (c *Catalog) GetTemplatesPath(definitions []string) []string {
 	allTemplates := []string{}

 	for _, t := range definitions {
-		paths, err := c.GetTemplatePath(t)
-		if err != nil {
-			gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
-		}
-		for _, path := range paths {
-			if _, ok := processed[path]; !ok {
-				processed[path] = true
-				allTemplates = append(allTemplates, path)
+		if strings.HasPrefix(t, "http") && (strings.HasSuffix(t, ".yaml") || strings.HasSuffix(t, ".yml")) {
+			if _, ok := processed[t]; !ok {
+				processed[t] = true
+				allTemplates = append(allTemplates, t)
+			}
+		} else {
+			paths, err := c.GetTemplatePath(t)
+			if err != nil {
+				gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
+			}
+			for _, path := range paths {
+				if _, ok := processed[path]; !ok {
+					processed[path] = true
+					allTemplates = append(allTemplates, path)
+				}
 			}
 		}
 	}
--- a/v2/pkg/catalog/loader/loader.go
+++ b/v2/pkg/catalog/loader/loader.go
@ -18,12 +18,13 @@ import (

 // Config contains the configuration options for the loader
 type Config struct {
-	Templates        []string
-	TemplateURLs     []string
-	Workflows        []string
-	WorkflowURLs     []string
-	ExcludeTemplates []string
-	IncludeTemplates []string
+	Templates                []string
+	TemplateURLs             []string
+	Workflows                []string
+	WorkflowURLs             []string
+	ExcludeTemplates         []string
+	IncludeTemplates         []string
+	RemoteTemplateDomainList []string

 	Tags              []string
 	ExcludeTags       []string
@ -58,25 +59,26 @@ type Store struct {
 // NewConfig returns a new loader config
 func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
 	loaderConfig := Config{
-		Templates:          options.Templates,
-		Workflows:          options.Workflows,
-		TemplateURLs:       options.TemplateURLs,
-		WorkflowURLs:       options.WorkflowURLs,
-		ExcludeTemplates:   options.ExcludedTemplates,
-		Tags:               options.Tags,
-		ExcludeTags:        options.ExcludeTags,
-		IncludeTemplates:   options.IncludeTemplates,
-		Authors:            options.Authors,
-		Severities:         options.Severities,
-		ExcludeSeverities:  options.ExcludeSeverities,
-		IncludeTags:        options.IncludeTags,
-		IncludeIds:         options.IncludeIds,
-		ExcludeIds:         options.ExcludeIds,
-		TemplatesDirectory: options.TemplatesDirectory,
-		Protocols:          options.Protocols,
-		ExcludeProtocols:   options.ExcludeProtocols,
-		Catalog:            catalog,
-		ExecutorOptions:    executerOpts,
+		Templates:                options.Templates,
+		Workflows:                options.Workflows,
+		RemoteTemplateDomainList: options.RemoteTemplateDomainList,
+		TemplateURLs:             options.TemplateURLs,
+		WorkflowURLs:             options.WorkflowURLs,
+		ExcludeTemplates:         options.ExcludedTemplates,
+		Tags:                     options.Tags,
+		ExcludeTags:              options.ExcludeTags,
+		IncludeTemplates:         options.IncludeTemplates,
+		Authors:                  options.Authors,
+		Severities:               options.Severities,
+		ExcludeSeverities:        options.ExcludeSeverities,
+		IncludeTags:              options.IncludeTags,
+		IncludeIds:               options.IncludeIds,
+		ExcludeIds:               options.ExcludeIds,
+		TemplatesDirectory:       options.TemplatesDirectory,
+		Protocols:                options.Protocols,
+		ExcludeProtocols:         options.ExcludeProtocols,
+		Catalog:                  catalog,
+		ExecutorOptions:          executerOpts,
 	}
 	return &loaderConfig
 }
@ -108,7 +110,7 @@ func New(config *Config) (*Store, error) {

 	urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0
 	if urlBasedTemplatesProvided {
-		remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs)
+		remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs, config.RemoteTemplateDomainList)
 		if err != nil {
 			return store, err
 		}
--- a/v2/pkg/catalog/loader/loader_test.go
+++ b/v2/pkg/catalog/loader/loader_test.go
@ -1,6 +1,7 @@
 package loader

 import (
+	"reflect"
 	"testing"

 	"github.com/stretchr/testify/require"
@ -38,3 +39,55 @@ func TestLoadTemplates(t *testing.T) {
 		require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
 	})
 }
+
+func TestRemoteTemplates(t *testing.T) {
+	var nilStringSlice []string
+	type args struct {
+		config *Config
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Store
+		wantErr bool
+	}{
+		{
+			name: "remote-templates-positive",
+			args: args{
+				config: &Config{
+					TemplateURLs:             []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
+					RemoteTemplateDomainList: []string{"localhost","raw.githubusercontent.com"},
+				},
+			},
+			want: &Store{
+				finalTemplates: []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
+			},
+			wantErr: false,
+		},
+		{
+			name: "remote-templates-negative",
+			args: args{
+				config: &Config{
+					TemplateURLs:             []string{"https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/technologies/tech-detect.yaml"},
+					RemoteTemplateDomainList: []string{"localhost"},
+				},
+			},
+			want: &Store{
+				finalTemplates: nilStringSlice,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := New(tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got.finalTemplates, tt.want.finalTemplates) {
+				t.Errorf("New() = %v, want %v", got.finalTemplates, tt.want.finalTemplates)
+			}
+		})
+	}
+}
--- a/v2/pkg/catalog/loader/remote_loader.go
+++ b/v2/pkg/catalog/loader/remote_loader.go
@ -4,9 +4,11 @@ import (
 	"bufio"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"

 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 )

 type ContentType string
@ -16,38 +18,38 @@ const (
 	Workflow ContentType = "Workflow"
 )

-type RemoteContentError struct {
+type RemoteContent struct {
 	Content []string
 	Type    ContentType
 	Error   error
 }

-func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) {
-	remoteContentErrorChannel := make(chan RemoteContentError)
+func getRemoteTemplatesAndWorkflows(templateURLs, workflowURLs, remoteTemplateDomainList []string) ([]string, []string, error) {
+	remoteContentChannel := make(chan RemoteContent)

 	for _, templateURL := range templateURLs {
-		go getRemoteContent(templateURL, remoteContentErrorChannel, Template)
+		go getRemoteContent(templateURL, remoteTemplateDomainList, remoteContentChannel, Template)
 	}
 	for _, workflowURL := range workflowURLs {
-		go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow)
+		go getRemoteContent(workflowURL, remoteTemplateDomainList, remoteContentChannel, Workflow)
 	}

 	var remoteTemplateList []string
 	var remoteWorkFlowList []string
 	var err error
 	for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ {
-		remoteContentError := <-remoteContentErrorChannel
-		if remoteContentError.Error != nil {
+		remoteContent := <-remoteContentChannel
+		if remoteContent.Error != nil {
 			if err != nil {
-				err = errors.New(remoteContentError.Error.Error() + ": " + err.Error())
+				err = errors.New(remoteContent.Error.Error() + ": " + err.Error())
 			} else {
-				err = remoteContentError.Error
+				err = remoteContent.Error
 			}
 		} else {
-			if remoteContentError.Type == Template {
-				remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...)
-			} else if remoteContentError.Type == Workflow {
-				remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...)
+			if remoteContent.Type == Template {
+				remoteTemplateList = append(remoteTemplateList, remoteContent.Content...)
+			} else if remoteContent.Type == Workflow {
+				remoteWorkFlowList = append(remoteWorkFlowList, remoteContent.Content...)
 			}
 		}
 	}
@ -55,17 +57,30 @@ func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string
 	return remoteTemplateList, remoteWorkFlowList, err
 }

-func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) {
+func getRemoteContent(URL string, remoteTemplateDomainList []string, remoteContentChannel chan<- RemoteContent, contentType ContentType) {
+	if err := validateRemoteRemplateURL(URL, remoteTemplateDomainList); err != nil {
+		remoteContentChannel <- RemoteContent{
+			Error: err,
+		}
+		return
+	}
+	if strings.HasPrefix(URL, "http") && (strings.HasSuffix(URL, ".yaml") || strings.HasSuffix(URL, ".yml")) {
+		remoteContentChannel <- RemoteContent{
+			Content: []string{URL},
+			Type:    contentType,
+		}
+		return
+	}
 	response, err := http.Get(URL)
 	if err != nil {
-		w <- RemoteContentError{
+		remoteContentChannel <- RemoteContent{
 			Error: err,
 		}
 		return
 	}
 	defer response.Body.Close()
 	if response.StatusCode < 200 || response.StatusCode > 299 {
-		w <- RemoteContentError{
+		remoteContentChannel <- RemoteContent{
 			Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode),
 		}
 		return
@ -78,18 +93,37 @@ func getRemoteContent(URL string, w chan<- RemoteContentError, contentType Conte
 		if text == "" {
 			continue
 		}
+		if utils.IsURL(text) {
+			if err := validateRemoteRemplateURL(text, remoteTemplateDomainList); err != nil {
+				remoteContentChannel <- RemoteContent{
+					Error: err,
+				}
+				return
+			}
+		}
 		templateList = append(templateList, text)
 	}

 	if err := scanner.Err(); err != nil {
-		w <- RemoteContentError{
+		remoteContentChannel <- RemoteContent{
 			Error: errors.Wrap(err, "get \"%s\""),
 		}
 		return
 	}

-	w <- RemoteContentError{
+	remoteContentChannel <- RemoteContent{
 		Content: templateList,
 		Type:    contentType,
 	}
 }
+
+func validateRemoteRemplateURL(inputURL string, remoteTemplateDomainList []string) error {
+	parsedURL, err := url.Parse(inputURL)
+	if err != nil {
+		return err
+	}
+	if !utils.StringSliceContains(remoteTemplateDomainList, parsedURL.Host) {
+		return errors.Errorf("Remote template URL host (%s) is not present in the `remote-template-domain` list in nuclei config", parsedURL.Host)
+	}
+	return nil
+}
--- a/v2/pkg/parsers/parser.go
+++ b/v2/pkg/parsers/parser.go
@ -2,8 +2,6 @@ package parsers

 import (
 	"fmt"
-	"io/ioutil"
-	"os"
 	"regexp"
 	"strings"

@ -129,14 +127,7 @@ func ParseTemplate(templatePath string) (*templates.Template, error) {
 	if value, err := parsedTemplatesCache.Has(templatePath); value != nil {
 		return value.(*templates.Template), err
 	}
-
-	f, err := os.Open(templatePath)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	data, err := ioutil.ReadAll(f)
+	data, err := utils.ReadFromPathOrURL(templatePath)
 	if err != nil {
 		return nil, err
 	}
--- a/v2/pkg/protocols/common/expressions/expressions.go
+++ b/v2/pkg/protocols/common/expressions/expressions.go
@ -6,9 +6,9 @@ import (
 	"github.com/Knetic/govaluate"

 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+	"github.com/projectdiscovery/stringsutil"
 )

 // Evaluate checks if the match contains a dynamic variable, for each
@ -33,13 +33,20 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
 }

 func evaluate(data string, base map[string]interface{}) (string, error) {
+	// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value
 	data = replacer.Replace(data, base)

+	// expressions can be:
+	// - simple: containing base values keys (variables)
+	// - complex: containing helper functions [ + variables]
+	// literals like {{2+2}} are not considered expressions
+	expressions := findExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base)))
 	dynamicValues := make(map[string]interface{})
-	for _, match := range findMatches(data) {
-		expr := generators.TrimDelimiters(match)
-
-		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
+	for _, expression := range expressions {
+		// replace variable placeholders with base values
+		expression = replacer.Replace(expression, base)
+		// turns expressions (either helper functions+base values or base values)
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions())
 		if err != nil {
 			continue
 		}
@ -47,19 +54,104 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
 		if err != nil {
 			continue
 		}
-		dynamicValues[expr] = result
+		dynamicValues[expression] = result
 	}
-	// Replacer dynamic values if any in raw request and parse  it
+	// Replacer dynamic values if any in raw request and parse it
 	return replacer.Replace(data, dynamicValues), nil
 }

-func findMatches(data string) []string {
-	var matches []string
-	for _, token := range strings.Split(data, marker.ParenthesisOpen) {
-		closingToken := strings.LastIndex(token, marker.ParenthesisClose)
-		if closingToken > 0 {
-			matches = append(matches, token[:closingToken])
+// maxIterations to avoid infinite loop
+const maxIterations = 250
+
+func findExpressions(data, OpenMarker, CloseMarker string, functions map[string]govaluate.ExpressionFunction) []string {
+	var (
+		iterations int
+		exps       []string
+	)
+	for {
+		// check if we reached the maximum number of iterations
+		if iterations > maxIterations {
+			break
+		}
+		iterations++
+		// attempt to find open markers
+		indexOpenMarker := strings.Index(data, OpenMarker)
+		// exits if not found
+		if indexOpenMarker < 0 {
+			break
+		}
+
+		indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)
+
+		shouldSearchCloseMarker := true
+		closeMarkerFound := false
+		innerData := data
+		var potentialMatch string
+		var indexCloseMarker, indexCloseMarkerOffset int
+		skip := indexOpenMarkerOffset
+		for shouldSearchCloseMarker {
+			// attempt to find close marker
+			indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)
+			// if no close markers are found exit
+			if indexCloseMarker < 0 {
+				shouldSearchCloseMarker = false
+				continue
+			}
+			indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)
+
+			potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
+			if isExpression(potentialMatch, functions) {
+				closeMarkerFound = true
+				shouldSearchCloseMarker = false
+				exps = append(exps, potentialMatch)
+			} else {
+				skip = indexCloseMarkerOffset
+			}
+		}
+
+		if closeMarkerFound {
+			// move after the close marker
+			data = data[indexCloseMarkerOffset:]
+		} else {
+			// move after the open marker
+			data = data[indexOpenMarkerOffset:]
 		}
 	}
-	return matches
+	return exps
+}
+
+func isExpression(data string, functions map[string]govaluate.ExpressionFunction) bool {
+	if _, err := govaluate.NewEvaluableExpression(data); err == nil {
+		return stringsutil.ContainsAny(data, getFunctionsNames(functions)...)
+	}
+
+	// check if it's a complex expression
+	_, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions())
+	return err == nil
+}
+
+func mapToFunctions(vars map[string]interface{}) map[string]govaluate.ExpressionFunction {
+	f := make(map[string]govaluate.ExpressionFunction)
+	for k := range vars {
+		f[k] = nil
+	}
+	return f
+}
+
+func mergeFunctions(m ...map[string]govaluate.ExpressionFunction) map[string]govaluate.ExpressionFunction {
+	o := make(map[string]govaluate.ExpressionFunction)
+	for _, mm := range m {
+		for k, v := range mm {
+			o[k] = v
+		}
+	}
+	return o
+}
+
+func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string {
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
 }
--- a/v2/pkg/protocols/common/expressions/expressions_test.go
+++ b/v2/pkg/protocols/common/expressions/expressions_test.go
@ -14,19 +14,19 @@ func TestEvaluate(t *testing.T) {
 	}{
 		{input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}},
 		{input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}},
-		// TODO #1501
-		//{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
-		//{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
-		//{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
-		//{input: `{{"{{" + '123*123' + "}}"}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+		{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
+		{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+		{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+		{input: `{{"{{" + '123*123' + "}}"}}`, expected: `{{"{{" + '123*123' + "}}"}}`, extra: map[string]interface{}{}},
+		{input: `{{a + '123*123' + b}}`, expected: `aa123*123bb`, extra: map[string]interface{}{"a": "aa", "b": "bb"}},
 		{input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}},
-		{input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}},
-		{input: `{{"1"+"1"}}`, expected: "11", extra: map[string]interface{}{}},
-		{input: `{{"1" + '*' + "1"}}`, expected: "1*1", extra: map[string]interface{}{}},
-		{input: `{{"a" + 'b' + "c"}}`, expected: "abc", extra: map[string]interface{}{}},
-		{input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}},
-		{input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}},
-		{input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}},
+		{input: `{{1+1}}`, expected: "{{1+1}}", extra: map[string]interface{}{}},
+		{input: `{{"1"+"1"}}`, expected: `{{"1"+"1"}}`, extra: map[string]interface{}{}},
+		{input: `{{"1" + '*' + "1"}}`, expected: `{{"1" + '*' + "1"}}`, extra: map[string]interface{}{}},
+		{input: `{{"a" + 'b' + "c"}}`, expected: `{{"a" + 'b' + "c"}}`, extra: map[string]interface{}{}},
+		{input: `{{10*2}}`, expected: `{{10*2}}`, extra: map[string]interface{}{}},
+		{input: `{{10/2}}`, expected: `{{10/2}}`, extra: map[string]interface{}{}},
+		{input: `{{10-2}}`, expected: `{{10-2}}`, extra: map[string]interface{}{}},
 		{input: "test", expected: "test", extra: map[string]interface{}{}},
 		{input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}},
 		{input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}},
--- a/v2/pkg/protocols/common/interactsh/interactsh.go
+++ b/v2/pkg/protocols/common/interactsh/interactsh.go
@ -75,7 +75,9 @@ type Options struct {
 	// Progress is the nuclei progress bar implementation.
 	Progress progress.Progress
 	// Debug specifies whether debugging output should be shown for interactsh-client
-	Debug bool
+	Debug         bool
+	DebugRequest  bool
+	DebugResponse bool
 	// DisableHttpFallback controls http retry in case of https failure for server url
 	DisableHttpFallback bool
 	// NoInteractsh disables the engine
@ -146,8 +148,8 @@ func (c *Client) firstTimeInitializeClient() error {
 	c.hostname = interactDomain

 	interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
-		if c.options.Debug {
-			debugPrintInteraction(interaction)
+		if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse {
+			c.debugPrintInteraction(interaction)
 		}
 		item := c.requests.Get(interaction.UniqueID)

@ -343,26 +345,53 @@ func HasMatchers(op *operators.Operators) bool {
 	return false
 }

-func debugPrintInteraction(interaction *server.Interaction) {
+// HasMarkers checks if the text contains interactsh markers
+func HasMarkers(data string) bool {
+	return strings.Contains(data, interactshURLMarker)
+}
+
+func (c *Client) debugPrintInteraction(interaction *server.Interaction) {
 	builder := &bytes.Buffer{}

 	switch interaction.Protocol {
 	case "dns":
-		builder.WriteString(fmt.Sprintf("[%s] Received DNS interaction (%s) from %s at %s", interaction.FullId, interaction.QType, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
-		builder.WriteString(fmt.Sprintf("\n-----------\nDNS Request\n-----------\n\n%s\n\n------------\nDNS Response\n------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
+		builder.WriteString(formatInteractionHeader("DNS", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+		if c.options.DebugRequest || c.options.Debug {
+			builder.WriteString(formatInteractionMessage("DNS Request", interaction.RawRequest))
+		}
+		if c.options.DebugResponse || c.options.Debug {
+			builder.WriteString(formatInteractionMessage("DNS Response", interaction.RawResponse))
+		}
 	case "http":
-		builder.WriteString(fmt.Sprintf("[%s] Received HTTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
-		builder.WriteString(fmt.Sprintf("\n------------\nHTTP Request\n------------\n\n%s\n\n-------------\nHTTP Response\n-------------\n\n%s\n\n", interaction.RawRequest, interaction.RawResponse))
+		builder.WriteString(formatInteractionHeader("HTTP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+		if c.options.DebugRequest || c.options.Debug {
+			builder.WriteString(formatInteractionMessage("HTTP Request", interaction.RawRequest))
+		}
+		if c.options.DebugResponse || c.options.Debug {
+			builder.WriteString(formatInteractionMessage("HTTP Response", interaction.RawResponse))
+		}
 	case "smtp":
-		builder.WriteString(fmt.Sprintf("[%s] Received SMTP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
-		builder.WriteString(fmt.Sprintf("\n------------\nSMTP Interaction\n------------\n\n%s\n\n", interaction.RawRequest))
+		builder.WriteString(formatInteractionHeader("SMTP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+		if c.options.DebugRequest || c.options.Debug || c.options.DebugResponse {
+			builder.WriteString(formatInteractionMessage("SMTP Interaction", interaction.RawRequest))
+		}
 	case "ldap":
-		builder.WriteString(fmt.Sprintf("[%s] Received LDAP interaction from %s at %s", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp.Format("2006-01-02 15:04:05")))
-		builder.WriteString(fmt.Sprintf("\n------------\nLDAP Interaction\n------------\n\n%s\n\n", interaction.RawRequest))
+		builder.WriteString(formatInteractionHeader("LDAP", interaction.FullId, interaction.RemoteAddress, interaction.Timestamp))
+		if c.options.DebugRequest || c.options.Debug || c.options.DebugResponse {
+			builder.WriteString(formatInteractionMessage("LDAP Interaction", interaction.RawRequest))
+		}
 	}
 	fmt.Fprint(os.Stderr, builder.String())
 }

+func formatInteractionHeader(protocol, ID, address string, at time.Time) string {
+	return fmt.Sprintf("[%s] Received %s interaction from %s at %s", ID, protocol, address, at.Format("2006-01-02 15:04:05"))
+}
+
+func formatInteractionMessage(key, value string) string {
+	return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value)
+}
+
 func hash(templateID, host string) string {
 	h := sha1.New()
 	h.Write([]byte(templateID))
--- a/v2/pkg/protocols/dns/dns.go
+++ b/v2/pkg/protocols/dns/dns.go
@ -234,6 +234,8 @@ func questionTypeToInt(questionType string) uint16 {
 		question = dns.TypeDS
 	case "AAAA":
 		question = dns.TypeAAAA
+	case "CAA":
+		question = dns.TypeCAA
 	}
 	return question
 }
--- a/v2/pkg/protocols/dns/dns_types.go
+++ b/v2/pkg/protocols/dns/dns_types.go
@ -31,6 +31,8 @@ const (
 	TXT
 	// name:AAAA
 	AAAA
+	// name:CAA
+	CAA
 	limit
 )

@ -45,6 +47,7 @@ var DNSRequestTypeMapping = map[DNSRequestType]string{
 	MX:    "MX",
 	TXT:   "TXT",
 	AAAA:  "AAAA",
+	CAA:   "CAA",
 }

 // GetSupportedDNSRequestTypes returns list of supported types
--- a/v2/pkg/protocols/dns/request.go
+++ b/v2/pkg/protocols/dns/request.go
@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 	"github.com/projectdiscovery/retryabledns"
 )

@ -27,7 +28,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
 func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	// Parse the URL and return domain if URL.
 	var domain string
-	if isURL(input) {
+	if utils.IsURL(input) {
 		domain = extractDomain(input)
 	} else {
 		domain = input
@ -125,18 +126,6 @@ func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols
 	}
 }

-// isURL tests a string to determine if it is a well-structured url or not.
-func isURL(toTest string) bool {
-	if _, err := url.ParseRequestURI(toTest); err != nil {
-		return false
-	}
-	u, err := url.Parse(toTest)
-	if err != nil || u.Scheme == "" || u.Host == "" {
-		return false
-	}
-	return true
-}
-
 // extractDomain extracts the domain name of a URL
 func extractDomain(theURL string) string {
 	u, err := url.Parse(theURL)
--- a/v2/pkg/protocols/headless/engine/rules.go
+++ b/v2/pkg/protocols/headless/engine/rules.go
@ -64,9 +64,7 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 	var rawResp strings.Builder
 	respPayloads := ctx.Response.Payload()
 	if respPayloads != nil {
-		rawResp.WriteString("HTTP/1.1 ")
-		rawResp.WriteString(fmt.Sprint(respPayloads.ResponseCode))
-		rawResp.WriteString(" " + respPayloads.ResponsePhrase + "\n")
+		rawResp.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\n", respPayloads.ResponseCode, respPayloads.ResponsePhrase))
 		for _, header := range respPayloads.ResponseHeaders {
 			rawResp.WriteString(header.Name + ": " + header.Value + "\n")
 		}
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@ -239,8 +239,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 	for {
 		// returns two values, error and skip, which skips the execution for the request instance.
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
-			hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
-
+			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 			generatedHttpRequest, err := generator.Make(reqURL, data, payloads, dynamicValue)
 			if err != nil {
 				if err == io.EOF {
@ -249,6 +248,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 				request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 				return true, err
 			}
+			hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
 			if reqURL == "" {
 				reqURL = generatedHttpRequest.URL()
 			}
@ -256,16 +256,16 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 			if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) {
 				return true, nil
 			}
-			var gotOutput bool
+			var gotMatches bool
 			request.options.RateLimiter.Take()

-			err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) {
+			err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
 				// Add the extracts to the dynamic values if any.
 				if event.OperatorsResult != nil {
-					gotOutput = true
+					gotMatches = event.OperatorsResult.Matched
 					gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues)
 				}
-				if hasInteractMarkers && request.options.Interactsh != nil {
+				if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
 					request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{
 						MakeResultFunc: request.MakeResultEvent,
 						Event:          event,
@ -292,7 +292,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 			request.options.Progress.IncrementRequests()

 			// If this was a match, and we want to stop at first match, skip all further requests.
-			if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput {
+			if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches {
 				return true, nil
 			}
 			return false, nil
@ -329,7 +329,7 @@ const drainReqSize = int64(8 * 1024)
 var errStopExecution = errors.New("stop execution due to unresolved variables")

 // executeRequest executes the actual generated request and returns error if occurred
-func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
+func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error {
 	request.setCustomHeaders(generatedRequest)

 	var (
@ -434,7 +434,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate

 		// If we have interactsh markers and request times out, still send
 		// a callback event so in case we receive an interaction, correlation is possible.
-		if hasInteractMarkers {
+		if hasInteractMatchers {
 			outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
 			if i := strings.LastIndex(hostname, ":"); i != -1 {
 				hostname = hostname[:i]
@ -558,7 +558,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
 			internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
 		})
-		if hasInteractMarkers {
+		if hasInteractMatchers {
 			event.UsesInteractsh = true
 		}

--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@ -2,8 +2,6 @@ package templates

 import (
 	"fmt"
-	"io/ioutil"
-	"os"
 	"reflect"
 	"strings"

@ -38,13 +36,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute

 	template := &Template{}

-	f, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	data, err := ioutil.ReadAll(f)
+	data, err := utils.ReadFromPathOrURL(filePath)
 	if err != nil {
 		return nil, err
 	}
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@ -1110,6 +1110,7 @@ func init() {
 		"MX",
 		"TXT",
 		"AAAA",
+		"CAA",
 	}

 	FILERequestDoc.Type = "file.Request"
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@ -23,6 +23,8 @@ type Options struct {
 	Templates goflags.StringSlice
 	// TemplateURLs specifies URLs to a list of templates to use
 	TemplateURLs goflags.StringSlice
+	// RemoteTemplates specifies list of allowed URLs to load remote templates from
+	RemoteTemplateDomainList goflags.StringSlice
 	// 	ExcludedTemplates  specifies the template/templates to exclude
 	ExcludedTemplates goflags.StringSlice
 	// CustomHeaders is the list of custom global headers to send with each request.
--- a/v2/pkg/utils/utils.go
+++ b/v2/pkg/utils/utils.go
@ -2,6 +2,10 @@ package utils

 import (
 	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
 	"strings"

 	"github.com/projectdiscovery/fileutil"
@ -37,3 +41,54 @@ func LoadFile(filename string) ([]string, error) {
 	}
 	return items, nil
 }
+
+// IsURL tests a string to determine if it is a well-structured url or not.
+func IsURL(input string) bool {
+	_, err := url.ParseRequestURI(input)
+	if err != nil {
+		return false
+	}
+
+	u, err := url.Parse(input)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return false
+	}
+
+	return true
+}
+
+// ReadFromPathOrURL reads and returns the contents of a file or url.
+func ReadFromPathOrURL(templatePath string) (data []byte, err error) {
+	if IsURL(templatePath) {
+		resp, err := http.Get(templatePath)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		data, err = ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		f, err := os.Open(templatePath)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		data, err = ioutil.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return
+}
+
+// StringSliceContains checks if a string slice contains a string.
+func StringSliceContains(slice []string, item string) bool {
+	for _, i := range slice {
+		if strings.EqualFold(i, item) {
+			return true
+		}
+	}
+	return false
+}