Merge branch 'dev' into new_dsl_functions

.github/workflows/release-binary.yml

@@ -18,7 +18,7 @@ jobs:
         with: 
           go-version: 1.18
 
-      - uses: goreleaser/goreleaser-action@v2
+      - uses: goreleaser/goreleaser-action@v3
         with: 
           args: "release --rm-dist"
           version: latest

Dockerfile

@@ -1,7 +1,7 @@
-FROM golang:1.18.2-alpine as build-env
+FROM golang:1.18.3-alpine as build-env
 RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
 
-FROM alpine:3.15.4
+FROM alpine:3.16.0
 RUN apk add --no-cache bind-tools ca-certificates chromium
 COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
 ENTRYPOINT ["nuclei"]

README.md

@@ -151,6 +151,7 @@ CONFIGURATIONS:
    -cc, -client-cert string    client certificate file (PEM-encoded) used for authenticating against scanned hosts
    -ck, -client-key string     client key file (PEM-encoded) used for authenticating against scanned hosts
    -ca, -client-ca string      client certificate authority file (PEM-encoded) used for authenticating against scanned hosts
+   -sml, -show-match-line      show match lines for file templates, works with extractors only
    -ztls                       use ztls library with autofallback to standard one for tls13
    -sni string                 tls sni hostname to use (default: input domain name)
 

integration_tests/file/data/test1.txt

@@ -0,0 +1,2 @@
+AAA
+BBB

integration_tests/file/data/test2.txt

@@ -0,0 +1,2 @@
+CCC
+DDD

integration_tests/file/data/test3.txt

@@ -0,0 +1,2 @@
+11 EE 11
+11 FF 11

integration_tests/file/extract.yaml

@@ -0,0 +1,16 @@
+id: file-extract
+
+info:
+  name: File with Extractor
+  author: pdteam
+  severity: info
+  tags: file
+
+file:
+  - extensions:
+      - all
+
+    extractors:
+      - type: regex
+        regex:
+          - "(?m)11\\s(EE|FF)\\s11"

integration_tests/file/matcher-with-and.yaml

@@ -0,0 +1,20 @@
+id: file-matcher-with-or
+
+info:
+  name: File Matcher With AND
+  author: pdteam
+  severity: info
+  tags: file
+
+file:
+  - extensions:
+      - all
+
+    matchers-condition: and
+    matchers:
+      - type: word
+        words:
+          - "CCC"
+      - type: word
+        words:
+          - "DDD"

integration_tests/file/matcher-with-or.yaml

@@ -0,0 +1,19 @@
+id: file-matcher-with-or
+
+info:
+  name: File Matcher With OR
+  author: pdteam
+  severity: info
+  tags: file
+
+file:
+  - extensions:
+      - all
+
+    matchers:
+      - type: word
+        words:
+          - "AA"
+      - type: word
+        words:
+          - "BB"

integration_tests/http/get-sni-unsafe.yaml

@@ -0,0 +1,18 @@
+id: basic-unsafe-get
+
+info:
+  name: Basic Unsafe GET Request with CLI SNI
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |+
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+    unsafe: true
+    matchers:
+      - type: word
+        words:
+          - "test-ok"

integration_tests/http/redirect-match-url.yaml

@@ -0,0 +1,18 @@
+id: redirect-match-url
+
+info:
+  name: Redirect Match URL
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    stop-at-first-match: true # Confirm stop-at-first-match
+    redirects: true # Confirm redirected URL matched value
+    max-redirects: 3
+    matchers:
+      - type: word
+        words:
+          - "This is test redirects matcher text"

v2/cmd/integration-test/file.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+)
+
+var fileTestcases = map[string]testutils.TestCase{
+	"file/matcher-with-or.yaml":  &fileWithOrMatcher{},
+	"file/matcher-with-and.yaml": &fileWithAndMatcher{},
+	"file/extract.yaml":          &fileWithExtractor{},
+}
+
+type fileWithOrMatcher struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *fileWithOrMatcher) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "file/data/", debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type fileWithAndMatcher struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *fileWithAndMatcher) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "file/data/", debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type fileWithExtractor struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *fileWithExtractor) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "file/data/", debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}

v2/cmd/integration-test/http.go

@@ -50,6 +50,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/variables.yaml":                           &httpVariables{},
 	"http/get-override-sni.yaml":                    &httpSniAnnotation{},
 	"http/get-sni.yaml":                             &customCLISNI{},
+	"http/redirect-match-url.yaml":                  &httpRedirectMatchURL{},
+	"http/get-sni-unsafe.yaml":                      &customCLISNIUnsafe{},
 }
 
 type httpInteractshRequest struct{}
@@ -856,3 +858,54 @@ func (h *httpSniAnnotation) Execute(filePath string) error {
 	}
 	return expectResultsCount(results, 1)
 }
+
+type httpRedirectMatchURL struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpRedirectMatchURL) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		http.Redirect(w, r, "/redirected", http.StatusFound)
+		_, _ = w.Write([]byte("This is test redirects matcher text"))
+	})
+	router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test redirects matcher text")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-no-meta")
+	if err != nil {
+		return err
+	}
+
+	if err := expectResultsCount(results, 1); err != nil {
+		return err
+	}
+	if results[0] != fmt.Sprintf("%s/redirected", ts.URL) {
+		return fmt.Errorf("mismatched url found: %s", results[0])
+	}
+	return nil
+}
+
+type customCLISNIUnsafe struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *customCLISNIUnsafe) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		if r.TLS.ServerName == "test" {
+			_, _ = w.Write([]byte("test-ok"))
+		} else {
+			_, _ = w.Write([]byte("test-ko"))
+		}
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-sni", "test")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}

v2/cmd/integration-test/integration-test.go

@@ -31,6 +31,7 @@ var (
 		"code":          codeTestcases,
 		"templatesPath": templatesPathTestCases,
 		"templatesDir":  templatesDirTestCases,
+		"file":          fileTestcases,
 	}
 )
 

v2/cmd/nuclei/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils/monitor"
 )
 
 var (
@@ -33,6 +34,11 @@ func main() {
 
 	runner.ParseOptions(options)
 
+	if options.HangMonitor {
+		cancel := monitor.NewStackMonitor(10 * time.Second)
+		defer cancel()
+	}
+
 	nucleiRunner, err := runner.New(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
@@ -148,6 +154,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"),
 		flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"),
 		flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"),
+		flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"),
 		flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"),
 		flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"),
 	)
@@ -198,6 +205,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),
 		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.BoolVarP(&options.HangMonitor, "hang-monitor", "hm", false, "enable nuclei hang monitoring"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
 		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
 		flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),

v2/go.mod

@@ -6,16 +6,16 @@ require (
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
 	github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
 	github.com/andygrunwald/go-jira v1.15.1
-	github.com/antchfx/htmlquery v1.2.4
+	github.com/antchfx/htmlquery v1.2.5
 	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/bluele/gcache v0.0.2
 	github.com/corpix/uarand v0.2.0
 	github.com/go-playground/validator/v10 v10.11.0
-	github.com/go-rod/rod v0.106.6
+	github.com/go-rod/rod v0.106.8
 	github.com/gobwas/ws v1.1.0
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/itchyny/gojq v0.12.7
+	github.com/itchyny/gojq v0.12.8
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
@@ -27,7 +27,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
 	github.com/projectdiscovery/cryptoutil v1.0.0
-	github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83
+	github.com/projectdiscovery/fastdialer v0.0.16-0.20220603192502-7d34c304eb65
 	github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
 	github.com/projectdiscovery/fileutil v0.0.0-20220427234316-40b2541a84b8
 	github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323
@@ -35,7 +35,7 @@ require (
 	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
 	github.com/projectdiscovery/interactsh v1.0.4
 	github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df
-	github.com/projectdiscovery/rawhttp v0.0.8-0.20220504112210-ae777c1ccd6b
+	github.com/projectdiscovery/rawhttp v0.0.8-0.20220526170355-03de6bb78f37
 	github.com/projectdiscovery/retryabledns v1.0.13
 	github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d
 	github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833
@@ -43,31 +43,32 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.4.0
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil/v3 v3.22.4
+	github.com/shirou/gopsutil/v3 v3.22.5
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.5.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
 	github.com/valyala/fasttemplate v1.2.1
 	github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95
-	github.com/xanzy/go-gitlab v0.65.0
+	github.com/xanzy/go-gitlab v0.68.0
 	github.com/ysmood/gson v0.7.1 // indirect
 	github.com/ysmood/leakless v0.7.0 // indirect
 	go.uber.org/atomic v1.9.0
 	go.uber.org/multierr v1.8.0
 	go.uber.org/ratelimit v0.2.0
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
-	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
+	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
 	golang.org/x/text v0.3.7
 	gopkg.in/yaml.v2 v2.4.0
 	moul.io/http2curl v1.0.0
 )
 
-require github.com/aws/aws-sdk-go v1.44.17
+require github.com/aws/aws-sdk-go v1.44.27
 
 require github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08
 
 require (
+	github.com/DataDog/gostackparse v0.5.0
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
 	github.com/docker/go-units v0.4.0
 	github.com/h2non/filetype v1.1.3
@@ -78,10 +79,10 @@ require (
 	github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d
 	github.com/projectdiscovery/sliceutil v0.0.0-20220511171050-c7d9bc5cadd9
 	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
-	github.com/projectdiscovery/wappalyzergo v0.0.42
+	github.com/projectdiscovery/wappalyzergo v0.0.45
 	github.com/stretchr/testify v1.7.1
 	github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -94,7 +95,7 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
-	github.com/antchfx/xpath v1.2.0 // indirect
+	github.com/antchfx/xpath v1.2.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
 	github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect
 	github.com/c4milo/unpackit v0.1.0 // indirect
@@ -118,8 +119,8 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/gosuri/uiprogress v0.0.1 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
 	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
 	github.com/itchyny/timefmt-go v0.1.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -160,12 +161,12 @@ require (
 	goftp.io/server/v2 v2.0.0 // indirect
 	golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
 )

v2/go.sum

@@ -36,6 +36,8 @@ git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmF
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DataDog/gostackparse v0.5.0 h1:jb72P6GFHPHz2W0onsN51cS3FkaMDcjb0QzgxxA4gDk=
+github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
 github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
@@ -71,11 +73,11 @@ github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5
 github.com/andygrunwald/go-jira v1.15.1 h1:6J9aYKb9sW8bxv3pBLYBrs0wdsFrmGI5IeTgWSKWKc8=
 github.com/andygrunwald/go-jira v1.15.1/go.mod h1:GIYN1sHOIsENWUZ7B4pDeT/nxEtrZpE8l0987O67ZR8=
 github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
-github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
+github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4=
-github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
+github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw=
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
+github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8=
-github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
 github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
 github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
@@ -86,8 +88,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.44.17 h1:of8MirZuVDat3BJgRbSwDO/GM/cgXh5Znf2tyEAv/vE=
+github.com/aws/aws-sdk-go v1.44.27 h1:8CMspeZSrewnbvAwgl8qo5R7orDLwQnTGBf/OKPiHxI=
-github.com/aws/aws-sdk-go v1.44.17/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.27/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -174,8 +176,8 @@ github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2B
 github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
 github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
-github.com/go-rod/rod v0.106.6 h1:zJorVPG7s8Xgbh7PkSySP4FNoo0OiougKaMb3j6zT6w=
+github.com/go-rod/rod v0.106.8 h1:pVMVz0jMtLVyx8FhJEEA6l+EY9Iw/nJTDYT/he4+UJc=
-github.com/go-rod/rod v0.106.6/go.mod h1:xkZOchuKqTOkMOBkrzb7uJpbKZRab1haPCWDvuZkS2U=
+github.com/go-rod/rod v0.106.8/go.mod h1:xkZOchuKqTOkMOBkrzb7uJpbKZRab1haPCWDvuZkS2U=
 github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
 github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
@@ -234,8 +236,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -268,12 +271,14 @@ github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJS
 github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
-github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
 github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
+github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E=
 github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@@ -289,8 +294,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
 github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
-github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
+github.com/itchyny/gojq v0.12.8 h1:Zxcwq8w4IeR8JJYEtoG2MWJZUv0RGY6QqJcO1cqV8+A=
-github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
+github.com/itchyny/gojq v0.12.8/go.mod h1:gE2kZ9fVRU0+JAksaTzjIlgnCa2akU+a1V0WXgJQN5c=
 github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
 github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
@@ -438,8 +443,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod
 github.com/projectdiscovery/cryptoutil v1.0.0 h1:5rQfnWDthJ5ZFcqze+rmT1N7l1HJQ6EB26MrjaYB7I0=
 github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4=
 github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
-github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 h1:1hzvl0lsWpvQ8nn1s9YMyBjO13/Z+f/T4W2jroOohfo=
+github.com/projectdiscovery/fastdialer v0.0.16-0.20220603192502-7d34c304eb65 h1:6nPONYe5Zas+0toFxP0vvDOxMwQIZo6geajd+FfwzMc=
-github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI=
+github.com/projectdiscovery/fastdialer v0.0.16-0.20220603192502-7d34c304eb65/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k=
 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew=
 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
 github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
@@ -481,8 +486,8 @@ github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUy
 github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d h1:WK/zpWYGuZhAudzVjbnCYk12L4SqH66jagBzHRm3eCo=
 github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
-github.com/projectdiscovery/rawhttp v0.0.8-0.20220504112210-ae777c1ccd6b h1:lsiBFKZfyRFOTmY6Ebn0sOcqEz5RKDq0fuqNmigRAYA=
+github.com/projectdiscovery/rawhttp v0.0.8-0.20220526170355-03de6bb78f37 h1:odvvszpfUvNq5UMzUhimete71/ph+XQzzL11C/u3lUc=
-github.com/projectdiscovery/rawhttp v0.0.8-0.20220504112210-ae777c1ccd6b/go.mod h1:kulxvr2aKZPB6jhq4ZQn/E+ArwRWczs1O6b1ZdcZAxo=
+github.com/projectdiscovery/rawhttp v0.0.8-0.20220526170355-03de6bb78f37/go.mod h1:kulxvr2aKZPB6jhq4ZQn/E+ArwRWczs1O6b1ZdcZAxo=
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
@@ -503,8 +508,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo
 github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
 github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
-github.com/projectdiscovery/wappalyzergo v0.0.42 h1:4DuJVpgesHSnUNVG+l62QIYEmcMXu3PLyhS1Wp75c30=
+github.com/projectdiscovery/wappalyzergo v0.0.45 h1:b2OJ4/FrQg6YBDwuK+xMJj/3+h6u+uHuZYURGYPqw3I=
-github.com/projectdiscovery/wappalyzergo v0.0.42/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
+github.com/projectdiscovery/wappalyzergo v0.0.45/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4=
@@ -528,8 +533,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
-github.com/shirou/gopsutil/v3 v3.22.4 h1:srAQaiX6jX/cYL6q29aE0m8lOskT9CurZ9N61YR3yoI=
+github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA=
-github.com/shirou/gopsutil/v3 v3.22.4/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM=
+github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -598,8 +603,8 @@ github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
 github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
-github.com/xanzy/go-gitlab v0.65.0 h1:9xSA9cRVhz3Z54JacIHdvWnNmNAoSz/BDnyMGOf3yIg=
+github.com/xanzy/go-gitlab v0.68.0 h1:b2iMQHgZ1V+NyRqLRJVv6RFfr4xnd/AASeS/PETYL0Y=
-github.com/xanzy/go-gitlab v0.65.0/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM=
+github.com/xanzy/go-gitlab v0.68.0/go.mod h1:o4yExCtdaqlM8YGdDJWuZoBmfxBsmA9TPEjs9mx1UO4=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -755,12 +760,12 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
+golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -768,8 +773,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
-golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -839,11 +844,10 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -859,8 +863,9 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
+golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -995,8 +1000,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1021,8 +1027,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

v2/internal/runner/runner.go

@@ -341,10 +341,9 @@ func (r *Runner) RunEnumeration() error {
 	if err != nil {
 		return errors.Wrap(err, "could not load templates from config")
 	}
-	store.Load()
 
 	if r.options.Validate {
-		if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
+		if err := store.ValidateTemplates(); err != nil {
 			return err
 		}
 		if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 && stats.GetValue(parsers.RuntimeWarningsStats) == 0 {
@@ -354,6 +353,7 @@ func (r *Runner) RunEnumeration() error {
 		}
 		return nil // exit
 	}
+	store.Load()
 
 	r.displayExecutionInfo(store)
 

v2/pkg/catalog/config/config.go

@@ -27,7 +27,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.7.1`
+const Version = `2.7.2`
 
 func getConfigDetails() (string, error) {
 	configDir, err := GetConfigDir()

v2/pkg/catalog/loader/loader.go

@@ -1,8 +1,9 @@
 package loader
 
 import (
-	"errors"
+	"os"
 
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
@@ -119,11 +120,18 @@ func New(config *Config) (*Store, error) {
 		store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...)
 	}
 
+	// Handle a dot as the current working directory
+	if len(store.finalTemplates) == 1 && store.finalTemplates[0] == "." {
+		currentDirectory, err := os.Getwd()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get current directory")
+		}
+		store.finalTemplates = []string{currentDirectory}
+	}
 	// Handle a case with no templates or workflows, where we use base directory
 	if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided {
 		store.finalTemplates = []string{config.TemplatesDirectory}
 	}
-
 	return store, nil
 }
 
@@ -157,13 +165,9 @@ func init() {
 
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
-func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error {
+func (store *Store) ValidateTemplates() error {
-	// consider all the templates by default if no templates passed by user
+	templatePaths := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
-	if len(templatesList) == 0 {
+	workflowPaths := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
-		templatesList = store.finalTemplates
-	}
-	templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
-	workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
 
 	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
 	filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)

v2/pkg/operators/common/dsl/dsl.go

@@ -510,6 +510,13 @@ func init() {
 		"to_string": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
 			return types.ToString(args[0]), nil
 		}),
+		"dec_to_hex": makeDslFunction(1, func(args ...interface{}) (interface{}, error) {
+			if number, ok := args[0].(float64); ok {
+				hexNum := strconv.FormatInt(int64(number), 16)
+				return types.ToString(hexNum), nil
+			}
+			return nil, fmt.Errorf("invalid number: %T", args[0])
+		}),
 	}
 
 	dslFunctions = make(map[string]dslFunction, len(tempDslFunctions))

v2/pkg/operators/common/dsl/dsl_test.go

@@ -112,6 +112,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
 	concat(args ...interface{}) string
 	contains(arg1, arg2 interface{}) interface{}
 	date(arg1 interface{}) interface{}
+	dec_to_hex(arg1 interface{}) interface{}
 	generate_java_gadget(arg1, arg2, arg3 interface{}) interface{}
 	gzip(arg1 interface{}) interface{}
 	gzip_decode(arg1 interface{}) interface{}
@@ -222,6 +223,7 @@ func TestDslExpressions(t *testing.T) {
 		`print_debug(1+2, "Hello")`:                        nil,
 		`to_number('4')`:                                   float64(4),
 		`to_string(4)`:                                     "4",
+		`dec_to_hex(7001)`:                                 "1b59",
 		`compare_versions('v1.0.0', '<1.1.1')`:             true,
 		`compare_versions('v1.1.1', '>v1.1.0')`:            true,
 		`compare_versions('v1.0.0', '>v0.0.1,<v1.0.1')`:    true,

v2/pkg/operators/matchers/match.go

@@ -180,12 +180,16 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
 
 		result, err := expression.Evaluate(data)
 		if err != nil {
-			gologger.Warning().Msgf(err.Error())
+			if strings.Contains(err.Error(), "No parameter") {
+				gologger.Warning().Msgf("[%s] %s", data["template-id"], err.Error())
+			} else {
+				gologger.Error().Label("WRN").Msgf("[%s] %s", data["template-id"], err.Error())
+			}
 			continue
 		}
 
 		if boolResult, ok := result.(bool); !ok {
-			gologger.Warning().Msgf("The return value of a DSL statement must return a boolean value.")
+			gologger.Error().Label("WRN").Msgf("[%s] The return value of a DSL statement must return a boolean value.", data["template-id"])
 			continue
 		} else if !boolResult {
 			// If we are in an AND request and a match failed,

v2/pkg/operators/operators.go

@@ -228,6 +228,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 					dynKeyName := fmt.Sprintf("%s%d", dynName, dynIndex)
 					dataDynamicValues[dynKeyName] = dynValue
 				}
+				dataDynamicValues[dynName] = dynValues
 			} else {
 				dataDynamicValues[dynName] = dynValues[0]
 			}

v2/pkg/output/output.go

@@ -275,12 +275,16 @@ func (w *StandardWriter) WriteFailure(event InternalEvent) error {
 		return nil
 	}
 	templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]))
+	var templateInfo model.Info
+	if event["template-info"] != nil {
+		templateInfo = event["template-info"].(model.Info)
+	}
 	data := &ResultEvent{
 		Template:      templatePath,
 		TemplateURL:   templateURL,
 		TemplateID:    types.ToString(event["template-id"]),
 		TemplatePath:  types.ToString(event["template-path"]),
-		Info:          event["template-info"].(model.Info),
+		Info:          templateInfo,
 		Type:          types.ToString(event["type"]),
 		Host:          types.ToString(event["host"]),
 		MatcherStatus: false,

v2/pkg/protocols/common/generators/maps.go

@@ -47,15 +47,14 @@ func MergeMapsMany(maps ...interface{}) map[string][]string {
 }
 
 // MergeMaps merges two maps into a new map
-func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
+func MergeMaps(maps ...map[string]interface{}) map[string]interface{} {
-	m := make(map[string]interface{}, len(m1)+len(m2))
+	merged := make(map[string]interface{})
-	for k, v := range m1 {
+	for _, m := range maps {
-		m[k] = v
+		for k, v := range m {
+			merged[k] = v
+		}
 	}
-	for k, v := range m2 {
+	return merged
-		m[k] = v
-	}
-	return m
 }
 
 // ExpandMapValues converts values from flat string to string slice

v2/pkg/protocols/common/generators/maps_test.go

@@ -7,10 +7,10 @@ import (
 )
 
 func TestMergeMapsMany(t *testing.T) {
-	got := MergeMapsMany(map[string]interface{}{"a": []string{"1", "2"}, "c": "5"}, map[string][]string{"b": []string{"3", "4"}})
+	got := MergeMapsMany(map[string]interface{}{"a": []string{"1", "2"}, "c": "5"}, map[string][]string{"b": {"3", "4"}})
 	require.Equal(t, map[string][]string{
-		"a": []string{"1", "2"},
+		"a": {"1", "2"},
-		"b": []string{"3", "4"},
+		"b": {"3", "4"},
-		"c": []string{"5"},
+		"c": {"5"},
 	}, got, "could not get correct merged map")
 }

v2/pkg/protocols/dns/dns.go

@@ -245,8 +245,8 @@ func classToInt(class string) uint16 {
 	return uint16(result)
 }
 
-// GenerateDNSVariables from a dns name
+// GenerateVariables from a dns name
-func GenerateDNSVariables(domain string) map[string]interface{} {
+func GenerateVariables(domain string) map[string]interface{} {
 	parsed, err := publicsuffix.Parse(strings.TrimSuffix(domain, "."))
 	if err != nil {
 		return map[string]interface{}{"FQDN": domain}

v2/pkg/protocols/dns/dns_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestGenerateDNSVariables(t *testing.T) {
-	vars := GenerateDNSVariables("www.projectdiscovery.io")
+	vars := GenerateVariables("www.projectdiscovery.io")
 	require.Equal(t, map[string]interface{}{
 		"FQDN": "www.projectdiscovery.io",
 		"RDN":  "projectdiscovery.io",

v2/pkg/protocols/dns/request.go

@@ -43,7 +43,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 	if err != nil {
 		return errors.Wrap(err, "could not build request")
 	}
-	vars := GenerateDNSVariables(domain)
+	vars := GenerateVariables(domain)
 	variablesMap := request.options.Variables.Evaluate(vars)
 	vars = generators.MergeMaps(variablesMap, vars)
 

v2/pkg/protocols/file/request.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@@ -200,7 +201,21 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath
 
 	scanner := bufio.NewScanner(reader)
 	buffer := []byte{}
-	scanner.Buffer(buffer, int(chunkSize))
+	if request.CompiledOperators.GetMatchersCondition() == matchers.ANDCondition {
+		scanner.Buffer(buffer, int(defaultMaxReadSize))
+		scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+			defaultMaxReadSizeInt := int(defaultMaxReadSize)
+			if len(data) > defaultMaxReadSizeInt {
+				return defaultMaxReadSizeInt, data[0:defaultMaxReadSizeInt], nil
+			}
+			if !atEOF {
+				return 0, nil, nil
+			}
+			return len(data), data, bufio.ErrFinalToken
+		})
+	} else {
+		scanner.Buffer(buffer, int(chunkSize))
+	}
 
 	var fileMatches []FileMatch
 	var opResult *operators.Result
@@ -288,18 +303,21 @@ func (request *Request) buildEvent(input, filePath string, fileMatches []FileMat
 	}
 
 	event := eventcreator.CreateEventWithOperatorResults(request, internalEvent, operatorResult)
-	for _, result := range event.Results {
+	// Annotate with line numbers if asked by the user
-		switch {
+	if request.options.Options.ShowMatchLine {
-		case result.MatcherName != "":
+		for _, result := range event.Results {
-			result.Lines = exprLines[result.MatcherName]
+			switch {
-		case result.ExtractorName != "":
+			case result.MatcherName != "":
-			result.Lines = exprLines[result.ExtractorName]
+				result.Lines = exprLines[result.MatcherName]
-		default:
+			case result.ExtractorName != "":
-			for _, extractedResult := range result.ExtractedResults {
+				result.Lines = exprLines[result.ExtractorName]
-				result.Lines = append(result.Lines, exprLines[extractedResult]...)
+			default:
+				for _, extractedResult := range result.ExtractedResults {
+					result.Lines = append(result.Lines, exprLines[extractedResult]...)
+				}
 			}
+			result.Lines = sliceutil.DedupeInt(result.Lines)
 		}
-		result.Lines = sliceutil.DedupeInt(result.Lines)
 	}
 	return event
 }

v2/pkg/protocols/headless/engine/http_client.go

@@ -26,6 +26,7 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
 	tlsConfig := &tls.Config{
 		Renegotiation:      tls.RenegotiateOnceAsClient,
 		InsecureSkipVerify: true,
+		MinVersion:         tls.VersionTLS10,
 	}
 
 	if options.SNI != "" {

v2/pkg/protocols/headless/engine/page_actions.go

@@ -423,13 +423,13 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi
 
 // WaitLoad waits for the page to load
 func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error {
-	p.page.Timeout(1 * time.Second).WaitNavigation(proto.PageLifecycleEventNameDOMContentLoaded)()
+	p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameDOMContentLoaded)()
 
 	// Wait for the window.onload event and also wait for the network requests
-	// to become idle for a maximum duration of 2 seconds. If the requests
+	// to become idle for a maximum duration of 3 seconds. If the requests
 	// do not finish,
 	if err := p.page.WaitLoad(); err != nil {
-		return errors.Wrap(err, "could not reset mouse")
+		return errors.Wrap(err, "could not wait load event")
 	}
 	_ = p.page.WaitIdle(1 * time.Second)
 	return nil

v2/pkg/protocols/headless/engine/page_actions_test.go

@@ -46,7 +46,7 @@ func TestActionScript(t *testing.T) {
 		<script>window.test = 'some-data';</script>
 	</html>`
 
-	timeout := 2 * time.Second
+	timeout := 15 * time.Second
 
 	t.Run("run-and-results", func(t *testing.T) {
 		actions := []*Action{

v2/pkg/protocols/headless/request.go

@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	httpProtocol "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 )
 
@@ -31,9 +32,11 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o
 	if request.options.Browser.UserAgent() == "" {
 		request.options.Browser.SetUserAgent(request.compiledUserAgent)
 	}
-	payloads := generators.BuildPayloadFromOptions(request.options.Options)
 
-	variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(metadata, payloads))
+	vars := GenerateVariables(inputURL)
+	payloads := generators.BuildPayloadFromOptions(request.options.Options)
+	values := generators.MergeMaps(vars, metadata, payloads)
+	variablesMap := request.options.Variables.Evaluate(values)
 	payloads = generators.MergeMaps(variablesMap, payloads)
 
 	if request.generator != nil {
@@ -141,3 +144,13 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
 		gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse)
 	}
 }
+
+// GenerateVariables will create default variables
+func GenerateVariables(URL string) map[string]interface{} {
+	parsed, err := url.Parse(URL)
+	if err != nil {
+		return nil
+	}
+
+	return httpProtocol.GenerateVariables(parsed, false)
+}

v2/pkg/protocols/http/build_request.go

@@ -93,7 +93,7 @@ func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues ma
 	}
 
 	values := generators.MergeMaps(
-		generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)),
+		generators.MergeMaps(dynamicValues, GenerateVariables(parsed, trailingSlash)),
 		generators.BuildPayloadFromOptions(r.request.options.Options),
 	)
 
@@ -156,7 +156,7 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam
 			return nil, fmt.Errorf("could not parse request URL: %w", err)
 		}
 		values = generators.MergeMaps(
-			generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
+			generators.MergeMaps(dynamicValues, GenerateVariables(parsed, false)),
 			values,
 		)
 
@@ -377,8 +377,8 @@ func setHeader(req *http.Request, name, value string) {
 	}
 }
 
-// generateVariables will create default variables after parsing a url
+// GenerateVariables will create default variables after parsing a url
-func generateVariables(parsed *url.URL, trailingSlash bool) map[string]interface{} {
+func GenerateVariables(parsed *url.URL, trailingSlash bool) map[string]interface{} {
 	domain := parsed.Host
 	if strings.Contains(parsed.Host, ":") {
 		domain = strings.Split(parsed.Host, ":")[0]
@@ -416,5 +416,5 @@ func generateVariables(parsed *url.URL, trailingSlash bool) map[string]interface
 		"File":     base,
 		"Scheme":   parsed.Scheme,
 	}
-	return generators.MergeMaps(httpVariables, dns.GenerateDNSVariables(domain))
+	return generators.MergeMaps(httpVariables, dns.GenerateVariables(domain))
 }

v2/pkg/protocols/http/build_request_test.go

@@ -27,7 +27,7 @@ func TestBaseURLWithTemplatePrefs(t *testing.T) {
 func TestVariables(t *testing.T) {
 	baseURL := "http://localhost:9001/test/123"
 	parsed, _ := url.Parse(baseURL)
-	values := generateVariables(parsed, true)
+	values := GenerateVariables(parsed, true)
 
 	require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
 	require.Equal(t, values["RootURL"], "http://localhost:9001", "incorrect rootURL")
@@ -40,7 +40,7 @@ func TestVariables(t *testing.T) {
 
 	baseURL = "https://example.com"
 	parsed, _ = url.Parse(baseURL)
-	values = generateVariables(parsed, false)
+	values = GenerateVariables(parsed, false)
 
 	require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
 	require.Equal(t, values["Host"], "example.com", "incorrect domain name")
@@ -52,7 +52,7 @@ func TestVariables(t *testing.T) {
 
 	baseURL = "ftp://foobar.com/"
 	parsed, _ = url.Parse(baseURL)
-	values = generateVariables(parsed, true)
+	values = GenerateVariables(parsed, true)
 
 	require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
 	require.Equal(t, values["Host"], "foobar.com", "incorrect domain name")

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -177,6 +177,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	tlsConfig := &tls.Config{
 		Renegotiation:      tls.RenegotiateOnceAsClient,
 		InsecureSkipVerify: true,
+		MinVersion:         tls.VersionTLS10,
 	}
 
 	if options.SNI != "" {

v2/pkg/protocols/http/operators.go

@@ -123,6 +123,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
 	data["body"] = body
 	data["content_length"] = resp.ContentLength
 	data["all_headers"] = headers
+	data["header"] = headers
 	data["duration"] = duration.Seconds()
 	data["template-id"] = request.options.TemplateID
 	data["template-info"] = request.options.TemplateInfo

v2/pkg/protocols/http/operators_test.go

@@ -41,7 +41,7 @@ func TestResponseToDSLMap(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 14, "could not get correct number of items in dsl map")
+	require.Len(t, event, 15, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 }
@@ -71,7 +71,7 @@ func TestHTTPOperatorMatch(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 14, "could not get correct number of items in dsl map")
+	require.Len(t, event, 15, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 
@@ -159,7 +159,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 14, "could not get correct number of items in dsl map")
+	require.Len(t, event, 15, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header")
 
@@ -286,7 +286,7 @@ func TestHTTPMakeResult(t *testing.T) {
 	matched := "http://example.com/test/?test=1"
 
 	event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
-	require.Len(t, event, 14, "could not get correct number of items in dsl map")
+	require.Len(t, event, 15, "could not get correct number of items in dsl map")
 	require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
 	require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
 

v2/pkg/protocols/http/request.go

@@ -406,6 +406,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		options.FollowRedirects = request.Redirects
 		options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes
 		options.ForceReadAllBody = request.ForceReadAllBody
+		options.SNI = request.options.Options.SNI
 		resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), options)
 	} else {
 		hostname = generatedRequest.request.URL.Host
@@ -533,7 +534,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 			return errors.Wrap(err, "could not read http response with redirect chain")
 		}
 	} else {
-		dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}}
+		dumpedResponse = []redirectedResponse{{resp: resp, fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}}
 	}
 
 	// if nuclei-project is enabled store the response if not previously done
@@ -554,6 +555,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		if generatedRequest.request != nil {
 			matchedURL = generatedRequest.request.URL.String()
 		}
+		// Give precedence to the final URL from response
+		if response.resp.Request != nil {
+			if responseURL := response.resp.Request.URL.String(); responseURL != "" {
+				matchedURL = responseURL
+			}
+		}
 		finalEvent := make(output.InternalEvent)
 
 		outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
@@ -597,6 +604,11 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, reqURL)
 
 		callback(event)
+
+		// Skip further responses if we have stop-at-first-match and a match
+		if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && len(event.Results) > 0 {
+			return nil
+		}
 	}
 	return nil
 }

v2/pkg/protocols/http/utils.go

@@ -142,7 +142,8 @@ func dump(req *generatedRequest, reqURL string) ([]byte, error) {
 
 		return dumpBytes, nil
 	}
-	return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(req.rawRequest.Data)), rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes})
+	rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}
+	return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
 }
 
 // handleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically)

v2/pkg/protocols/ssl/ssl.go

@@ -233,16 +233,18 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
 	data["matched"] = addressToDial
 	data["not_after"] = float64(certNotAfter)
 	data["ip"] = request.dialer.GetDialedIP(hostname)
-
+	data["template-path"] = requestOptions.TemplatePath
+	data["template-id"] = requestOptions.TemplateID
+	data["template-info"] = requestOptions.TemplateInfo
 	event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
 	if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
 		msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input)
 		if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
-		gologger.Debug().Msg(msg)
+			gologger.Debug().Msg(msg)
-		gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
+			gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
 		}
 		if requestOptions.Options.StoreResponse {
-		request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
+			request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
 		}
 	}
 	callback(event)

v2/pkg/protocols/websocket/websocket.go

@@ -87,7 +87,7 @@ type Input struct {
 }
 
 const (
-	parseUrlErrorMessage = "could not parse input url"
+	parseUrlErrorMessage                   = "could not parse input url"
 	evaluateTemplateExpressionErrorMessage = "could not evaluate template expressions"
 )
 
@@ -190,7 +190,11 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
 		}
 		header.Set(key, string(finalData))
 	}
-	tlsConfig := &tls.Config{InsecureSkipVerify: true, ServerName: hostname}
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true,
+		ServerName:         hostname,
+		MinVersion:         tls.VersionTLS10,
+	}
 	if requestOptions.Options.SNI != "" {
 		tlsConfig.ServerName = requestOptions.Options.SNI
 	}

v2/pkg/types/types.go

@@ -174,6 +174,8 @@ type Options struct {
 	TemplatesVersion bool
 	// TemplateList lists available templates
 	TemplateList bool
+	// HangMonitor enables nuclei hang monitoring
+	HangMonitor bool
 	// Stdin specifies whether stdin input was given to the process
 	Stdin bool
 	// StopAtFirstMatch stops processing template at first full match (this may break chained requests)
@@ -206,6 +208,8 @@ type Options struct {
 	ClientCAFile string
 	// Use ZTLS library
 	ZTLS bool
+	// ShowMatchLine enables display of match line number
+	ShowMatchLine bool
 	// EnablePprof enables exposing pprof runtime information with a webserver.
 	EnablePprof bool
 	// StoreResponse stores received response to output directory

v2/pkg/utils/monitor/monitor.go

@@ -0,0 +1,139 @@
+// Package monitor implements a goroutine based monitoring for
+// detecting stuck scanner processes and dumping stack and other
+// relevant information for investigation.
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/DataDog/gostackparse"
+	"github.com/projectdiscovery/gologger"
+	"github.com/rs/xid"
+)
+
+// Agent is an agent for monitoring hanging programs
+type Agent struct {
+	cancel    context.CancelFunc
+	lastStack []string
+
+	goroutineCount   int
+	currentIteration int // number of times we've checked hang
+}
+
+const defaultMonitorIteration = 6
+
+// NewStackMonitor returns a new stack monitor instance
+func NewStackMonitor(interval time.Duration) context.CancelFunc {
+	ctx, cancel := context.WithCancel(context.Background())
+	ticker := time.NewTicker(interval)
+
+	monitor := &Agent{cancel: cancel}
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				ticker.Stop()
+			case <-ticker.C:
+				monitor.monitorWorker()
+			default:
+				continue
+			}
+		}
+	}()
+	return cancel
+}
+
+// monitorWorker is a worker for monitoring running goroutines
+func (s *Agent) monitorWorker() {
+	current := runtime.NumGoroutine()
+	if current != s.goroutineCount {
+		s.goroutineCount = current
+		s.currentIteration = 0
+		return
+	}
+	s.currentIteration++
+
+	if s.currentIteration == defaultMonitorIteration-1 {
+		lastStackTrace := generateStackTraceSlice(getStack(true))
+		s.lastStack = lastStackTrace
+		return
+	}
+
+	// cancel the monitoring goroutine if we discover
+	// we've been stuck for some iterations.
+	if s.currentIteration == defaultMonitorIteration {
+		currentStack := getStack(true)
+
+		// Bail out if the stacks don't match from previous iteration
+		newStack := generateStackTraceSlice(currentStack)
+		if !compareStringSliceEqual(s.lastStack, newStack) {
+			s.currentIteration = 0
+			return
+		}
+		s.cancel()
+		stackTraceFile := fmt.Sprintf("nuclei-stacktrace-%s.dump", xid.New().String())
+		gologger.Error().Msgf("Detected hanging goroutine (count=%d/%d) = %s\n", current, s.goroutineCount, stackTraceFile)
+		if err := ioutil.WriteFile(stackTraceFile, currentStack, os.ModePerm); err != nil {
+			gologger.Error().Msgf("Could not write stack trace for goroutines: %s\n", err)
+		}
+		os.Exit(1) // exit forcefully if we've been stuck
+	}
+}
+
+// getStack returns full stack trace of the program
+var getStack = func(all bool) []byte {
+	for i := 1024 * 1024; ; i *= 2 {
+		buf := make([]byte, i)
+		if n := runtime.Stack(buf, all); n < i {
+			return buf[:n-1]
+		}
+	}
+}
+
+// generateStackTraceSlice returns a list of current stack in string slice format
+func generateStackTraceSlice(stack []byte) []string {
+	goroutines, _ := gostackparse.Parse(bytes.NewReader(stack))
+
+	var builder strings.Builder
+	var stackList []string
+	for _, goroutine := range goroutines {
+		builder.WriteString(goroutine.State)
+		builder.WriteString("|")
+
+		for _, frame := range goroutine.Stack {
+			builder.WriteString(frame.Func)
+			builder.WriteString(";")
+		}
+		stackList = append(stackList, builder.String())
+		builder.Reset()
+	}
+	return stackList
+}
+
+// compareStringSliceEqual compares two string slices for equality without order
+func compareStringSliceEqual(first, second []string) bool {
+	if len(first) != len(second) {
+		return false
+	}
+	diff := make(map[string]int, len(first))
+	for _, x := range first {
+		diff[x]++
+	}
+	for _, y := range second {
+		if _, ok := diff[y]; !ok {
+			return false
+		}
+		diff[y] -= 1
+		if diff[y] == 0 {
+			delete(diff, y)
+		}
+	}
+	return len(diff) == 0
+}

v2/pkg/utils/monitor/monitor_test.go

@@ -0,0 +1,15 @@
+package monitor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMonitorCompareStringSliceEqual(t *testing.T) {
+	value := compareStringSliceEqual([]string{"a", "b"}, []string{"b", "a"})
+	require.True(t, value, "could not get correct value")
+
+	value = compareStringSliceEqual([]string{"a", "c"}, []string{"b", "a"})
+	require.False(t, value, "could get incorrect value")
+}