Merge pull request #922 from projectdiscovery/dev

Hotfix release
2021-08-10 02:19:40 +05:30 · 2021-08-10 02:19:40 +05:30 · 701c999f8a
parent 985a7c0f66 3d49094d07
commit 701c999f8a
31 changed files with 617 additions and 179 deletions
--- a/.github/workflows/functional-test.yml
+++ b/.github/workflows/functional-test.yml
@ -0,0 +1,25 @@
+name: 🧪 Functional Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Functional Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Functional Tests
+        run: |
+          chmod +x run.sh
+          bash run.sh
+        working-directory: v2/cmd/functional-test
--- a/.gitignore
+++ b/.gitignore
@ -8,3 +8,6 @@ bin
 v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
 v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
 v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
+v2/cmd/functional-test/nuclei_dev
+v2/cmd/functional-test/nuclei_main
+v2/cmd/functional-test/functional-test
--- a/.golangci.yml
+++ b/.golangci.yml
@ -1,108 +0,0 @@
-linters-settings:
-  dupl:
-    threshold: 100
-  exhaustive:
-    default-signifies-exhaustive: false
-  # funlen:
-  #   lines: 100
-  #   statements: 50
-  #goconst:
-  #  min-len: 2
-  #  min-occurrences: 2
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - dupImport # https://github.com/go-critic/go-critic/issues/845
-      - ifElseChain
-  # gocyclo:
-  #   min-complexity: 15
-  goimports:
-    local-prefixes: github.com/golangci/golangci-lint
-  golint:
-    min-confidence: 0
-  gomnd:
-    settings:
-      mnd:
-        # don't include the "operation" and "assign"
-        checks: argument,case,condition,return
-  govet:
-    check-shadowing: true
-    settings:
-      printf:
-        funcs:
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
-  # lll:
-  #   line-length: 140
-  maligned:
-    suggest-new: true
-  misspell:
-    locale: US
-  nolintlint:
-    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
-    allow-unused: false # report any unused nolint directives
-    require-explanation: false # don't require an explanation for nolint directives
-    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
-
-linters:
-  # please, do not use `enable-all`: it's deprecated and will be removed soon.
-  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
-  disable-all: true
-  enable:
-    #- bodyclose
-    - deadcode
-    - dogsled
-    - errcheck
-    - exhaustive
-    - gochecknoinits
-    #- goconst
-    - gocritic
-    - gofmt
-    - goimports
-    #- gomnd
-    - goprintffuncname
-    - gosimple
-    - govet
-    - ineffassign
-    - interfacer
-    - maligned
-    - misspell
-    - nakedret
-    - noctx
-    - nolintlint
-    - rowserrcheck
-    - scopelint
-    - staticcheck
-    - structcheck
-    - stylecheck
-    - typecheck
-    - unconvert
-    - unparam
-    - unused
-    - varcheck
-    - whitespace
-    - revive
-
-  # don't enable:
-  # - depguard
-  # - asciicheck
-  # - funlen
-  # - gochecknoglobals
-  # - gocognit
-  # - gocyclo
-  # - godot
-  # - godox
-  # - goerr113
-  # - gosec
-  # - lll
-  # - nestif
-  # - prealloc
-  # - testpackage
-  # - wsl
--- a/2
+++ b/2
@ -1,4 +1,4 @@
-FROM golang:1.16.6-alpine as build-env
+FROM golang:1.16.7-alpine as build-env
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei

 FROM alpine:latest
--- a/README.md
+++ b/README.md
@ -82,7 +82,7 @@ Usage:

 Flags:
 TARGET:
-   -u, -target string  target URL/host to scan
+   -u, -target string[]  target URLs/hosts to scan
   -l, -list string    path to file containing a list of target URLs/hosts to scan (one per line)

 TEMPLATES:
--- a/README_CN.md
+++ b/README_CN.md
@ -97,7 +97,7 @@ nuclei -h
 |burp-collaborator-biid|使用burp-collaborator插件|nuclei -burp-collaborator-biid XXXX|
 |c|并行的最大模板数量(默认10)|nuclei -c 10|
 |l|对URL列表进行测试|nuclei -l urls.txt|
-|target|对目标进行测试|nuclei -target hxxps://example.com|
+|target|对目标进行测试|nuclei -target hxxps://example.com -target hxxps://example2.com|
 |t|要检测的模板种类|nuclei -t git-core.yaml -t cves/|
 |no-color|输出不显示颜色|nuclei -no-color|
 |no-meta|不显示匹配的元数据|nuclei -no-meta|
--- a/v2/cmd/functional-test/main.go
+++ b/v2/cmd/functional-test/main.go
@ -0,0 +1,79 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+)
+
+var (
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+	errored = false
+
+	mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
+	devNucleiBinary  = flag.String("dev", "", "Dev Branch Nuclei Binary")
+	testcases        = flag.String("testcases", "", "Test cases file for nuclei functional tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	file, err := os.Open(*testcases)
+	if err != nil {
+		return errors.Wrap(err, "could not open test cases")
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		if err := runIndividualTestCase(text); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, text)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase string) error {
+	parts := strings.Fields(testcase)
+
+	var finalArgs []string
+	if len(parts) > 1 {
+		finalArgs = parts[1:]
+	}
+	mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run nuclei main test")
+	}
+	devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run nuclei dev test")
+	}
+	if mainOutput == devOutput {
+		return nil
+	}
+	return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
+}
--- a/v2/cmd/functional-test/run.sh
+++ b/v2/cmd/functional-test/run.sh
@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Building functional-test binary'
+go build
+
+echo 'Building Nuclei binary from current branch'
+go build -o nuclei_dev ../nuclei
+
+echo 'Installing latest release of nuclei'
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+
+echo 'Starting Nuclei functional test'
+./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt
--- a/v2/cmd/functional-test/testcases.txt
+++ b/v2/cmd/functional-test/testcases.txt
@ -0,0 +1,51 @@
+{{binary}}
+{{binary}} -tags cve
+{{binary}} -tags cve,exposure
+{{binary}} -tags cve,exposure -tags token
+{{binary}} -tags cve,exposure -tags token,logs
+{{binary}} -tags "cve","exposure" -tags "token","logs"
+{{binary}} -tags 'cve','exposure' -tags 'token','logs'
+{{binary}} -tags cve -severity high
+{{binary}} -tags cve,exposure -severity high,critical
+{{binary}} -tags cve,exposure -severity "high,critical,medium"
+{{binary}} -tags cve -author geeknik
+{{binary}} -tags cve -author geeknik,pdteam
+{{binary}} -tags cve -author geeknik -severity high
+{{binary}} -tags cve
+{{binary}} -tags cve,exposure
+{{binary}} -tags cve,exposure -tags token
+{{binary}} -tags cve,exposure -tags token,logs
+{{binary}} -tags "cve","exposure" -tags "token","logs"
+{{binary}} -tags 'cve','exposure' -tags 'token','logs'
+{{binary}} -tags cve -severity high
+{{binary}} -tags cve,exposure -severity high,critical
+{{binary}} -tags cve,exposure -severity "high,critical,medium"
+{{binary}} -tags cve -author geeknik
+{{binary}} -tags cve -author geeknik,pdteam
+{{binary}} -tags cve -author geeknik -severity high
+{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
+{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity "high,critical"
+{{binary}} -tags cve -etags ssrf
+{{binary}} -tags cve,exposure -etags ssrf,config
+{{binary}} -tags cve,exposure -etags ssrf,config -severity high
+{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
+{{binary}} -tags cve,dos,fuzz
+{{binary}} -tags cve -include-tags dos,fuzz
+{{binary}} -tags cve -exclude-tags cve2020
+{{binary}} -tags cve -exclude-templates cves/2020/
+{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
+{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
+{{binary}} -t cves/
+{{binary}} -t cves/ -t exposures/
+{{binary}} -t cves/ -t exposures/ -tags config
+{{binary}} -t cves/ -t exposures/ -tags config,ssrf
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
+{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
+{{binary}} -w workflows
+{{binary}} -w workflows -author geeknik,pdteam
+{{binary}} -w workflows -severity high,critical
+{{binary}} -w workflows -author geeknik,pdteam  -severity high,critical
--- a/v2/cmd/integration-test/integration-test.go
+++ b/v2/cmd/integration-test/integration-test.go
@ -13,6 +13,8 @@ var (
 	debug      = os.Getenv("DEBUG") == "true"
 	customTest = os.Getenv("TEST")
 	protocol   = os.Getenv("PROTO")
+
+	errored = false
 )

 func main() {
@ -36,13 +38,16 @@ func main() {
 				err := test.Execute(file)
 				if err != nil {
 					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err)
-					os.Exit(1)
+					errored = true
 				} else {
 					fmt.Printf("%s Test \"%s\" passed!\n", success, file)
 				}
 			}
 		}
 	}
+	if errored {
+		os.Exit(1)
+	}
 }

 func errIncorrectResultsCount(results []string) error {
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@ -42,8 +42,8 @@ func readConfig() {
 on extensive configurability, massive extensibility and ease of use.`)

 	createGroup(flagSet, "input", "Target",
-		flagSet.StringVarP(&options.Target, "target", "u", "", "target URL/host to scan"),
-		flagSet.StringVarP(&options.Targets, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+		flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan"),
+		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
 	)

 	createGroup(flagSet, "templates", "Templates",
@ -92,9 +92,12 @@ on extensive configurability, massive extensibility and ease of use.`)

 		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),

+		flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"),
+
 		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
 		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
 		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
+		flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "Enable environment variables support"),
 	)

 	createGroup(flagSet, "interactsh", "interactsh",
@ -109,6 +112,7 @@ on extensive configurability, massive extensibility and ease of use.`)

 	createGroup(flagSet, "rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
 	)
--- a/v2/go.mod
+++ b/v2/go.mod
@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/andygrunwald/go-jira v1.13.0
+	github.com/antchfx/htmlquery v1.2.3
 	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/c4milo/unpackit v0.1.0 // indirect
@ -17,29 +18,30 @@ require (
 	github.com/gosuri/uiprogress v0.0.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
-	github.com/json-iterator/go v1.1.10
+	github.com/itchyny/gojq v0.12.4
+	github.com/json-iterator/go v1.1.11
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karrick/godirwalk v1.16.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mattn/go-runewidth v0.0.10 // indirect
-	github.com/miekg/dns v1.1.38
+	github.com/miekg/dns v1.1.43
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/owenrumney/go-sarif v1.0.4
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
 	github.com/projectdiscovery/fastdialer v0.0.8
-	github.com/projectdiscovery/goflags v0.0.6
+	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/interactsh v0.0.3
+	github.com/projectdiscovery/interactsh v0.0.4
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.10
 	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
-	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
+	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rs/xid v1.2.1
+	github.com/rs/xid v1.3.0
 	github.com/segmentio/ksuid v1.0.3
 	github.com/shirou/gopsutil/v3 v3.21.5
 	github.com/spaolacci/murmur3 v1.1.0
@ -52,11 +54,9 @@ require (
 	github.com/xanzy/go-gitlab v0.44.0
 	go.uber.org/atomic v1.7.0
 	go.uber.org/multierr v1.6.0
-	go.uber.org/ratelimit v0.1.0
-	golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
-	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
+	go.uber.org/ratelimit v0.2.0
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
 	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
--- a/v2/go.sum
+++ b/v2/go.sum
@ -31,6 +31,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
+git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
@ -40,8 +42,14 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
 github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
 github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
+github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
+github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
+github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
 github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
 github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
 github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
@ -102,6 +110,7 @@ github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsH
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -136,8 +145,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@ -178,20 +188,26 @@ github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKgh
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
+github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
+github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
+github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
+github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
 github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
-github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
-github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
+github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
+github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
 github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
 github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
@ -212,22 +228,22 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
+github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
 github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
+github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@ -258,15 +274,14 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS
 github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
 github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
 github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
-github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
-github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
-github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
+github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
+github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
-github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
+github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8=
+github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
 github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
 github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
@ -275,10 +290,8 @@ github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQY
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
 github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
-github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
@ -287,8 +300,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
-github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
 github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@ -305,7 +318,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -364,19 +376,16 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
-go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
+go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
+go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
-golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -429,6 +438,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@ -437,8 +447,10 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -455,8 +467,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -474,6 +486,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -486,13 +499,14 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk=
+golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@ -125,7 +125,7 @@ func New(options *types.Options) (*Runner, error) {
 		os.Exit(0)
 	}

-	if (len(options.Templates) == 0 || !options.NewTemplates || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
+	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}
 	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
@ -137,11 +137,23 @@ func New(options *types.Options) (*Runner, error) {
 	runner.inputCount = 0
 	dupeCount := 0

-	// Handle single target
-	if options.Target != "" {
-		runner.inputCount++
-		// nolint:errcheck // ignoring error
-		runner.hostMap.Set(options.Target, nil)
+	// Handle multiple target
+	if len(options.Targets) != 0 {
+		for _, target := range options.Targets {
+			url := strings.TrimSpace(target)
+			if url == "" {
+				continue
+			}
+
+			if _, ok := runner.hostMap.Get(url); ok {
+				dupeCount++
+				continue
+			}
+
+			runner.inputCount++
+			// nolint:errcheck // ignoring error
+			runner.hostMap.Set(url, nil)
+		}
 	}

 	// Handle stdin
@ -152,19 +164,21 @@ func New(options *types.Options) (*Runner, error) {
 			if url == "" {
 				continue
 			}
+
 			if _, ok := runner.hostMap.Get(url); ok {
 				dupeCount++
 				continue
 			}
+
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
 			runner.hostMap.Set(url, nil)
 		}
 	}

-	// Handle taget file
-	if options.Targets != "" {
-		input, inputErr := os.Open(options.Targets)
+	// Handle target file
+	if options.TargetsFilePath != "" {
+		input, inputErr := os.Open(options.TargetsFilePath)
 		if inputErr != nil {
 			return nil, errors.Wrap(inputErr, "could not open targets file")
 		}
@ -236,7 +250,9 @@ func New(options *types.Options) (*Runner, error) {
 		}
 	}

-	if options.RateLimit > 0 {
+	if options.RateLimitMinute > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second))
+	} else if options.RateLimit > 0 {
 		runner.ratelimiter = ratelimit.New(options.RateLimit)
 	} else {
 		runner.ratelimiter = ratelimit.NewUnlimited()
--- a/v2/internal/testutils/integration.go
+++ b/v2/internal/testutils/integration.go
@ -1,9 +1,11 @@
 package testutils

 import (
+	"errors"
 	"net"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 )

@ -30,6 +32,23 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) (
 	return parts, nil
 }

+var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: (\d+)`)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) {
+	cmd := exec.Command(nucleiBinary, args...)
+
+	data, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	matches := templateLoaded.FindAllStringSubmatch(string(data), -1)
+	if len(matches) == 0 {
+		return "", errors.New("no matches found")
+	}
+	return matches[0][1], nil
+}
+
 // RunNucleiWorkflowAndGetResults returns a list of results for a workflow
 func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
 	cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent")
--- a/v2/internal/testutils/testutils.go
+++ b/v2/internal/testutils/testutils.go
@ -45,8 +45,8 @@ var DefaultOptions = &types.Options{
 	RateLimit:          150,
 	ProjectPath:        "",
 	Severity:           []string{},
-	Target:             "",
-	Targets:            "",
+	Targets:            []string{},
+	TargetsFilePath:    "",
 	Output:             "",
 	ProxyURL:           "",
 	ProxySocksURL:      "",
--- a/v2/pkg/catalog/config/config.go
+++ b/v2/pkg/catalog/config/config.go
@ -28,7 +28,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"

 // Version is the current version of nuclei
-const Version = `2.4.2`
+const Version = `2.4.3`

 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()
--- a/v2/pkg/operators/extractors/compile.go
+++ b/v2/pkg/operators/extractors/compile.go
@ -4,6 +4,8 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"github.com/itchyny/gojq"
 )

 // CompileExtractors performs the initial setup operation on a extractor
@ -28,6 +30,18 @@ func (e *Extractor) CompileExtractors() error {
 		e.KVal[i] = strings.ToLower(kval)
 	}

+	for _, query := range e.JSON {
+		query, err := gojq.Parse(query)
+		if err != nil {
+			return fmt.Errorf("could not parse json: %s", query)
+		}
+		compiled, err := gojq.Compile(query)
+		if err != nil {
+			return fmt.Errorf("could not compile json: %s", query)
+		}
+		e.jsonCompiled = append(e.jsonCompiled, compiled)
+	}
+
 	// Setup the part of the request to match, if any.
 	if e.Part == "" {
 		e.Part = "body"
--- a/v2/pkg/operators/extractors/extract.go
+++ b/v2/pkg/operators/extractors/extract.go
@ -1,6 +1,12 @@
 package extractors

 import (
+	"strings"
+
+	"encoding/json"
+
+	"github.com/antchfx/htmlquery"
+
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )

@ -42,3 +48,70 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
 	}
 	return results
 }
+
+// ExtractHTML extracts items from text using XPath selectors
+func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	doc, err := htmlquery.Parse(strings.NewReader(corpus))
+	if err != nil {
+		return results
+	}
+	for _, k := range e.XPath {
+		nodes, err := htmlquery.QueryAll(doc, k)
+		if err != nil {
+			continue
+		}
+		for _, node := range nodes {
+			var value string
+
+			if e.Attribute != "" {
+				value = htmlquery.SelectAttr(node, e.Attribute)
+			} else {
+				value = htmlquery.InnerText(node)
+			}
+			if _, ok := results[value]; !ok {
+				results[value] = struct{}{}
+			}
+		}
+	}
+	return results
+}
+
+// ExtractJSON extracts text from a corpus using JQ queries and returns it
+func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
+	results := make(map[string]struct{})
+
+	var jsonObj interface{}
+
+	err := json.Unmarshal([]byte(corpus), &jsonObj)
+
+	if err != nil {
+		return results
+	}
+
+	for _, k := range e.jsonCompiled {
+		iter := k.Run(jsonObj)
+		for {
+			v, ok := iter.Next()
+			if !ok {
+				break
+			}
+			if _, ok := v.(error); ok {
+				break
+			}
+			var result string
+			if res, err := types.JSONScalarToString(v); err == nil {
+				result = res
+			} else if res, err := json.Marshal(v); err == nil {
+				result = string(res)
+			} else {
+				result = types.ToString(v)
+			}
+			if _, ok := results[result]; !ok {
+				results[result] = struct{}{}
+			}
+		}
+	}
+	return results
+}
--- a/v2/pkg/operators/extractors/extractors.go
+++ b/v2/pkg/operators/extractors/extractors.go
@ -1,6 +1,10 @@
 package extractors

-import "regexp"
+import (
+	"regexp"
+
+	"github.com/itchyny/gojq"
+)

 // Extractor is used to extract part of response using a regex.
 type Extractor struct {
@ -21,6 +25,15 @@ type Extractor struct {
 	// KVal are the kval to be present in the response headers/cookies
 	KVal []string `yaml:"kval,omitempty"`

+	// XPath are the Xpath selectors for the extractor
+	XPath []string `yaml:"xpath"`
+	// Attribute is an optional attribute to extract from response XPath
+	Attribute string `yaml:"attribute"`
+	// JSON are the json pattern required to be present in the response
+	JSON []string `yaml:"json"`
+	// jsonCompiled is the compiled variant
+	jsonCompiled []*gojq.Code
+
 	// Part is the part of the request to match
 	//
 	// By default, matching is performed in request body.
@ -37,12 +50,18 @@ const (
 	RegexExtractor ExtractorType = iota + 1
 	// KValExtractor extracts responses with key:value
 	KValExtractor
+	// XPathExtractor extracts responses with Xpath selectors
+	XPathExtractor
+	// JSONExtractor extracts responses with json
+	JSONExtractor
 )

 // ExtractorTypes is an table for conversion of extractor type from string.
 var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
 	"kval":  KValExtractor,
+	"xpath": XPathExtractor,
+	"json":  JSONExtractor,
 }

 // GetType returns the type of the matcher
--- a/v2/pkg/protocols/common/generators/env.go
+++ b/v2/pkg/protocols/common/generators/env.go
@ -0,0 +1,28 @@
+package generators
+
+import (
+	"os"
+
+	"github.com/projectdiscovery/stringsutil"
+)
+
+var envVars map[string]interface{}
+
+func parseEnvVars() map[string]interface{} {
+	sliceEnvVars := os.Environ()
+	parsedEnvVars := make(map[string]interface{}, len(sliceEnvVars))
+	for _, envVar := range sliceEnvVars {
+		key, val := stringsutil.Before(envVar, "="), stringsutil.After(envVar, "=")
+		parsedEnvVars[key] = val
+	}
+	return parsedEnvVars
+}
+
+// EnvVars returns a map with all environment variables into a map
+func EnvVars() map[string]interface{} {
+	if envVars == nil {
+		envVars = parseEnvVars()
+	}
+
+	return envVars
+}
--- a/v2/pkg/protocols/common/generators/maps.go
+++ b/v2/pkg/protocols/common/generators/maps.go
@ -1,6 +1,8 @@
 package generators

-import "strings"
+import (
+	"strings"
+)

 // MergeMaps merges two maps into a new map
 func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
--- a/v2/pkg/protocols/common/generators/slice.go
+++ b/v2/pkg/protocols/common/generators/slice.go
@ -0,0 +1,16 @@
+package generators
+
+import "github.com/projectdiscovery/stringsutil"
+
+// SliceToMap converts a slice of strings to map of string splitting each item at sep as "key sep value"
+func SliceToMap(s []string, sep string) map[string]interface{} {
+	m := make(map[string]interface{})
+	for _, sliceItem := range s {
+		key := stringsutil.Before(sliceItem, sep)
+		value := stringsutil.After(sliceItem, sep)
+		if key != "" {
+			m[key] = value
+		}
+	}
+	return m
+}
--- a/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java
+++ b/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java
@ -1,6 +1,6 @@
 import java.io.*;

-class Deserialize{  
+class Deserialize {  
    public static void main(String args[]) {
        FileInputStream fileIn = null;
        ObjectInputStream in = null;
--- a/v2/pkg/protocols/http/build_request.go
+++ b/v2/pkg/protocols/http/build_request.go
@ -62,6 +62,16 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	parsedString := parsed.String()
 	values["BaseURL"] = parsedString

+	// merge with vars
+	if !r.options.Options.Vars.IsEmpty() {
+		values = generators.MergeMaps(values, r.options.Options.Vars.AsMap())
+	}
+
+	// merge with env vars
+	if r.options.Options.EnvironmentVariables {
+		values = generators.MergeMaps(values, generators.EnvVars())
+	}
+
 	// If data contains \n it's a raw request, process it like raw. Else
 	// continue with the template based request flow.
 	if isRawRequest {
--- a/v2/pkg/protocols/http/http.go
+++ b/v2/pkg/protocols/http/http.go
@ -82,12 +82,19 @@ func (r *Request) GetID() string {

 // Compile compiles the protocol request for further execution.
 func (r *Request) Compile(options *protocols.ExecuterOptions) error {
-	client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{
+	connectionConfiguration := &httpclientpool.Configuration{
 		Threads:         r.Threads,
 		MaxRedirects:    r.MaxRedirects,
 		FollowRedirects: r.Redirects,
 		CookieReuse:     r.CookieReuse,
-	})
+	}
+
+	// if the headers contain "Connection" we need to disable the automatic keep alive of the standard library
+	if _, hasConnectionHeader := r.Headers["Connection"]; hasConnectionHeader {
+		connectionConfiguration.Connection = &httpclientpool.ConnectionConfiguration{DisableKeepAlive: false}
+	}
+
+	client, err := httpclientpool.Get(options.Options, connectionConfiguration)
 	if err != nil {
 		return errors.Wrap(err, "could not get dns client")
 	}
--- a/v2/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go
@ -50,6 +50,12 @@ func Init(options *types.Options) error {
 	return nil
 }

+// // Configuration contains the custom configuration options for a connection
+type ConnectionConfiguration struct {
+	// DisableKeepAlive of the connection
+	DisableKeepAlive bool
+}
+
 // Configuration contains the custom configuration options for a client
 type Configuration struct {
 	// Threads contains the threads for the client
@ -60,6 +66,8 @@ type Configuration struct {
 	CookieReuse bool
 	// FollowRedirects specifies whether to follow redirects
 	FollowRedirects bool
+	// Connection defines custom connection configuration
+	Connection *ConnectionConfiguration
 }

 // Hash returns the hash of the configuration to allow client pooling
@ -74,10 +82,17 @@ func (c *Configuration) Hash() string {
 	builder.WriteString(strconv.FormatBool(c.FollowRedirects))
 	builder.WriteString("r")
 	builder.WriteString(strconv.FormatBool(c.CookieReuse))
+	builder.WriteString("c")
+	builder.WriteString(strconv.FormatBool(c.Connection != nil))
 	hash := builder.String()
 	return hash
 }

+// HasCustomOptions checks whether the configuration requires custom settings
+func (c *Configuration) HasStandardOptions() bool {
+	return c.Threads == 0 && c.MaxRedirects == 0 && !c.FollowRedirects && !c.CookieReuse && c.Connection == nil
+}
+
 // GetRawHTTP returns the rawhttp request client
 func GetRawHTTP(options *types.Options) *rawhttp.Client {
 	if rawhttpClient == nil {
@ -90,7 +105,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {

 // Get creates or gets a client for the protocol based on custom configuration
 func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) {
-	if configuration.Threads == 0 && configuration.MaxRedirects == 0 && !configuration.FollowRedirects && !configuration.CookieReuse {
+	if configuration.HasStandardOptions() {
 		return normalClient, nil
 	}
 	return wrappedGet(options, configuration)
@ -140,6 +155,11 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	followRedirects := configuration.FollowRedirects
 	maxRedirects := configuration.MaxRedirects

+	// override connection's settings if required
+	if configuration.Connection != nil {
+		disableKeepAlives = configuration.Connection.DisableKeepAlive
+	}
+
 	transport := &http.Transport{
 		DialContext:         Dialer.Dial,
 		MaxIdleConns:        maxIdleConns,
--- a/v2/pkg/protocols/http/operators.go
+++ b/v2/pkg/protocols/http/operators.go
@ -54,6 +54,10 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
 		return extractor.ExtractRegex(item)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
+	case extractors.XPathExtractor:
+		return extractor.ExtractHTML(item)
+	case extractors.JSONExtractor:
+		return extractor.ExtractJSON(item)
 	}
 	return nil
 }
--- a/v2/pkg/protocols/http/operators_test.go
+++ b/v2/pkg/protocols/http/operators_test.go
@ -168,6 +168,47 @@ func TestHTTPOperatorExtract(t *testing.T) {
 		require.Greater(t, len(data), 0, "could not extractor kval valid response")
 		require.Equal(t, map[string]struct{}{"Test-Response": {}}, data, "could not extract correct kval data")
 	})
+
+	t.Run("json", func(t *testing.T) {
+		event["body"] = exampleJSONResponseBody
+
+		t.Run("jq-simple", func(t *testing.T) {
+			extractor := &extractors.Extractor{
+				Type: "json",
+				JSON: []string{".batters | .batter | .[] | .id"},
+			}
+			err = extractor.CompileExtractors()
+			require.Nil(t, err, "could not compile json extractor")
+
+			data := request.Extract(event, extractor)
+			require.Greater(t, len(data), 0, "could not extractor json valid response")
+			require.Equal(t, map[string]struct{}{"1001": {}, "1002": {}, "1003": {}, "1004": {}}, data, "could not extract correct json data")
+		})
+		t.Run("jq-array", func(t *testing.T) {
+			extractor := &extractors.Extractor{
+				Type: "json",
+				JSON: []string{".array"},
+			}
+			err = extractor.CompileExtractors()
+			require.Nil(t, err, "could not compile json extractor")
+
+			data := request.Extract(event, extractor)
+			require.Greater(t, len(data), 0, "could not extractor json valid response")
+			require.Equal(t, map[string]struct{}{"[\"hello\",\"world\"]": {}}, data, "could not extract correct json data")
+		})
+		t.Run("jq-object", func(t *testing.T) {
+			extractor := &extractors.Extractor{
+				Type: "json",
+				JSON: []string{".batters"},
+			}
+			err = extractor.CompileExtractors()
+			require.Nil(t, err, "could not compile json extractor")
+
+			data := request.Extract(event, extractor)
+			require.Greater(t, len(data), 0, "could not extractor json valid response")
+			require.Equal(t, map[string]struct{}{"{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]}": {}}, data, "could not extract correct json data")
+		})
+	})
 }

 func TestHTTPMakeResult(t *testing.T) {
@ -305,3 +346,63 @@ const exampleResponseBody = `
 </body>
 </html>
 `
+
+const exampleJSONResponseBody = `
+{
+  "id": "0001",
+  "type": "donut",
+  "name": "Cake",
+  "ppu": 0.55,
+  "array": ["hello", "world"],
+  "batters": {
+    "batter": [
+      {
+        "id": "1001",
+        "type": "Regular"
+      },
+      {
+        "id": "1002",
+        "type": "Chocolate"
+      },
+      {
+        "id": "1003",
+        "type": "Blueberry"
+      },
+      {
+        "id": "1004",
+        "type": "Devil's Food"
+      }
+    ]
+  },
+  "topping": [
+    {
+      "id": "5001",
+      "type": "None"
+    },
+    {
+      "id": "5002",
+      "type": "Glazed"
+    },
+    {
+      "id": "5005",
+      "type": "Sugar"
+    },
+    {
+      "id": "5007",
+      "type": "Powdered Sugar"
+    },
+    {
+      "id": "5006",
+      "type": "Chocolate with Sprinkles"
+    },
+    {
+      "id": "5003",
+      "type": "Chocolate"
+    },
+    {
+      "id": "5004",
+      "type": "Maple"
+    }
+  ]
+}
+`
--- a/v2/pkg/types/interfaces.go
+++ b/v2/pkg/types/interfaces.go
@ -8,6 +8,23 @@ import (
 	"strings"
 )

+// JSONScalarToString converts an interface coming from json to string
+// Inspired from: https://github.com/cli/cli/blob/09b09810dd812e3ede54b59ad9d6912b946ac6c5/pkg/export/template.go#L72
+func JSONScalarToString(input interface{}) (string, error) {
+	switch tt := input.(type) {
+	case string:
+		return ToString(tt), nil
+	case float64:
+		return ToString(tt), nil
+	case nil:
+		return ToString(tt), nil
+	case bool:
+		return ToString(tt), nil
+	default:
+		return "", fmt.Errorf("cannot convert type to string: %v", tt)
+	}
+}
+
 // ToString converts an interface to string in a quick way
 func ToString(data interface{}) string {
 	switch s := data.(type) {
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@ -18,6 +18,8 @@ type Options struct {
 	ExcludedTemplates goflags.StringSlice
 	// CustomHeaders is the list of custom global headers to send with each request.
 	CustomHeaders goflags.StringSlice
+	// Vars is the list of custom global vars
+	Vars goflags.RuntimeMap
 	// Severity filters templates based on their severity and only run the matching ones.
 	Severity goflags.NormalizedStringSlice
 	// Author filters templates based on their author and only run the matching ones.
@ -32,10 +34,10 @@ type Options struct {
 	ProjectPath string
 	// InteractshURL is the URL for the interactsh server.
 	InteractshURL string
-	// Target is a single URL/Domain to scan using a template
-	Target string
-	// Targets specifies the targets to scan using templates.
-	Targets string
+	// Target URLs/Domains to scan using a template
+	Targets goflags.StringSlice
+	// TargetsFilePath specifies the targets from a file to scan using templates.
+	TargetsFilePath string
 	// Output is the file to write found results to.
 	Output string
 	// ProxyURL is the URL for the proxy server
@ -70,6 +72,8 @@ type Options struct {
 	Retries int
 	// Rate-Limit is the maximum number of requests per specified target
 	RateLimit int
+	// Rate-Limit is the maximum number of requests per minute for specified target
+	RateLimitMinute int
 	// PageTimeout is the maximum time to wait for a page in seconds
 	PageTimeout int
 	// InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time.
@ -141,4 +145,6 @@ type Options struct {
 	UpdateNuclei bool
 	// NoUpdateTemplates disables checking for nuclei templates updates
 	NoUpdateTemplates bool
+	// EnvironmentVariables enables support for environment variables
+	EnvironmentVariables bool
 }