Merge branch 'dev' into issue-2188-reporting-client

2023-02-20 15:26:16 +05:30 · 2023-02-20 15:26:16 +05:30 · ba7fcd08ff
parent 0618ccb923 8f151b58c2
commit ba7fcd08ff
32 changed files with 1332 additions and 265 deletions
--- a/.github/workflows/publish-docs.yaml
+++ b/.github/workflows/publish-docs.yaml
@ -14,6 +14,7 @@ jobs:
        uses: actions/checkout@v3
        with: 
          fetch-depth: 0
+          ref: ${{ github.head_ref }}

      - name: "Set up Go"
        uses: actions/setup-go@v3
--- a/4
+++ b/4
@ -1,8 +1,8 @@
-FROM golang:1.19.0-alpine as build-env
+FROM golang:1.20.1-alpine as build-env
 RUN apk add build-base
 RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

-FROM alpine:3.17.1
+FROM alpine:3.17.2
 RUN apk add --no-cache bind-tools ca-certificates chromium
 COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
 ENTRYPOINT ["nuclei"]
--- a/README.md
+++ b/README.md
@ -406,7 +406,18 @@ Examples of using Nuclei From Go Code to run templates on targets are provided i

 ### Credits

-Thanks to all the amazing community [contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors). Do also check out the below similar open-source projects that may fit in your workflow:
+Thanks to all the amazing [community contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors) and keeping this project updated. :heart:
+
+If you have an idea or some kind of improvement, you are welcome to contribute and participate in the Project, feel free to send your PR.
+
+<p align="center">
+<a href="https://github.com/projectdiscovery/nuclei/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=projectdiscovery/nuclei&max=500">
+</a>
+</p>
+
+
+Do also check out the below similar open-source projects that may fit in your workflow:

 [FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)

--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@ -132,7 +132,6 @@ dns:
          regex:
            - ec2-[-\d]+\.compute[-\d]*\.amazonaws\.com
            - ec2-[-\d]+\.[\w\d\-]+\.compute[-\d]*\.amazonaws\.com
-          dsl: []
    name: '{{FQDN}}'
    type: CNAME
    class: inet
@ -165,7 +164,6 @@ file:
        - type: regex
          regex:
            - amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
-          dsl: []
    extensions:
        - all
    archive: false
@ -2077,6 +2075,19 @@ attribute: href
 ```


+</div>
+
+<hr />
+
+<div class="dd">
+
+<code>dsl</code>  <i>[]string</i>
+
+</div>
+<div class="dt">
+
+Extracts using DSL expressions.
+
 </div>

 <hr />
@ -2515,7 +2526,6 @@ extractors:
      regex:
        - ec2-[-\d]+\.compute[-\d]*\.amazonaws\.com
        - ec2-[-\d]+\.[\w\d\-]+\.compute[-\d]*\.amazonaws\.com
-      dsl: []
 name: '{{FQDN}}'
 type: CNAME
 class: inet
@ -2837,7 +2847,6 @@ extractors:
    - type: regex
      regex:
        - amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
-      dsl: []
 extensions:
    - all
 archive: false
--- a/integration_tests/http/dsl-functions.yaml
+++ b/integration_tests/http/dsl-functions.yaml
@ -96,6 +96,8 @@ requests:
        83: {{join(" ", uniq("ab", "cd", "12", "34", "12", "cd"))}}
        84: {{split("ab,cd,efg", ",")}}
        85: {{split("ab,cd,efg", ",", 2)}}
+        86: {{ip_format('127.0.0.1', 3)}}
+        87: {{ip_format('127.0.1.0', 11)}}

    extractors:
      - type: regex
--- a/integration_tests/run.sh
+++ b/integration_tests/run.sh
@ -3,7 +3,7 @@
 echo "::group::Build nuclei"
 rm integration-test nuclei 2>/dev/null
 cd ../v2/cmd/nuclei
-go build
+go build -race .
 mv nuclei ../../../integration_tests/nuclei 
 echo "::endgroup::"

--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@ -132,8 +132,7 @@
    },
    "extractors.Extractor": {
      "required": [
-        "type",
-        "DSL"
+        "type"
      ],
      "properties": {
        "name": {
@ -187,11 +186,13 @@
          "title": "optional attribute to extract from xpath",
          "description": "Optional attribute to extract from response XPath"
        },
-        "DSL": {
+        "dsl": {
          "items": {
            "type": "string"
          },
-          "type": "array"
+          "type": "array",
+          "title": "dsl expressions to extract",
+          "description": "Optional attribute to extract from response dsl"
        },
        "part": {
          "type": "string",
--- a/v2/cmd/docgen/docgen.go
+++ b/v2/cmd/docgen/docgen.go
@ -21,6 +21,11 @@ func main() {
 	if err != nil {
 		log.Fatalf("Could not encode docs: %s\n", err)
 	}
+
+	if len(os.Args) < 3 {
+		log.Fatalf("syntax: %s md-docs-file jsonschema-file\n", os.Args[0])
+	}
+
 	err = os.WriteFile(os.Args[1], data, 0644)
 	if err != nil {
 		log.Fatalf("Could not write docs: %s\n", err)
--- a/v2/cmd/integration-test/http.go
+++ b/v2/cmd/integration-test/http.go
@ -19,7 +19,7 @@ import (
 )

 var httpTestcases = map[string]testutils.TestCase{
-	//"http/raw-unsafe-request.yaml":                  &httpRawUnsafeRequest{},
+	// "http/raw-unsafe-request.yaml":                  &httpRawUnsafeRequest{},
 	"http/get-headers.yaml":                         &httpGetHeaders{},
 	"http/get-query-string.yaml":                    &httpGetQueryString{},
 	"http/get-redirects.yaml":                       &httpGetRedirects{},
@ -297,7 +297,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error {
 	resultPart = stringsutil.TrimPrefixAny(resultPart, "/", " ", "[")

 	extracted := strings.Split(resultPart, ",")
-	numberOfDslFunctions := 85
+	numberOfDslFunctions := 87
 	if len(extracted) != numberOfDslFunctions {
 		return errors.New("incorrect number of results")
 	}
--- a/v2/go.mod
+++ b/v2/go.mod
@ -25,11 +25,11 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.12
 	github.com/projectdiscovery/fastdialer v0.0.22
-	github.com/projectdiscovery/hmap v0.0.6
+	github.com/projectdiscovery/hmap v0.0.7
 	github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
-	github.com/projectdiscovery/rawhttp v0.1.7
-	github.com/projectdiscovery/retryabledns v1.0.20
-	github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a
+	github.com/projectdiscovery/rawhttp v0.1.9
+	github.com/projectdiscovery/retryabledns v1.0.21
+	github.com/projectdiscovery/retryablehttp-go v1.0.11
 	github.com/projectdiscovery/stringsutil v0.0.2
 	github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
 	github.com/remeh/sizedwaitgroup v1.0.0
@ -41,12 +41,12 @@ require (
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
 	github.com/valyala/fasttemplate v1.2.2
-	github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37
-	github.com/xanzy/go-gitlab v0.79.0
+	github.com/weppos/publicsuffix-go v0.20.0
+	github.com/xanzy/go-gitlab v0.80.2
 	go.uber.org/multierr v1.9.0
-	golang.org/x/net v0.5.0
-	golang.org/x/oauth2 v0.4.0
-	golang.org/x/text v0.6.0
+	golang.org/x/net v0.6.0
+	golang.org/x/oauth2 v0.5.0
+	golang.org/x/text v0.7.0
 	gopkg.in/yaml.v2 v2.4.0
 	moul.io/http2curl v1.0.0
 )
@ -59,7 +59,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.18.12
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.12
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.3
 	github.com/docker/go-units v0.5.0
 	github.com/fatih/structs v1.1.0
 	github.com/go-git/go-git/v5 v5.5.2
@ -72,16 +72,16 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/goflags v0.1.6
-	github.com/projectdiscovery/gologger v1.1.7
-	github.com/projectdiscovery/httpx v1.2.5
+	github.com/projectdiscovery/gologger v1.1.8
+	github.com/projectdiscovery/httpx v1.2.7
 	github.com/projectdiscovery/nvd v1.0.9
 	github.com/projectdiscovery/ratelimit v0.0.6
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
-	github.com/projectdiscovery/tlsx v1.0.2
+	github.com/projectdiscovery/tlsx v1.0.5
 	github.com/projectdiscovery/uncover v1.0.2
-	github.com/projectdiscovery/utils v0.0.6
-	github.com/projectdiscovery/wappalyzergo v0.0.79
+	github.com/projectdiscovery/utils v0.0.9
+	github.com/projectdiscovery/wappalyzergo v0.0.81
 	github.com/stretchr/testify v1.8.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v3 v3.0.1
@ -89,45 +89,84 @@ require (

 require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/bits-and-blooms/bitset v1.3.1 // indirect
 	github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
+	github.com/cloudflare/cfssl v1.6.3 // indirect
 	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
+	github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/getsentry/sentry-go v0.16.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.6.1 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
+	github.com/fullstorydev/grpcurl v1.8.1 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
+	github.com/golang/mock v1.5.0 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/certificate-transparency-go v1.1.2-0.20210511102531-373a877eec92 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jhump/protoreflect v1.8.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/karlseguin/expect v1.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/projectdiscovery/asnmap v0.0.1 // indirect
 	github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
-	github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
-	github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a // indirect
-	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
-	github.com/tidwall/btree v1.4.3 // indirect
+	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/buntdb v1.2.10 // indirect
-	github.com/tidwall/gjson v1.14.3 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
-	github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
+	github.com/urfave/cli v1.22.5 // indirect
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.0-alpha.0 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/etcdctl/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/server/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 // indirect
+	go.etcd.io/etcd/v3 v3.5.0-alpha.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
+	google.golang.org/genproto v0.0.0-20220804142021-4e6b2dfa6612 // indirect
+	google.golang.org/grpc v1.51.0 // indirect
+	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )

 require (
@ -146,9 +185,9 @@ require (
 	github.com/caddyserver/certmagic v0.16.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
-	github.com/cockroachdb/errors v1.9.0 // indirect
-	github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
-	github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230207164304-7d1e4ba7ffd0 // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
@ -185,14 +224,14 @@ require (
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mholt/acmez v1.0.4 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.21 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.22 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect
 	github.com/projectdiscovery/mapcidr v1.0.3
-	github.com/projectdiscovery/networkpolicy v0.0.3
+	github.com/projectdiscovery/networkpolicy v0.0.4
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
@ -207,15 +246,15 @@ require (
 	github.com/ysmood/gson v0.7.3 // indirect
 	github.com/ysmood/leakless v0.8.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
-	github.com/zmap/zcrypto v0.0.0-20230113044912-682e75113af0 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
+	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
+	github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/zap v1.23.0 // indirect
 	goftp.io/server/v2 v2.0.0 // indirect
 	golang.org/x/crypto v0.5.0 // indirect
-	golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
-	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771
+	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.5.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
@ -252,7 +291,7 @@ require (
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/projectdiscovery/fileutil v0.0.3
 	github.com/projectdiscovery/iputil v0.0.2 // indirect
-	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/src-d/gcfg v1.4.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
--- a/v2/go.sum
+++ b/v2/go.sum
--- a/v2/pkg/catalog/config/config.go
+++ b/v2/pkg/catalog/config/config.go
@ -32,7 +32,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"

 // Version is the current version of nuclei
-const Version = `2.8.8`
+const Version = `2.8.9`

 var customConfigDirectory string

--- a/v2/pkg/operators/common/dsl/dsl.go
+++ b/v2/pkg/operators/common/dsl/dsl.go
@ -38,6 +38,7 @@ import (
 	"github.com/spaolacci/murmur3"

 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/mapcidr"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/randomip"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -773,22 +774,14 @@ func init() {
 				return argStr[start:end], nil
 			},
 		),
-		"aes_cbc": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
-			key := []byte(types.ToString(args[0]))
-			cleartext := []byte(types.ToString(args[1]))
-			block, _ := aes.NewCipher(key)
-			blockSize := block.BlockSize()
-			n := blockSize - len(cleartext)%blockSize
-			temp := bytes.Repeat([]byte{byte(n)}, n)
-			cleartext = append(cleartext, temp...)
-			iv := make([]byte, 16)
-			if _, err := crand.Read(iv); err != nil {
-				return nil, err
-			}
-			blockMode := cipher.NewCBCEncrypter(block, iv)
-			ciphertext := make([]byte, len(cleartext))
-			blockMode.CryptBlocks(ciphertext, cleartext)
-			ciphertext = append(iv, ciphertext...)
+		"aes_cbc": makeDslFunction(3, func(args ...interface{}) (interface{}, error) {
+			bKey := []byte(args[1].(string))
+			bIV := []byte(args[2].(string))
+			bPlaintext := pkcs5padding([]byte(args[0].(string)), aes.BlockSize, len(args[0].(string)))
+			block, _ := aes.NewCipher(bKey)
+			ciphertext := make([]byte, len(bPlaintext))
+			mode := cipher.NewCBCEncrypter(block, bIV)
+			mode.CryptBlocks(ciphertext, bPlaintext)
 			return ciphertext, nil
 		}),
 		"aes_gcm": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
@ -923,6 +916,20 @@ func init() {

 			return buf.String(), nil
 		}),
+		"ip_format": makeDslFunction(2, func(args ...interface{}) (interface{}, error) {
+			ipFormat, err := strconv.ParseInt(types.ToString(args[1]), 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			if ipFormat <= 0 || ipFormat > 11 {
+				return nil, fmt.Errorf("invalid format, format must be in range 1-11")
+			}
+			formattedIps := mapcidr.AlterIP(types.ToString(args[0]), []string{types.ToString(args[1])}, 3, false)
+			if len(formattedIps) == 0 {
+				return nil, fmt.Errorf("no formatted IP returned")
+			}
+			return formattedIps[0], nil
+		}),
 	}

 	dslFunctions = make(map[string]dslFunction, len(tempDslFunctions))
@ -1199,6 +1206,12 @@ func toChunks(input string, chunkSize int) []string {
 	return chunks
 }

+func pkcs5padding(ciphertext []byte, blockSize int, after int) []byte {
+	padding := (blockSize - len(ciphertext)%blockSize)
+	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
+	return append(ciphertext, padtext...)
+}
+
 type CompilationError struct {
 	DslSignature string
 	WrappedError error
--- a/v2/pkg/operators/common/dsl/dsl_test.go
+++ b/v2/pkg/operators/common/dsl/dsl_test.go
@ -93,7 +93,7 @@ func TestDslFunctionSignatures(t *testing.T) {
 }

 func TestGetPrintableDslFunctionSignatures(t *testing.T) {
-	expected := `	aes_cbc(arg1, arg2 interface{}) interface{}
+	expected := `	aes_cbc(arg1, arg2, arg3 interface{}) interface{}
 	aes_gcm(arg1, arg2 interface{}) interface{}
 	base64(arg1 interface{}) interface{}
 	base64_decode(arg1 interface{}) interface{}
@ -117,6 +117,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
 	hmac(arg1, arg2, arg3 interface{}) interface{}
 	html_escape(arg1 interface{}) interface{}
 	html_unescape(arg1 interface{}) interface{}
+	ip_format(arg1, arg2 interface{}) interface{}
 	join(separator string, elements ...interface{}) string
 	join(separator string, elements []interface{}) string
 	json_minify(arg1 interface{}) interface{}
@ -177,7 +178,7 @@ func TestGetPrintableDslFunctionSignatures(t *testing.T) {
 	assert.Equal(t, expected, signatures)

 	coloredSignatures := GetPrintableDslFunctionSignatures(false)
-	require.Contains(t, coloredSignatures, `[93maes_cbc[0m(arg1, arg2 [38;5;208minterface{}[0m)[38;5;208m interface{}[0m`, "could not get colored signatures")
+	require.Contains(t, coloredSignatures, `[93maes_cbc[0m(arg1, arg2, arg3 [38;5;208minterface{}[0m)[38;5;208m interface{}[0m`, "could not get colored signatures")
 }

 func TestDslExpressions(t *testing.T) {
@ -270,6 +271,10 @@ func TestDslExpressions(t *testing.T) {
 		`join(", ", split(hex_encode("abcdefg"), 2))`:             "61, 62, 63, 64, 65, 66, 67",
 		`json_minify("{  \"name\":  \"John Doe\",   \"foo\":  \"bar\"     }")`: "{\"foo\":\"bar\",\"name\":\"John Doe\"}",
 		`json_prettify("{\"foo\":\"bar\",\"name\":\"John Doe\"}")`:             "{\n    \"foo\": \"bar\",\n    \"name\": \"John Doe\"\n}",
+		`ip_format('127.0.0.1', '1')`:                                          "127.0.0.1",
+		`ip_format('127.0.0.1', '3')`:                                          "0177.0.0.01",
+		`ip_format('127.0.0.1', '5')`:                                          "281472812449793",
+		`ip_format('127.0.1.0', '11')`:                                         "127.0.256",
 	}

 	testDslExpressionScenarios(t, dslExpressions)
--- a/v2/pkg/operators/extractors/extractors.go
+++ b/v2/pkg/operators/extractors/extractors.go
@ -90,7 +90,7 @@ type Extractor struct {

 	// description: |
 	//   Extracts using DSL expressions.
-	DSL         []string
+	DSL         []string `yaml:"dsl,omitempty" jsonschema:"title=dsl expressions to extract,description=Optional attribute to extract from response dsl"`
 	dslCompiled []*govaluate.EvaluableExpression

 	// description: |
--- a/v2/pkg/protocols/common/helpers/deserialization/java.go
+++ b/v2/pkg/protocols/common/helpers/deserialization/java.go
@ -135,7 +135,7 @@ func generateDNSPayload(URL string) []byte {
 	buffer.WriteString(string(rune(len(hostname))))
 	buffer.WriteString(hostname)

-	middle, _ := hex.DecodeString("74000071007E0005740005")
+	middle, _ := hex.DecodeString("74000071007E0005740004")
 	buffer.Write(middle)
 	buffer.WriteString(parsed.Scheme)

--- a/v2/pkg/protocols/common/interactsh/interactsh.go
+++ b/v2/pkg/protocols/common/interactsh/interactsh.go
@ -24,6 +24,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils/atomcache"
 	"github.com/projectdiscovery/retryablehttp-go"
 )

@ -32,13 +33,13 @@ type Client struct {
 	// interactsh is a client for interactsh server.
 	interactsh *client.Client
 	// requests is a stored cache for interactsh-url->request-event data.
-	requests *ccache.Cache
+	requests *atomcache.Cache
 	// interactions is a stored cache for interactsh-interaction->interactsh-url data
-	interactions *ccache.Cache
+	interactions *atomcache.Cache
 	// matchedTemplates is a stored cache to track matched templates
-	matchedTemplates *ccache.Cache
+	matchedTemplates *atomcache.Cache
 	// interactshURLs is a stored cache to track track multiple interactsh markers
-	interactshURLs *ccache.Cache
+	interactshURLs *atomcache.Cache

 	options          *Options
 	eviction         time.Duration
@ -51,7 +52,7 @@ type Client struct {

 	firstTimeGroup sync.Once
 	generated      uint32 // decide to wait if we have a generated url
-	matched        bool
+	matched        atomic.Bool
 }

 var (
@ -108,14 +109,14 @@ const defaultMaxInteractionsCount = 5000
 func New(options *Options) (*Client, error) {
 	configure := ccache.Configure()
 	configure = configure.MaxSize(options.CacheSize)
-	cache := ccache.New(configure)
+	cache := atomcache.NewWithCache(ccache.New(configure))

 	interactionsCfg := ccache.Configure()
 	interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount)
-	interactionsCache := ccache.New(interactionsCfg)
+	interactionsCache := atomcache.NewWithCache(ccache.New(interactionsCfg))

-	matchedTemplateCache := ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))
-	interactshURLCache := ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))
+	matchedTemplateCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)))
+	interactshURLCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount)))

 	interactClient := &Client{
 		eviction:         options.Eviction,
@ -172,7 +173,6 @@ func (c *Client) firstTimeInitializeClient() error {

 	interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
 		item := c.requests.Get(interaction.UniqueID)
-
 		if item == nil {
 			// If we don't have any request for this ID, add it to temporary
 			// lru cache, so we can correlate when we get an add request.
@ -230,7 +230,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
 	}

 	if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) {
-		c.matched = true
+		c.matched.Store(true)
 		if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch {
 			c.matchedTemplates.Set(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration)
 		}
@ -239,17 +239,17 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
 }

 // URL returns a new URL that can be interacted with
-func (c *Client) URL() string {
+func (c *Client) URL() (string, error) {
 	c.firstTimeGroup.Do(func() {
 		if err := c.firstTimeInitializeClient(); err != nil {
 			gologger.Error().Msgf("Could not initialize interactsh client: %s", err)
 		}
 	})
 	if c.interactsh == nil {
-		return ""
+		return "", errors.New("interactsh client not initialized")
 	}
 	atomic.CompareAndSwapUint32(&c.generated, 0, 1)
-	return c.interactsh.URL()
+	return c.interactsh.URL(), nil
 }

 // Close closes the interactsh clients after waiting for cooldown period.
@ -262,9 +262,8 @@ func (c *Client) Close() bool {
 		c.interactsh.Close()
 	}

-	closeCache := func(cc *ccache.Cache) {
+	closeCache := func(cc *atomcache.Cache) {
 		if cc != nil {
-			cc.Clear()
 			cc.Stop()
 		}
 	}
@ -273,31 +272,41 @@ func (c *Client) Close() bool {
 	closeCache(c.matchedTemplates)
 	closeCache(c.interactshURLs)

-	return c.matched
+	return c.matched.Load()
 }

-// ReplaceMarkers replaces the {{interactsh-url}} placeholders to actual
-// URLs pointing to interactsh-server.
-//
-// It accepts data to replace as well as the URL to replace placeholders
-// with generated uniquely for each request.
-func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) {
-	for interactshURLMarkerRegex.Match([]byte(data)) {
-		url := c.URL()
-		interactshURLs = append(interactshURLs, url)
-		interactshURLMarker := interactshURLMarkerRegex.FindString(data)
-		if interactshURLMarker != "" {
+// ReplaceMarkers replaces the default {{interactsh-url}} placeholders with interactsh urls
+func (c *Client) Replace(data string, interactshURLs []string) (string, []string) {
+	return c.ReplaceWithMarker(data, interactshURLMarkerRegex, interactshURLs)
+}
+
+// ReplaceMarkers replaces the placeholders with interactsh urls and appends them to interactshURLs
+func (c *Client) ReplaceWithMarker(data string, regex *regexp.Regexp, interactshURLs []string) (string, []string) {
+	for _, interactshURLMarker := range regex.FindAllString(data, -1) {
+		if url, err := c.NewURLWithData(interactshURLMarker); err == nil {
+			interactshURLs = append(interactshURLs, url)
 			data = strings.Replace(data, interactshURLMarker, url, 1)
-			urlIndex := strings.Index(url, ".")
-			if urlIndex == -1 {
-				continue
-			}
-			c.interactshURLs.Set(url, interactshURLMarker, defaultInteractionDuration)
 		}
 	}
 	return data, interactshURLs
 }

+func (c *Client) NewURL() (string, error) {
+	return c.NewURLWithData("")
+}
+
+func (c *Client) NewURLWithData(data string) (string, error) {
+	url, err := c.URL()
+	if err != nil {
+		return "", err
+	}
+	if url == "" {
+		return "", errors.New("empty interactsh url")
+	}
+	c.interactshURLs.Set(url, data, defaultInteractionDuration)
+	return url, nil
+}
+
 // MakePlaceholders does placeholders for interact URLs and other data to a map
 func (c *Client) MakePlaceholders(urls []string, data map[string]interface{}) {
 	data["interactsh-server"] = c.getInteractServerHostname()
--- a/v2/pkg/protocols/common/variables/variables.go
+++ b/v2/pkg/protocols/common/variables/variables.go
@ -56,7 +56,7 @@ func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{},
 	variables.ForEach(func(key string, value interface{}) {
 		valueString := types.ToString(value)
 		if strings.Contains(valueString, "interactsh-url") {
-			valueString, interactURLs = interact.ReplaceMarkers(valueString, interactURLs)
+			valueString, interactURLs = interact.Replace(valueString, interactURLs)
 		}
 		result[key] = evaluateVariableValue(valueString, generators.MergeMaps(values, result), result)
 	})
--- a/v2/pkg/protocols/headless/engine/engine.go
+++ b/v2/pkg/protocols/headless/engine/engine.go
@ -14,6 +14,7 @@ import (

 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	fileutil "github.com/projectdiscovery/utils/file"
+	reflectutil "github.com/projectdiscovery/utils/reflect"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )

@ -53,6 +54,12 @@ func New(options *types.Options) (*Browser, error) {
 		chromeLauncher = chromeLauncher.NoSandbox(true)
 	}

+	// we need to set go rod internal value with reflection
+	launcherInternalBrowser := reflectutil.GetUnexportedField(chromeLauncher, "browser")
+	if internalInstance, ok := launcherInternalBrowser.(*launcher.Browser); ok {
+		internalInstance.IgnoreCerts = true
+	}
+
 	executablePath, err := os.Executable()
 	if err != nil {
 		return nil, err
--- a/v2/pkg/protocols/headless/engine/page_actions.go
+++ b/v2/pkg/protocols/headless/engine/page_actions.go
@ -613,7 +613,7 @@ func (p *Page) getActionArgWithValues(action *Action, arg string, values map[str
 	argValue = replaceWithValues(argValue, values)
 	if p.instance.interactsh != nil {
 		var interactshURLs []string
-		argValue, interactshURLs = p.instance.interactsh.ReplaceMarkers(argValue, p.InteractshURLs)
+		argValue, interactshURLs = p.instance.interactsh.Replace(argValue, p.InteractshURLs)
 		p.addInteractshURL(interactshURLs...)
 	}
 	return argValue
--- a/v2/pkg/protocols/http/build_request.go
+++ b/v2/pkg/protocols/http/build_request.go
@ -74,9 +74,9 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	isRawRequest := len(r.request.Raw) > 0
 	// replace interactsh variables with actual interactsh urls
 	if r.options.Interactsh != nil {
-		reqData, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(reqData, []string{})
+		reqData, r.interactshURLs = r.options.Interactsh.Replace(reqData, []string{})
 		for payloadName, payloadValue := range payloads {
-			payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs)
+			payloads[payloadName], r.interactshURLs = r.options.Interactsh.Replace(types.ToString(payloadValue), r.interactshURLs)
 		}
 	} else {
 		for payloadName, payloadValue := range payloads {
@ -133,8 +133,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	finalparams := parsed.Params
 	finalparams.Merge(reqURL.Params)
 	reqURL.Params = finalparams
-
-	return r.generateHttpRequest(ctx, reqURL.String(), finalVars, payloads)
+	return r.generateHttpRequest(ctx, reqURL, finalVars, payloads)
 }

 // selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
@ -205,19 +204,23 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 	if err != nil {
 		return nil, ErrEvalExpression.Wrap(err).WithTag("self-contained")
 	}
-	return r.generateHttpRequest(ctx, data, values, payloads)
+	urlx, err := urlutil.ParseURL(data, true)
+	if err != nil {
+		return nil, errorutil.NewWithErr(err).Msgf("failed to parse %v in self contained request", data).WithTag("self-contained")
+	}
+	return r.generateHttpRequest(ctx, urlx, values, payloads)
 }

 // generateHttpRequest generates http request from request data from template and variables
 // finalVars = contains all variables including generator and protocol specific variables
 // generatorValues = contains variables used in fuzzing or other generator specific values
-func (r *requestGenerator) generateHttpRequest(ctx context.Context, data string, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
+func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
 	method, err := expressions.Evaluate(r.request.Method.String(), finalVars)
 	if err != nil {
 		return nil, ErrEvalExpression.Wrap(err).Msgf("failed to evaluate while generating http request")
 	}
 	// Build a request on the specified URL
-	req, err := retryablehttp.NewRequestWithContext(ctx, method, data, nil)
+	req, err := retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
 	if err != nil {
 		return nil, err
 	}
@ -254,8 +257,11 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		// Todo: sync internally upon writing latest request byte
 		body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second)
 	}
-
-	req, err := retryablehttp.NewRequestWithContext(ctx, rawRequestData.Method, rawRequestData.FullURL, body)
+	urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
+	if err != nil {
+		return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
+	}
+	req, err := retryablehttp.NewRequestFromURLWithContext(ctx, rawRequestData.Method, urlx, body)
 	if err != nil {
 		return nil, err
 	}
@ -281,9 +287,10 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		interactshURLs: r.interactshURLs,
 	}

-	if reqWithAnnotations, cancelFunc, hasAnnotations := r.request.parseAnnotations(rawRequest, req); hasAnnotations {
-		generatedRequest.request = reqWithAnnotations
-		generatedRequest.customCancelFunction = cancelFunc
+	if reqWithOverrides, hasAnnotations := r.request.parseAnnotations(rawRequest, req); hasAnnotations {
+		generatedRequest.request = reqWithOverrides.request
+		generatedRequest.customCancelFunction = reqWithOverrides.cancelFunc
+		generatedRequest.interactshURLs = append(generatedRequest.interactshURLs, reqWithOverrides.interactshURLs...)
 	}

 	return generatedRequest, nil
@ -294,7 +301,7 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
 	// Set the header values requested
 	for header, value := range r.request.Headers {
 		if r.options.Interactsh != nil {
-			value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs)
+			value, r.interactshURLs = r.options.Interactsh.Replace(value, r.interactshURLs)
 		}
 		value, err := expressions.Evaluate(value, values)
 		if err != nil {
@ -315,7 +322,7 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
 	if r.request.Body != "" {
 		body := r.request.Body
 		if r.options.Interactsh != nil {
-			body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs)
+			body, r.interactshURLs = r.options.Interactsh.Replace(r.request.Body, r.interactshURLs)
 		}
 		body, err := expressions.Evaluate(body, values)
 		if err != nil {
--- a/v2/pkg/protocols/http/fuzz/parts.go
+++ b/v2/pkg/protocols/http/fuzz/parts.go
@ -66,7 +66,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
 	var req *retryablehttp.Request
 	var err error
 	if input.BaseRequest == nil {
-		req, err = retryablehttp.NewRequest(http.MethodGet, parsed.String(), nil)
+		req, err = retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil)
 		if err != nil {
 			return err
 		}
@ -98,7 +98,7 @@ func (rule *Rule) executeEvaluate(input *ExecuteRuleInput, key, value, payload s
 		"value": value,
 	})
 	firstpass, _ := expressions.Evaluate(payload, values)
-	interactData, interactshURLs := rule.options.Interactsh.ReplaceMarkers(firstpass, interactshURLs)
+	interactData, interactshURLs := rule.options.Interactsh.Replace(firstpass, interactshURLs)
 	evaluated, _ := expressions.Evaluate(interactData, values)
 	replaced := rule.executeReplaceRule(input, value, evaluated)
 	return replaced, interactshURLs
--- a/v2/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go
@ -59,7 +59,29 @@ func Init(options *types.Options) error {
 type ConnectionConfiguration struct {
 	// DisableKeepAlive of the connection
 	DisableKeepAlive bool
-	Cookiejar        *cookiejar.Jar
+	cookiejar        *cookiejar.Jar
+	mu               sync.RWMutex
+}
+
+func (cc *ConnectionConfiguration) SetCookieJar(cookiejar *cookiejar.Jar) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	cc.cookiejar = cookiejar
+}
+
+func (cc *ConnectionConfiguration) GetCookieJar() *cookiejar.Jar {
+	cc.mu.RLock()
+	defer cc.mu.RUnlock()
+
+	return cc.cookiejar
+}
+
+func (cc *ConnectionConfiguration) HasCookieJar() bool {
+	cc.mu.RLock()
+	defer cc.mu.RUnlock()
+
+	return cc.cookiejar != nil
 }

 // Configuration contains the custom configuration options for a client
@ -244,8 +266,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	}

 	var jar *cookiejar.Jar
-	if configuration.Connection != nil && configuration.Connection.Cookiejar != nil {
-		jar = configuration.Connection.Cookiejar
+	if configuration.Connection != nil && configuration.Connection.HasCookieJar() {
+		jar = configuration.Connection.GetCookieJar()
 	} else if configuration.CookieReuse {
 		if jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}); err != nil {
 			return nil, errors.Wrap(err, "could not create cookiejar")
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@ -512,6 +512,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			}
 			resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
 		} else if generatedRequest.request != nil {
+			// hot fix to avoid double url encoding (should only be called once)
+			generatedRequest.request.Prepare()
 			resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
 		}
 	} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
@ -555,13 +557,14 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			httpclient := request.httpClient
 			if input.CookieJar != nil {
 				connConfiguration := request.connConfiguration
-				connConfiguration.Connection.Cookiejar = input.CookieJar
+				connConfiguration.Connection.SetCookieJar(input.CookieJar)
 				client, err := httpclientpool.Get(request.options.Options, connConfiguration)
 				if err != nil {
 					return errors.Wrap(err, "could not get http client")
 				}
 				httpclient = client
 			}
+			generatedRequest.request.Prepare()
 			resp, err = httpclient.Do(generatedRequest.request)
 		}
 	}
@ -570,6 +573,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		formedURL = input.MetaInput.Input
 	}

+	// converts whitespace and other chars that cannot be printed to url encoded values
+	formedURL = urlutil.URLEncodeWithEscapes(formedURL)
+
 	// Dump the requests containing all headers
 	if !generatedRequest.original.Race {
 		var dumpError error
--- a/v2/pkg/protocols/http/request_annotations.go
+++ b/v2/pkg/protocols/http/request_annotations.go
@ -47,12 +47,14 @@ func parseFlowAnnotations(rawRequest string) (flowMark, bool) {
 	return fm, hasFlowOveride
 }

+type annotationOverrides struct {
+	request        *retryablehttp.Request
+	cancelFunc     context.CancelFunc
+	interactshURLs []string
+}
+
 // parseAnnotations and override requests settings
-func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (*retryablehttp.Request, context.CancelFunc, bool) {
-	var (
-		modified   bool
-		cancelFunc context.CancelFunc
-	)
+func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) {
 	// parse request for known ovverride annotations

 	// @Host:target
@ -89,8 +91,14 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 			value = value[:idxForwardSlash]
 		}

-		if stringsutil.EqualFoldAny(value, "request.host") {
+		switch value {
+		case "request.host":
 			value = request.Host
+		case "interactsh-url":
+			if interactshURL, err := r.options.Interactsh.NewURLWithData("interactsh-url"); err == nil {
+				value = interactshURL
+			}
+			overrides.interactshURLs = append(overrides.interactshURLs, value)
 		}
 		ctx := context.WithValue(request.Context(), fastdialer.SniName, value)
 		request = request.Clone(ctx)
@ -106,16 +114,19 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 			value := strings.TrimSpace(duration[1])
 			if parsed, err := time.ParseDuration(value); err == nil {
 				//nolint:govet // cancelled automatically by withTimeout
-				ctx, cancelFunc = context.WithTimeout(context.Background(), parsed)
+				ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed)
 				request = request.Clone(ctx)
 			}
 		} else {
 			//nolint:govet // cancelled automatically by withTimeout
-			ctx, cancelFunc = context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second)
+			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second)
 			request = request.Clone(ctx)
 		}
 	}
-	return request, cancelFunc, modified
+
+	overrides.request = request
+
+	return
 }

 func isHostPort(value string) bool {
--- a/v2/pkg/protocols/http/request_annotations_test.go
+++ b/v2/pkg/protocols/http/request_annotations_test.go
@ -22,10 +22,10 @@ func TestRequestParseAnnotationsTimeout(t *testing.T) {
 		httpReq, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
 		require.Nil(t, err, "could not create http request")

-		newRequest, cancelFunc, modified := request.parseAnnotations(rawRequest, httpReq)
-		require.NotNil(t, cancelFunc, "could not initialize valid cancel function")
+		overrides, modified := request.parseAnnotations(rawRequest, httpReq)
+		require.NotNil(t, overrides.cancelFunc, "could not initialize valid cancel function")
 		require.True(t, modified, "could not get correct modified value")
-		_, deadlined := newRequest.Context().Deadline()
+		_, deadlined := overrides.request.Context().Deadline()
 		require.True(t, deadlined, "could not get set request deadline")
 	})

@ -39,10 +39,10 @@ func TestRequestParseAnnotationsTimeout(t *testing.T) {
 		httpReq, err := retryablehttp.NewRequestWithContext(context.Background(), http.MethodGet, "https://example.com", nil)
 		require.Nil(t, err, "could not create http request")

-		newRequest, cancelFunc, modified := request.parseAnnotations(rawRequest, httpReq)
-		require.Nil(t, cancelFunc, "cancel function should be nil")
+		newRequestWithOverrides, modified := request.parseAnnotations(rawRequest, httpReq)
+		require.Nil(t, newRequestWithOverrides.cancelFunc, "cancel function should be nil")
 		require.False(t, modified, "could not get correct modified value")
-		_, deadlined := newRequest.Context().Deadline()
+		_, deadlined := newRequestWithOverrides.request.Context().Deadline()
 		require.False(t, deadlined, "could not get set request deadline")
 	})
 }
--- a/v2/pkg/protocols/network/request.go
+++ b/v2/pkg/protocols/network/request.go
@ -154,7 +154,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac

 		if request.options.Interactsh != nil {
 			var transformedData string
-			transformedData, interactshURLs = request.options.Interactsh.ReplaceMarkers(string(data), []string{})
+			transformedData, interactshURLs = request.options.Interactsh.Replace(string(data), []string{})
 			data = []byte(transformedData)
 		}

--- a/v2/pkg/protocols/protocols.go
+++ b/v2/pkg/protocols/protocols.go
@ -77,6 +77,9 @@ type ExecuterOptions struct {

 	Operators []*operators.Operators // only used by offlinehttp module

+	// DoNotCache bool disables optional caching of the templates structure
+	DoNotCache bool
+
 	Colorizer      aurora.Aurora
 	WorkflowLoader model.WorkflowLoader
 	ResumeCfg      *types.ResumeCfg
--- a/v2/pkg/templates/compile.go
+++ b/v2/pkg/templates/compile.go
@ -3,7 +3,6 @@ package templates
 import (
 	"fmt"
 	"io"
-	"net/http"
 	"reflect"

 	"github.com/pkg/errors"
@ -15,6 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/retryablehttp-go"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )

@ -34,13 +34,17 @@ func init() {
 //
 //nolint:gocritic // this cannot be passed by pointer
 func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
-	if value, err := parsedTemplatesCache.Has(filePath); value != nil {
-		return value.(*Template), err
+	if !options.DoNotCache {
+		if value, err := parsedTemplatesCache.Has(filePath); value != nil {
+			return value.(*Template), err
+		}
 	}

 	var reader io.ReadCloser
 	if utils.IsURL(filePath) {
-		resp, err := http.Get(filePath)
+		//todo:instead of creating a new client each time, a default one should be reused (same as the standard library)
+		// use retryablehttp (tls verification is enabled by default in the standard library)
+		resp, err := retryablehttp.DefaultClient().Get(filePath)
 		if err != nil {
 			return nil, err
 		}
@ -67,7 +71,9 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 		template.CompiledWorkflow.Options = &options
 	}
 	template.Path = filePath
-	parsedTemplatesCache.Store(filePath, template, err)
+	if !options.DoNotCache {
+		parsedTemplatesCache.Store(filePath, template, err)
+	}
 	return template, nil
 }

--- a/v2/pkg/templates/compile_test.go
+++ b/v2/pkg/templates/compile_test.go
@ -2,6 +2,7 @@ package templates_test

 import (
 	"context"
+	"fmt"
 	"log"
 	"testing"
 	"time"
@ -80,7 +81,7 @@ func Test_ParseFromURL(t *testing.T) {
 	}
 	setup()
 	got, err := templates.Parse(filePath, nil, executerOpts)
-	require.Nil(t, err, "could not parse template")
+	require.Nilf(t, err, "could not parse template (%s)", fmt.Sprint(err))
 	require.Equal(t, expectedTemplate.ID, got.ID)
 	require.Equal(t, expectedTemplate.Info, got.Info)
 	require.Equal(t, expectedTemplate.TotalRequests, got.TotalRequests)
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@ -811,7 +811,7 @@ func init() {
 			FieldName: "extractors",
 		},
 	}
-	EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11)
+	EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 12)
 	EXTRACTORSExtractorDoc.Fields[0].Name = "name"
 	EXTRACTORSExtractorDoc.Fields[0].Type = "string"
 	EXTRACTORSExtractorDoc.Fields[0].Note = ""
@ -868,26 +868,31 @@ func init() {
 	EXTRACTORSExtractorDoc.Fields[7].Comments[encoder.LineComment] = "Attribute is an optional attribute to extract from response XPath."

 	EXTRACTORSExtractorDoc.Fields[7].AddExample("", "href")
-	EXTRACTORSExtractorDoc.Fields[8].Name = "part"
-	EXTRACTORSExtractorDoc.Fields[8].Type = "string"
+	EXTRACTORSExtractorDoc.Fields[8].Name = "dsl"
+	EXTRACTORSExtractorDoc.Fields[8].Type = "[]string"
 	EXTRACTORSExtractorDoc.Fields[8].Note = ""
-	EXTRACTORSExtractorDoc.Fields[8].Description = "Part is the part of the request response to extract data from.\n\nEach protocol exposes a lot of different parts which are well\ndocumented in docs for each request type."
-	EXTRACTORSExtractorDoc.Fields[8].Comments[encoder.LineComment] = "Part is the part of the request response to extract data from."
-
-	EXTRACTORSExtractorDoc.Fields[8].AddExample("", "body")
-
-	EXTRACTORSExtractorDoc.Fields[8].AddExample("", "raw")
-	EXTRACTORSExtractorDoc.Fields[9].Name = "internal"
-	EXTRACTORSExtractorDoc.Fields[9].Type = "bool"
+	EXTRACTORSExtractorDoc.Fields[8].Description = "Extracts using DSL expressions."
+	EXTRACTORSExtractorDoc.Fields[8].Comments[encoder.LineComment] = "Extracts using DSL expressions."
+	EXTRACTORSExtractorDoc.Fields[9].Name = "part"
+	EXTRACTORSExtractorDoc.Fields[9].Type = "string"
 	EXTRACTORSExtractorDoc.Fields[9].Note = ""
-	EXTRACTORSExtractorDoc.Fields[9].Description = "Internal, when set to true will allow using the value extracted\nin the next request for some protocols (like HTTP)."
-	EXTRACTORSExtractorDoc.Fields[9].Comments[encoder.LineComment] = "Internal, when set to true will allow using the value extracted"
-	EXTRACTORSExtractorDoc.Fields[10].Name = "case-insensitive"
+	EXTRACTORSExtractorDoc.Fields[9].Description = "Part is the part of the request response to extract data from.\n\nEach protocol exposes a lot of different parts which are well\ndocumented in docs for each request type."
+	EXTRACTORSExtractorDoc.Fields[9].Comments[encoder.LineComment] = "Part is the part of the request response to extract data from."
+
+	EXTRACTORSExtractorDoc.Fields[9].AddExample("", "body")
+
+	EXTRACTORSExtractorDoc.Fields[9].AddExample("", "raw")
+	EXTRACTORSExtractorDoc.Fields[10].Name = "internal"
 	EXTRACTORSExtractorDoc.Fields[10].Type = "bool"
 	EXTRACTORSExtractorDoc.Fields[10].Note = ""
-	EXTRACTORSExtractorDoc.Fields[10].Description = "CaseInsensitive enables case-insensitive extractions. Default is false."
-	EXTRACTORSExtractorDoc.Fields[10].Comments[encoder.LineComment] = "CaseInsensitive enables case-insensitive extractions. Default is false."
-	EXTRACTORSExtractorDoc.Fields[10].Values = []string{
+	EXTRACTORSExtractorDoc.Fields[10].Description = "Internal, when set to true will allow using the value extracted\nin the next request for some protocols (like HTTP)."
+	EXTRACTORSExtractorDoc.Fields[10].Comments[encoder.LineComment] = "Internal, when set to true will allow using the value extracted"
+	EXTRACTORSExtractorDoc.Fields[11].Name = "case-insensitive"
+	EXTRACTORSExtractorDoc.Fields[11].Type = "bool"
+	EXTRACTORSExtractorDoc.Fields[11].Note = ""
+	EXTRACTORSExtractorDoc.Fields[11].Description = "CaseInsensitive enables case-insensitive extractions. Default is false."
+	EXTRACTORSExtractorDoc.Fields[11].Comments[encoder.LineComment] = "CaseInsensitive enables case-insensitive extractions. Default is false."
+	EXTRACTORSExtractorDoc.Fields[11].Values = []string{
 		"false",
 		"true",
 	}
--- a/v2/pkg/utils/atomcache/atomcache.go
+++ b/v2/pkg/utils/atomcache/atomcache.go
@ -0,0 +1,62 @@
+package atomcache
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/karlseguin/ccache"
+)
+
+type Cache struct {
+	*ccache.Cache
+	Closed atomic.Bool
+	mu     sync.RWMutex
+}
+
+func NewWithCache(c *ccache.Cache) *Cache {
+	return &Cache{Cache: c}
+}
+
+func (c *Cache) Get(key string) *ccache.Item {
+	if c.Closed.Load() {
+		return nil
+	}
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.Cache.Get(key)
+}
+
+func (c *Cache) Set(key string, value interface{}, duration time.Duration) {
+	if c.Closed.Load() {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.Cache.Set(key, value, duration)
+}
+
+func (c *Cache) Delete(key string) bool {
+	if c.Closed.Load() {
+		return false
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	return c.Cache.Delete(key)
+}
+
+func (c *Cache) Stop() {
+	if c.Closed.Load() {
+		return
+	}
+	c.Closed.Store(true)
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.Cache.Stop()
+}