Merge branch 'dev' into remote-template-workflow-lists

# Conflicts:
#	v2/cmd/nuclei/main.go

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,15 @@
+blank_issues_enabled: false
+
+contact_links:
+
+  - name: Ask an question / advise on using nuclei
+    url: https://github.com/projectdiscovery/nuclei/discussions/categories/q-a
+    about: Ask a question or request support for using nuclei
+
+  - name: Share idea / feature to discuss for nuclei
+    url: https://github.com/projectdiscovery/nuclei/discussions/categories/ideas
+    about: Share idea / feature to discuss for nuclei
+
+  - name: Connect with PD Team (Discord)
+    url: https://discord.gg/projectdiscovery
+    about: Connect with PD Team for direct communication

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,14 +1,21 @@
 ---
 name: Feature request
-about: Suggest an idea for this project
+about: Request feature to implement in this project
-title: "[feature]"
+title: ""
-labels: ''
+labels: 'Type: Enhancement'
 assignees: ''
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
+<!--
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+1. Please make sure to provide a detailed description with all the relevant information that might be required to start working on this feature.
+2. In case you are not sure about your request or whether the particular feature is already supported or not, please start a discussion instead.
+3. GitHub Discussion: https://github.com/projectdiscovery/nuclei/discussions/categories/ideas
+4. Join our discord server at https://discord.gg/projectdiscovery to discuss the idea on the #nuclei channel.
+-->
 
-**Describe the solution you'd like**
+### Please describe your feature request:
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of feature to implement -->
+
+### Describe the use case of this feature:
+<!-- A clear and concise description of the feature request's motivation and the use-cases in which it could be useful. -->

.github/ISSUE_TEMPLATE/issue-report.md

@@ -1,18 +1,36 @@
 ---
 name: Issue report
-about: Create a report to help us improve
+about: Create a report to help us to improve the project
-title: "[issue]"
+labels: 'Type: Bug'
-labels: ''
-assignees: ''
 
 ---
 
-**Describe the bug**
+<!-- 
-A clear and concise description of what the bug is.
+1. Please search to see if an issue already exists for the bug you encountered.
+2. For support requests, FAQs or "How to" questions, please use the GitHub Discussions section instead - https://github.com/projectdiscovery/nuclei/discussions or
+3. Join our discord server at https://discord.gg/projectdiscovery and post the question on the #nuclei channel.
+-->
 
-**Nuclei version**
+<!-- ISSUES MISSING IMPORTANT INFORMATION MAY BE CLOSED WITHOUT INVESTIGATION. -->
-Please share the version of the nuclei you are running with `nuclei -version` 
+
+### Nuclei version:
+<!-- You can find current version of nuclei with "nuclei -version" -->
+<!-- We only accept issues that are reproducible on the latest version of nuclei. -->
+<!-- You can find the latest version of project at https://github.com/projectdiscovery/nuclei/releases/ -->
+
+### Current Behavior:
+<!-- A concise description of what you're experiencing. -->
+
+### Expected Behavior:
+<!-- A concise description of what you expected to happen. -->
+
+### Steps To Reproduce:
+<!--
+Example: steps to reproduce the behavior:
+1. Run 'nuclei -t ... -u ..'
+2. See error...
+-->
 
 
-**Screenshot of the error or bug**
+### Anything else:
-please add the screenshot showing bug or issue you are facing.
+<!-- Links? References? Screnshots? Anything that will give us more context about the issue that you are encountering! -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,13 @@
+## Proposed changes
+
+<!-- Describe the overall picture of your modifications to help maintainers understand the pull request. PRs are required to be associated to their related issue tickets or feature request. -->
+
+
+## Checklist
+
+<!-- Put an "x" in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code. -->
+
+- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/nuclei/tree/dev) branch
+- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] I have added necessary documentation (if appropriate)

.github/workflows/release-binary.yml

@@ -1,30 +1,26 @@
 name: 🎉 Release Binary
 on:
   create:
-    tags:
-      - v*
   workflow_dispatch:
 
 jobs: 
   release: 
     runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+
     steps: 
-      - 
+      - uses: actions/checkout@v2
-        name: "Check out code"
-        uses: actions/checkout@v2
         with: 
           fetch-depth: 0
-      - 
+
-        name: "Set up Go"
+      - uses: actions/setup-go@v2
-        uses: actions/setup-go@v2
         with: 
           go-version: 1.17
-      - 
+
-        env: 
+      - uses: goreleaser/goreleaser-action@v2
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        name: "Create release on GitHub"
-        uses: goreleaser/goreleaser-action@v2
         with: 
           args: "release --rm-dist"
           version: latest
           workdir: v2/
+        env: 
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

README.md

@@ -44,6 +44,8 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 
 # Install Nuclei
 
+Nuclei requires **go1.17** to install successfully. Run the following command to install the latest version -
+
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
 ```
@@ -88,30 +90,29 @@ TARGET:
    -l, -list string      path to file containing a list of target URLs/hosts to scan (one per line)
 
 TEMPLATES:
-   -tl                      list all available templates
    -t, -templates string[]  template or template directory paths to include in the scan
-   -w, -workflows string[]  list of workflows to run
+   -nt, -new-templates      run only new templates added in latest nuclei-templates release
-   -nt, -new-templates      run newly added templates only
+   -w, -workflows string[]  workflow or workflow directory paths to include in the scan
    -validate                validate the passed templates to nuclei
+   -tl                      list all available templates
 
 FILTERING:
-   -tags string[]                         execute a subset of templates that contain the provided tags
+   -tags string[]                    execute a subset of templates that contain the provided tags
-   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
+   -etags, -exclude-tags string[]    exclude templates with the provided tags
-   -etags, -exclude-tags string[]         exclude templates with the provided tags
+   -itags, -include-tags string[]    tags from the default deny list that permit executing more intrusive templates
-   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
+   -et, -exclude-templates string[]  template or template directory paths to exclude
-   -exclude-templates, -exclude string[]  template or template directory paths to exclude
+   -it, -include-templates string[]  templates to be executed even if they are excluded either by default or configuration
-   -severity, -impact value[]             Templates to run based on severity. Possible values: info, low, medium, high, critical
+   -s, -severity value[]             Templates to run based on severity. Possible values - info,low,medium,high,critical
-   -author string[]                       execute templates that are (co-)created by the specified authors
+   -es, -exclude-severity value[]    Templates to exclude based on severity. Possible values - info,low,medium,high,critical
+   -a, -author string[]              execute templates that are (co-)created by the specified authors
 
 OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -silent                       display findings only
-   -v, -verbose                  show verbose output
-   -vv                           display extra verbose information
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
-   -nm, -no-meta                 don't display match metadata in CLI output
+   -nm, -no-meta                 don't display match metadata
    -nts, -no-timestamp           don't display timestamp metadata in CLI output
    -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
    -me, -markdown-export string  directory to export results in markdown format
@@ -123,37 +124,39 @@ CONFIGURATIONS:
    -H, -header string[]        custom headers in header:value format
    -V, -var value              custom vars in var=value format
    -r, -resolvers string       file containing resolver list for nuclei
-   -system-resolvers           use system DNS resolving as error fallback
+   -sr, -system-resolvers      use system DNS resolving as error fallback
    -passive                    enable passive HTTP response processing mode
-   -env-vars                   enable environment variables support
+   -ev, -env-vars              enable environment variables to be used in template
 
 INTERACTSH:
-   -no-interactsh                     disable interactsh server for OOB testing
+   -iserver, -interactsh-server string  interactsh server url for self-hosted instance (default "https://interactsh.com")
-   -interactsh-url string             interactsh server url for self-hosted instance (default "https://interactsh.com")
+   -itoken, -interactsh-token string    authentication token for self-hosted interactsh server
-   -interactsh-token string           authentication token for self-hosted interactsh server
+   -interactions-cache-size int         number of requests to keep in the interactions cache (default 5000)
-   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int           number of seconds to wait before evicting requests from cache (default 60)
-   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int      number of seconds to wait before each interaction poll request (default 5)
-   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int    extra time for interaction polling before exiting (default 5)
-   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+   -ni, -no-interactsh                  disable interactsh server for OAST testing, exclude OAST based templates
 
 RATE-LIMIT:
    -rl, -rate-limit int          maximum number of requests to send per second (default 150)
    -rlm, -rate-limit-minute int  maximum number of requests to send per minute
    -bs, -bulk-size int           maximum number of hosts to be analyzed in parallel per template (default 25)
-   -c, -concurrency int          maximum number of templates to be executed in parallel (default 10)
+   -c, -concurrency int          maximum number of templates to be executed in parallel (default 25)
 
 OPTIMIZATIONS:
    -timeout int               time to wait in seconds before timeout (default 5)
    -retries int               number of times to retry a failed request (default 1)
-   -max-host-error int        max errors for a host before skipping from scan (default 30)
+   -mhe, -max-host-error int  max errors for a host before skipping from scan (default 30)
    -project                   use a project folder to avoid sending same request multiple times
-   -project-path string       set a specific project path (default "$TMPDIR/")
+   -project-path string       set a specific project path
    -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+   -stream                    Stream mode - start elaborating without sorting the input
 
 HEADLESS:
-   -headless          enable templates that require headless browser support
+   -headless            enable templates that require headless browser support
-   -page-timeout int  seconds to wait for each page in headless mode (default 20)
+   -page-timeout int    seconds to wait for each page in headless mode (default 20)
-   -show-browser      show the browser on the screen when running templates with headless mode
+   -sb, -show-browser   show the browser on the screen when running templates with headless mode
+   -sc, -system-chrome  Use local installed chrome browser instead of nuclei installed
 
 DEBUG:
    -debug                     show all requests and responses
@@ -161,22 +164,24 @@ DEBUG:
    -debug-resp                show all received responses
    -proxy, -proxy-url string  URL of the HTTP proxy server
    -proxy-socks-url string    URL of the SOCKS proxy server
-   -trace-log string          file to write sent requests trace log
+   -tlog, -trace-log string   file to write sent requests trace log
    -version                   show nuclei version
+   -v, -verbose               show verbose output
+   -vv                        display extra verbose information
    -tv, -templates-version    shows the version of the installed nuclei-templates
 
 UPDATE:
-   -update                        update nuclei to the latest released version
+   -update                        update nuclei engine to the latest released version
-   -ut, -update-templates         update the community templates to latest released version
+   -ut, -update-templates         update nuclei-templates to latest released version
-   -nut, -no-update-templates     do not check for nuclei-templates updates
+   -ud, -update-directory string  overwrite the default directory to install nuclei-templates
-   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+   -duc, -disable-update-check    disable automatic nuclei/templates update check
 
 STATISTICS:
    -stats                    display statistics about the running scan
-   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -sj, -stats-json          write statistics data to an output file in JSONL(ines) format
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
-   -metrics                  expose nuclei metrics on a port
+   -m, -metrics              expose nuclei metrics on a port
-   -metrics-port int         port to expose nuclei metrics on (default 9092)
+   -mp, -metrics-port int    port to expose nuclei metrics on (default 9092)
 ```
 
 ### Running Nuclei

SYNTAX-REFERENCE.md

@@ -230,6 +230,19 @@ Workflows is a list of workflows to execute for a template.
 
 <hr />
 
+<div class="dd">
+
+<code>self-contained</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+Self Contained marks Requests for the template as self-contained
+
+</div>
+
+<hr />
+
 
 
 
@@ -823,14 +836,14 @@ in a combined manner allowing multirequest based matchers.
 
 Attack is the type of payload combinations to perform.
 
-Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates
+batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates
 permutations and combinations for all payloads.
 
 
 Valid values:
 
 
-  - <code>sniper</code>
+  - <code>batteringram</code>
 
   - <code>pitchfork</code>
 
@@ -2312,14 +2325,14 @@ host:
 
 Attack is the type of payload combinations to perform.
 
-Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates
+Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates
 permutations and combinations for all payloads.
 
 
 Valid values:
 
 
-  - <code>sniper</code>
+  - <code>batteringram</code>
 
   - <code>pitchfork</code>
 
@@ -2379,6 +2392,31 @@ read-size: 2048
 ```
 
 
+</div>
+
+<hr />
+
+<div class="dd">
+
+<code>read-all</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+ReadAll determines if the data stream should be read till the end regardless of the size
+
+Default value for read-all is false.
+
+
+
+Examples:
+
+
+```yaml
+read-all: false
+```
+
+
 </div>
 
 <hr />

integration_tests/http/interactsh.yaml

@@ -0,0 +1,19 @@
+id: interactsh-integration-test
+
+info:
+  name: Interactsh Integration Test
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      url: 'http://{{interactsh-url}}'
+
+    matchers:
+      - type: word
+        part: interactsh_protocol # Confirms the HTTP Interaction
+        words:
+          - "http"

integration_tests/http/self-contained.yaml

@@ -0,0 +1,18 @@
+id: example-self-contained-input
+
+info:
+  name: example-self-contained
+  author: pd-team
+  severity: info
+
+self-contained: true
+requests:
+  - raw:
+      - |
+        GET http://localhost:5431/ HTTP/1.1
+        Host: {{Hostname}}
+
+    matchers:
+      - type: word
+        words:
+          - This is self-contained response

integration_tests/network/self-contained.yaml

@@ -0,0 +1,16 @@
+id: example-self-contained-input
+
+info:
+  name: example-self-contained
+  author: pd-team
+  severity: info
+
+self-contained: true
+network:
+  - host:
+      - "localhost:5431"
+
+    matchers:
+      - type: word
+        words:
+          - "Authentication successful"

nuclei-jsonschema.json

@@ -606,7 +606,7 @@
         },
         "attack": {
           "enum": [
-            "sniper",
+            "batteringram",
             "pitchfork",
             "clusterbomb"
           ],
@@ -777,7 +777,7 @@
         },
         "attack": {
           "enum": [
-            "sniper",
+            "batteringram",
             "pitchfork",
             "clusterbomb"
           ],
@@ -809,6 +809,11 @@
           "title": "size of network response to read",
           "description": "Size of response to read at the end. Default is 1024 bytes"
         },
+        "read-all": {
+          "type": "boolean",
+          "title": "read all response stream",
+          "description": "Read all response stream till the server stops sending"
+        },
         "matchers": {
           "items": {
             "$ref": "#/definitions/matchers.Matcher"
@@ -845,6 +850,7 @@
       ],
       "properties": {
         "id": {
+          "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$",
           "type": "string",
           "title": "id of the template",
           "description": "The Unique ID for the template",
@@ -911,6 +917,11 @@
           "type": "array",
           "title": "list of workflows to execute",
           "description": "List of workflows to execute for template"
+        },
+        "self-contained": {
+          "type": "boolean",
+          "title": "mark requests as self-contained",
+          "description": "Mark Requests for the template as self-contained"
         }
       },
       "additionalProperties": false,

v2/cmd/functional-test/run.sh

@@ -7,7 +7,7 @@ echo 'Building Nuclei binary from current branch'
 go build -o nuclei_dev ../nuclei
 
 echo 'Installing latest release of nuclei'
-GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
 echo 'Starting Nuclei functional test'
 ./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt

v2/cmd/integration-test/http.go

@@ -31,6 +31,34 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
 	"http/request-condition.yaml":     &httpRequestCondition{},
 	"http/request-condition-new.yaml": &httpRequestCondition{},
+	"http/interactsh.yaml":            &httpInteractshRequest{},
+	"http/self-contained.yaml":        &httpRequestSelContained{},
+}
+
+type httpInteractshRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpInteractshRequest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		value := r.Header.Get("url")
+		if value != "" {
+			if resp, _ := http.DefaultClient.Get(value); resp != nil {
+				resp.Body.Close()
+			}
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
 }
 
 type httpGetHeaders struct{}
@@ -493,3 +521,35 @@ func (h *httpRequestCondition) Execute(filePath string) error {
 	}
 	return nil
 }
+
+type httpRequestSelContained struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpRequestSelContained) Execute(filePath string) error {
+	router := httprouter.New()
+	var routerErr error
+
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte("This is self-contained response"))
+	})
+	server := &http.Server{
+		Addr:    fmt.Sprintf("localhost:%d", defaultStaticPort),
+		Handler: router,
+	}
+	go func() {
+		_ = server.ListenAndServe()
+	}()
+	defer server.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/integration-test/network.go

@@ -7,11 +7,14 @@ import (
 )
 
 var networkTestcases = map[string]testutils.TestCase{
-	"network/basic.yaml":      &networkBasic{},
+	"network/basic.yaml":          &networkBasic{},
-	"network/hex.yaml":        &networkBasic{},
+	"network/hex.yaml":            &networkBasic{},
-	"network/multi-step.yaml": &networkMultiStep{},
+	"network/multi-step.yaml":     &networkMultiStep{},
+	"network/self-contained.yaml": &networkRequestSelContained{},
 }
 
+const defaultStaticPort = 5431
+
 type networkBasic struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -94,3 +97,28 @@ func (h *networkMultiStep) Execute(filePath string) error {
 	}
 	return nil
 }
+
+type networkRequestSelContained struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *networkRequestSelContained) Execute(filePath string) error {
+	var routerErr error
+
+	ts := testutils.NewTCPServer(func(conn net.Conn) {
+		defer conn.Close()
+
+		_, _ = conn.Write([]byte("Authentication successful"))
+	}, defaultStaticPort)
+	defer ts.Close()
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -7,7 +7,7 @@
 
 # github contains configuration options for github issue tracker
 #github: 
-#  # base-url is the optional self-hosted github application url
+#  # base-url (optional) is the self-hosted github application url
 #  base-url: ""
 #  # username is the username of the github user
 #  username: ""
@@ -17,12 +17,14 @@
 #  token: ""
 #  # project-name is the name of the repository.
 #  project-name: ""
-#  # issue-label is the label of the created issue type
+#  # issue-label (optional) is the label of the created issue type
 #  issue-label: ""
+#  # severity-as-label (optional) sets the sevetiry as the label of the created issue type
+#  severity-as-label: false
 
 # gitlab contains configuration options for gitlab issue tracker
 #gitlab: 
-#  # base-url is the optional self-hosted gitlab application url
+#  # base-url (optional) is the self-hosted gitlab application url
 #  base-url: ""
 #  # username is the username of the gitlab user
 #  username: ""
@@ -30,14 +32,16 @@
 #  token: ""
 #  # project-id is the ID of the repository.
 #  project-id: ""
-#  # issue-label is the label of the created issue type
+#  # issue-label (optional) is the label of the created issue type
 #  issue-label: ""
+#  # severity-as-label (optional) sets the sevetiry as the label of the created issue type
+#  severity-as-label: false
 
 # jira contains configuration options for jira issue tracker
 #jira:
-#  # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
+#  # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
 #  cloud: true
-#  # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
+#  # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created
 #  update-existing: false
 #  # URL is the jira application url
 #  url: ""
@@ -60,9 +64,9 @@
 #  port: 9200
 #  # IndexName is the name of the elasticsearch index
 #  index-name: nuclei
-#  # SSL enables ssl for elasticsearch connection
+#  # SSL (optional) enables ssl for elasticsearch connection
 #  ssl: false
-#  # SSLVerification disables SSL verification for elasticsearch
+#  # SSLVerification (optional) disables SSL verification for elasticsearch
 #  ssl-verification: false
 #  # Username for the elasticsearch instance
 #  username: test

v2/cmd/nuclei/main.go

@@ -53,45 +53,35 @@ on extensive configurability, massive extensibility and ease of use.`)
 	)
 
 	createGroup(flagSet, "templates", "Templates",
-		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
-
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"),
 		flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"),
-		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"),
+		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
+		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"),
 		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"),
-
-		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
+		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
 	)
 
 	createGroup(flagSet, "filters", "Filtering",
 		flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"),
-		flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
+		flagSet.NormalizedStringSliceVarP(&options.IncludeTags, "include-tags", "itags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list
 		flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"),
-
+		flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
-		flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
+		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"),
-		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
+		flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
-
+		flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
-		flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
+		flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"),
-		flagSet.VarP(&options.ExcludeSeverities, "exclude-impact", "exclude-severity", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
-		flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
 	)
 
 	createGroup(flagSet, "output", "Output",
 		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
-
 		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
-		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
-		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
-
 		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
-
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"),
 		flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"),
-
 		flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
 		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
 	)
@@ -99,50 +89,46 @@ on extensive configurability, massive extensibility and ease of use.`)
 	createGroup(flagSet, "configs", "Configurations",
 		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
 		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
-
 		flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
-
 		flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"),
-
 		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
-		flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
+		flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
 		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
-		flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"),
+		flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"),
 	)
 
 	createGroup(flagSet, "interactsh", "interactsh",
-		flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"),
+		flagSet.StringVarP(&options.InteractshURL, "interactsh-server", "iserver", "https://interactsh.com", "interactsh server url for self-hosted instance"),
-		flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interactsh.com", "interactsh server url for self-hosted instance"),
+		flagSet.StringVarP(&options.InteractshToken, "interactsh-token", "itoken", "", "authentication token for self-hosted interactsh server"),
-		flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"),
 		flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
 		flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
 		flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
 		flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
+		flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"),
 	)
 
 	createGroup(flagSet, "rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
 		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
-		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"),
+		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
 	)
 
 	createGroup(flagSet, "optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
 		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
-		flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"),
+		flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"),
-
 		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
 		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
-
 		flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"),
+		flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input"),
 	)
 
 	createGroup(flagSet, "headless", "Headless",
 		flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"),
 		flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
-		flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"),
+		flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"),
-		flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "Use local installed chrome browser instead of nuclei installed"),
+		flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "Use local installed chrome browser instead of nuclei installed"),
 	)
 
 	createGroup(flagSet, "debug", "Debug",
@@ -154,27 +140,26 @@ on extensive configurability, massive extensibility and ease of use.`)
 		   TODO should auto-set the HTTP_PROXY variable for the process? */
 		flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"),
 		flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"),
-
+		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
-		flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"),
-
 		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
 		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
 	)
 
 	createGroup(flagSet, "update", "Update",
-		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"),
+		flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei engine to the latest released version"),
-		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"),
+		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"),
-		flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"),
+		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default directory to install nuclei-templates"),
-		flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"),
+		flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"),
 	)
 
 	createGroup(flagSet, "stats", "Statistics",
 		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
-		flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"),
+		flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "write statistics data to an output file in JSONL(ines) format"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
-
+		flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"),
-		flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"),
+		flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"),
-		flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"),
 	)
 
 	_ = flagSet.Parse()

v2/go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
 	github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e
+	github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
 	github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5
 	github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240
 	github.com/projectdiscovery/gologger v1.1.4
@@ -40,7 +41,7 @@ require (
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a
 	github.com/projectdiscovery/retryablehttp-go v1.0.2
-	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
+	github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d
 	github.com/projectdiscovery/yamldoc-go v1.0.2
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
@@ -64,6 +65,7 @@ require (
 	golang.org/x/text v0.3.7
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
+	moul.io/http2curl v1.0.0
 )
 
 require (
@@ -73,6 +75,9 @@ require (
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
 	github.com/antchfx/xpath v1.1.6 // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/bits-and-blooms/bitset v1.2.0 // indirect
+	github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
@@ -86,6 +91,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/css v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
 	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
@@ -95,6 +101,7 @@ require (
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.15 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -104,6 +111,7 @@ require (
 	github.com/projectdiscovery/mapcidr v0.0.8 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/tklauser/go-sysconf v0.3.7 // indirect
 	github.com/tklauser/numcpus v0.2.3 // indirect
 	github.com/trivago/tgo v1.0.7 // indirect

v2/go.sum

@@ -96,16 +96,23 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA=
+github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
@@ -318,8 +325,11 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@@ -393,6 +403,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
@@ -468,6 +479,8 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY=
+github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -562,7 +575,10 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i
 github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ=
 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
+github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew=
+github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
 github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
+github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA=
 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
 github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
@@ -607,8 +623,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB
 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
 github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
-github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw=
+github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4=
 github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ=
 github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -647,6 +664,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -662,8 +681,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -1199,6 +1220,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
+moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
+moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
 mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

v2/internal/runner/banner.go

@@ -20,6 +20,6 @@ func showBanner() {
 	gologger.Print().Msgf("%s\n", banner)
 	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
+	gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
-	gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/options.go

@@ -22,11 +22,6 @@ func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
-	// if VerboseVerbose is set, it implicitly enables the Verbose option as well
-	if options.VerboseVerbose {
-		options.Verbose = true
-	}
-
 	// Read the inputs and configure the logging
 	configureOutput(options)
 
@@ -127,7 +122,7 @@ func isValidURL(urlString string) bool {
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose || options.VerboseVerbose {
+	if options.Verbose {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
 	if options.Debug {

v2/internal/runner/processor.go

@@ -7,11 +7,20 @@ import (
 	"go.uber.org/atomic"
 )
 
+// processSelfContainedTemplates execute a self-contained template.
+func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool {
+	match, err := template.Executer.Execute("")
+	if err != nil {
+		gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err)
+	}
+	return match
+}
+
 // processTemplateWithList execute a template against the list of user provided targets
 func (r *Runner) processTemplateWithList(template *templates.Template) bool {
 	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
-	r.hostMap.Scan(func(k, _ []byte) error {
+	processItem := func(k, _ []byte) error {
 		URL := string(k)
 
 		// Skip if the host has had errors
@@ -29,7 +38,13 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool {
 			results.CAS(false, match)
 		}(URL)
 		return nil
-	})
+	}
+	if r.options.Stream {
+		_ = r.hostMapStream.Scan(processItem)
+	} else {
+		r.hostMap.Scan(processItem)
+	}
+
 	wg.Wait()
 	return results.Load()
 }
@@ -39,7 +54,7 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
 	results := &atomic.Bool{}
 	wg := sizedwaitgroup.New(r.options.BulkSize)
 
-	r.hostMap.Scan(func(k, _ []byte) error {
+	processItem := func(k, _ []byte) error {
 		URL := string(k)
 
 		// Skip if the host has had errors
@@ -53,7 +68,14 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
 			results.CAS(false, match)
 		}(URL)
 		return nil
-	})
+	}
+
+	if r.options.Stream {
+		_ = r.hostMapStream.Scan(processItem)
+	} else {
+		r.hostMap.Scan(processItem)
+	}
+
 	wg.Wait()
 	return results.Load()
 }

v2/internal/runner/runner.go

@@ -16,6 +16,8 @@ import (
 	"go.uber.org/ratelimit"
 	"gopkg.in/yaml.v2"
 
+	"github.com/projectdiscovery/filekv"
+	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
@@ -45,6 +47,7 @@ import (
 // Runner is a client for running the enumeration process.
 type Runner struct {
 	hostMap         *hybrid.HybridMap
+	hostMapStream   *filekv.FileDB
 	output          output.Writer
 	interactsh      *interactsh.Client
 	inputCount      int64
@@ -119,6 +122,20 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	runner.hostMap = hm
 
+	if options.Stream {
+		fkvOptions := filekv.DefaultOptions
+		if tmpFileName, err := fileutil.GetTempFileName(); err != nil {
+			return nil, errors.Wrap(err, "could not create temporary input file")
+		} else {
+			fkvOptions.Path = tmpFileName
+		}
+		fkv, err := filekv.Open(fkvOptions)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create temporary unsorted input file")
+		}
+		runner.hostMapStream = fkv
+	}
+
 	runner.inputCount = 0
 	dupeCount := 0
 
@@ -138,6 +155,9 @@ func New(options *types.Options) (*Runner, error) {
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
 			runner.hostMap.Set(url, nil)
+			if options.Stream {
+				_ = runner.hostMapStream.Set([]byte(url), nil)
+			}
 		}
 	}
 
@@ -158,6 +178,9 @@ func New(options *types.Options) (*Runner, error) {
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
 			runner.hostMap.Set(url, nil)
+			if options.Stream {
+				_ = runner.hostMapStream.Set([]byte(url), nil)
+			}
 		}
 	}
 
@@ -180,6 +203,9 @@ func New(options *types.Options) (*Runner, error) {
 			runner.inputCount++
 			// nolint:errcheck // ignoring error
 			runner.hostMap.Set(url, nil)
+			if options.Stream {
+				_ = runner.hostMapStream.Set([]byte(url), nil)
+			}
 		}
 		input.Close()
 	}
@@ -290,6 +316,9 @@ func (r *Runner) Close() {
 	if r.projectFile != nil {
 		r.projectFile.Close()
 	}
+	if r.options.Stream {
+		r.hostMapStream.Close()
+	}
 	protocolinit.Close()
 }
 
@@ -511,7 +540,9 @@ func (r *Runner) RunEnumeration() error {
 		go func(template *templates.Template) {
 			defer wgtemplates.Done()
 
-			if len(template.Workflows) > 0 {
+			if template.SelfContained {
+				results.CAS(false, r.processSelfContainedTemplates(template))
+			} else if len(template.Workflows) > 0 {
 				results.CAS(false, r.processWorkflowWithList(template))
 			} else {
 				results.CAS(false, r.processTemplateWithList(template))

v2/internal/testutils/integration.go

@@ -92,10 +92,14 @@ type TCPServer struct {
 }
 
 // NewTCPServer creates a new TCP server from a handler
-func NewTCPServer(handler func(conn net.Conn)) *TCPServer {
+func NewTCPServer(handler func(conn net.Conn), port ...int) *TCPServer {
 	server := &TCPServer{}
 
-	l, err := net.Listen("tcp", "127.0.0.1:0")
+	var gotPort int
+	if len(port) > 0 {
+		gotPort = port[0]
+	}
+	l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", gotPort))
 	if err != nil {
 		panic(err)
 	}

v2/pkg/catalog/config/config.go

@@ -26,7 +26,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.5.3-dev`
+const Version = `2.5.4-dev`
 
 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()

v2/pkg/operators/common/dsl/dsl.go

@@ -224,6 +224,15 @@ var functions = map[string]govaluate.ExpressionFunction{
 		}
 		return rand.Intn(max-min) + min, nil
 	},
+	"unixtime": func(args ...interface{}) (interface{}, error) {
+		seconds := 0
+		if len(args) >= 1 {
+			seconds = int(args[0].(float64))
+		}
+		now := time.Now()
+		offset := now.Add(time.Duration(seconds) * time.Second)
+		return offset.Unix(), nil
+	},
 	// Time Functions
 	"waitfor": func(args ...interface{}) (interface{}, error) {
 		seconds := args[0].(float64)

v2/pkg/output/output.go

@@ -59,15 +59,15 @@ type InternalWrappedEvent struct {
 // ResultEvent is a wrapped result event for a single nuclei output.
 type ResultEvent struct {
 	// TemplateID is the ID of the template for the result.
-	TemplateID string `json:"templateID"`
+	TemplateID string `json:"template-id"`
 	// TemplatePath is the path of template
 	TemplatePath string `json:"-"`
 	// Info contains information block of the template for the result.
 	Info model.Info `json:"info,inline"`
 	// MatcherName is the name of the matcher matched if any.
-	MatcherName string `json:"matcher_name,omitempty"`
+	MatcherName string `json:"matcher-name,omitempty"`
 	// ExtractorName is the name of the extractor matched if any.
-	ExtractorName string `json:"extractor_name,omitempty"`
+	ExtractorName string `json:"extractor-name,omitempty"`
 	// Type is the type of the result event.
 	Type string `json:"type"`
 	// Host is the host input on which match was found.
@@ -75,9 +75,9 @@ type ResultEvent struct {
 	// Path is the path input on which match was found.
 	Path string `json:"path,omitempty"`
 	// Matched contains the matched input in its transformed form.
-	Matched string `json:"matched,omitempty"`
+	Matched string `json:"matched-at,omitempty"`
 	// ExtractedResults contains the extraction result from the inputs.
-	ExtractedResults []string `json:"extracted_results,omitempty"`
+	ExtractedResults []string `json:"extracted-results,omitempty"`
 	// Request is the optional, dumped request for the match.
 	Request string `json:"request,omitempty"`
 	// Response is the optional, dumped response for the match.
@@ -90,7 +90,9 @@ type ResultEvent struct {
 	Timestamp time.Time `json:"timestamp"`
 	// Interaction is the full details of interactsh interaction.
 	Interaction *server.Interaction `json:"interaction,omitempty"`
-
+	// CURLCommand is an optional curl command to reproduce the request
+	// Only applicable if the report is for HTTP.
+	CURLCommand         string         `json:"curl-command,omitempty"`
 	FileToIndexPosition map[string]int `json:"-"`
 }
 

v2/pkg/parsers/parser.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 
@@ -17,7 +18,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 )
 
-const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+const (
+	mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+	invalidFieldFormatTemplate    = "invalid field format for '%s' (allowed format is %s)"
+)
 
 // LoadTemplate returns true if the template is valid and matches the filtering criteria.
 func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
@@ -30,12 +34,12 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
 		return false, nil
 	}
 
-	templateInfo := template.Info
+	if validationError := validateTemplateFields(template); validationError != nil {
-	if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		stats.Increment(SyntaxErrorStats)
 		return false, validationError
 	}
 
-	return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
+	return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags)
 }
 
 // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
@@ -45,10 +49,8 @@ func LoadWorkflow(templatePath string) (bool, error) {
 		return false, templateParseError
 	}
 
-	templateInfo := template.Info
-
 	if len(template.Workflows) > 0 {
-		if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		if validationError := validateTemplateFields(template); validationError != nil {
 			return false, validationError
 		}
 		return true, nil
@@ -71,18 +73,29 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode
 	return match, err
 }
 
-func validateMandatoryInfoFields(info *model.Info) error {
+func validateTemplateFields(template *templates.Template) error {
-	if info == nil {
+	info := template.Info
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "info")
+
-	}
+	var errors []string
 
 	if utils.IsBlank(info.Name) {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name"))
 	}
 
 	if info.Authors.IsEmpty() {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author"))
 	}
+
+	if template.ID == "" {
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id"))
+	} else if !templateIDRegexp.MatchString(template.ID) {
+		errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()))
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf(strings.Join(errors, ", "))
+	}
+
 	return nil
 }
 
@@ -90,6 +103,7 @@ var (
 	parsedTemplatesCache *cache.Templates
 	ShouldValidate       bool
 	fieldErrorRegexp     = regexp.MustCompile(`not found in`)
+	templateIDRegexp     = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`)
 )
 
 const (

v2/pkg/parsers/parser_test.go

@@ -0,0 +1,110 @@
+package parsers
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplate(t *testing.T) {
+	origTemplatesCache := parsedTemplatesCache
+	defer func() { parsedTemplatesCache = origTemplatesCache }()
+
+	tt := []struct {
+		name        string
+		template    *templates.Template
+		templateErr error
+
+		expectedErr error
+	}{
+		{
+			name: "valid",
+			template: &templates.Template{
+				ID: "CVE-2021-27330",
+				Info: model.Info{
+					Name:    "Valid template",
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+		},
+		{
+			name:        "emptyTemplate",
+			template:    &templates.Template{},
+			expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"),
+		},
+		{
+			name: "emptyNameWithInvalidID",
+			template: &templates.Template{
+				ID: "invalid id",
+				Info: model.Info{
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+			expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"),
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr)
+
+			tagFilter := filter.New(&filter.Config{})
+			success, err := LoadTemplate(tc.name, tagFilter, nil)
+			if tc.expectedErr == nil {
+				require.NoError(t, err)
+				require.True(t, success)
+			} else {
+				require.Equal(t, tc.expectedErr, err)
+				require.False(t, success)
+			}
+		})
+	}
+
+	t.Run("invalidTemplateID", func(t *testing.T) {
+		tt := []struct {
+			id      string
+			success bool
+		}{
+			{id: "A-B-C", success: true},
+			{id: "A-B-C-1", success: true},
+			{id: "CVE_2021_27330", success: true},
+			{id: "ABC DEF", success: false},
+			{id: "_-__AAA_", success: false},
+			{id: " CVE-2021-27330", success: false},
+			{id: "CVE-2021-27330 ", success: false},
+			{id: "CVE-2021-27330-", success: false},
+			{id: "-CVE-2021-27330-", success: false},
+			{id: "CVE-2021--27330", success: false},
+			{id: "CVE-2021+27330", success: false},
+		}
+		for i, tc := range tt {
+			name := fmt.Sprintf("regexp%d", i)
+			t.Run(name, func(t *testing.T) {
+				template := &templates.Template{
+					ID: tc.id,
+					Info: model.Info{
+						Name:    "Valid template",
+						Authors: stringslice.StringSlice{Value: "Author"},
+					},
+				}
+				parsedTemplatesCache.Store(name, template, nil)
+
+				tagFilter := filter.New(&filter.Config{})
+				success, err := LoadTemplate(name, tagFilter, nil)
+				if tc.success {
+					require.NoError(t, err)
+					require.True(t, success)
+				} else {
+					require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err)
+					require.False(t, success)
+				}
+			})
+		}
+	})
+}

v2/pkg/protocols/common/generators/generators.go

@@ -14,7 +14,7 @@ type Generator struct {
 type Type int
 
 const (
-	// Sniper replaces one iteration of the payload with a value.
+	// Batteringram replaces same payload into all of the defined payload positions at once.
 	BatteringRam Type = iota + 1
 	// PitchFork replaces variables with positional value from multiple wordlists
 	PitchFork
@@ -43,10 +43,10 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
 	generator.Type = payloadType
 	generator.payloads = compiled
 
-	// Validate the sniper/batteringram payload set
+	// Validate the batteringram payload set
 	if payloadType == BatteringRam {
 		if len(payloads) != 1 {
-			return nil, errors.New("sniper/batteringram must have single payload set")
+			return nil, errors.New("batteringram must have single payload set")
 		}
 	}
 	return generator, nil

v2/pkg/protocols/common/generators/options.go

@@ -0,0 +1,20 @@
+package generators
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// BuildPayloadFromOptions returns a map with the payloads provided via CLI
+func BuildPayloadFromOptions(options *types.Options) map[string]interface{} {
+	m := make(map[string]interface{})
+	// merge with vars
+	if !options.Vars.IsEmpty() {
+		m = MergeMaps(m, options.Vars.AsMap())
+	}
+
+	// merge with env vars
+	if options.EnvironmentVariables {
+		m = MergeMaps(EnvVars(), m)
+	}
+	return m
+}

v2/pkg/protocols/headless/engine/http_client.go

@@ -1,12 +1,18 @@
 package engine
 
 import (
+	"context"
 	"crypto/tls"
+	"fmt"
+	"net"
 	"net/http"
+	"net/http/cookiejar"
+	"net/url"
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"golang.org/x/net/proxy"
 )
 
 // newhttpClient creates a new http client for headless communication with a timeout
@@ -22,5 +28,40 @@ func newhttpClient(options *types.Options) *http.Client {
 			InsecureSkipVerify: true,
 		},
 	}
-	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}
+
+	if options.ProxyURL != "" {
+		if proxyURL, err := url.Parse(options.ProxyURL); err == nil {
+			transport.Proxy = http.ProxyURL(proxyURL)
+		}
+	} else if options.ProxySocksURL != "" {
+		var proxyAuth *proxy.Auth
+
+		socksURL, proxyErr := url.Parse(options.ProxySocksURL)
+		if proxyErr == nil {
+			proxyAuth = &proxy.Auth{}
+			proxyAuth.User = socksURL.User.Username()
+			proxyAuth.Password, _ = socksURL.User.Password()
+		}
+		dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
+		dc := dialer.(interface {
+			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+		})
+		if proxyErr == nil {
+			transport.DialContext = dc.DialContext
+		}
+	}
+
+	jar, _ := cookiejar.New(nil)
+
+	httpclient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Duration(options.Timeout*3) * time.Second,
+		Jar:       jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// the browser should follow redirects not us
+			return http.ErrUseLastResponse
+		},
+	}
+
+	return httpclient
 }

v2/pkg/protocols/headless/engine/page.go

@@ -8,7 +8,7 @@ import (
 	"github.com/go-rod/rod/lib/proto"
 )
 
-// Page is a single page in an isolated browser instanace
+// Page is a single page in an isolated browser instance
 type Page struct {
 	page     *rod.Page
 	rules    []requestRule

v2/pkg/protocols/headless/engine/page_actions_test.go

@@ -2,9 +2,11 @@ package engine
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -16,18 +18,7 @@ import (
 )
 
 func TestActionNavigate(t *testing.T) {
-	_ = protocolstate.Init(&types.Options{})
+	response := `
-
-	browser, err := New(&types.Options{ShowBrowser: false})
-	require.Nil(t, err, "could not create browser")
-	defer browser.Close()
-
-	instance, err := browser.NewInstance()
-	require.Nil(t, err, "could not create browser instance")
-	defer instance.Close()
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintln(w, `
 		<html>
 		<head>
 			<title>Nuclei Test Page</title>
@@ -35,270 +26,429 @@ func TestActionNavigate(t *testing.T) {
 		<body>
 			<h1>Nuclei Test</h1>
 		</body>
-	</html>`)
+	</html>`
-	}))
-	defer ts.Close()
-
-	parsed, err := url.Parse(ts.URL)
-	require.Nil(t, err, "could not parse URL")
 
 	actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}}
-	_, page, err := instance.Run(parsed, actions, 20*time.Second)
-	require.Nil(t, err, "could not run page actions")
-	defer page.Close()
 
-	require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+	})
 }
 
 func TestActionScript(t *testing.T) {
-	_ = protocolstate.Init(&types.Options{})
+	response := `
+		<html>
+		<head>
+			<title>Nuclei Test Page</title>
+		</head>
+		<body>Nuclei Test Page</body>
+		<script>window.test = 'some-data';</script>
+	</html>`
 
-	browser, err := New(&types.Options{ShowBrowser: false})
+	timeout := 2 * time.Second
-	require.Nil(t, err, "could not create browser")
-	defer browser.Close()
-
-	instance, err := browser.NewInstance()
-	require.Nil(t, err, "could not create browser instance")
 
 	t.Run("run-and-results", func(t *testing.T) {
-		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			fmt.Fprintln(w, `
-			<html>
-			<head>
-				<title>Nuclei Test Page</title>
-			</head>
-			<body>Nuclei Test Page</body>
-			<script>window.test = 'some-data';</script>
-		</html>`)
-		}))
-		defer ts.Close()
-
-		parsed, err := url.Parse(ts.URL)
-		require.Nil(t, err, "could not parse URL")
-
 		actions := []*Action{
 			{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
 			{ActionType: "waitload"},
 			{ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}},
 		}
-		out, page, err := instance.Run(parsed, actions, 20*time.Second)
+		testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
-		require.Nil(t, err, "could not run page actions")
+			require.Nil(t, err, "could not run page actions")
-		defer page.Close()
+			require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
-
+			require.Equal(t, "some-data", out["test"], "could not run js and get results correctly")
-		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+		})
-		require.Equal(t, "some-data", out["test"], "could not run js and get results correctly")
 	})
 
 	t.Run("hook", func(t *testing.T) {
-		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			fmt.Fprintln(w, `
-			<html>
-			<head>
-				<title>Nuclei Test Page</title>
-			</head>
-			<body>Nuclei Test Page</body>
-		</html>`)
-		}))
-		defer ts.Close()
-
-		parsed, err := url.Parse(ts.URL)
-		require.Nil(t, err, "could not parse URL")
-
 		actions := []*Action{
 			{ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}},
 			{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
 			{ActionType: "waitload"},
 			{ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}},
 		}
-		out, page, err := instance.Run(parsed, actions, 20*time.Second)
+		testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
-		require.Nil(t, err, "could not run page actions")
+			require.Nil(t, err, "could not run page actions")
-		defer page.Close()
+			require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
-
+			require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook")
-		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+		})
-		require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook")
 	})
 }
 
 func TestActionClick(t *testing.T) {
-	_ = protocolstate.Init(&types.Options{})
+	response := `
-
-	browser, err := New(&types.Options{ShowBrowser: false})
-	require.Nil(t, err, "could not create browser")
-	defer browser.Close()
-
-	instance, err := browser.NewInstance()
-	require.Nil(t, err, "could not create browser instance")
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintln(w, `
 		<html>
-		<head>
+			<head>
-			<title>Nuclei Test Page</title>
+				<title>Nuclei Test Page</title>
-		</head>
+			</head>
-		<body>Nuclei Test Page</body>
+			<body>Nuclei Test Page</body>
-		<button onclick='this.setAttribute("a", "ok")'>click me</button>
+			<button onclick='this.setAttribute("a", "ok")'>click me</button>
-	</html>`)
+		</html>`
-	}))
-	defer ts.Close()
-
-	parsed, err := url.Parse(ts.URL)
-	require.Nil(t, err, "could not parse URL")
 
 	actions := []*Action{
 		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
 		{ActionType: "waitload"},
 		{ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
 	}
-	_, page, err := instance.Run(parsed, actions, 20*time.Second)
-	require.Nil(t, err, "could not run page actions")
-	defer page.Close()
 
-	require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
-	el := page.Page().MustElement("button")
+		require.Nil(t, err, "could not run page actions")
-	val := el.MustAttribute("a")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
-	require.Equal(t, "ok", *val, "could not click button")
+		el := page.Page().MustElement("button")
+		val := el.MustAttribute("a")
+		require.Equal(t, "ok", *val, "could not click button")
+	})
 }
 
 func TestActionRightClick(t *testing.T) {
-	_ = protocolstate.Init(&types.Options{})
+	response := `
-
-	browser, err := New(&types.Options{ShowBrowser: false})
-	require.Nil(t, err, "could not create browser")
-	defer browser.Close()
-
-	instance, err := browser.NewInstance()
-	require.Nil(t, err, "could not create browser instance")
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintln(w, `
 		<html>
-		<head>
+			<head>
-			<title>Nuclei Test Page</title>
+				<title>Nuclei Test Page</title>
-		</head>
+			</head>
-		<body>Nuclei Test Page</body>
+			<body>Nuclei Test Page</body>
-		<button id="test" onrightclick=''>click me</button>
+			<button id="test" onrightclick=''>click me</button>
-		<script>
+			<script>
-		elm = document.getElementById("test");
+				elm = document.getElementById("test");
-		elm.onmousedown = function(event) {
+				elm.onmousedown = function(event) {
-			if (event.which == 3) {
+					if (event.which == 3) {
-				elm.setAttribute("a", "ok")
+						elm.setAttribute("a", "ok")
-			}
+					}
-		}
+				}
-		</script>
+			</script>
-	</html>`)
+		</html>`
-	}))
-	defer ts.Close()
-
-	parsed, err := url.Parse(ts.URL)
-	require.Nil(t, err, "could not parse URL")
 
 	actions := []*Action{
 		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
 		{ActionType: "waitload"},
 		{ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
 	}
-	_, page, err := instance.Run(parsed, actions, 20*time.Second)
-	require.Nil(t, err, "could not run page actions")
-	defer page.Close()
 
-	require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
-	el := page.Page().MustElement("button")
+		require.Nil(t, err, "could not run page actions")
-	val := el.MustAttribute("a")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
-	require.Equal(t, "ok", *val, "could not click button")
+		el := page.Page().MustElement("button")
+		val := el.MustAttribute("a")
+		require.Equal(t, "ok", *val, "could not click button")
+	})
 }
 
 func TestActionTextInput(t *testing.T) {
-	_ = protocolstate.Init(&types.Options{})
+	response := `
-
-	browser, err := New(&types.Options{ShowBrowser: false})
-	require.Nil(t, err, "could not create browser")
-	defer browser.Close()
-
-	instance, err := browser.NewInstance()
-	require.Nil(t, err, "could not create browser instance")
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintln(w, `
 		<html>
-		<head>
+			<head>
-			<title>Nuclei Test Page</title>
+				<title>Nuclei Test Page</title>
-		</head>
+			</head>
-		<body>Nuclei Test Page</body>
+			<body>Nuclei Test Page</body>
-		<input type="text" onchange="this.setAttribute('event', 'input-change')">
+			<input type="text" onchange="this.setAttribute('event', 'input-change')">
-	</html>`)
+		</html>`
-	}))
-	defer ts.Close()
-
-	parsed, err := url.Parse(ts.URL)
-	require.Nil(t, err, "could not parse URL")
 
 	actions := []*Action{
 		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
 		{ActionType: "waitload"},
 		{ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}},
 	}
-	_, page, err := instance.Run(parsed, actions, 20*time.Second)
-	require.Nil(t, err, "could not run page actions")
-	defer page.Close()
 
-	require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
-	el := page.Page().MustElement("input")
+		require.Nil(t, err, "could not run page actions")
-	val := el.MustAttribute("event")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
-	require.Equal(t, "input-change", *val, "could not get input change")
+		el := page.Page().MustElement("input")
-	require.Equal(t, "test", el.MustText(), "could not get input change value")
+		val := el.MustAttribute("event")
+		require.Equal(t, "input-change", *val, "could not get input change")
+		require.Equal(t, "test", el.MustText(), "could not get input change value")
+	})
 }
 
 func TestActionHeadersChange(t *testing.T) {
-	_ = protocolstate.Init(&types.Options{})
-
-	browser, err := New(&types.Options{ShowBrowser: false})
-	require.Nil(t, err, "could not create browser")
-	defer browser.Close()
-
-	instance, err := browser.NewInstance()
-	require.Nil(t, err, "could not create browser instance")
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Header.Get("Test") == "Hello" {
-			fmt.Fprintln(w, `found`)
-		}
-	}))
-	defer ts.Close()
-
-	parsed, err := url.Parse(ts.URL)
-	require.Nil(t, err, "could not parse URL")
-
 	actions := []*Action{
 		{ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
 		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
 		{ActionType: "waitload"},
 	}
-	_, page, err := instance.Run(parsed, actions, 20*time.Second)
-	require.Nil(t, err, "could not run page actions")
-	defer page.Close()
 
-	require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
+	handler := func(w http.ResponseWriter, r *http.Request) {
-}
+		if r.Header.Get("Test") == "Hello" {
+			_, _ = fmt.Fprintln(w, `found`)
+		}
+	}
 
-func TestActionWaitVisible(t *testing.T) {
+	testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
-	t.Run("wait for an element being visible", func(t *testing.T) {
+		require.Nil(t, err, "could not run page actions")
-		testWaitVisible(t, 2*time.Second, func(page *Page, err error) {
+		require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
-			require.Nil(t, err, "could not run page actions")
-
-			page.Page().MustElement("button").MustVisible()
-			page.Close()
-		})
-	})
-
-	t.Run("timeout because of element not visible", func(t *testing.T) {
-		testWaitVisible(t, time.Second/2, func(page *Page, err error) {
-			require.Error(t, err)
-			require.Contains(t, err.Error(), "Element did not appear in the given amount of time")
-		})
 	})
 }
 
-func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page, err error)) {
+func TestActionScreenshot(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>Nuclei Test Page</body>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+		{ActionType: "screenshot", Data: map[string]string{"to": "test"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+		el := page.Page()
+		require.FileExists(t, "test.png", el, "could not get screenshot file")
+		_ = os.Remove("test.png")
+	})
+}
+
+func TestActionTimeInput(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>Nuclei Test Page</body>
+			<input type="date">
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+		{ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+		el := page.Page().MustElement("input")
+		require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value")
+	})
+}
+
+func TestActionSelectInput(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>
+				<select name="test" id="test">
+				  <option value="test1">Test1</option>
+				  <option value="test2">Test2</option>
+				</select>
+			</body>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+		{ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		el := page.Page().MustElement("select")
+		require.Equal(t, "Test2", el.MustText(), "could not get input change value")
+	})
+}
+
+func TestActionFilesInput(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>Nuclei Test Page</body>
+			<input type="file">
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+		{ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
+		el := page.Page().MustElement("input")
+		require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file")
+	})
+}
+
+func TestActionWaitLoad(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<button id="test">Wait for me!</button>
+			<script>
+				window.onload = () => document.querySelector('#test').style.color = 'red';
+			</script>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		el := page.Page().MustElement("button")
+		style, attributeErr := el.Attribute("style")
+		require.Nil(t, attributeErr)
+		require.Equal(t, "color: red;", *style, "could not get color")
+	})
+}
+
+func TestActionGetResource(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>
+				<img id="test" src="https://nuclei.projectdiscovery.io/static/logo.png">
+			</body>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, len(out["src"]), 3159, "could not find resource")
+	})
+}
+
+func TestActionExtract(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<button id="test">Wait for me!</button>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "Wait for me!", out["extract"], "could not extract text")
+	})
+}
+
+func TestActionSetMethod(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource")
+	})
+}
+
+func TestActionAddHeader(t *testing.T) {
+	actions := []*Action{
+		{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+	}
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Test") == "Hello" {
+			_, _ = fmt.Fprintln(w, `found`)
+		}
+	}
+
+	testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
+	})
+}
+
+func TestActionDeleteHeader(t *testing.T) {
+	actions := []*Action{
+		{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}},
+		{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}},
+		{ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}},
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+	}
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" {
+			_, _ = fmt.Fprintln(w, `header deleted`)
+		}
+	}
+
+	testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly")
+	})
+}
+
+func TestActionSetBody(t *testing.T) {
+	actions := []*Action{
+		{ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}},
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+	}
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		body, _ := ioutil.ReadAll(r.Body)
+		_, _ = fmt.Fprintln(w, string(body))
+	}
+
+	testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
+	})
+}
+
+func TestActionKeyboard(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>
+				<input type="text" name="test" id="test">
+			</body>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitload"},
+		{ActionType: "click", Data: map[string]string{"selector": "input"}},
+		{ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		el := page.Page().MustElement("input")
+		require.Equal(t, "Test2", el.MustText(), "could not get input change value")
+	})
+}
+
+func TestActionSleep(t *testing.T) {
 	response := `
 		<html>
 			<head>
@@ -310,6 +460,59 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page
 			</script>
 		</html>`
 
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "sleep", Data: map[string]string{"duration": "2"}},
+	}
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.Nil(t, err, "could not run page actions")
+		require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button")
+	})
+}
+
+func TestActionWaitVisible(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<button style="display:none" id="test">Wait for me!</button>
+			<script>
+				setTimeout(() => document.querySelector('#test').style.display = '', 1000);
+			</script>
+		</html>`
+
+	actions := []*Action{
+		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}},
+	}
+
+	t.Run("wait for an element being visible", func(t *testing.T) {
+		testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) {
+			require.Nil(t, err, "could not run page actions")
+
+			page.Page().MustElement("button").MustVisible()
+		})
+	})
+
+	t.Run("timeout because of element not visible", func(t *testing.T) {
+		testHeadlessSimpleResponse(t, response, actions, time.Second/2, func(page *Page, err error, out map[string]string) {
+			require.Error(t, err)
+			require.Contains(t, err.Error(), "Element did not appear in the given amount of time")
+		})
+	})
+}
+
+func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) {
+	t.Helper()
+	testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = fmt.Fprintln(w, response)
+	}, assert)
+}
+
+func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) {
+	t.Helper()
 	_ = protocolstate.Init(&types.Options{})
 
 	browser, err := New(&types.Options{ShowBrowser: false})
@@ -318,19 +521,17 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page
 
 	instance, err := browser.NewInstance()
 	require.Nil(t, err, "could not create browser instance")
+	defer instance.Close()
 
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(handler))
-		fmt.Fprintln(w, response)
-	}))
 	defer ts.Close()
 
 	parsed, err := url.Parse(ts.URL)
 	require.Nil(t, err, "could not parse URL")
+	extractedData, page, err := instance.Run(parsed, actions, timeout)
+	assert(page, err, extractedData)
 
-	actions := []*Action{
+	if page != nil {
-		{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
+		page.Close()
-		{ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}},
 	}
-	_, page, err := instance.Run(parsed, actions, timeout)
-	assert(page, err)
 }

v2/pkg/protocols/headless/engine/rules.go

@@ -8,6 +8,9 @@ import (
 
 // routingRuleHandler handles proxy rule for actions related to request/response modification
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
+	// usually browsers don't use chunked transfer encoding so we set the content-length nevertheless
+	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
+
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue

v2/pkg/protocols/http/build_request.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
@@ -20,6 +21,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
 )
@@ -38,9 +40,22 @@ type generatedRequest struct {
 	dynamicValues   map[string]interface{}
 }
 
+func (g *generatedRequest) URL() string {
+	if g.request != nil {
+		return g.request.URL.String()
+	}
+	if g.rawRequest != nil {
+		return g.rawRequest.FullURL
+	}
+	return ""
+}
+
 // Make creates a http request for the provided input.
 // It returns io.EOF as error when all the requests have been exhausted.
 func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
+	if r.request.SelfContained {
+		return r.makeSelfContainedRequest(dynamicValues, interactURL)
+	}
 	// We get the next payload for the request.
 	data, payloads, ok := r.nextValue()
 	if !ok {
@@ -48,6 +63,14 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	}
 	ctx := context.Background()
 
+	if interactURL != "" {
+		data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
+
+		for payloadName, payloadValue := range payloads {
+			payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL)
+		}
+	}
+
 	parsed, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
@@ -55,22 +78,22 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 
 	data, parsed = baseURLWithTemplatePrefs(data, parsed)
 
-	trailingSlash := false
 	isRawRequest := len(r.request.Raw) > 0
+
+	// If the request is not a raw request, and the URL input path is suffixed with
+	// a trailing slash, and our Input URL is also suffixed with a trailing slash,
+	// mark trailingSlash bool as true which will be later used during variable generation
+	// to generate correct path removed slash which would otherwise generate // invalid sequence.
+	// TODO: Figure out a cleaner way to do this sanitization.
+	trailingSlash := false
 	if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") {
 		trailingSlash = true
 	}
-	values := generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash))
 
-	// merge with vars
+	values := generators.MergeMaps(
-	if !r.options.Options.Vars.IsEmpty() {
+		generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)),
-		values = generators.MergeMaps(values, r.options.Options.Vars.AsMap())
+		generators.BuildPayloadFromOptions(r.request.options.Options),
-	}
+	)
-
-	// merge with env vars
-	if r.options.Options.EnvironmentVariables {
-		values = generators.MergeMaps(generators.EnvVars(), values)
-	}
 
 	// If data contains \n it's a raw request, process it like raw. Else
 	// continue with the template based request flow.
@@ -80,6 +103,48 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
 }
 
+func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
+	// We get the next payload for the request.
+	data, payloads, ok := r.nextValue()
+	if !ok {
+		return nil, io.EOF
+	}
+	ctx := context.Background()
+
+	isRawRequest := r.request.isRaw()
+
+	// If the request is a raw request, get the URL from the request
+	// header and use it to make the request.
+	if isRawRequest {
+		// Get the hostname from the URL section to build the request.
+		reader := bufio.NewReader(strings.NewReader(data))
+		s, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, fmt.Errorf("could not read request: %s", err)
+		}
+
+		parts := strings.Split(s, " ")
+		if len(parts) < 3 {
+			return nil, fmt.Errorf("malformed request supplied")
+		}
+		parsed, err := url.Parse(parts[1])
+		if err != nil {
+			return nil, fmt.Errorf("could not parse request URL: %s", err)
+		}
+		values := generators.MergeMaps(
+			generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
+			generators.BuildPayloadFromOptions(r.request.options.Options),
+		)
+
+		return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
+	}
+	values := generators.MergeMaps(
+		dynamicValues,
+		generators.BuildPayloadFromOptions(r.request.options.Options),
+	)
+	return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
+}
+
 // Total returns the total number of requests for the generator
 func (r *requestGenerator) Total() int {
 	if r.payloadIterator != nil {

v2/pkg/protocols/http/http.go

@@ -46,13 +46,13 @@ type Request struct {
 	// description: |
 	//   Attack is the type of payload combinations to perform.
 	//
-	//   Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates
+	//   batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates
 	//   permutations and combinations for all payloads.
 	// values:
-	//   - "sniper"
+	//   - "batteringram"
 	//   - "pitchfork"
 	//   - "clusterbomb"
-	AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"`
+	AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"`
 	// description: |
 	//   Method is the HTTP Request Method.
 	// values:
@@ -137,6 +137,10 @@ type Request struct {
 	rawhttpClient *rawhttp.Client
 	dynamicValues map[string]interface{}
 
+	// description: |
+	//   SelfContained specifies if the request is self contained.
+	SelfContained bool `yaml:"-" json:"-"`
+
 	// description: |
 	//   CookieReuse is an optional setting that enables cookie reuse for
 	//   all requests defined in raw section.
@@ -180,6 +184,10 @@ func (request *Request) GetID() string {
 	return request.ID
 }
 
+func (request *Request) isRaw() bool {
+	return len(request.Raw) > 0
+}
+
 // Compile compiles the protocol request for further execution.
 func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	connectionConfiguration := &httpclientpool.Configuration{

v2/pkg/protocols/http/operators.go

@@ -147,6 +147,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
+		CURLCommand:      types.ToString(wrapped.InternalEvent["curl-command"]),
 	}
 	return data
 }

v2/pkg/protocols/http/raw/raw.go

@@ -90,8 +90,10 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
 			return nil, fmt.Errorf("could not parse request URL: %s", parseErr)
 		}
 
-		rawRequest.Path = parts[1]
+		rawRequest.Path = parsed.Path
-		rawRequest.Headers["Host"] = parsed.Host
+		if _, ok := rawRequest.Headers["Host"]; !ok {
+			rawRequest.Headers["Host"] = parsed.Host
+		}
 	} else if len(parts) > 1 {
 		rawRequest.Path = parts[1]
 	}
@@ -104,7 +106,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
 	if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") {
 		parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/")
 	}
-	rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path)
+	if parsedURL.Path != rawRequest.Path {
+		rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path)
+	}
 	if strings.HasSuffix(rawRequest.Path, "//") {
 		rawRequest.Path = strings.TrimSuffix(rawRequest.Path, "/")
 	}

v2/pkg/protocols/http/request.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
+	"moul.io/http2curl"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -100,6 +101,9 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
 		if err == io.EOF {
 			break
 		}
+		if reqURL == "" {
+			reqURL = generatedHttpRequest.URL()
+		}
 		if err != nil {
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 			return err
@@ -160,6 +164,9 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
 		if err == io.EOF {
 			break
 		}
+		if reqURL == "" {
+			reqURL = generatedHttpRequest.URL()
+		}
 		if err != nil {
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 			return err
@@ -215,6 +222,9 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
 		if err == io.EOF {
 			break
 		}
+		if reqURL == "" {
+			reqURL = generatedHttpRequest.URL()
+		}
 		if err != nil {
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 			return err
@@ -373,6 +383,16 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		resp.Body.Close()
 	}()
 
+	var curlCommand string
+	if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil {
+		bodyBytes, _ := generatedRequest.request.BodyBytes()
+		resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
+		command, _ := http2curl.GetCurlCommand(resp.Request)
+		if err == nil && command != nil {
+			curlCommand = command.String()
+		}
+	}
+
 	gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL)
 	request.options.Output.Request(request.options.TemplateID, formedURL, "http", err)
 
@@ -429,7 +449,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data)
 
 	// Decode gbk response content-types
-	if contentType := strings.ToLower(resp.Header.Get("Content-Type")); contentType != "" && (strings.Contains(contentType, "gbk") || strings.Contains(contentType, "gb2312")) {
+	// gb18030 supersedes gb2312
+	if isContentTypeGbk(resp.Header.Get("Content-Type")) {
 		dumpedResponse, err = decodegbk(dumpedResponse)
 		if err != nil {
 			return errors.Wrap(err, "could not gbk decode")
@@ -438,6 +459,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 		if err != nil {
 			return errors.Wrap(err, "could not gbk decode")
 		}
+
+		// the uncompressed body needs to be decoded to standard utf8
+		data, err = decodegbk(data)
+		if err != nil {
+			return errors.Wrap(err, "could not gbk decode")
+		}
 	}
 
 	// if nuclei-project is enabled store the response if not previously done
@@ -460,6 +487,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	if i := strings.LastIndex(hostname, ":"); i != -1 {
 		hostname = hostname[:i]
 	}
+	outputEvent["curl-command"] = curlCommand
 	outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
 	outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
 	for k, v := range previousEvent {

v2/pkg/protocols/http/utils.go

@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
 	"github.com/projectdiscovery/rawhttp"
+	"github.com/projectdiscovery/stringsutil"
 	"golang.org/x/text/encoding/simplifiedchinese"
 	"golang.org/x/text/transform"
 )
@@ -135,3 +136,9 @@ func decodegbk(s []byte) ([]byte, error) {
 	}
 	return d, nil
 }
+
+// isContentTypeGbk checks if the content-type header is gbk
+func isContentTypeGbk(contentType string) bool {
+	contentType = strings.ToLower(contentType)
+	return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030")
+}

v2/pkg/protocols/network/network.go

@@ -35,13 +35,13 @@ type Request struct {
 	// description: |
 	//   Attack is the type of payload combinations to perform.
 	//
-	//   Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates
+	//   Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates
 	//   permutations and combinations for all payloads.
 	// values:
-	//   - "sniper"
+	//   - "batteringram"
 	//   - "pitchfork"
 	//   - "clusterbomb"
-	AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"`
+	AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"`
 	// description: |
 	//   Payloads contains any payloads for the current request.
 	//
@@ -60,6 +60,17 @@ type Request struct {
 	// examples:
 	//   - value: "2048"
 	ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"`
+	// description: |
+	//   ReadAll determines if the data stream should be read till the end regardless of the size
+	//
+	//   Default value for read-all is false.
+	// examples:
+	//   - value: false
+	ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"`
+
+	// description: |
+	//   SelfContained specifies if the request is self contained.
+	SelfContained bool `yaml:"-" json:"-"`
 
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`

v2/pkg/protocols/network/request.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"net"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -26,7 +27,14 @@ var _ protocols.Request = &Request{}
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	address, err := getAddress(input)
+	var address string
+	var err error
+
+	if request.SelfContained {
+		address = ""
+	} else {
+		address, err = getAddress(input)
+	}
 	if err != nil {
 		request.options.Output.Request(request.options.TemplateID, input, "network", err)
 		request.options.Progress.IncrementFailedRequestsBy(1)
@@ -41,6 +49,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 			}
 			actualAddress = net.JoinHostPort(actualAddress, kv.port)
 		}
+		if input != "" {
+			input = actualAddress
+		}
 
 		if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil {
 			gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
@@ -59,6 +70,8 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho
 		return err
 	}
 
+	payloads := generators.BuildPayloadFromOptions(request.options.Options)
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 
@@ -67,12 +80,13 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho
 			if !ok {
 				break
 			}
+			value = generators.MergeMaps(value, payloads)
 			if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
 				return err
 			}
 		}
 	} else {
-		value := make(map[string]interface{})
+		value := generators.MergeMaps(map[string]interface{}{}, payloads)
 		if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
 			return err
 		}
@@ -86,6 +100,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 		conn     net.Conn
 		err      error
 	)
+
 	request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address})
 
 	if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil {
@@ -186,13 +201,48 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	if request.ReadSize != 0 {
 		bufferSize = request.ReadSize
 	}
-	final := make([]byte, bufferSize)
+
-	n, err := conn.Read(final)
+	var (
-	if err != nil && err != io.EOF {
+		final []byte
-		request.options.Output.Request(request.options.TemplateID, address, "network", err)
+		n     int
-		return errors.Wrap(err, "could not read from server")
+	)
+
+	if request.ReadAll {
+		readInterval := time.NewTimer(time.Second * 1)
+		// stop the timer and drain the channel
+		closeTimer := func(t *time.Timer) {
+			if !t.Stop() {
+				<-t.C
+			}
+		}
+	read_socket:
+		for {
+			select {
+			case <-readInterval.C:
+				closeTimer(readInterval)
+				break read_socket
+			default:
+				buf := make([]byte, bufferSize)
+				nBuf, err := conn.Read(buf)
+				if err != nil && !os.IsTimeout(err) {
+					request.options.Output.Request(request.options.TemplateID, address, "network", err)
+					closeTimer(readInterval)
+					return errors.Wrap(err, "could not read from server")
+				}
+				responseBuilder.Write(buf[:nBuf])
+				final = append(final, buf...)
+				n += nBuf
+			}
+		}
+	} else {
+		final = make([]byte, bufferSize)
+		n, err = conn.Read(final)
+		if err != nil && err != io.EOF {
+			request.options.Output.Request(request.options.TemplateID, address, "network", err)
+			return errors.Wrap(err, "could not read from server")
+		}
+		responseBuilder.Write(final[:n])
 	}
-	responseBuilder.Write(final[:n])
 
 	response := responseBuilder.String()
 	outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)

v2/pkg/protocols/offlinehttp/operators.go

@@ -98,6 +98,13 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
 	for k, v := range extra {
 		data[k] = v
 	}
+	for _, cookie := range resp.Cookies() {
+		data[strings.ToLower(cookie.Name)] = cookie.Value
+	}
+	for k, v := range resp.Header {
+		k = strings.ToLower(strings.TrimSpace(k))
+		data[k] = strings.Join(v, " ")
+	}
 
 	data["path"] = host
 	data["matched"] = matched
@@ -106,13 +113,6 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
 	data["content_length"] = resp.ContentLength
 	data["status_code"] = resp.StatusCode
 	data["body"] = body
-	for _, cookie := range resp.Cookies() {
-		data[strings.ToLower(cookie.Name)] = cookie.Value
-	}
-	for k, v := range resp.Header {
-		k = strings.ToLower(strings.TrimSpace(k))
-		data[k] = strings.Join(v, " ")
-	}
 	data["all_headers"] = headers
 	data["duration"] = duration.Seconds()
 	data["template-id"] = request.options.TemplateID

v2/pkg/reporting/exporters/es/elasticsearch.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	"time"
 
 	"encoding/base64"
@@ -22,9 +23,9 @@ type Options struct {
 	IP string `yaml:"ip"`
 	// Port is the port of elasticsearch instance
 	Port int `yaml:"port"`
-	// SSL enables ssl for elasticsearch connection
+	// SSL (optional) enables ssl for elasticsearch connection
 	SSL bool `yaml:"ssl"`
-	// SSLVerification disables SSL verification for elasticsearch
+	// SSLVerification (optional) disables SSL verification for elasticsearch
 	SSLVerification bool `yaml:"ssl-verification"`
 	// Username for the elasticsearch instance
 	Username string `yaml:"username"`
@@ -49,6 +50,10 @@ type Exporter struct {
 // New creates and returns a new exporter for elasticsearch
 func New(option *Options) (*Exporter, error) {
 	var ei *Exporter
+	err := validateOptions(option)
+	if err != nil {
+		return nil, err
+	}
 
 	client := &http.Client{
 		Timeout: 5 * time.Second,
@@ -81,6 +86,31 @@ func New(option *Options) (*Exporter, error) {
 	return ei, nil
 }
 
+func validateOptions(options *Options) error {
+	errs := []string{}
+	if options.IP == "" {
+		errs = append(errs, "IP")
+	}
+	if options.Port == 0 {
+		errs = append(errs, "Port")
+	}
+	if options.Username == "" {
+		errs = append(errs, "Username")
+	}
+	if options.Password == "" {
+		errs = append(errs, "Password")
+	}
+	if options.IndexName == "" {
+		errs = append(errs, "IndexName")
+	}
+
+	if len(errs) > 0 {
+		return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
+	}
+
+	return nil
+}
+
 // Export exports a passed result event to elasticsearch
 func (i *Exporter) Export(event *output.ResultEvent) error {
 	// creating a request

v2/pkg/reporting/format/format.go

@@ -131,6 +131,13 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the
 			}
 		}
 	}
+	builder.WriteString("\n")
+
+	if event.CURLCommand != "" {
+		builder.WriteString("\n**CURL Command**\n```\n")
+		builder.WriteString(event.CURLCommand)
+		builder.WriteString("\n```")
+	}
 
 	builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version))
 	data := builder.String()

v2/pkg/reporting/trackers/github/github.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"golang.org/x/oauth2"
 
@@ -22,22 +23,29 @@ type Integration struct {
 
 // Options contains the configuration options for github issue tracker client
 type Options struct {
-	// BaseURL is the optional self-hosted github application url
+	// BaseURL (optional) is the self-hosted github application url
 	BaseURL string `yaml:"base-url"`
 	// Username is the username of the github user
 	Username string `yaml:"username"`
-	// Owner is the owner name of the repository for issues.
+	// Owner (manadatory) is the owner name of the repository for issues.
 	Owner string `yaml:"owner"`
 	// Token is the token for github account.
 	Token string `yaml:"token"`
 	// ProjectName is the name of the repository.
 	ProjectName string `yaml:"project-name"`
-	// IssueLabel is the label of the created issue type
+	// IssueLabel (optional) is the label of the created issue type
 	IssueLabel string `yaml:"issue-label"`
+	// SeverityAsLabel (optional) sends the severity as the label of the created
+	// issue.
+	SeverityAsLabel bool `yaml:"severity-as-label"`
 }
 
 // New creates a new issue tracker integration client based on options.
 func New(options *Options) (*Integration, error) {
+	err := validateOptions(options)
+	if err != nil {
+		return nil, err
+	}
 	ctx := context.Background()
 	ts := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: options.Token},
@@ -50,21 +58,53 @@ func New(options *Options) (*Integration, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "could not parse custom baseurl")
 		}
+		if !strings.HasSuffix(parsed.Path, "/") {
+			parsed.Path += "/"
+		}
 		client.BaseURL = parsed
 	}
 	return &Integration{client: client, options: options}, nil
 }
 
+func validateOptions(options *Options) error {
+	errs := []string{}
+	if options.Username == "" {
+		errs = append(errs, "Username")
+	}
+	if options.Owner == "" {
+		errs = append(errs, "Owner")
+	}
+	if options.Token == "" {
+		errs = append(errs, "Token")
+	}
+	if options.ProjectName == "" {
+		errs = append(errs, "ProjectName")
+	}
+
+	if len(errs) > 0 {
+		return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
+	}
+
+	return nil
+}
+
 // CreateIssue creates an issue in the tracker
 func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 	summary := format.Summary(event)
 	description := format.MarkdownDescription(event)
+	labels := []string{}
 	severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
+	if i.options.SeverityAsLabel && severityLabel != "" {
+		labels = append(labels, severityLabel)
+	}
+	if label := i.options.IssueLabel; label != "" {
+		labels = append(labels, label)
+	}
 
 	req := &github.IssueRequest{
 		Title:     &summary,
 		Body:      &description,
-		Labels:    &[]string{i.options.IssueLabel, severityLabel},
+		Labels:    &labels,
 		Assignees: &[]string{i.options.Username},
 	}
 	_, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req)

v2/pkg/reporting/trackers/gitlab/gitlab.go

@@ -2,6 +2,9 @@ package gitlab
 
 import (
 	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
@@ -17,7 +20,7 @@ type Integration struct {
 
 // Options contains the configuration options for gitlab issue tracker client
 type Options struct {
-	// BaseURL is the optional self-hosted gitlab application url
+	// BaseURL (optional) is the self-hosted gitlab application url
 	BaseURL string `yaml:"base-url"`
 	// Username is the username of the gitlab user
 	Username string `yaml:"username"`
@@ -27,10 +30,17 @@ type Options struct {
 	ProjectName string `yaml:"project-name"`
 	// IssueLabel is the label of the created issue type
 	IssueLabel string `yaml:"issue-label"`
+	// SeverityAsLabel (optional) sends the severity as the label of the created
+	// issue.
+	SeverityAsLabel bool `yaml:"severity-as-label"`
 }
 
 // New creates a new issue tracker integration client based on options.
 func New(options *Options) (*Integration, error) {
+	err := validateOptions(options)
+	if err != nil {
+		return nil, err
+	}
 	gitlabOpts := []gitlab.ClientOptionFunc{}
 	if options.BaseURL != "" {
 		gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL))
@@ -46,16 +56,42 @@ func New(options *Options) (*Integration, error) {
 	return &Integration{client: git, userID: user.ID, options: options}, nil
 }
 
+func validateOptions(options *Options) error {
+	errs := []string{}
+	if options.Username == "" {
+		errs = append(errs, "Username")
+	}
+	if options.Token == "" {
+		errs = append(errs, "Token")
+	}
+	if options.ProjectName == "" {
+		errs = append(errs, "ProjectName")
+	}
+
+	if len(errs) > 0 {
+		return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
+	}
+
+	return nil
+}
+
 // CreateIssue creates an issue in the tracker
 func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 	summary := format.Summary(event)
 	description := format.MarkdownDescription(event)
+	labels := []string{}
 	severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
+	if i.options.SeverityAsLabel && severityLabel != "" {
+		labels = append(labels, severityLabel)
+	}
+	if label := i.options.IssueLabel; label != "" {
+		labels = append(labels, label)
+	}
 
 	_, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{
 		Title:       &summary,
 		Description: &description,
-		Labels:      gitlab.Labels{i.options.IssueLabel, severityLabel},
+		Labels:      labels,
 		AssigneeIDs: []int{i.userID},
 	})
 

v2/pkg/reporting/trackers/jira/jira.go

@@ -2,6 +2,7 @@ package jira
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"strings"
@@ -22,9 +23,9 @@ type Integration struct {
 
 // Options contains the configuration options for jira client
 type Options struct {
-	// Cloud value is set to true when Jira cloud is used
+	// Cloud value (optional) is set to true when Jira cloud is used
 	Cloud bool `yaml:"cloud"`
-	// UpdateExisting value if true, the existing opened issue is updated
+	// UpdateExisting value (optional) if true, the existing opened issue is updated
 	UpdateExisting bool `yaml:"update-existing"`
 	// URL is the URL of the jira server
 	URL string `yaml:"url"`
@@ -36,12 +37,19 @@ type Options struct {
 	Token string `yaml:"token"`
 	// ProjectName is the name of the project.
 	ProjectName string `yaml:"project-name"`
-	// IssueType is the name of the created issue type
+	// IssueType (optional) is the name of the created issue type
 	IssueType string `yaml:"issue-type"`
+	// SeverityAsLabel (optional) sends the severity as the label of the created
+	// issue.
+	SeverityAsLabel bool `yaml:"severity-as-label"`
 }
 
 // New creates a new issue tracker integration client based on options.
 func New(options *Options) (*Integration, error) {
+	err := validateOptions(options)
+	if err != nil {
+		return nil, err
+	}
 	username := options.Email
 	if !options.Cloud {
 		username = options.AccountID
@@ -57,10 +65,42 @@ func New(options *Options) (*Integration, error) {
 	return &Integration{jira: jiraClient, options: options}, nil
 }
 
+func validateOptions(options *Options) error {
+	errs := []string{}
+	if options.URL == "" {
+		errs = append(errs, "URL")
+	}
+	if options.AccountID == "" {
+		errs = append(errs, "AccountID")
+	}
+	if options.Email == "" {
+		errs = append(errs, "Email")
+	}
+	if options.Token == "" {
+		errs = append(errs, "Token")
+	}
+	if options.ProjectName == "" {
+		errs = append(errs, "ProjectName")
+	}
+
+	if len(errs) > 0 {
+		return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
+	}
+
+	return nil
+}
+
 // CreateNewIssue creates a new issue in the tracker
 func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
 	summary := format.Summary(event)
-	severityLabel := fmt.Sprintf("Severity:%s", event.Info.SeverityHolder.Severity.String())
+	labels := []string{}
+	severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
+	if i.options.SeverityAsLabel && severityLabel != "" {
+		labels = append(labels, severityLabel)
+	}
+	if label := i.options.IssueType; label != "" {
+		labels = append(labels, label)
+	}
 
 	fields := &jira.IssueFields{
 		Assignee:    &jira.User{AccountID: i.options.AccountID},
@@ -69,7 +109,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
 		Type:        jira.IssueType{Name: i.options.IssueType},
 		Project:     jira.Project{Key: i.options.ProjectName},
 		Summary:     summary,
-		Labels:      []string{severityLabel},
+		Labels:      labels,
 	}
 	// On-prem version of Jira server does not use AccountID
 	if !i.options.Cloud {
@@ -244,6 +284,13 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th
 			}
 		}
 	}
+	builder.WriteString("\n")
+
+	if event.CURLCommand != "" {
+		builder.WriteString("\n*CURL Command*\n{code}\n")
+		builder.WriteString(event.CURLCommand)
+		builder.WriteString("\n{code}")
+	}
 	builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei v%s](https://github.com/projectdiscovery/nuclei)", config.Version))
 	data := builder.String()
 	return data

v2/pkg/templates/compile.go

@@ -144,6 +144,21 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 	}
 	template.Path = filePath
 
+	template.parseSelfContainedRequests()
+
 	parsedTemplatesCache.Store(filePath, template, err)
 	return template, nil
 }
+
+// parseSelfContainedRequests parses the self contained template requests.
+func (t *Template) parseSelfContainedRequests() {
+	if !t.SelfContained {
+		return
+	}
+	for _, request := range t.RequestsHTTP {
+		request.SelfContained = true
+	}
+	for _, request := range t.RequestsNetwork {
+		request.SelfContained = true
+	}
+}

v2/pkg/templates/templates.go

@@ -27,7 +27,7 @@ type Template struct {
 	// examples:
 	//   - name: ID Example
 	//     value: "\"CVE-2021-19520\""
-	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"`
+	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"`
 	// description: |
 	//   Info contains metadata information about the template.
 	// examples:
@@ -62,6 +62,10 @@ type Template struct {
 	workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"`
 	CompiledWorkflow   *workflows.Workflow `yaml:"-" json:"-" jsonschema:"-"`
 
+	// description: |
+	//   Self Contained marks Requests for the template as self-contained
+	SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"`
+
 	// TotalRequests is the total number of requests for the template.
 	TotalRequests int `yaml:"-" json:"-"`
 	// Executer is the actual template executor for running template requests

v2/pkg/templates/templates_doc.go

@@ -31,7 +31,7 @@ func init() {
 	TemplateDoc.Type = "Template"
 	TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and"
 	TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template."
-	TemplateDoc.Fields = make([]encoder.Doc, 8)
+	TemplateDoc.Fields = make([]encoder.Doc, 9)
 	TemplateDoc.Fields[0].Name = "id"
 	TemplateDoc.Fields[0].Type = "string"
 	TemplateDoc.Fields[0].Note = ""
@@ -84,6 +84,11 @@ func init() {
 	TemplateDoc.Fields[7].Note = ""
 	TemplateDoc.Fields[7].Description = "Workflows is a list of workflows to execute for a template."
 	TemplateDoc.Fields[7].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
+	TemplateDoc.Fields[8].Name = "self-contained"
+	TemplateDoc.Fields[8].Type = "bool"
+	TemplateDoc.Fields[8].Note = ""
+	TemplateDoc.Fields[8].Description = "Self Contained marks Requests for the template as self-contained"
+	TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
 
 	MODELInfoDoc.Type = "model.Info"
 	MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template"
@@ -317,10 +322,10 @@ func init() {
 	HTTPRequestDoc.Fields[7].Name = "attack"
 	HTTPRequestDoc.Fields[7].Type = "string"
 	HTTPRequestDoc.Fields[7].Note = ""
-	HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads."
+	HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads."
 	HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform."
 	HTTPRequestDoc.Fields[7].Values = []string{
-		"sniper",
+		"batteringram",
 		"pitchfork",
 		"clusterbomb",
 	}
@@ -838,7 +843,7 @@ func init() {
 			FieldName: "network",
 		},
 	}
-	NETWORKRequestDoc.Fields = make([]encoder.Doc, 9)
+	NETWORKRequestDoc.Fields = make([]encoder.Doc, 10)
 	NETWORKRequestDoc.Fields[0].Name = "id"
 	NETWORKRequestDoc.Fields[0].Type = "string"
 	NETWORKRequestDoc.Fields[0].Note = ""
@@ -854,10 +859,10 @@ func init() {
 	NETWORKRequestDoc.Fields[2].Name = "attack"
 	NETWORKRequestDoc.Fields[2].Type = "string"
 	NETWORKRequestDoc.Fields[2].Note = ""
-	NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads."
+	NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads."
 	NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform."
 	NETWORKRequestDoc.Fields[2].Values = []string{
-		"sniper",
+		"batteringram",
 		"pitchfork",
 		"clusterbomb",
 	}
@@ -878,22 +883,29 @@ func init() {
 	NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end"
 
 	NETWORKRequestDoc.Fields[5].AddExample("", 2048)
-	NETWORKRequestDoc.Fields[6].Name = "matchers"
+	NETWORKRequestDoc.Fields[6].Name = "read-all"
-	NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher"
+	NETWORKRequestDoc.Fields[6].Type = "bool"
 	NETWORKRequestDoc.Fields[6].Note = ""
-	NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+	NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false."
-	NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+	NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size"
-	NETWORKRequestDoc.Fields[7].Name = "extractors"
+
-	NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+	NETWORKRequestDoc.Fields[6].AddExample("", false)
+	NETWORKRequestDoc.Fields[7].Name = "matchers"
+	NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher"
 	NETWORKRequestDoc.Fields[7].Note = ""
-	NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+	NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
-	NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+	NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
-	NETWORKRequestDoc.Fields[8].Name = "matchers-condition"
+	NETWORKRequestDoc.Fields[8].Name = "extractors"
-	NETWORKRequestDoc.Fields[8].Type = "string"
+	NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor"
 	NETWORKRequestDoc.Fields[8].Note = ""
-	NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
-	NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
-	NETWORKRequestDoc.Fields[8].Values = []string{
+	NETWORKRequestDoc.Fields[9].Name = "matchers-condition"
+	NETWORKRequestDoc.Fields[9].Type = "string"
+	NETWORKRequestDoc.Fields[9].Note = ""
+	NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[9].Values = []string{
 		"and",
 		"or",
 	}

v2/pkg/types/types.go

@@ -150,6 +150,8 @@ type Options struct {
 	Stdin bool
 	// StopAtFirstMatch stops processing template at first full match (this may break chained requests)
 	StopAtFirstMatch bool
+	// Stream the input without sorting
+	Stream bool
 	// NoMeta disables display of metadata for the matches
 	NoMeta bool
 	// NoTimestamp disables display of timestamp for the matcher