Merge branch 'v3-beta', remote-tracking branch 'origin' into dev

.github/workflows/build-test.yml

@@ -12,7 +12,7 @@ jobs:
     name: Test Builds
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest, windows-latest, macOS-latest]
 
     runs-on: ${{ matrix.os }}
@@ -22,6 +22,11 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
 
+      - name: Set up Python   # required for running python code in py-snippet.yaml integration test
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
       - name: Check out code
         uses: actions/checkout@v3
 
@@ -56,6 +61,10 @@ jobs:
         run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
         working-directory: v2/cmd/nuclei/
 
-      - name: Example Code Tests
-        run: go build .
-        working-directory: v2/examples/
+      - name: Example SDK Simple
+        run: go run .
+        working-directory: v2/examples/simple/
+
+      - name: Example SDK Advanced
+        run: go run .
+        working-directory: v2/examples/advanced/

.github/workflows/functional-test.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/lint-test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Checkout code
         uses: actions/checkout@v3

.github/workflows/performance-test.yaml

@@ -11,7 +11,7 @@ jobs:
     name: Test Performance
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest, macOS-latest]
 
     runs-on: ${{ matrix.os }}

.github/workflows/publish-docs.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v4
         with: 
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Generate YAML Syntax Documentation
         id: generate-docs

.github/workflows/release-binary.yml

@@ -17,7 +17,7 @@ jobs:
 
       - uses: actions/setup-go@v4
         with: 
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - uses: goreleaser/goreleaser-action@v4
         with: 

.github/workflows/release-test.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       
       - name: release test
         uses: goreleaser/goreleaser-action@v4

.github/workflows/template-validate.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with: 
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Template Validation
         run: |

.gitignore

@@ -17,4 +17,11 @@ v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
 v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
 *.exe
 v2/.gitignore
+v2/pkg/js/devtools/bindgen/cmd/bindgen
+v2/pkg/js/devtools/jsdocgen/jsdocgen
+bindgen
+jsdocgen
+nuclei
+v2/scrapefuncs
 *.DS_Store
+v2/pkg/protocols/headless/engine/.cache

DEBUG.md

@@ -34,9 +34,12 @@ When this flag is provided, nuclei will log all errors to the file specified. Th
 
 Nuclei was built with some environment variables in mind to help with debugging. These environment variables can be set to enable debugging of a particular component/functionality for nuclei.
 
-| Environment Variable   | Description                                              |
-| ---------------------- | -------------------------------------------------------- |
-| `DEBUG=true`           | Enables Printing Stack Traces for all errors             |
-| `SHOW_DSL_ERRORS=true` | Enables Printing DSL Errors (that are hidden by default) |
+| Environment Variable             | Description                                              |
+| -------------------------------- | -------------------------------------------------------- |
+| `DEBUG=true`                     | Enables Printing Stack Traces for all errors             |
+| `SHOW_DSL_ERRORS=true`           | Enables Printing DSL Errors (that are hidden by default) |
+| `HIDE_TEMPLATE_SIG_WARNING=true` | Hides Template Signature Verification Warnings           |
+| `NUCLEI_LOG_ALL=true`            | Log All Events that were skipped in verbose mode         |
+
 
 

Dockerfile

@@ -1,5 +1,5 @@
 # Build
-FROM golang:1.20.6-alpine AS build-env
+FROM golang:1.21-alpine AS build-env
 RUN apk add build-base
 WORKDIR /app
 COPY . /app

README.md

@@ -37,7 +37,7 @@
 
 ---
 
-Nuclei is used to send requests across targets based on a template, leading to zero false positives and providing fast scanning on a large number of hosts. Nuclei offers scanning for a variety of protocols, including TCP, DNS, HTTP, SSL, File, Whois, Websocket, Headless etc. With powerful and flexible templating, Nuclei can be used to model all kinds of security checks.
+Nuclei is used to send requests across targets based on a template, leading to zero false positives and providing fast scanning on a large number of hosts. Nuclei offers scanning for a variety of protocols, including TCP, DNS, HTTP, SSL, File, Whois, Websocket, Headless, Code etc. With powerful and flexible templating, Nuclei can be used to model all kinds of security checks.
 
 We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 300** security researchers and engineers.
 
@@ -56,7 +56,7 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 
 # Install Nuclei
 
-Nuclei requires **go1.20** to install successfully. Run the following command to install the latest version -
+Nuclei requires **go1.21** to install successfully. Run the following command to install the latest version -
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
@@ -146,8 +146,8 @@ FILTERING:
    -em, -exclude-matchers string[]    template matchers to exclude in result
    -s, -severity value[]              templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
    -es, -exclude-severity value[]     templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
-   -pt, -type value[]                 templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois
-   -ept, -exclude-type value[]        templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois
+   -pt, -type value[]                 templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code
+   -ept, -exclude-type value[]        templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code
    -tc, -template-condition string[]  templates to run based on expression condition
 
 OUTPUT:
@@ -379,7 +379,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
 
 ### Using Nuclei From Go Code
 
-Examples of using Nuclei From Go Code to run templates on targets are provided in the [examples](v2/examples/) folder.
+Complete guide of using Nuclei as Library/SDK is available at [lib](v2/lib/README.md)
 
 
 ### Resources

README_CN.md

@@ -52,7 +52,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主
 
 # 安装Nuclei
 
-Nuclei需要**go1.20**才能安装成功。执行下列命令安装最新版本的Nuclei
+Nuclei需要**go1**才能安装成功。执行下列命令安装最新版本的Nuclei
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

README_ID.md

@@ -52,7 +52,7 @@ Kami memiliki [repositori khusus](https://github.com/projectdiscovery/nuclei-tem
 
 # Instalasi Nuclei
 
-Nuclei membutuhkan **go1.20** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
+Nuclei membutuhkan **go1.21** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

README_KR.md

@@ -50,7 +50,7 @@ Nuclei는 템플릿을 기반으로 대상 간에 요청을 보내기 위해 사
 
 # 설치
 
-Nuclei를 성공적으로 설치하기 위해서 **go1.20**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다.
+Nuclei를 성공적으로 설치하기 위해서 **go1.21**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다.
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

SYNTAX-REFERENCE.md

@@ -2820,28 +2820,6 @@ StopAtFirstMatch stops the execution of the requests and template as soon as a m
 </div>
 <div class="dt">
 
-Fuzzing describes schema to fuzz headless requests
-
-</div>
-
-<hr />
-
-<div class="dd">
-
-<code>cookie-reuse</code>  <i>bool</i>
-
-</div>
-<div class="dt">
-
-CookieReuse is an optional setting that enables cookie reuse
-
-</div>
-
-<hr />
-
-
-
-
 
 ## engine.Action
 Action is an action taken by the browser to reach a navigation

docs/getting-started/running.mdx

@@ -178,6 +178,25 @@ export AZURE_CONTAINER_NAME=templates
 
 </AccordionGroup>
 
+Environment variables can also be provided to disable download from default and custom template locations:
+
+```bash
+# Disable download from the default nuclei-templates project
+export DISABLE_NUCLEI_TEMPLATES_PUBLIC_DOWNLOAD=true
+
+# Disable download from public / private GitHub project(s)
+export DISABLE_NUCLEI_TEMPLATES_GITHUB_DOWNLOAD=true
+
+# Disable download from public / private GitLab project(s)
+export DISABLE_NUCLEI_TEMPLATES_GITLAB_DOWNLOAD=true
+
+# Disable download from public / private AWS Bucket(s)
+export DISABLE_NUCLEI_TEMPLATES_AWS_DOWNLOAD=true
+
+# Disable download from public / private Azure Blob Storage
+export DISABLE_NUCLEI_TEMPLATES_AZURE_DOWNLOAD=true
+```
+
 Once the environment variables are set, following command to download the custom templates:
 
 ```bash

docs/mint.json

@@ -57,7 +57,7 @@
             "template-guide/introduction",
             "template-guide/template-details",
             {
-               "group":"HTTP",
+               "group":"HTTPs",
                "pages":[
                   "template-guide/http/base-http",
                   "template-guide/http/raw-http",
@@ -71,6 +71,8 @@
             "template-guide/network",
             "template-guide/dns",
             "template-guide/file",
+            "template-guide/javascript",
+            "template-guide/code",
             {
                "group":"Operators",
                "pages":[
@@ -140,11 +142,5 @@
    "feedback":{
       "thumbsRating":true,
       "suggestEdit":true
-   },
-   "api":{
-      "auth":{
-         "method":"key",
-         "name":"X-API-KEY"
-      }
    }
 }

docs/template-guide/code.mdx

@@ -0,0 +1,342 @@
+---
+title: "Code"
+---
+
+## Code Requests (beta)
+
+Nuclei enables the execution of external code on the host operating system. This feature allows security researchers, pentesters, and developers to extend the capabilities of Nuclei and perform complex actions beyond the scope of regular supported protocol-based testing.
+
+By leveraging this capability, Nuclei can interact with the underlying operating system and execute custom scripts or commands, opening up a wide range of possibilities. It enables users to perform tasks such as system-level configurations, file operations, network interactions, and more. This level of control and flexibility empowers users to tailor their security testing workflows according to their specific requirements.
+
+However, it's important to exercise caution while utilizing this feature, as executing external code on the host operating system carries inherent risks. It is crucial to ensure that the executed code is secure, thoroughly tested, and does not pose any unintended consequences or security risks to the target system.
+
+## Template Signing (beta)
+
+Template signing via the private-public key mechanism is a crucial aspect of ensuring the integrity and authenticity of templates. This mechanism involves the use of asymmetric cryptography, specifically  ECDSA algorithm, to create a secure and verifiable signature.
+
+In this process, a template author generates a private key that remains confidential and securely stored. The corresponding public key is then shared with the template consumers. When a template is created or modified, the author signs it using their private key, generating a unique signature that is attached to the template.
+
+Template consumers can verify the authenticity and integrity of a signed template by using the author's public key. By applying the appropriate cryptographic algorithm (ECDSA), they can validate the signature and ensure that the template has not been tampered with since it was signed. This provides a level of trust, as any modifications or unauthorized changes to the template would result in a failed verification process.
+
+By employing the private-public key mechanism, template signing adds an additional layer of security and trust to the template ecosystem. It helps establish the identity of the template author and ensures that the templates used in various systems are genuine and have not been altered maliciously.
+
+### What does signing a template mean ?
+
+Template signing is a mechanism to ensure the integrity and authenticity of templates. The primary goal is to provide template writers/consumers a way to trust crowdsource/custom templates ensuring that they are not tampered
+
+All Official nuclei templates include a digital signature in them and are verified by nuclei while loading templates using ProjectDiscovery's public key shipped with nuclei binary itself.
+
+Individuals / Organizations running nuclei in their work environment can generate their own key-pair with `nuclei` and sign their custom templates with their private key, thus ensuring that only authorized templates are being used in their environment.
+
+This also allows entities to fully utilize the power of new protocols like `code` and `javascript` without worrying about malicious custom templates being used in their environment.
+
+**Points to note**
+
+- Template signing is optional for all protocols except `code`.
+
+- Code File References (ex: `source: protocols/code/pyfile.py`) are allowed and content of these files is included in the template digest
+
+- Payload File References (ex: `payloads: protocols/http/params.txt`) are not included in the template digest as it is treated as a payload/helper and not actual code that is being executed
+
+- Template Signing is deterministic while both signing and verifying a template i.e if a code file is referenced in a template that is present outside of templates directory with `-lfa` flag then verification will fail if same template is used without `-lfa` flag. (Note this only applies to `-lfa` i.e local file access flag only)
+
+## How to sign custom templates
+
+Simplest and recommended way to generate key-pair and signing/verfifying templates is to use `nuclei` itself.
+
+#### When Signing a template if key-pair does not exist then nuclei will prompt user to generate a new key-pair with options 
+
+```console
+$ ./nuclei -t my-template.yaml -sign -v
+[INF] Generating new key-pair for signing templates
+[*] Enter User/Organization Name (exit to abort) : projectdiscovery/nuclei-templates
+[*] Enter passphrase (exit to abort): 
+[*] Enter same passphrase again: 
+[INF] Successfully generated new key-pair for signing templates
+
+```
+
+> Note: Passphrase is optional and can be left blank when used private key is encrypted with passphrase using PEMCipherAES256 Algo
+
+#### Signing a template with existing key-pair
+
+```console
+$ ./nuclei -t ~/nuclei-templates/http -sign -v
+[INF] All templates signatures were elaborated success=6464 failed=0
+```
+
+### Template Digest
+
+When a template is signed, a digest is generated and added to the template. This digest is a hash of the template content and is used to verify the integrity of the template. If the template is modified after signing, the digest will change, and the signature verification will fail which happens during template loading.
+
+```yaml
+# digest: 4a0a00473045022100eb01da6b97893e7868c584f330a0cd52df9bddac005860bb8595ba5b8aed58c9022050043feac68d69045cf320cba9298a2eb2e792ea4720d045d01e803de1943e7d:4a3eb6b4988d95847d4203be25ed1d46
+```
+
+It is in the format of `signature:fragment` where signature is digital signature of template which is used to verify integrity of template
+and fragment is a metadata generated by md5 hashing public key to disable re-signing of code templates not written by you.
+
+fragment is meant to act like a speed bump to prevent mass-signing of code protocol templates to prevent any unintended misuse.
+
+### Where are keys stored ?
+
+key-pair generated by nuclei are stored in 2 files in `$config/nuclei/keys` directory where `$config` is system specific config directory
+
+```
+$ la ~/.config/nuclei/keys 
+total 16
+-rw-------  1 tarun  staff   251B Oct  4 21:45 nuclei-user-private-key.pem # encrypted private key with passphrase
+-rw-------  1 tarun  staff   572B Oct  4 21:45 nuclei-user.crt # self signed certificate which includes public key and identifier (i.e user/org name)
+```
+
+### Sharing and Using Public Key
+
+Public key is stored in $config/nuclei/keys/nuclei-user.crt and can be shared with other users / organizations to verify templates signed by you.
+
+#### Using Public Key
+
+- A simple way to use public key is to copy it to $config/nuclei/keys directory of other user's machine
+
+- Another way is to use environment variable `NUCLEI_USER_CERTIFICATE=xxx` to specify path of public key or content of public key directly
+
+```console
+$ export NUCLEI_USER_CERTIFICATE=path/to/nuclei-user.crt
+```
+
+or 
+
+```console
+$ export NUCLEI_USER_CERTIFICATE=$(cat path/to/nuclei-user.crt)
+```
+
+#### Verifying Templates
+
+Everytime `nuclei` is run, it loads user certificate (aka public key) from above locations and uses it to verify templates.
+
+`nuclei` also prints identifier of public key being used and warns user of unsigned custom templates
+
+```
+[INF] Executing 6219 signed templates from projectdiscovery/nuclei-templates
+[WRN] Executing 687 unsigned templates. Use with caution.
+```
+
+### Managing Private Key
+
+Private key is stored in $config/nuclei/keys/nuclei-user-private-key.pem and is encrypted with passphrase if provided while generating key-pair.
+
+It is not used/loaded by default by nuclei and is only used on demand i.e when signing templates using `-sign` flag
+
+Some Users might want to store / backup or move private key to different location or machine and `nuclei` doesn't enforce any restrictions on that.
+
+#### Using Private Key
+
+- A simple way to use private key is to copy it to $config/nuclei/keys directory of other user's machine
+
+- Another way is to use environment variable `NUCLEI_USER_PRIVATE_KEY=xxx` to specify path of private key or content of private key directly
+
+```console
+$ export NUCLEI_USER_PRIVATE_KEY=path/to/nuclei-user-private-key.pem
+```
+
+or
+
+```console
+$ export NUCLEI_USER_PRIVATE_KEY=$(cat path/to/nuclei-user-private-key.pem)
+```
+
+> Note: You are responsible for securing and managing private key and nuclei has no accountability for any loss of private key
+
+
+## Code
+
+In the context of template creation, a code block is used to indicate the start of the requests for the template. This block marks the beginning of the code-related instructions.
+
+```yaml
+# Start the requests for the template right here
+code:
+```
+
+To execute the code, a list of engines is specified, which are searched sequentially until a valid one is found on the system. The engine names must match the corresponding binary names on the system.
+
+```yaml
+- engine:
+    - py
+    - python3
+```
+
+The code to be executed can be provided either as an external file or as a code snippet directly within the template.
+
+For an external file:
+
+```yaml
+source: protocols/code/pyfile.py
+```
+
+For a code snippet:
+```yaml
+source: |
+      import sys
+      print("hello from " + sys.stdin.read())
+```
+
+The target is passed to the template via stdin, and the output of the executed code is available for further processing in matchers and extractors. In the case of the Code protocol, the body part represents all data printed to stdout during the execution of the code.
+
+#### Matchers / Extractor Parts
+
+Valid `part` values supported by **Code** protocol for Matchers / Extractor are - 
+    
+| Value    | Description                                          |
+| -------- | ---------------------------------------------------- |
+| response | execution output (trailing whitespaces are filtered) |
+| stderr   | Raw Stderr Output(if any)                            |
+
+
+#### **Example Code Template**
+
+The provided example demonstrates the execution of a Python script within the template. The specified engines are searched in the given order, and the code snippet is executed accordingly. Additionally, a matcher is included to check if the code's stdout contains the phrase "hello from input." (input must be passed as target with nuclei)
+
+```yaml
+id: py-code-snippet
+
+info:
+  name: py-code-snippet
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    py-code-snippet
+
+code:
+  - engine:
+      - py
+      - python3
+    source: |
+      import sys
+      print("hello from " + sys.stdin.read())
+    
+    matchers:
+      - type: word
+        words:
+          - "hello from input"
+# digest: 4a0a00473045022067a69eb337ffa56d1c8e2cc57b7f74a5eb3294e6f366c9074778b2da3f1d795d02210096d6acda6acd2fe0ff005b08a9c0b72b63f599532ec6493f44b8518265d0e5fd:4a3eb6b4988d95847d4203be25ed1d46
+```
+
+### Optional Fields for Code Protocol
+
+Apart from required fields mentioned above, Code protocol also supports following optional fields to further customize the execution of code.
+
+#### Args
+
+Args are arguments that are sent to engine while executing the code. For example if we want to bypass execution policy in powershell for specific template this can be done by adding following args to the template.
+
+```yaml
+  - engine:
+      - powershell
+      - powershell.exe
+    args:
+      - -ExecutionPolicy
+      - Bypass
+      - -File
+```
+
+#### Pattern
+
+Pattern field can be used to customize name / extension of temporary file while executing a code snippet in a template
+
+```yaml
+    pattern: "*.ps1"
+```
+
+adding `pattern: "*.ps1"` will make sure that name of temporary file given pattern
+
+
+### Example Code Template with Args and Pattern
+
+Below is a example code template where we are executing a powershell script while customizing behaviour of execution policy and setting pattern to `*.ps1`
+
+```yaml
+id: ps1-code-snippet
+
+info:
+  name: ps1-code-snippet
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    ps1-code-snippet
+
+code:
+  - engine:
+      - powershell
+      - powershell.exe
+    args:
+      - -ExecutionPolicy
+      - Bypass
+      - -File
+    pattern: "*.ps1"
+    source: |
+      $stdin = [Console]::In
+      $line = $stdin.ReadLine()
+      Write-Host "hello from $line"
+    
+    matchers:
+      - type: word
+        words:
+          - "hello from input"
+# digest: 4a0a00473045022100eb01da6b97893e7868c584f330a0cd52df9bddac005860bb8595ba5b8aed58c9022050043feac68d69045cf320cba9298a2eb2e792ea4720d045d01e803de1943e7d:4a3eb6b4988d95847d4203be25ed1d46
+```
+
+For more examples, please refer to example [code-templates](https://github.com/projectdiscovery/nuclei/blob/3a5f9d626ea7b632ccca601b658acd9758f8f01b/integration_tests/protocols/code) in integration tests.
+
+
+## FAQ
+
+### I got this error when running a template . What does it mean ?
+
+```
+./nuclei -u scanme.sh -t simple-code.yaml 
+
+                     __     _
+   ____  __  _______/ /__  (_)
+  / __ \/ / / / ___/ / _ \/ /
+ / / / / /_/ / /__/ /  __/ /
+/_/ /_/\__,_/\___/_/\___/_/   v3.0.0-dev
+
+		projectdiscovery.io
+
+[WRN] Found 1 unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)
+[INF] Current nuclei version: v3.0.0-dev (development)
+[INF] Current nuclei-templates version: v9.6.4 (latest)
+[WRN] Executing 1 unsigned templates. Use with caution.
+[INF] Targets loaded for current scan: 1
+[INF] No results found. Better luck next time!
+[FTL] Could not run nuclei: no templates provided for scan
+```
+
+Here `simple-code.yaml` is a code protocol template which is not signed or content of template has been modified after signing which indicates loss of integrity of template.
+If you are template writer then you can go ahead and sign the template using `-sign` flag and if you are template consumer then you should carefully examine the template before signing it.
+
+### What does `re-signing code templates are not allowed for security reasons` error mean?
+
+```bash
+nuclei -u scanme.sh -t simple-code.yaml -sign
+
+[ERR] could not sign 'simple-code.yaml': [signer:RUNTIME] re-signing code templates are not allowed for security reasons.
+[INF] All templates signatures were elaborated success=0 failed=1
+```
+
+The error message `re-signing code templates are not allowed for security reasons` comes from the Nuclei engine. This error indicates that a code template initially signed by another user and someone is trying to re-sign it.
+
+This measure was implemented to prevent running untrusted templates unknowingly, which might lead to potential security issues.
+When you encounter this error, it suggests that you're dealing with a template that has been signed by another user Likely, the original signer is not you or the team from projectdiscovery.
+
+By default, Nuclei disallows executing code templates that are signed by anyone other than you or from the public templates provided by projectdiscovery/nuclei-templates.
+
+This is done to prevent potential security abuse using code templates.
+
+To resolve this error:
+  1.  Open and thoroughly examine the code template for any modifications.
+  2.  Manually remove the existing digest signature from the template.
+  3.  Sign the template again.
+
+This way, you can ensure that only templates verified and trusted by you (or projectdiscovery) are run, thus maintaining a secure environment.

docs/template-guide/javascript.mdx

@@ -0,0 +1,218 @@
+---
+title: "Javascript"
+---
+
+## Introduction
+
+Nuclei and its community thrives on its ability to write exploits/checks in fast and simple way in YAML format and we aim to make **nuclei templates** as standard for writing security checks and that comes with understanding its limitations and addressing them as well as expanding its capabilities. It is already possible to write most complex HTTP, DNS, SSL protocol exploits / checks with increasing support and a powerful and easy to use DSL in nuclei engine but we understand this may not be enough for addressing / writing vulnerabilities across all protocols as well as other non-remote domains of security like local privilege escalation checks, kernel etc.
+
+To address this and expand to other domains of security, Nuclei v3 ships with a embedded runtime for javascript that is tailored for **Nuclei** with the help of **[Goja](https://github.com/dop251/goja)**.
+
+
+## Features
+
+- **Provider/Driver specific exploit**
+
+Some vulnerabilities are specific to software/driver, example a redis buffer overflow exploit or a exploit of specific VPN software or anything that's not a IETF standard protocol. since they are not standard protocols and it doesn't make much sense to add them as a protocol in nuclei.
+Such exploits cannot be written using 'network' protocol or Very complex to write, such exploits can be written by exposing required library in nuclei (if not already present) and writing actual exploit in javascript protocol minus the boilerplate and scaling issues and other goodies of nuclei
+
+- **Non Network Checks**
+
+Security is not limited to network and nuclei also doesn't intend to limit itself to network only. There are lot of security checks that are not network related like
+
+1. local privilege escalation checks
+2. kernel exploits
+3. account misconfigurations
+4. system misconfigurations etc
+
+- **Complex network protocol exploits**
+
+Some network exploits are very complex to write due to nature of protocol or exploit itself example [CVE-2020-0796](https://nvd.nist.gov/vuln/detail/cve-2020-0796) where you have to manually construct a packet. such exploits are usually written in python but now can be written in javascript protocol itself
+
+- **Multi Step Exploits**
+
+Ldap / kerberos exploits usually involves multi step process of authentication and then exploitation etc and not easy to write in YAML based DSL
+
+- **Scalable and maintainable exploits**
+
+One off exploits written in code are not scalable and maintainable due to nature of language , boilerplate code and lot of other factors. The goal here is to only write **bare minimum** code required to run exploit and let nuclei engine handle the rest
+
+- **Leveraging turing complete language**
+
+While YAML based DSL is powerful and easy to use it is not turing complete and has its own limitations. Javascript is turing complete thus users who are already familiar with javascript can write network and other exploits without learning new DSL or hacking around existing DSL.
+
+## Goja
+
+Goja is ECMAScript/Javascript engine/runtime written in pure go and has full support for ECMAScript 5.1. It is fast, can be used in goroutines and has very small memory footprint which makes it good fit for embedding in nuclei and provides additional layer of security and flexibility due to nature of javascript language and its implementation.
+
+This does not break any nuclei design principle nor does it change how nuclei works and is dependency free. It complements nuclei engine by adding existing turing complete language (i.e javascript) instead of re-inventing the wheel by creating new DSL (domain specific language)
+
+## Requirements
+
+- A bare minimum knowledge of javascript (loops, functions , arrays is enough) is required to write javascript protocol template
+- Nuclei v3.0.0 or above
+
+
+## API Reference
+
+API reference of all exposed modules and functions can be found [here](https://projectdiscovery.github.io/js-proto-docs/).
+
+## Javascript Protocol
+
+Javascript protocol is new protocol added in nuclei v3 to allow writing exploits / checks in javascript language but internally are executed in go. And this javscript is tailored towards nuclei ecosystem this means
+
+- It is not intended to fit / imported with any existing javascript libraries or frameworks outside of nuclei ecosystem.
+- Nuclei Engine provides a set of functions, libraries that are tailor made for writing exploits / checks and only adds required/necessary functionality to compliment existing YAML based DSL.
+- It is not intended to be used as general purpose javascript runtime and does not replace matchers or extractors or any existing functionality of nuclei.
+- Javascript Protocol is intended to bridge gap between network protocol to add any new xyz protocol while adding lot of other functionalities.
+- Nuclei v3.0.0 ships with **15+ libraries (ssh,ftp,rdp,kerberos,redis)** tailored for writing exploits/checks in javascript and will be continiously expanded in future.
+
+Here is a simple example of javascript protocol template
+
+```yaml
+id: ssh-server-fingerprint
+
+info:
+  name: Fingerprint SSH Server Software
+  author: Ice3man543,tarunKoyalwar
+  severity: info
+  
+
+javascript:
+  - code: |
+      var m = require("nuclei/ssh");
+      var c = m.SSHClient();
+      var response = c.ConnectSSHInfoMode(Host, Port);
+      to_json(response);
+    args:
+      Host: "{{Host}}"
+      Port: "22"
+
+    extractors:
+      - type: json
+        json:
+          - '.ServerID.Raw'
+```
+
+In above nuclei template we are fingerprinting SSH Server Software by connecting in Non-Auth mode and extracting server banner.  Lets break down the template.
+
+### Code
+
+Code contains actual javascript code that is executed by nuclei engine at runtime In above template we are
+
+- importing `nuclei/ssh` module/library
+- creating a new instance of `SSHClient` object
+- connecting to SSH server in `Info` mode
+- converting response to json
+
+### Args
+
+Args can be simply understood as variables in javascript that are passed at runtime and support DSL usage
+
+
+### **Output**
+
+Value of Last expression is returned as output of javascript protocol template and can be used in matchers / extractors. If server returns an error instead then `error` variable is exposed in matcher/extractor with error message.
+
+### Example
+
+**SSH Password Bruteforce Template**
+
+```yaml
+id: ssh-brute
+
+info:
+  name: SSH Credential Stuffing
+  author: tarunKoyalwar
+  severity: critical
+  
+
+javascript:
+  - pre-condition: |
+      var m = require("nuclei/ssh");
+      var c = m.SSHClient();
+      var response = c.ConnectSSHInfoMode(Host, Port);
+      // only bruteforce if ssh server allows password based authentication
+      response["UserAuth"].includes("password")
+
+    code: |
+      var m = require("nuclei/ssh");
+      var c = m.SSHClient();
+      c.Connect(Host,Port,Username,Password);
+
+    args:
+      Host: "{{Host}}"
+      Port: "22"
+      Username: "{{usernames}}"
+      Password: "{{passwords}}"
+
+    threads: 10
+    attack: clusterbomb
+    payloads:
+      usernames: helpers/wordlists/wp-users.txt
+      passwords: helpers/wordlists/wp-passwords.txt
+
+    stop-at-first-match: true
+    matchers:
+      - type: dsl
+        dsl:
+          - "response == true"
+          - "success == true"
+        condition: and
+```
+
+In above nuclei template we are bruteforcing ssh server with list of usernames and passwords. We can tell that this might not have been possible to achieve with network template Let's break down the template.
+
+### Pre-Condition
+
+`pre-condition` is a optional javascript code that is executed before running "code" and acts as pre-condition to exploit. In above template before attempting to bruteforce we are checking if 
+- address is actually a ssh server
+- ssh server is configured to allow password based authentication
+
+**Furthur explaination** 
+
+- If pre-condition returns `true` only then code is executed otherwise it is skipped
+- In code section we import `nuclei/ssh` module and create a new instance of `SSHClient` object
+- and then we attempt to connect to ssh server with username and password
+- this template uses [payloads](https://docs.nuclei.sh/template-guide/http/http-payloads) to launch a clusterbomb attack with 10 threads and exits on first match
+
+Looking at this template now we can tell that javascript template is very powerful to write multi step and protocol/vendor specific exploits which is primary goal of javascript protocol.
+
+
+### Init
+
+`init` is a optional javascript code that can be used to initialize template and it is executed just after compiling template and before running it on any target. Although rarely needed, it can be used to load and preprocess data before running template on any target.
+For example in below code block we are loading all ssh private keys from `nuclei-templates/helpers` directory and storing them as a variable in payloads with name `keys`, if we were loading private keys from 'pre-condition' code block then it would have been loaded for every target which is not ideal.
+
+```
+variables:
+  keysDir: "helpers/"  # load all private keys from this directory
+
+javascript:
+    # init field can be used to make any preperations before the actual exploit
+    # here we are reading all private keys from helpers folder and storing them in a list
+  - init: |
+      let m = require('nuclei/fs');
+      let privatekeys = m.ReadFilesFromDir(keysDir)
+      updatePayload('keys',privatekeys)
+
+    payloads:
+      # 'keys' will be updated by actual private keys after init is executed
+      keys: 
+        - key1
+        - key2
+```
+
+Two special functions that are available in init block are
+
+
+| Function                   | Description                              |
+| -------------------------- | ---------------------------------------- |
+| `updatePayload(key,value)` | updates payload with given key and value |
+| `set(key,value)`           | sets a variable with given key and value |
+
+A collection of javascript protocol templates can be found [here](https://github.com/projectdiscovery/nuclei-templates/pull/8206).
+
+## Contributing
+
+If you want to add a new module or function to nuclei javascript runtime please open a PR with your changes, refer [Contributing](https://github.com/projectdiscovery/nuclei/blob/7085b47c19dbe3a70b3d4eb23177995bda5c285a/v2/pkg/js/CONTRIBUTE.md) for more details.

docs/template-guide/workflows.mdx

@@ -14,7 +14,7 @@ Workflows can be defined with `workflows` attribute, following the `template` /
 
 ```yaml
 workflows:
-  - template: technologies/template-to-execute.yaml
+  - template: http/technologies/template-to-execute.yaml
 ```
 
 **Type of workflows**
@@ -30,10 +30,10 @@ A workflow that runs all config related templates on the list of give URLs.
 
 ```yaml
 workflows:
-  - template: files/git-config.yaml
-  - template: files/svn-config.yaml
-  - template: files/env-file.yaml
-  - template: files/backup-files.yaml
+  - template: http/exposures/configs/git-config.yaml
+  - template: http/exposures/configs/exposed-svn.yaml
+  - template: http/vulnerabilities/generic/generic-env.yaml
+  - template: http/exposures/backups/zip-backup-files.yaml
   - tags: xss,ssrf,cve,lfi
 ```
 
@@ -41,9 +41,8 @@ A workflow that runs specific list of checks defined for your project.
 
 ```yaml
 workflows:
-  - template: cves/
-  - template: exposed-tokens/
-  - template: exposures/
+  - template: http/cves/
+  - template: http/exposures/
   - tags: exposures
 ```
 ### Conditional Workflows
@@ -56,7 +55,7 @@ A workflow that executes subtemplates when base template gets matched.
 
 ```yaml
 workflows:
-  - template: technologies/jira-detect.yaml
+  - template: http/technologies/jira-detect.yaml
     subtemplates:
       - tags: jira
       - template: exploits/jira/
@@ -68,7 +67,7 @@ A workflow that executes subtemplates when a matcher of base template is found i
 
 ```yaml
 workflows:
-  - template: technologies/tech-detect.yaml
+  - template: http/technologies/tech-detect.yaml
     matchers:
       - name: vbulletin
         subtemplates:
@@ -89,15 +88,15 @@ A workflow showcasing chain of template executions that run only if the previous
 
 ```yaml
 workflows:
-  - template: technologies/tech-detect.yaml
+  - template: http/technologies/tech-detect.yaml
     matchers:
       - name: lotus-domino
         subtemplates:
-          - template: technologies/lotus-domino-version.yaml
+          - template: http/technologies/lotus-domino-version.yaml
             subtemplates:
-              - template: cves/xx-yy-zz.yaml
+              - template: http/cves/2020/xx-yy-zz.yaml
                 subtemplates:
-                  - template: cves/xx-xx-xx.yaml
+                  - template: http/cves/2020/xx-xx-xx.yaml
 ```
 
 Conditional workflows are great examples of performing checks and vulnerability detection in most efficient manner instead of spraying all the templates on all the targets and generally come with good ROI on your time and is gentle for the targets as well.

integration_tests/flow/conditional-flow-negative.yaml

@@ -0,0 +1,27 @@
+id: ghost-blog-detection
+info:
+  name: Ghost blog detection
+  author: pdteam
+  severity: info
+
+
+flow: dns() && http()
+
+dns:
+  - name: "{{FQDN}}"
+    type: CNAME
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"

integration_tests/flow/conditional-flow.yaml

@@ -0,0 +1,27 @@
+id: ghost-blog-detection
+info:
+  name: Ghost blog detection
+  author: pdteam
+  severity: info
+
+
+flow: dns() && http()
+
+dns:
+  - name: "{{FQDN}}"
+    type: CNAME
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"

integration_tests/flow/dns-ns-probe.yaml

@@ -0,0 +1,42 @@
+id: dns-ns-probe
+
+info:
+  name: Nuclei flow dns ns probe
+  author: pdteam
+  severity: info
+  description: Description of the Template
+  reference: https://example-reference-link
+
+flow: |
+  dns("fetch-ns");
+  for(let ns of template["nameservers"]) {
+    set("nameserver",ns);
+    dns("probe-ns");
+  };
+
+dns:
+  - id: "fetch-ns"
+    name: "{{FQDN}}"
+    type: NS
+    matchers:
+      - type: word
+        words:
+          - "IN\tNS"
+    extractors:
+      - type: regex
+        internal: true
+        name: "nameservers"
+        group: 1
+        regex:
+          - "IN\tNS\t(.+)"
+
+  - id: "probe-ns"
+    name: "{{nameserver}}"
+    type: A
+    class: inet
+    retries: 3
+    recursion: true
+    extractors:
+      - type: dsl
+        dsl:
+          - "a"

integration_tests/flow/iterate-values-flow.yaml

@@ -0,0 +1,35 @@
+id: extract-emails
+
+info:
+  name: Extract Email IDs from Response
+  author: pdteam
+  severity: info
+
+
+flow: |
+  http(1)
+  for(let email of template["emails"]) {
+    set("email",email);
+    http(2);
+  }
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    extractors:
+      - type: regex
+        name: emails
+        internal: true
+        regex:
+          - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
+
+  - method: GET
+    path:
+      - "{{BaseURL}}/user/{{base64(email)}}"
+  
+    matchers:
+      - type: word
+        words:
+          - "Welcome"

integration_tests/protocols/code/ps1-snippet.yaml

@@ -0,0 +1,28 @@
+id: ps1-code-snippet
+
+info:
+  name: ps1-code-snippet
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    ps1-code-snippet
+
+code:
+  - engine:
+      - powershell
+      - powershell.exe
+    args:
+      - -ExecutionPolicy
+      - Bypass
+      - -File
+    pattern: "*.ps1"
+    source: |
+      $stdin = [Console]::In
+      $line = $stdin.ReadLine()
+      Write-Host "hello from $line"
+    
+    matchers:
+      - type: word
+        words:
+          - "hello from input"

integration_tests/protocols/code/py-env-var.yaml

@@ -0,0 +1,23 @@
+id: py-code-snippet
+
+info:
+  name: py-code-snippet
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    py-code-snippet
+
+code:
+  - engine:
+      - py
+      - python3
+    source: |
+      import sys,os
+      print("hello from " + sys.stdin.read() + " " + os.getenv('baz'))
+    
+    matchers:
+      - type: word
+        words:
+          - "hello from input baz"
+# digest: 4a0a00473045022100eb01da6b97893e7868c584f330a0cd52df9bddac005860bb8595ba5b8aed58c9022050043feac68d69045cf320cba9298a2eb2e792ea4720d045d01e803de1943e7d:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -0,0 +1,21 @@
+id: py-file
+
+info:
+  name: py-file
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    py-file
+
+code:
+  - engine:
+      - py
+      - python3
+    source: protocols/code/pyfile.py
+    
+    matchers:
+      - type: word
+        words:
+          - "hello from input"
+# digest: 4a0a00473045022100863e07e45c5fa8d808022dfd60679145e17b4ad6c97b493ef28adaf586407dc3022001f2b2d6e565123c0ef51921862352b0b5499b4adfbf5a92af20eb77107c4920:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -0,0 +1,29 @@
+id: testcode
+
+info:
+  name: testcode
+  author: testcode
+  severity: info
+  tags: code
+  description: |
+    testcode
+
+variables:
+  i: "{{interactsh-url}}"
+
+code:
+  - engine:
+      - py
+      - python3
+    # Simulate interactsh interaction
+    source: |
+      import os
+      from urllib.request import urlopen
+      urlopen("http://" + os.getenv('i'))
+
+    matchers:
+      - type: word
+        part: interactsh_protocol
+        words:
+          - "http"
+# digest: 4a0a004730450220785cbdcb0925c922fb34055b3b9277dec165e2f3ba938f5fd7488d400b11a1f5022100dc67027e9e8d6f249c8fc68d61866d636b137bd28e6870a716fbbe969f8b672b:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-nosig.yaml

@@ -0,0 +1,21 @@
+id: py-nosig
+
+info:
+  name: py-nosig
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    Python code without signature
+
+code:
+  - engine:
+      - py
+      - python3
+    source: |
+      print("py unsigned code")
+    
+    matchers:
+      - type: word
+        words:
+          - "py unsigned code"

integration_tests/protocols/code/py-snippet.yaml

@@ -0,0 +1,24 @@
+id: py-code-snippet
+
+info:
+  name: py-code-snippet
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    py-code-snippet
+
+code:
+  - engine:
+      - py
+      - python3
+      - python
+    source: |
+      import sys
+      print("hello from " + sys.stdin.read())
+    
+    matchers:
+      - type: word
+        words:
+          - "hello from input"
+# digest: 4a0a00473045022067a69eb337ffa56d1c8e2cc57b7f74a5eb3294e6f366c9074778b2da3f1d795d02210096d6acda6acd2fe0ff005b08a9c0b72b63f599532ec6493f44b8518265d0e5fd:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/pyfile.py

@@ -0,0 +1,2 @@
+import sys
+print("hello from " + sys.stdin.read())

integration_tests/protocols/code/unsigned.yaml

@@ -0,0 +1,21 @@
+id: unsigned-code-snippet
+
+info:
+  name: unsigned-code-snippet
+  author: pdteam
+  severity: info
+  tags: code
+  description: |
+    unsigned-code-snippet
+
+code:
+  - engine:
+      - py
+      - python3
+    source: |
+      print("unsigned code")
+    
+    matchers:
+      - type: word
+        words:
+          - "unsigned code"

integration_tests/protocols/headless/file-upload.yaml

@@ -15,7 +15,7 @@ headless:
       args:
         by: xpath
         xpath: /html/body/form/input[1]
-        value: headless/file-upload.yaml
+        value: protocols/headless/file-upload.yaml
     - action: sleep
       args:
         duration: 2

integration_tests/protocols/http/matcher-status.yaml

@@ -1,4 +1,4 @@
-id: matchet-status
+id: matcher-status
 
 info:
   name: Test Matcher Status