Merge branch 'dev' into feature-ldap

dev
Sandeep Singh 2021-12-25 12:49:44 +05:30 committed by GitHub
commit 8fbe451d54
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 1470 additions and 238 deletions

View File

@ -7,7 +7,10 @@ on:
jobs:
build:
name: Test Builds
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
uses: actions/setup-go@v2

View File

@ -8,7 +8,10 @@ on:
jobs:
functional:
name: Functional Test
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
uses: actions/setup-go@v2
@ -23,5 +26,5 @@ jobs:
GH_ACTION: true
run: |
chmod +x run.sh
bash run.sh
working-directory: v2/cmd/functional-test
bash run.sh ${{ matrix.os }}
working-directory: v2/cmd/functional-test

View File

@ -247,6 +247,19 @@ Websocket contains the Websocket request to make in the template.
<div class="dd">
<code>whois</code> <i>[]<a href="#whoisrequest">whois.Request</a></i>
</div>
<div class="dt">
WHOIS contains the WHOIS request to make in the template.
</div>
<hr />
<div class="dd">
<code>workflows</code> <i>[]<a href="#workflowsworkflowtemplate">workflows.WorkflowTemplate</a></i>
</div>
@ -284,6 +297,24 @@ Stop execution once first match is found
<hr />
<div class="dd">
<code>signature</code> <i><a href="#httpsignaturetypeholder">http.SignatureTypeHolder</a></i>
</div>
<div class="dt">
Signature is the request signature method
Valid values:
- <code>AWS</code>
</div>
<hr />
@ -1169,6 +1200,24 @@ max-size: 2048
<div class="dd">
<code>signature</code> <i><a href="#signaturetypeholder">SignatureTypeHolder</a></i>
</div>
<div class="dt">
Signature is the request signature method
Valid values:
- <code>AWS</code>
</div>
<hr />
<div class="dd">
<code>cookie-reuse</code> <i>bool</i>
</div>
@ -1320,6 +1369,8 @@ Appears in:
- <code><a href="#websocketrequest">websocket.Request</a>.matchers</code>
- <code><a href="#whoisrequest">whois.Request</a>.matchers</code>
@ -1713,6 +1764,8 @@ Appears in:
- <code><a href="#websocketrequest">websocket.Request</a>.extractors</code>
- <code><a href="#whoisrequest">whois.Request</a>.extractors</code>
@ -2127,6 +2180,20 @@ Enum Values:
## SignatureTypeHolder
SignatureTypeHolder is used to hold internal type of the signature
Appears in:
- <code><a href="#httprequest">http.Request</a>.signature</code>
## dns.Request
Request contains a DNS protocol request to be made from a template
@ -2561,7 +2628,7 @@ extensions:
</div>
<div class="dt">
ExtensionDenylist is the list of file extensions to deny during matching.
DenyList is the list of file, directories or extensions to deny during matching.
By default, it contains some non-interesting extensions that are hardcoded
in nuclei.
@ -3599,6 +3666,106 @@ name: prefix
## whois.Request
Request is a request for the WHOIS protocol
Appears in:
- <code><a href="#template">Template</a>.whois</code>
<hr />
<div class="dd">
<code>matchers</code> <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
</div>
<div class="dt">
Matchers contains the detection mechanism for the request to identify
whether the request was successful by doing pattern matching
on request/responses.
Multiple matchers can be combined with `matcher-condition` flag
which accepts either `and` or `or` as argument.
</div>
<hr />
<div class="dd">
<code>extractors</code> <i>[]<a href="#extractorsextractor">extractors.Extractor</a></i>
</div>
<div class="dt">
Extractors contains the extraction mechanism for the request to identify
and extract parts of the response.
</div>
<hr />
<div class="dd">
<code>matchers-condition</code> <i>string</i>
</div>
<div class="dt">
MatchersCondition is the condition between the matchers. Default is OR.
Valid values:
- <code>and</code>
- <code>or</code>
</div>
<hr />
<div class="dd">
<code>query</code> <i>string</i>
</div>
<div class="dt">
Query contains query for the request
</div>
<hr />
<div class="dd">
<code>server</code> <i>string</i>
</div>
<div class="dt">
description: |
Optional WHOIS server URL.
If present, specifies the WHOIS server to execute the Request on.
Otherwise, nil enables bootstrapping
</div>
<hr />
## workflows.WorkflowTemplate
Appears in:
@ -3730,3 +3897,17 @@ Subtemplates are run if the name of matcher matches.
## http.SignatureTypeHolder
SignatureTypeHolder is used to hold internal type of the signature
Appears in:
- <code><a href="#template">Template</a>.signature</code>

View File

@ -9,7 +9,7 @@ self-contained: true
requests:
- raw:
- |
GET http://localhost:5431/ HTTP/1.1
GET http://127.0.0.1:5431/ HTTP/1.1
Host: {{Hostname}}
matchers:

View File

@ -8,7 +8,7 @@ info:
self-contained: true
network:
- host:
- "localhost:5431"
- "127.0.0.1:5431"
matchers:
- type: word

View File

@ -0,0 +1,14 @@
id: basic-whois-example
info:
name: test template for WHOIS
author: pdteam
severity: info
whois:
- query: "{{Host}}"
extractors:
- type: kval
kval:
- "expiration date"
- "registrar"

View File

@ -472,8 +472,8 @@
"type": "string"
},
"type": "array",
"title": "extensions to deny match",
"description": "List of file extensions to deny during matching"
"title": "denylist",
"description": "List of files"
},
"id": {
"type": "string",
@ -741,6 +741,12 @@
"title": "maximum http response body size",
"description": "Maximum size of http response body to read in bytes"
},
"signature": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/http.SignatureTypeHolder",
"title": "signature is the http request signature method",
"description": "Signature is the HTTP Request signature Method"
},
"cookie-reuse": {
"type": "boolean",
"title": "optional cookie reuse enable",
@ -790,6 +796,14 @@
"additionalProperties": false,
"type": "object"
},
"http.SignatureTypeHolder": {
"enum": [
"AWS"
],
"type": "string",
"title": "type of the signature",
"description": "Type of the signature"
},
"network.Input": {
"properties": {
"data": {
@ -1026,6 +1040,47 @@
"additionalProperties": false,
"type": "object"
},
"whois.Request": {
"properties": {
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
},
"type": "array",
"title": "matchers to run on response",
"description": "Detection mechanism to identify whether the request was successful by doing pattern matching"
},
"extractors": {
"items": {
"$ref": "#/definitions/extractors.Extractor"
},
"type": "array",
"title": "extractors to run on response",
"description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response"
},
"matchers-condition": {
"enum": [
"and",
"or"
],
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
},
"query": {
"type": "string",
"title": "query for the WHOIS request",
"description": "Query contains query for the request"
},
"server": {
"type": "string",
"title": "server url to execute the WHOIS request on",
"description": "Server contains the server url to execute the WHOIS request on"
}
},
"additionalProperties": false,
"type": "object"
},
"templates.Template": {
"required": [
"id",
@ -1110,6 +1165,15 @@
"title": "websocket requests to make",
"description": "Websocket requests to make for the template"
},
"whois": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/whois.Request"
},
"type": "array",
"title": "whois requests to make",
"description": "WHOIS requests to make for the template"
},
"workflows": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
@ -1128,6 +1192,11 @@
"type": "boolean",
"title": "stop at first match",
"description": "Stop at first match for the template"
},
"signature": {
"$ref": "#/definitions/http.SignatureTypeHolder",
"title": "signature is the http request signature method",
"description": "Signature is the HTTP Request signature Method"
}
},
"additionalProperties": false,

View File

@ -71,7 +71,7 @@ func getCVEData(client *nvd.Client, filePath, data string) {
cveName := matches[0][1]
severityMatches := severityRegex.FindAllStringSubmatch(data, 1)
if len(matches) == 0 {
if len(severityMatches) == 0 {
return
}
severityValue := severityMatches[0][1]

17
v2/cmd/functional-test/run.sh Normal file → Executable file
View File

@ -1,16 +1,23 @@
#!/bin/bash
# reading os type from arguments
CURRENT_OS=$1
if [ "${CURRENT_OS}" == "windows-latest" ];then
extension=.exe
fi
echo "::group::Building functional-test binary"
go build
go build -o functional-test$extension
echo "::endgroup::"
echo "::group::Building Nuclei binary from current branch"
go build -o nuclei_dev ../nuclei
go build -o nuclei_dev$extension ../nuclei
echo "::endgroup::"
echo "::group::Installing latest release of nuclei"
GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
echo "::group::Building latest release of nuclei"
go build -o nuclei$extension -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
echo "::endgroup::"
echo 'Starting Nuclei functional test'
./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt
./functional-test$extension -main ./nuclei$extension -dev ./nuclei_dev$extension -testcases testcases.txt

View File

@ -26,6 +26,7 @@ var (
"loader": loaderTestcases,
"websocket": websocketTestCases,
"headless": headlessTestcases,
"whois": whoisTestCases,
}
)

View File

@ -0,0 +1,20 @@
package main
import (
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var whoisTestCases = map[string]testutils.TestCase{
"whois/basic.yaml": &whoisBasic{},
}
type whoisBasic struct{}
// Execute executes a test case and returns an error if occurred
func (h *whoisBasic) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://example.com", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}

View File

@ -1,61 +1,55 @@
# to specify which severities should be reported
#allow-list:
# severity: critical, high
# to specify which severities should be excluded from reporting
# severity: high, critical
#deny-list:
# severity: info, low, medium
# severity: low
#
# GitHub contains configuration options for GitHub issue tracker
#GitHub:
# # base-url (optional) is the self-hosted GitHub application url
# base-url: ""
#github:
# # base-url is the optional self-hosted GitHub application url
# base-url: https://localhost:8443/github
# # username is the username of the GitHub user
# username: ""
# # owner is the owner name of the repository for issues.
# owner: ""
# # token is the token for GitHub account.
# token: ""
# # project-name is the name of the repository.
# project-name: ""
# # issue-label (optional) is the label of the created issue type
# issue-label: ""
# # severity-as-label (optional) sets the severity as the label of the created issue type
# severity-as-label: false
# GitLab contains configuration options for GitLab issue tracker
#GitLab:
# # base-url (optional) is the self-hosted GitLab application url
# base-url: ""
# username: test-username
# # owner is the owner name of the repository for issues
# owner: test-owner
# # token is the token for GitHub account
# token: test-token
# # project-name is the name of the repository
# project-name: test-project
# # issue-label is the label of the created issue type
# issue-label: bug
#
# GitLab contains configuration options for gitlab issue tracker
#gitlab:
# # base-url is the optional self-hosted GitLab application url
# base-url: https://localhost:8443/gitlab
# # username is the username of the GitLab user
# username: ""
# # token is the token for GitLab account.
# token: ""
# # project-id is the ID of the repository.
# project-id: ""
# # issue-label (optional) is the label of the created issue type
# issue-label: ""
# # severity-as-label (optional) sets the severity as the label of the created issue type
# severity-as-label: false
# username: test-username
# # token is the token for GitLab account
# token: test-token
# # project-name is the name/id of the project(repository)
# project-name: "1234"
# # issue-label is the label of the created issue type
# issue-label: bug
#
# Jira contains configuration options for Jira issue tracker
#Jira:
# # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
#jira:
# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
# cloud: true
# # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created
# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
# update-existing: false
# # URL is the Jira application URL
# url: ""
# # URL is the jira application url
# url: https://localhost/jira
# # account-id is the account-id of the Jira user or username in case of on-prem Jira
# account-id: ""
# account-id: test-account-id
# # email is the email of the user for Jira instance
# email: ""
# email: test@test.com
# # token is the token for Jira instance or password in case of on-prem Jira
# token: ""
# token: test-token
# # project-name is the name of the project.
# project-name: ""
# project-name: test-project-name
# # issue-type is the name of the created issue type
# issue-type: ""
# issue-type: bug
#
# elasticsearch contains configuration options for elasticsearch exporter
#elasticsearch:
# # IP for elasticsearch instance
@ -64,11 +58,11 @@
# port: 9200
# # IndexName is the name of the elasticsearch index
# index-name: nuclei
# # SSL (optional) enables ssl for elasticsearch connection
# # SSL enables ssl for elasticsearch connection
# ssl: false
# # SSLVerification (optional) disables SSL verification for elasticsearch
# # SSLVerification disables SSL verification for elasticsearch
# ssl-verification: false
# # Username for the elasticsearch instance
# username: test
# # Password is the password for elasticsearch instance
# password: test
# password: test

View File

@ -3,7 +3,6 @@ module github.com/projectdiscovery/nuclei/v2
go 1.17
require (
github.com/Ice3man543/nvd v1.0.8
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
github.com/andygrunwald/go-jira v1.14.0
@ -47,7 +46,6 @@ require (
github.com/shirou/gopsutil/v3 v3.21.9
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.4.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.0
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
github.com/valyala/fasttemplate v1.2.1
@ -58,21 +56,31 @@ require (
go.uber.org/atomic v1.9.0
go.uber.org/multierr v1.7.0
go.uber.org/ratelimit v0.2.0
golang.org/x/net v0.0.0-20211020060615-d418f374d309
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
golang.org/x/text v0.3.7
gopkg.in/yaml.v2 v2.4.0
moul.io/http2curl v1.0.0
)
require github.com/aws/aws-sdk-go v1.42.3
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
require (
github.com/Ice3man543/nvd v1.0.8
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
github.com/stretchr/testify v1.7.0
)
require (
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
github.com/Mzack9999/ldapserver v1.0.2-0.20211214172138-8f1cdd128383 // indirect
github.com/PuerkitoBio/goquery v1.6.0 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
github.com/andybalholm/cascadia v1.1.0 // indirect
github.com/antchfx/xpath v1.2.0 // indirect
@ -103,6 +111,7 @@ require (
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
github.com/itchyny/timefmt-go v0.1.3 // indirect
github.com/jasonlvhit/gocron v0.0.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
@ -110,6 +119,7 @@ require (
github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
@ -133,6 +143,7 @@ require (
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

View File

@ -73,9 +73,12 @@ github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA=
github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
@ -104,6 +107,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.42.3 h1:lBKr3tQ06m1uykiychMNKLK1bRfOzaIEQpsI/S3QiNc=
github.com/aws/aws-sdk-go v1.42.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
@ -404,10 +409,16 @@ github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0=
github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@ -508,6 +519,7 @@ github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tB
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
@ -558,6 +570,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7
github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8=
github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 h1:3Xn/CN6GVY+7mVuGgt5bfp0F9JwcWqnvwfb23Jf8Vxg=
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7/go.mod h1:inRbqVxN7ri77yTJY3ZtGtKegIFa3Qnarh7Xp9P7LgY=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@ -865,8 +879,9 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -957,8 +972,8 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1221,6 +1236,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -1246,6 +1262,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

View File

@ -29,6 +29,10 @@ func ParseOptions(options *types.Options) {
// Show the user the banner
showBanner()
if !filepath.IsAbs(options.TemplatesDirectory) {
cwd, _ := os.Getwd()
options.TemplatesDirectory = filepath.Join(cwd, options.TemplatesDirectory)
}
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", config.Version)
os.Exit(0)

View File

@ -73,7 +73,7 @@ func New(options *types.Options) (*Runner, error) {
options.NoUpdateTemplates = true
}
if err := runner.updateTemplates(); err != nil {
gologger.Warning().Msgf("Could not update templates: %s\n", err)
gologger.Error().Msgf("Could not update templates: %s\n", err)
}
if options.Headless {
if engine.MustDisableSandbox() {

View File

@ -119,7 +119,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
if err := config.WriteConfiguration(r.templatesConfig); err != nil {
return err
}
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String())
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory)
return nil
}
@ -135,13 +135,13 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
return config.WriteConfiguration(r.templatesConfig)
}
if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil {
if err := r.updateTemplatesWithVersion(latestVersion, currentVersion, r, ctx); err != nil {
return err
}
return nil
}
func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
func (r *Runner) updateTemplatesWithVersion(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
if latestVersion.GT(currentVersion) {
gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String())
gologger.Info().Msgf("Downloading latest release...")
@ -163,7 +163,7 @@ func updateTemplates(latestVersion semver.Version, currentVersion semver.Version
if err := config.WriteConfiguration(runner.templatesConfig); err != nil {
return err
}
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String())
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s) to %s. GoodLuck!\n", latestVersion.String(), r.templatesConfig.TemplatesDirectory)
}
return nil
}
@ -200,10 +200,6 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
return readErr
}
r.templatesConfig = configuration
if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") {
r.options.TemplatesDirectory = configuration.TemplatesDirectory
}
}
return nil
}

View File

@ -119,7 +119,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions")
}
func TestCalculateTemplateAbsolutePath(t *testing.T) {
func TestCalculateTemplateAbsolutePathPositiveScenario(t *testing.T) {
configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates")
defer os.RemoveAll(configuredTemplateDirectory)
@ -136,24 +136,6 @@ func TestCalculateTemplateAbsolutePath(t *testing.T) {
require.False(t, skipFile)
}
})
t.Run("negative scenarios", func(t *testing.T) {
filePathsFromZip := []string{
"./../nuclei-templates/../cve/test.yaml",
"nuclei-templates/../cve/test.yaml",
"nuclei-templates/cve/../test.yaml",
"nuclei-templates/././../cve/test.yaml",
"nuclei-templates/.././../cve/test.yaml",
"nuclei-templates/.././../cve/../test.yaml",
}
for _, filePathFromZip := range filePathsFromZip {
calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory)
require.Nil(t, err)
require.True(t, skipFile)
require.Equal(t, "", calculatedTemplateAbsPath)
}
})
}
func zipFromDirectory(zipPath, directory string) error {

View File

@ -0,0 +1,34 @@
//go:build !windows
package runner
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestCalculateTemplateAbsolutePathNegativeScenario(t *testing.T) {
configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates")
defer os.RemoveAll(configuredTemplateDirectory)
t.Run("negative scenarios", func(t *testing.T) {
filePathsFromZip := []string{
"./../nuclei-templates/../cve/test.yaml",
"nuclei-templates/../cve/test.yaml",
"nuclei-templates/cve/../test.yaml",
"nuclei-templates/././../cve/test.yaml",
"nuclei-templates/.././../cve/test.yaml",
"nuclei-templates/.././../cve/../test.yaml",
}
for _, filePathFromZip := range filePathsFromZip {
calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory)
require.Nil(t, err)
require.True(t, skipFile)
require.Equal(t, "", calculatedTemplateAbsPath)
}
})
}

View File

@ -26,7 +26,7 @@ type Config struct {
const nucleiConfigFilename = ".templates-config.json"
// Version is the current version of nuclei
const Version = `2.5.6-dev`
const Version = `2.5.8-dev`
func getConfigDetails() (string, error) {
homeDir, err := os.UserHomeDir()

View File

@ -154,7 +154,7 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
// MatchDSL matches on a generic map result
func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
logExpressionEvaluationFailure := func (matcherName string, err error) {
logExpressionEvaluationFailure := func(matcherName string, err error) {
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error())
}

View File

@ -1,12 +1,20 @@
package projectfile
import (
"fmt"
"net/http"
"regexp"
"github.com/pkg/errors"
"github.com/projectdiscovery/hmap/store/hybrid"
)
var (
ErrNotFound = errors.New("not found")
regexUserAgent = regexp.MustCompile(`(?mi)\r\nUser-Agent: .+\r\n`)
regexDefaultInteract = regexp.MustCompile(`(?mi)[a-zA-Z1-9%.]+interact.sh`)
)
type Options struct {
Path string
Cleanup bool
@ -31,15 +39,22 @@ func New(options *Options) (*ProjectFile, error) {
return &p, nil
}
func (pf *ProjectFile) cleanupData(data []byte) []byte {
// ignore all user agents
data = regexUserAgent.ReplaceAll(data, []byte("\r\n"))
// ignore interact markers
return regexDefaultInteract.ReplaceAll(data, []byte(""))
}
func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
reqHash, err := hash(req)
reqHash, err := hash(pf.cleanupData(req))
if err != nil {
return nil, err
}
data, ok := pf.hm.Get(reqHash)
if !ok {
return nil, fmt.Errorf("not found")
return nil, ErrNotFound
}
var httpRecord HTTPRecord
@ -52,7 +67,7 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
}
func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error {
reqHash, err := hash(req)
reqHash, err := hash(pf.cleanupData(req))
if err != nil {
return err
}

View File

@ -16,49 +16,66 @@ func ContainsUnresolvedVariables(items ...string) error {
if len(matches) == 0 {
return nil
}
errorString := &strings.Builder{}
errorString.WriteString("unresolved variables found: ")
for i, match := range matches {
var unresolvedVariables []string
for _, match := range matches {
if len(match) < 2 {
continue
}
errorString.WriteString(match[1])
if i != len(matches)-1 {
errorString.WriteString(",")
}
unresolvedVariables = append(unresolvedVariables, match[1])
}
errorMessage := errorString.String()
return errors.New(errorMessage)
return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ","))
}
return nil
}
// ContainsVariablesWithNames returns an error with variable names if the passed
// input contains unresolved {{<pattern-here>}} variables within the provided list
func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error {
for _, data := range items {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
errorString := &strings.Builder{}
errorString.WriteString("unresolved variables with values found: ")
for i, match := range matches {
var unresolvedVariables []string
for _, match := range matches {
if len(match) < 2 {
continue
}
matchName := match[1]
if _, ok := names[matchName]; !ok {
errorString.WriteString(matchName)
if i != len(matches)-1 {
errorString.WriteString(",")
}
unresolvedVariables = append(unresolvedVariables, matchName)
}
}
errorMessage := errorString.String()
return errors.New(errorMessage)
return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
}
return nil
}
// ContainsVariablesWithIgnoreList returns an error with variable names if the passed
// input contains unresolved {{<pattern-here>}} other than the ones listed in the ignore list
func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...string) error {
var unresolvedVariables []string
for _, data := range items {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
for _, match := range matches {
if len(match) < 2 {
continue
}
matchName := match[1]
if _, ok := skipNames[matchName]; ok {
continue
}
unresolvedVariables = append(unresolvedVariables, matchName)
}
}
if len(unresolvedVariables) > 0 {
return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
}
return nil
}

View File

@ -165,6 +165,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false)
if !matched || result == nil {
return false // if we don't match, return

View File

@ -6,6 +6,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -23,6 +24,9 @@ func Init(options *types.Options) error {
if err := httpclientpool.Init(options); err != nil {
return err
}
if err := signerpool.Init(options); err != nil {
return err
}
return networkclientpool.Init(options)
}

View File

@ -1,6 +1,7 @@
package file
import (
"path/filepath"
"strings"
"github.com/pkg/errors"
@ -19,13 +20,13 @@ type Request struct {
// - value: '[]string{".txt", ".go", ".json"}'
Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"`
// description: |
// ExtensionDenylist is the list of file extensions to deny during matching.
// DenyList is the list of file, directories or extensions to deny during matching.
//
// By default, it contains some non-interesting extensions that are hardcoded
// in nuclei.
// examples:
// - value: '[]string{".avi", ".mov", ".mp3"}'
ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"`
DenyList []string `yaml:"denylist,omitempty" jsonschema:"title=denylist, directories and extentions to deny match,description=List of files, directories and extensions to deny during matching"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"`
@ -41,9 +42,9 @@ type Request struct {
CompiledOperators *operators.Operators `yaml:"-"`
// cache any variables that may be needed for operation.
options *protocols.ExecuterOptions
extensions map[string]struct{}
extensionDenylist map[string]struct{}
options *protocols.ExecuterOptions
extensions map[string]struct{}
denyList map[string]struct{}
// description: |
// NoRecursive specifies whether to not do recursive checks if folders are provided.
@ -89,7 +90,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
request.options = options
request.extensions = make(map[string]struct{})
request.extensionDenylist = make(map[string]struct{})
request.denyList = make(map[string]struct{})
for _, extension := range request.Extensions {
if extension == "all" {
@ -101,17 +102,17 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
request.extensions[extension] = struct{}{}
}
}
for _, extension := range defaultDenylist {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
// process default denylist (extensions)
for _, excludeItem := range defaultDenylist {
if !strings.HasPrefix(excludeItem, ".") {
excludeItem = "." + excludeItem
}
request.extensionDenylist[extension] = struct{}{}
request.denyList[excludeItem] = struct{}{}
}
for _, extension := range request.ExtensionDenylist {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
}
request.extensionDenylist[extension] = struct{}{}
for _, excludeItem := range request.DenyList {
request.denyList[excludeItem] = struct{}{}
// also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d)
request.denyList[filepath.Clean(excludeItem)] = struct{}{}
}
return nil
}

View File

@ -16,11 +16,11 @@ func TestFileCompile(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -29,7 +29,7 @@ func TestFileCompile(t *testing.T) {
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist")
require.Contains(t, request.denyList, ".go", "could not get .go in denylist")
require.NotContains(t, request.extensions, ".go", "could get .go in allowlist")
require.True(t, request.allExtensions, "could not get correct allExtensions")
}

View File

@ -7,7 +7,7 @@ import (
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
"github.com/projectdiscovery/folderutil"
"github.com/projectdiscovery/gologger"
)
@ -51,7 +51,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string
return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
for _, match := range matches {
if !request.validatePath(match) {
if !request.validatePath(absPath, match) {
continue
}
if _, ok := processed[match]; !ok {
@ -73,7 +73,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str
return false, nil
}
if _, ok := processed[absPath]; !ok {
if !request.validatePath(absPath) {
if !request.validatePath(absPath, absPath) {
return false, nil
}
processed[absPath] = struct{}{}
@ -93,7 +93,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
if d.IsDir() {
return nil
}
if !request.validatePath(path) {
if !request.validatePath(absPath, path) {
return nil
}
if _, ok := processed[path]; !ok {
@ -107,7 +107,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin
}
// validatePath validates a file path for blacklist and whitelist options
func (request *Request) validatePath(item string) bool {
func (request *Request) validatePath(absPath, item string) bool {
extension := filepath.Ext(item)
if len(request.extensions) > 0 {
@ -117,9 +117,81 @@ func (request *Request) validatePath(item string) bool {
return false
}
}
if _, ok := request.extensionDenylist[extension]; ok {
gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)
if matchingRule, ok := request.isInDenyList(absPath, item); ok {
gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule)
return false
}
return true
}
func (request *Request) isInDenyList(absPath, item string) (string, bool) {
extension := filepath.Ext(item)
// check for possible deny rules
// - extension is in deny list
if _, ok := request.denyList[extension]; ok {
return extension, true
}
// - full path is in deny list
if _, ok := request.denyList[item]; ok {
return item, true
}
// file is in a forbidden subdirectory
filename := filepath.Base(item)
fullPathWithoutFilename := strings.TrimSuffix(item, filename)
relativePathWithFilename := strings.TrimPrefix(item, absPath)
relativePath := strings.TrimSuffix(relativePathWithFilename, filename)
// - filename is in deny list
if _, ok := request.denyList[filename]; ok {
return filename, true
}
// - relative path is in deny list
if _, ok := request.denyList[relativePath]; ok {
return relativePath, true
}
// relative path + filename are in the forbidden list
if _, ok := request.denyList[relativePathWithFilename]; ok {
return relativePathWithFilename, true
}
// root path + relative path are in the forbidden list
if _, ok := request.denyList[fullPathWithoutFilename]; ok {
return fullPathWithoutFilename, true
}
// check any progressive combined part of the relative and absolute path with filename for matches within rules prefixes
if pathTreeItem, ok := request.isAnyChunkInDenyList(relativePath, false); ok {
return pathTreeItem, true
}
if pathTreeItem, ok := request.isAnyChunkInDenyList(item, true); ok {
return pathTreeItem, true
}
return "", false
}
func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) {
var paths []string
if splitWithUtils {
pathInfo, _ := folderutil.NewPathInfo(path)
paths, _ = pathInfo.Paths()
} else {
pathTree := strings.Split(path, string(os.PathSeparator))
for i := range pathTree {
paths = append(paths, filepath.Join(pathTree[:i]...))
}
}
for _, pathTreeItem := range paths {
if _, ok := request.denyList[pathTreeItem]; ok {
return pathTreeItem, true
}
}
return "", false
}

View File

@ -19,11 +19,11 @@ func TestFindInputPaths(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,

View File

@ -20,11 +20,11 @@ func TestResponseToDSLMap(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -45,11 +45,11 @@ func TestFileOperatorMatch(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -133,11 +133,11 @@ func TestFileOperatorExtract(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -240,11 +240,11 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
DenyList: []string{".go"},
Operators: operators.Operators{
MatchersCondition: matcherCondition,
Matchers: matchers,

View File

@ -23,11 +23,11 @@ func TestFileExecuteWithResults(t *testing.T) {
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all"},
ExtensionDenylist: []string{".go"},
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"all"},
DenyList: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",

View File

@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -121,13 +122,32 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam
if len(parts) < 3 {
return nil, fmt.Errorf("malformed request supplied")
}
payloads := generators.BuildPayloadFromOptions(r.request.options.Options)
// in case cases (eg requests signing, some variables uses default values if missing)
if defaultList := GetVariablesDefault(r.request.Signature.Value); defaultList != nil {
payloads = generators.MergeMaps(defaultList, payloads)
}
parts[1] = replacer.Replace(parts[1], payloads)
// the url might contain placeholders with ignore list
if ignoreList := GetVariablesNamesSkipList(r.request.Signature.Value); ignoreList != nil {
if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil {
return nil, err
}
} else { // the url might contain placeholders
if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
return nil, err
}
}
parsed, err := url.Parse(parts[1])
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
}
values := generators.MergeMaps(
generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
generators.BuildPayloadFromOptions(r.request.options.Options),
payloads,
)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)

View File

@ -128,6 +128,12 @@ type Request struct {
// SelfContained specifies if the request is self-contained.
SelfContained bool `yaml:"-" json:"-"`
// description: |
// Signature is the request signature method
// values:
// - "AWS"
Signature SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
// description: |
// CookieReuse is an optional setting that enables cookie reuse for
// all requests defined in raw section.

View File

@ -8,7 +8,7 @@ import (
"io"
"io/ioutil"
"net/url"
"path/filepath"
"path"
"strings"
"github.com/projectdiscovery/rawhttp/client"
@ -30,7 +30,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
rawRequest := &Request{
Headers: make(map[string]string),
}
parsedURL, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("could not parse request URL: %w", err)
@ -143,7 +143,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
}
func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) []byte {
fixedPath := filepath.Join(baseURL.Path, requestPath)
fixedPath := path.Join(baseURL.Path, requestPath)
fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1)
return fixed
}

View File

@ -29,7 +29,10 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/stringsutil"
)
@ -336,24 +339,27 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
err error
)
// Dump request for variables checks
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
if !generatedRequest.original.Race {
var dumpError error
// TODO: dump is currently not working with post-processors - somehow it alters the signature
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
if dumpError != nil {
return dumpError
}
dumpedRequestString := string(dumpedRequest)
// Check if are there any unresolved variables. If yes, skip unless overridden by user.
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
return errStopExecution
}
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", dumpedRequestString)
if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
return errStopExecution
}
} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
return errStopExecution
}
}
}
var formedURL string
@ -391,9 +397,27 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
}
}
if resp == nil {
if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
return errSignature
}
resp, err = request.httpClient.Do(generatedRequest.request)
}
}
// Dump the requests containing all headers
if !generatedRequest.original.Race {
var dumpError error
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
if dumpError != nil {
return dumpError
}
dumpedRequestString := string(dumpedRequest)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", dumpedRequestString)
}
}
if err != nil {
// rawhttp doesn't support draining response bodies.
if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil {
@ -533,6 +557,40 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
return nil
}
// handleSignature of the http request
func (request *Request) handleSignature(generatedRequest *generatedRequest) error {
switch request.Signature.Value {
case AWSSignature:
var awsSigner signer.Signer
payloads := request.options.Options.Vars.AsMap()
awsAccessKeyId := types.ToString(payloads["aws-id"])
awsSecretAccessKey := types.ToString(payloads["aws-secret"])
awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey}
service := types.ToString(payloads["service"])
region := types.ToString(payloads["region"])
// if region is empty use default value
if region == "" {
region = types.ToString(signer.AwsDefaultVars["region"])
}
awsSignatureArguments := signer.AwsSignatureArguments{
Service: types.ToString(service),
Region: types.ToString(region),
Time: time.Now(),
}
awsSigner, err := signerpool.Get(request.options.Options, &signerpool.Configuration{SignerArgs: awsSignerArgs})
if err != nil {
return err
}
err = awsSigner.SignHTTP(generatedRequest.request.Request, awsSignatureArguments)
if err != nil {
return err
}
}
return nil
}
// setCustomHeaders sets the custom headers for generated request
func (request *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range request.customHeaders {

View File

@ -0,0 +1,107 @@
package http
import (
"encoding/json"
"github.com/alecthomas/jsonschema"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
)
// SignatureType is the type of signature
type SignatureType int
// Supported values for the SignatureType
const (
AWSSignature SignatureType = iota + 1
signatureLimit
)
// signatureTypeMappings is a table for conversion of signature type from string.
var signatureTypeMappings = map[SignatureType]string{
AWSSignature: "AWS",
}
func GetSupportedSignaturesTypes() []SignatureType {
var result []SignatureType
for index := SignatureType(1); index < signatureLimit; index++ {
result = append(result, index)
}
return result
}
func toSignatureType(valueToMap string) (SignatureType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range signatureTypeMappings {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("invalid signature type: " + valueToMap)
}
func (t SignatureType) String() string {
return signatureTypeMappings[t]
}
// SignatureTypeHolder is used to hold internal type of the signature
type SignatureTypeHolder struct {
Value SignatureType
}
func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "type of the signature",
Description: "Type of the signature",
}
for _, types := range GetSupportedSignaturesTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *SignatureTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toSignatureType(marshalledTypes)
if err != nil {
return err
}
holder.Value = computedType
return nil
}
func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.Value.String())
}
func (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) {
return holder.Value.String(), nil
}
var ErrNoIgnoreList = errors.New("uknown signature types")
// GetVariablesNamesSkipList depending on the signature type
func GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {
switch signature {
case AWSSignature:
return signer.AwsSkipList
default:
return nil
}
}
// GetVariablesNamesSkipList depending on the signature type
func GetVariablesDefault(signature SignatureType) map[string]interface{} {
switch signature {
case AWSSignature:
return signer.AwsDefaultVars
default:
return nil
}
}

View File

@ -0,0 +1,140 @@
package signer
import (
"bytes"
"context"
"errors"
"io/ioutil"
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws/credentials"
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
)
type AwsSigner struct {
creds *credentials.Credentials
signer *v4.Signer
}
type AwsSignerArgs struct {
AwsId string
AwsSecretToken string
}
func (awsSignerArgs AwsSignerArgs) Validate() error {
if awsSignerArgs.AwsId == "" {
return errors.New("empty id")
}
if awsSignerArgs.AwsSecretToken == "" {
return errors.New("empty token")
}
return nil
}
type AwsSignatureArguments struct {
Service string
Region string
Time time.Time
}
func (awsSignatureArguments AwsSignatureArguments) Validate() error {
if awsSignatureArguments.Region == "" {
return errors.New("empty region")
}
if awsSignatureArguments.Service == "" {
return errors.New("empty service")
}
return nil
}
func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) {
if err := args.Validate(); err != nil {
return nil, err
}
creds := credentials.NewStaticCredentials(args.AwsId, args.AwsSecretToken, "")
if creds == nil {
return nil, errors.New("couldn't create the credentials structure")
}
signer := v4.NewSigner(creds)
return &AwsSigner{creds: creds, signer: signer}, nil
}
func NewAwsSignerFromEnv() (*AwsSigner, error) {
creds := credentials.NewEnvCredentials()
if creds == nil {
return nil, errors.New("couldn't create the credentials structure")
}
signer := v4.NewSigner(creds)
return &AwsSigner{creds: creds, signer: signer}, nil
}
func NewAwsSignerFromFile() (*AwsSigner, error) {
creds := credentials.NewSharedCredentials("", "")
if creds == nil {
return nil, errors.New("couldn't create the credentials structure")
}
signer := v4.NewSigner(creds)
return &AwsSigner{creds: creds, signer: signer}, nil
}
func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) error {
signatureArgs, err := awsSigner.checkSignatureArgs(args)
if err != nil {
return err
}
awsSigner.prepareRequest(request)
var body *bytes.Reader
if request.Body != nil {
bodyBytes, err := ioutil.ReadAll(request.Body)
if err != nil {
return err
}
request.Body.Close()
body = bytes.NewReader(bodyBytes)
}
if _, err := awsSigner.signer.Sign(request, body, signatureArgs.Service, signatureArgs.Region, signatureArgs.Time); err != nil {
return err
}
return nil
}
func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error) {
signatureArgs, err := awsSigner.checkSignatureArgs(args)
if err != nil {
return nil, err
}
reqClone := request.Clone(context.Background())
awsSigner.prepareRequest(reqClone)
err = awsSigner.SignHTTP(reqClone, signatureArgs)
if err != nil {
return nil, err
}
headers := make(map[string]string)
headers["X-Amz-Date"] = reqClone.Header.Get("X-Amz-Date")
headers["Authorization"] = reqClone.Header.Get("Authorization")
return headers, nil
}
func (awsSigner *AwsSigner) checkSignatureArgs(args interface{}) (AwsSignatureArguments, error) {
if signatureArgs, ok := args.(AwsSignatureArguments); ok {
return signatureArgs, signatureArgs.Validate()
}
return AwsSignatureArguments{}, errors.New("wrong signature type")
}
func (awsSigner *AwsSigner) prepareRequest(request *http.Request) {
request.Header.Del("Host")
}
var AwsSkipList = map[string]interface{}{
"region": struct{}{},
}
var AwsDefaultVars = map[string]interface{}{
"region": "us-east-2",
}

View File

@ -0,0 +1,40 @@
package signer
import (
"errors"
"net/http"
)
type Signer interface {
SignHTTP(request *http.Request, args interface{}) error
CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error)
}
type SignerArgs interface {
Validate() error
}
type SignatureArguments interface {
Validate() error
}
func NewSigner(args SignerArgs) (signer Signer, err error) {
switch signerArgs := args.(type) {
case AwsSignerArgs:
awsSigner, err := NewAwsSigner(signerArgs)
if err != nil {
// $HOME/.aws/credentials
awsSigner, err = NewAwsSignerFromFile()
if err != nil {
// env variables
awsSigner, err = NewAwsSignerFromEnv()
if err != nil {
return nil, err
}
}
}
return awsSigner, err
default:
return nil, errors.New("unknown signature arguments type")
}
}

View File

@ -0,0 +1,57 @@
package signerpool
import (
"fmt"
"strings"
"sync"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
var (
poolMutex *sync.RWMutex
clientPool map[string]signer.Signer
)
// Init initializes the clientpool implementation
func Init(options *types.Options) error {
poolMutex = &sync.RWMutex{}
clientPool = make(map[string]signer.Signer)
return nil
}
// Configuration contains the custom configuration options for a client
type Configuration struct {
SignerArgs signer.SignerArgs
}
// Hash returns the hash of the configuration to allow client pooling
func (c *Configuration) Hash() string {
builder := &strings.Builder{}
builder.WriteString(fmt.Sprintf("%v", c.SignerArgs))
hash := builder.String()
return hash
}
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (signer.Signer, error) {
hash := configuration.Hash()
poolMutex.RLock()
if client, ok := clientPool[hash]; ok {
poolMutex.RUnlock()
return client, nil
}
poolMutex.RUnlock()
client, err := signer.NewSigner(configuration.SignerArgs)
if err != nil {
return nil, err
}
poolMutex.Lock()
clientPool[hash] = client
poolMutex.Unlock()
return client, nil
}

View File

@ -0,0 +1,193 @@
package whois
import (
"net/url"
"strings"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/openrdap/rdap"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Request is a request for the WHOIS protocol
type Request struct {
// Operators for the current request go here.
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
// description: |
// Query contains query for the request
Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"`
// description: |
// Optional WHOIS server URL.
//
// If present, specifies the WHOIS server to execute the Request on.
// Otherwise, nil enables bootstrapping
Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"`
// cache any variables that may be needed for operation.
client *rdap.Client
options *protocols.ExecuterOptions
parsedServerURL *url.URL
}
// Compile compiles the request generators preparing any requests possible.
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
var err error
if request.Server != "" {
request.parsedServerURL, err = url.Parse(request.Server)
if err != nil {
return errors.Wrap(err, "failed to parse server URL")
}
}
request.options = options
request.client = &rdap.Client{}
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
request.CompiledOperators = compiled
}
return nil
}
// Requests returns the total number of requests the rule will perform
func (request *Request) Requests() int {
return 1
}
// GetID returns the ID for the request if any.
func (request *Request) GetID() string {
return ""
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// generate variables
variables := generateVariables(input)
// and replace placeholders
query := replacer.Replace(request.Query, variables)
// build an rdap request
rdapReq := rdap.NewAutoRequest(query)
rdapReq.Server = request.parsedServerURL
res, err := request.client.Do(rdapReq)
if err != nil {
return errors.Wrap(err, "could not make whois request")
}
gologger.Verbose().Msgf("Sent WHOIS request to %s", query)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Msgf("[%s] Dumped WHOIS request for %s", request.options.TemplateID, query)
}
data := make(map[string]interface{})
var response interface{}
switch rdapReq.Type {
case rdap.DomainRequest:
// convert the rdap response to a whois style response (for domain request type only)
whoisResp := res.ToWhoisStyleResponse()
for k, v := range whoisResp.Data {
data[strings.ToLower(k)] = strings.Join(v, ",")
}
response = whoisResp
default:
response = res.Object
}
jsonData, _ := jsoniter.Marshal(response)
jsonDataString := string(jsonData)
data["type"] = request.Type().String()
data["host"] = query
data["response"] = jsonDataString
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false))
}
callback(event)
return nil
}
// Match performs matching operation for a matcher on model and returns:
// true and a list of matched snippets if the matcher type is supports it
// otherwise false and an empty string slice
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
return protocols.MakeDefaultMatchFunc(data, matcher)
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
return protocols.MakeDefaultExtractFunc(data, matcher)
}
// MakeResultEvent creates a result event from internal wrapped event
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
// GetCompiledOperators returns a list of the compiled operators
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["response"]),
}
return data
}
// Type returns the type of the protocol request
func (request *Request) Type() templateTypes.ProtocolType {
return templateTypes.WHOISProtocol
}
// generateVariables will create default variables after parsing a url
func generateVariables(input string) map[string]interface{} {
var domain string
parsed, err := url.Parse(input)
if err != nil {
return map[string]interface{}{"Input": input}
}
domain = parsed.Host
if domain == "" {
domain = input
}
if strings.Contains(domain, ":") {
domain = strings.Split(domain, ":")[0]
}
return map[string]interface{}{
"Input": input,
"Hostname": parsed.Host,
"Host": domain,
}
}

View File

@ -108,6 +108,11 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
// parseSelfContainedRequests parses the self contained template requests.
func (template *Template) parseSelfContainedRequests() {
if template.Signature.Value.String() != "" {
for _, request := range template.RequestsHTTP {
request.Signature = template.Signature
}
}
if !template.SelfContained {
return
}
@ -128,7 +133,8 @@ func (template *Template) Requests() int {
len(template.RequestsHeadless) +
len(template.Workflows) +
len(template.RequestsSSL) +
len(template.RequestsWebsocket)
len(template.RequestsWebsocket) +
len(template.RequestsWHOIS)
}
// compileProtocolRequests compiles all the protocol requests for the template
@ -166,6 +172,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
case len(template.RequestsWebsocket) > 0:
requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket)
case len(template.RequestsWHOIS) > 0:
requests = template.convertRequestToProtocolsRequest(template.RequestsWHOIS)
}
template.Executer = executer.NewExecuter(requests, &options)
return nil

View File

@ -2,6 +2,9 @@
package templates
import (
"encoding/json"
validate "github.com/go-playground/validator/v10"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
@ -11,8 +14,11 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"go.uber.org/multierr"
"gopkg.in/yaml.v2"
)
// Template is a YAML input file which defines all the requests and
@ -66,6 +72,9 @@ type Template struct {
// Websocket contains the Websocket request to make in the template.
RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"`
// description: |
// WHOIS contains the WHOIS request to make in the template.
RequestsWHOIS []*whois.Request `yaml:"whois,omitempty" json:"whois,omitempty" jsonschema:"title=whois requests to make,description=WHOIS requests to make for the template"`
// description: |
// Workflows is a yaml based workflow declaration code.
workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"`
@ -78,6 +87,12 @@ type Template struct {
// Stop execution once first match is found
StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"`
// description: |
// Signature is the request signature method
// values:
// - "AWS"
Signature http.SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
// TotalRequests is the total number of requests for the template.
TotalRequests int `yaml:"-" json:"-"`
// Executer is the actual template executor for running template requests
@ -96,6 +111,7 @@ var TemplateProtocols = []string{
"workflow",
"ssl",
"websocket",
"whois",
}
// Type returns the type of the template
@ -117,7 +133,47 @@ func (template *Template) Type() types.ProtocolType {
return types.SSLProtocol
case len(template.RequestsWebsocket) > 0:
return types.WebsocketProtocol
case len(template.RequestsWHOIS) > 0:
return types.WHOISProtocol
default:
return types.InvalidProtocol
}
}
// MarshalYAML forces recursive struct validation during marshal operation
func (template *Template) MarshalYAML() ([]byte, error) {
out, marshalErr := yaml.Marshal(template)
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
}
// MarshalYAML forces recursive struct validation after unmarshal operation
func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error {
type Alias Template
alias := &Alias{}
err := unmarshal(alias)
if err != nil {
return err
}
*template = Template(*alias)
return validate.New().Struct(template)
}
// MarshalJSON forces recursive struct validation during marshal operation
func (template *Template) MarshalJSON() ([]byte, error) {
out, marshalErr := json.Marshal(template)
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
}
// UnmarshalJSON forces recursive struct validation after unmarshal operation
func (template *Template) UnmarshalJSON(data []byte) error {
type Alias Template
alias := &Alias{}
err := json.Unmarshal(data, alias)
if err != nil {
return err
}
*template = Template(*alias)
return validate.New().Struct(template)
}

View File

@ -21,6 +21,7 @@ var (
ExtractorTypeHolderDoc encoder.Doc
GENERATORSAttackTypeHolderDoc encoder.Doc
HTTPMethodTypeHolderDoc encoder.Doc
SignatureTypeHolderDoc encoder.Doc
DNSRequestDoc encoder.Doc
DNSRequestTypeHolderDoc encoder.Doc
FILERequestDoc encoder.Doc
@ -33,15 +34,17 @@ var (
SSLRequestDoc encoder.Doc
WEBSOCKETRequestDoc encoder.Doc
WEBSOCKETInputDoc encoder.Doc
WHOISRequestDoc encoder.Doc
WORKFLOWSWorkflowTemplateDoc encoder.Doc
WORKFLOWSMatcherDoc encoder.Doc
HTTPSignatureTypeHolderDoc encoder.Doc
)
func init() {
TemplateDoc.Type = "Template"
TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and"
TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template."
TemplateDoc.Fields = make([]encoder.Doc, 12)
TemplateDoc.Fields = make([]encoder.Doc, 14)
TemplateDoc.Fields[0].Name = "id"
TemplateDoc.Fields[0].Type = "string"
TemplateDoc.Fields[0].Note = ""
@ -99,21 +102,34 @@ func init() {
TemplateDoc.Fields[8].Note = ""
TemplateDoc.Fields[8].Description = "Websocket contains the Websocket request to make in the template."
TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Websocket contains the Websocket request to make in the template."
TemplateDoc.Fields[9].Name = "workflows"
TemplateDoc.Fields[9].Type = "[]workflows.WorkflowTemplate"
TemplateDoc.Fields[9].Name = "whois"
TemplateDoc.Fields[9].Type = "[]whois.Request"
TemplateDoc.Fields[9].Note = ""
TemplateDoc.Fields[9].Description = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[9].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[10].Name = "self-contained"
TemplateDoc.Fields[10].Type = "bool"
TemplateDoc.Fields[9].Description = "WHOIS contains the WHOIS request to make in the template."
TemplateDoc.Fields[9].Comments[encoder.LineComment] = "WHOIS contains the WHOIS request to make in the template."
TemplateDoc.Fields[10].Name = "workflows"
TemplateDoc.Fields[10].Type = "[]workflows.WorkflowTemplate"
TemplateDoc.Fields[10].Note = ""
TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[11].Name = "stop-at-first-match"
TemplateDoc.Fields[10].Description = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template."
TemplateDoc.Fields[11].Name = "self-contained"
TemplateDoc.Fields[11].Type = "bool"
TemplateDoc.Fields[11].Note = ""
TemplateDoc.Fields[11].Description = "Stop execution once first match is found"
TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Stop execution once first match is found"
TemplateDoc.Fields[11].Description = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained"
TemplateDoc.Fields[12].Name = "stop-at-first-match"
TemplateDoc.Fields[12].Type = "bool"
TemplateDoc.Fields[12].Note = ""
TemplateDoc.Fields[12].Description = "Stop execution once first match is found"
TemplateDoc.Fields[12].Comments[encoder.LineComment] = "Stop execution once first match is found"
TemplateDoc.Fields[13].Name = "signature"
TemplateDoc.Fields[13].Type = "http.SignatureTypeHolder"
TemplateDoc.Fields[13].Note = ""
TemplateDoc.Fields[13].Description = "Signature is the request signature method"
TemplateDoc.Fields[13].Comments[encoder.LineComment] = "Signature is the request signature method"
TemplateDoc.Fields[13].Values = []string{
"AWS",
}
MODELInfoDoc.Type = "model.Info"
MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template"
@ -372,7 +388,7 @@ func init() {
Value: "HTTP response headers in name:value format",
},
}
HTTPRequestDoc.Fields = make([]encoder.Doc, 27)
HTTPRequestDoc.Fields = make([]encoder.Doc, 28)
HTTPRequestDoc.Fields[0].Name = "matchers"
HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher"
HTTPRequestDoc.Fields[0].Note = ""
@ -492,51 +508,59 @@ func init() {
HTTPRequestDoc.Fields[17].Comments[encoder.LineComment] = "MaxSize is the maximum size of http response body to read in bytes."
HTTPRequestDoc.Fields[17].AddExample("Read max 2048 bytes of the response", 2048)
HTTPRequestDoc.Fields[18].Name = "cookie-reuse"
HTTPRequestDoc.Fields[18].Type = "bool"
HTTPRequestDoc.Fields[18].Name = "signature"
HTTPRequestDoc.Fields[18].Type = "SignatureTypeHolder"
HTTPRequestDoc.Fields[18].Note = ""
HTTPRequestDoc.Fields[18].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
HTTPRequestDoc.Fields[19].Name = "redirects"
HTTPRequestDoc.Fields[18].Description = "Signature is the request signature method"
HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "Signature is the request signature method"
HTTPRequestDoc.Fields[18].Values = []string{
"AWS",
}
HTTPRequestDoc.Fields[19].Name = "cookie-reuse"
HTTPRequestDoc.Fields[19].Type = "bool"
HTTPRequestDoc.Fields[19].Note = ""
HTTPRequestDoc.Fields[19].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
HTTPRequestDoc.Fields[20].Name = "pipeline"
HTTPRequestDoc.Fields[19].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
HTTPRequestDoc.Fields[20].Name = "redirects"
HTTPRequestDoc.Fields[20].Type = "bool"
HTTPRequestDoc.Fields[20].Note = ""
HTTPRequestDoc.Fields[20].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
HTTPRequestDoc.Fields[21].Name = "unsafe"
HTTPRequestDoc.Fields[20].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
HTTPRequestDoc.Fields[21].Name = "pipeline"
HTTPRequestDoc.Fields[21].Type = "bool"
HTTPRequestDoc.Fields[21].Note = ""
HTTPRequestDoc.Fields[21].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
HTTPRequestDoc.Fields[22].Name = "race"
HTTPRequestDoc.Fields[21].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
HTTPRequestDoc.Fields[22].Name = "unsafe"
HTTPRequestDoc.Fields[22].Type = "bool"
HTTPRequestDoc.Fields[22].Note = ""
HTTPRequestDoc.Fields[22].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
HTTPRequestDoc.Fields[23].Name = "req-condition"
HTTPRequestDoc.Fields[22].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
HTTPRequestDoc.Fields[23].Name = "race"
HTTPRequestDoc.Fields[23].Type = "bool"
HTTPRequestDoc.Fields[23].Note = ""
HTTPRequestDoc.Fields[23].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
HTTPRequestDoc.Fields[24].Name = "stop-at-first-match"
HTTPRequestDoc.Fields[23].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
HTTPRequestDoc.Fields[24].Name = "req-condition"
HTTPRequestDoc.Fields[24].Type = "bool"
HTTPRequestDoc.Fields[24].Note = ""
HTTPRequestDoc.Fields[24].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[25].Name = "skip-variables-check"
HTTPRequestDoc.Fields[24].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
HTTPRequestDoc.Fields[25].Name = "stop-at-first-match"
HTTPRequestDoc.Fields[25].Type = "bool"
HTTPRequestDoc.Fields[25].Note = ""
HTTPRequestDoc.Fields[25].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[26].Name = "iterate-all"
HTTPRequestDoc.Fields[25].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HTTPRequestDoc.Fields[26].Name = "skip-variables-check"
HTTPRequestDoc.Fields[26].Type = "bool"
HTTPRequestDoc.Fields[26].Note = ""
HTTPRequestDoc.Fields[26].Description = "IterateAll iterates all the values extracted from internal extractors"
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
HTTPRequestDoc.Fields[26].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
HTTPRequestDoc.Fields[27].Name = "iterate-all"
HTTPRequestDoc.Fields[27].Type = "bool"
HTTPRequestDoc.Fields[27].Note = ""
HTTPRequestDoc.Fields[27].Description = "IterateAll iterates all the values extracted from internal extractors"
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
MATCHERSMatcherDoc.Type = "matchers.Matcher"
MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol."
@ -570,6 +594,10 @@ func init() {
TypeName: "websocket.Request",
FieldName: "matchers",
},
{
TypeName: "whois.Request",
FieldName: "matchers",
},
}
MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13)
MATCHERSMatcherDoc.Fields[0].Name = "type"
@ -731,6 +759,10 @@ func init() {
TypeName: "websocket.Request",
FieldName: "extractors",
},
{
TypeName: "whois.Request",
FieldName: "extractors",
},
}
EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11)
EXTRACTORSExtractorDoc.Fields[0].Name = "name"
@ -892,6 +924,17 @@ func init() {
"PURGE",
}
SignatureTypeHolderDoc.Type = "SignatureTypeHolder"
SignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature"
SignatureTypeHolderDoc.Description = "SignatureTypeHolder is used to hold internal type of the signature"
SignatureTypeHolderDoc.AppearsIn = []encoder.Appearance{
{
TypeName: "http.Request",
FieldName: "signature",
},
}
SignatureTypeHolderDoc.Fields = make([]encoder.Doc, 0)
DNSRequestDoc.Type = "dns.Request"
DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template"
DNSRequestDoc.Description = "Request contains a DNS protocol request to be made from a template"
@ -1139,8 +1182,8 @@ func init() {
FILERequestDoc.Fields[4].Name = "denylist"
FILERequestDoc.Fields[4].Type = "[]string"
FILERequestDoc.Fields[4].Note = ""
FILERequestDoc.Fields[4].Description = "ExtensionDenylist is the list of file extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei."
FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "ExtensionDenylist is the list of file extensions to deny during matching."
FILERequestDoc.Fields[4].Description = "DenyList is the list of file, directories or extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei."
FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "DenyList is the list of file, directories or extensions to deny during matching."
FILERequestDoc.Fields[4].AddExample("", []string{".avi", ".mov", ".mp3"})
FILERequestDoc.Fields[5].Name = "id"
@ -1645,6 +1688,46 @@ func init() {
WEBSOCKETInputDoc.Fields[1].AddExample("", "prefix")
WHOISRequestDoc.Type = "whois.Request"
WHOISRequestDoc.Comments[encoder.LineComment] = " Request is a request for the WHOIS protocol"
WHOISRequestDoc.Description = "Request is a request for the WHOIS protocol"
WHOISRequestDoc.AppearsIn = []encoder.Appearance{
{
TypeName: "Template",
FieldName: "whois",
},
}
WHOISRequestDoc.Fields = make([]encoder.Doc, 5)
WHOISRequestDoc.Fields[0].Name = "matchers"
WHOISRequestDoc.Fields[0].Type = "[]matchers.Matcher"
WHOISRequestDoc.Fields[0].Note = ""
WHOISRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
WHOISRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
WHOISRequestDoc.Fields[1].Name = "extractors"
WHOISRequestDoc.Fields[1].Type = "[]extractors.Extractor"
WHOISRequestDoc.Fields[1].Note = ""
WHOISRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
WHOISRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
WHOISRequestDoc.Fields[2].Name = "matchers-condition"
WHOISRequestDoc.Fields[2].Type = "string"
WHOISRequestDoc.Fields[2].Note = ""
WHOISRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR."
WHOISRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
WHOISRequestDoc.Fields[2].Values = []string{
"and",
"or",
}
WHOISRequestDoc.Fields[3].Name = "query"
WHOISRequestDoc.Fields[3].Type = "string"
WHOISRequestDoc.Fields[3].Note = ""
WHOISRequestDoc.Fields[3].Description = "Query contains query for the request"
WHOISRequestDoc.Fields[3].Comments[encoder.LineComment] = "Query contains query for the request"
WHOISRequestDoc.Fields[4].Name = "server"
WHOISRequestDoc.Fields[4].Type = "string"
WHOISRequestDoc.Fields[4].Note = ""
WHOISRequestDoc.Fields[4].Description = "description: |\n Optional WHOIS server URL.\n\n If present, specifies the WHOIS server to execute the Request on.\n Otherwise, nil enables bootstrapping"
WHOISRequestDoc.Fields[4].Comments[encoder.LineComment] = " description: |"
WORKFLOWSWorkflowTemplateDoc.Type = "workflows.WorkflowTemplate"
WORKFLOWSWorkflowTemplateDoc.Comments[encoder.LineComment] = ""
WORKFLOWSWorkflowTemplateDoc.Description = ""
@ -1708,6 +1791,17 @@ func init() {
WORKFLOWSMatcherDoc.Fields[1].Note = ""
WORKFLOWSMatcherDoc.Fields[1].Description = "Subtemplates are run if the name of matcher matches."
WORKFLOWSMatcherDoc.Fields[1].Comments[encoder.LineComment] = "Subtemplates are run if the name of matcher matches."
HTTPSignatureTypeHolderDoc.Type = "http.SignatureTypeHolder"
HTTPSignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature"
HTTPSignatureTypeHolderDoc.Description = "SignatureTypeHolder is used to hold internal type of the signature"
HTTPSignatureTypeHolderDoc.AppearsIn = []encoder.Appearance{
{
TypeName: "Template",
FieldName: "signature",
},
}
HTTPSignatureTypeHolderDoc.Fields = make([]encoder.Doc, 0)
}
// GetTemplateDoc returns documentation for the file templates_doc.go.
@ -1728,6 +1822,7 @@ func GetTemplateDoc() *encoder.FileDoc {
&ExtractorTypeHolderDoc,
&GENERATORSAttackTypeHolderDoc,
&HTTPMethodTypeHolderDoc,
&SignatureTypeHolderDoc,
&DNSRequestDoc,
&DNSRequestTypeHolderDoc,
&FILERequestDoc,
@ -1740,8 +1835,10 @@ func GetTemplateDoc() *encoder.FileDoc {
&SSLRequestDoc,
&WEBSOCKETRequestDoc,
&WEBSOCKETInputDoc,
&WHOISRequestDoc,
&WORKFLOWSWorkflowTemplateDoc,
&WORKFLOWSMatcherDoc,
&HTTPSignatureTypeHolderDoc,
},
}
}

View File

@ -34,6 +34,8 @@ const (
SSLProtocol
// name:websocket
WebsocketProtocol
// name:whois
WHOISProtocol
limit
InvalidProtocol
)
@ -49,6 +51,7 @@ var protocolMappings = map[ProtocolType]string{
WorkflowProtocol: "workflow",
SSLProtocol: "ssl",
WebsocketProtocol: "websocket",
WHOISProtocol: "whois",
}
func GetSupportedProtocolTypes() ProtocolTypes {