mirror of https://github.com/daffainfo/nuclei.git
Merge branch 'dev' into issue-1594-regexp
mzack
commit
64bea4da8c
|
|
@ -24,6 +24,7 @@ jobs:
|
|||
- name: Functional Tests
|
||||
env:
|
||||
GH_ACTION: true
|
||||
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||
run: |
|
||||
chmod +x run.sh
|
||||
bash run.sh ${{ matrix.os }}
|
||||
|
|
|
|||
|
|
@ -115,8 +115,8 @@ FILTERING:
|
|||
-eid, -exclude-id string[] templates to exclude based on template ids (comma-separated, file)
|
||||
-it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration
|
||||
-et, -exclude-templates string[] template or template directory to exclude (comma-separated, file)
|
||||
-s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical
|
||||
-es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical
|
||||
-s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
|
||||
-es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
|
||||
-pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
|
||||
-ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
|
||||
|
||||
|
|
|
|||
|
|
@ -633,6 +633,8 @@ Enum Values:
|
|||
- <code>high</code>
|
||||
|
||||
- <code>critical</code>
|
||||
|
||||
- <code>unknown</code>
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
|
@ -3218,6 +3220,34 @@ Steps is the list of actions to run for headless request
|
|||
|
||||
<div class="dd">
|
||||
|
||||
<code>user_agent</code> <i><a href="#useragentuseragentholder">userAgent.UserAgentHolder</a></i>
|
||||
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
descriptions: |
|
||||
User-Agent is the type of user-agent to use for the request.
|
||||
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>custom_user_agent</code> <i>string</i>
|
||||
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
description: |
|
||||
If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request.
|
||||
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>matchers</code> <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
|
||||
|
||||
</div>
|
||||
|
|
@ -3430,6 +3460,48 @@ Enum Values:
|
|||
|
||||
|
||||
|
||||
## userAgent.UserAgentHolder
|
||||
UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes
|
||||
|
||||
Appears in:
|
||||
|
||||
|
||||
- <code><a href="#headlessrequest">headless.Request</a>.user_agent</code>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<hr />
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code></code> <i>UserAgent</i>
|
||||
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
|
||||
|
||||
|
||||
Enum Values:
|
||||
|
||||
|
||||
- <code>random</code>
|
||||
|
||||
- <code>off</code>
|
||||
|
||||
- <code>default</code>
|
||||
|
||||
- <code>custom</code>
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## ssl.Request
|
||||
Request is a request for the SSL protocol
|
||||
|
||||
|
|
|
|||
|
|
@ -103,7 +103,8 @@
|
|||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"critical"
|
||||
"critical",
|
||||
"unknown"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "severity of the template",
|
||||
|
|
@ -119,6 +120,16 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"userAgent.UserAgentHolder": {
|
||||
"enum": [
|
||||
"off",
|
||||
"default",
|
||||
"custom"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "userAgent for the headless",
|
||||
"description": "userAgent for the headless http request"
|
||||
},
|
||||
"extractors.Extractor": {
|
||||
"required": [
|
||||
"type"
|
||||
|
|
@ -531,6 +542,17 @@
|
|||
"title": "list of actions for headless request",
|
||||
"description": "List of actions to run for headless request"
|
||||
},
|
||||
"user_agent": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/userAgent.UserAgentHolder",
|
||||
"title": "user agent for the headless request",
|
||||
"description": "User agent for the headless request"
|
||||
},
|
||||
"custom_user_agent": {
|
||||
"type": "string",
|
||||
"title": "custom user agent for the headless request",
|
||||
"description": "Custom user agent for the headless request"
|
||||
},
|
||||
"matchers": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/matchers.Matcher"
|
||||
|
|
|
|||
|
|
@ -3,36 +3,48 @@ before:
|
|||
- go mod tidy
|
||||
|
||||
builds:
|
||||
- env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- windows
|
||||
- linux
|
||||
- darwin
|
||||
goarch:
|
||||
- amd64
|
||||
- 386
|
||||
- arm
|
||||
- arm64
|
||||
|
||||
- main: cmd/nuclei/main.go
|
||||
binary: nuclei
|
||||
id: nuclei-cli
|
||||
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
|
||||
goos: [windows,linux,darwin]
|
||||
goarch: [amd64,386,arm,arm64]
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: '386'
|
||||
goarch: 386
|
||||
- goos: windows
|
||||
goarch: 'arm'
|
||||
goarch: arm
|
||||
- goos: windows
|
||||
goarch: 'arm64'
|
||||
|
||||
binary: '{{ .ProjectName }}'
|
||||
main: cmd/nuclei/main.go
|
||||
goarch: arm64
|
||||
|
||||
flags:
|
||||
- -trimpath
|
||||
|
||||
- main: cmd/cve-annotate/main.go
|
||||
binary: cve-annotate
|
||||
id: annotate
|
||||
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
|
||||
goos: [linux]
|
||||
goarch: [amd64]
|
||||
|
||||
archives:
|
||||
- format: zip
|
||||
id: nuclei
|
||||
builds: [nuclei-cli]
|
||||
replacements:
|
||||
darwin: macOS
|
||||
|
||||
- format: zip
|
||||
id: annotate
|
||||
builds: [annotate]
|
||||
name_template: "{{ .Binary }}"
|
||||
|
||||
checksum:
|
||||
algorithm: sha256
|
||||
|
|
|
|||
10
v2/go.mod
10
v2/go.mod
|
|
@ -15,7 +15,7 @@ require (
|
|||
github.com/go-rod/rod v0.102.0
|
||||
github.com/gobwas/ws v1.1.0
|
||||
github.com/google/go-github v17.0.0+incompatible
|
||||
github.com/itchyny/gojq v0.12.6
|
||||
github.com/itchyny/gojq v0.12.7
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/karlseguin/ccache v2.0.3+incompatible
|
||||
|
|
@ -43,7 +43,7 @@ require (
|
|||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.3.0
|
||||
github.com/segmentio/ksuid v1.0.4
|
||||
github.com/shirou/gopsutil/v3 v3.22.1
|
||||
github.com/shirou/gopsutil/v3 v3.22.2
|
||||
github.com/spaolacci/murmur3 v1.1.0
|
||||
github.com/spf13/cast v1.4.1
|
||||
github.com/syndtr/goleveldb v1.0.0
|
||||
|
|
@ -54,7 +54,7 @@ require (
|
|||
github.com/ysmood/gson v0.6.4 // indirect
|
||||
github.com/ysmood/leakless v0.7.0 // indirect
|
||||
go.uber.org/atomic v1.9.0
|
||||
go.uber.org/multierr v1.7.0
|
||||
go.uber.org/multierr v1.8.0
|
||||
go.uber.org/ratelimit v0.2.0
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
|
||||
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
|
||||
|
|
@ -63,7 +63,7 @@ require (
|
|||
moul.io/http2curl v1.0.0
|
||||
)
|
||||
|
||||
require github.com/aws/aws-sdk-go v1.43.7
|
||||
require github.com/aws/aws-sdk-go v1.43.9
|
||||
|
||||
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
|
||||
|
||||
|
|
@ -155,7 +155,7 @@ require (
|
|||
goftp.io/server/v2 v2.0.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
|
||||
golang.org/x/mod v0.4.2 // indirect
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
|
||||
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
|
|
|
|||
19
v2/go.sum
19
v2/go.sum
|
|
@ -83,8 +83,8 @@ github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3st
|
|||
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.43.7 h1:Gbs53KxXJWbO3txoVkevf56bhdDFqRisl7MQQ6581vc=
|
||||
github.com/aws/aws-sdk-go v1.43.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go v1.43.9 h1:k1S/29Bp2QD5ZopnGzIn0Sp63yyt3WH1JRE2OOU3Aig=
|
||||
github.com/aws/aws-sdk-go v1.43.9/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
|
|
@ -281,8 +281,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
|
|||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
|
||||
github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
|
||||
github.com/itchyny/gojq v0.12.6 h1:VjaFn59Em2wTxDNGcrRkDK9ZHMNa8IksOgL13sLL4d0=
|
||||
github.com/itchyny/gojq v0.12.6/go.mod h1:ZHrkfu7A+RbZLy5J1/JKpS4poEqrzItSTGDItqsfP0A=
|
||||
github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
|
||||
github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
|
||||
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
|
||||
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
|
||||
github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
|
||||
|
|
@ -505,8 +505,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
|
|||
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
|
||||
github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
|
||||
github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
|
||||
github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
|
||||
github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
|
|
@ -627,8 +627,9 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
|||
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
|
||||
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
|
||||
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
|
||||
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
|
||||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
|
||||
|
|
@ -814,11 +815,11 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import (
|
|||
"github.com/google/go-github/github"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
"github.com/projectdiscovery/folderutil"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
|
|
@ -223,7 +224,16 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
|
|||
|
||||
// getLatestReleaseFromGithub returns the latest release from GitHub
|
||||
func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) {
|
||||
gitHubClient := github.NewClient(nil)
|
||||
var tc *http.Client
|
||||
if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
|
||||
ctx := context.Background()
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: token},
|
||||
)
|
||||
tc = oauth2.NewClient(ctx, ts)
|
||||
}
|
||||
|
||||
gitHubClient := github.NewClient(tc)
|
||||
|
||||
release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ const (
|
|||
High
|
||||
// name:critical
|
||||
Critical
|
||||
// name:unknown
|
||||
Unknown
|
||||
limit
|
||||
)
|
||||
|
||||
|
|
@ -33,6 +35,7 @@ var severityMappings = map[Severity]string{
|
|||
Medium: "medium",
|
||||
High: "high",
|
||||
Critical: "critical",
|
||||
Unknown: "unknown",
|
||||
}
|
||||
|
||||
func GetSupportedSeverities() Severities {
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ func TestYamlUnmarshalFail(t *testing.T) {
|
|||
|
||||
func TestGetSupportedSeverities(t *testing.T) {
|
||||
severities := GetSupportedSeverities()
|
||||
assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical})
|
||||
assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical, Unknown})
|
||||
}
|
||||
|
||||
func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
package userAgent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/jsonschema"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type UserAgent int
|
||||
|
||||
// name:UserAgent
|
||||
const (
|
||||
// name:random
|
||||
Random UserAgent = iota
|
||||
// name:off
|
||||
Off
|
||||
// name:default
|
||||
Default
|
||||
// name:custom
|
||||
Custom
|
||||
limit
|
||||
)
|
||||
|
||||
var userAgentMappings = map[UserAgent]string{
|
||||
Random: "random",
|
||||
Off: "off",
|
||||
Default: "default",
|
||||
Custom: "custom",
|
||||
}
|
||||
|
||||
func GetSupportedUserAgentOptions() []UserAgent {
|
||||
var result []UserAgent
|
||||
for index := UserAgent(1); index < limit; index++ {
|
||||
result = append(result, index)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toUserAgent(valueToMap string) (UserAgent, error) {
|
||||
normalizedValue := normalizeValue(valueToMap)
|
||||
for key, currentValue := range userAgentMappings {
|
||||
if normalizedValue == currentValue {
|
||||
return key, nil
|
||||
}
|
||||
}
|
||||
return -1, errors.New("Invalid userAgent: " + valueToMap)
|
||||
}
|
||||
|
||||
func normalizeValue(value string) string {
|
||||
return strings.TrimSpace(strings.ToLower(value))
|
||||
}
|
||||
|
||||
func (userAgent UserAgent) String() string {
|
||||
return userAgentMappings[userAgent]
|
||||
}
|
||||
|
||||
// UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes
|
||||
type UserAgentHolder struct {
|
||||
Value UserAgent `mapping:"true"`
|
||||
}
|
||||
|
||||
func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Type {
|
||||
gotType := &jsonschema.Type{
|
||||
Type: "string",
|
||||
Title: "userAgent for the headless",
|
||||
Description: "userAgent for the headless http request",
|
||||
}
|
||||
for _, userAgent := range GetSupportedUserAgentOptions() {
|
||||
gotType.Enum = append(gotType.Enum, userAgent.String())
|
||||
}
|
||||
return gotType
|
||||
}
|
||||
|
||||
func (userAgentHolder *UserAgentHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var marshalledUserAgent string
|
||||
if err := unmarshal(&marshalledUserAgent); err != nil {
|
||||
return err
|
||||
}
|
||||
computedUserAgent, err := toUserAgent(marshalledUserAgent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
userAgentHolder.Value = computedUserAgent
|
||||
return nil
|
||||
}
|
||||
|
||||
func (userAgentHolder *UserAgentHolder) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(userAgentHolder.Value.String())
|
||||
}
|
||||
|
||||
func (userAgentHolder UserAgentHolder) MarshalYAML() (interface{}, error) {
|
||||
return userAgentHolder.Value.String(), nil
|
||||
}
|
||||
|
|
@ -8,7 +8,6 @@ import (
|
|||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/corpix/uarand"
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/pkg/errors"
|
||||
|
|
@ -89,9 +88,6 @@ func New(options *types.Options) (*Browser, error) {
|
|||
customAgent = parts[1]
|
||||
}
|
||||
}
|
||||
if customAgent == "" {
|
||||
customAgent = uarand.GetRandom()
|
||||
}
|
||||
|
||||
httpclient, err := newHttpClient(options)
|
||||
if err != nil {
|
||||
|
|
@ -116,6 +112,16 @@ func MustDisableSandbox() bool {
|
|||
return runtime.GOOS == "linux" && os.Geteuid() == 0
|
||||
}
|
||||
|
||||
// SetUserAgent sets custom user agent to the browser
|
||||
func (b *Browser) SetUserAgent(customUserAgent string) {
|
||||
b.customAgent = customUserAgent
|
||||
}
|
||||
|
||||
// UserAgent fetch the currently set custom user agent
|
||||
func (b *Browser) UserAgent() string {
|
||||
return b.customAgent
|
||||
}
|
||||
|
||||
// Close closes the browser engine
|
||||
func (b *Browser) Close() {
|
||||
b.engine.Close()
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
package headless
|
||||
|
||||
import (
|
||||
"github.com/corpix/uarand"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/projectdiscovery/fileutil"
|
||||
useragent "github.com/projectdiscovery/nuclei/v2/pkg/model/types/userAgent"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
|
|
@ -33,6 +35,15 @@ type Request struct {
|
|||
// Steps is the list of actions to run for headless request
|
||||
Steps []*engine.Action `yaml:"steps,omitempty" jsonschema:"title=list of actions for headless request,description=List of actions to run for headless request"`
|
||||
|
||||
// descriptions: |
|
||||
// User-Agent is the type of user-agent to use for the request.
|
||||
UserAgent useragent.UserAgentHolder `yaml:"user_agent,omitempty" jsonschema:"title=user agent for the headless request,description=User agent for the headless request"`
|
||||
|
||||
// description: |
|
||||
// If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request.
|
||||
CustomUserAgent string `yaml:"custom_user_agent,omitempty" jsonschema:"title=custom user agent for the headless request,description=Custom user agent for the headless request"`
|
||||
compiledUserAgent string
|
||||
|
||||
// Operators for the current request go here.
|
||||
operators.Operators `yaml:",inline,omitempty"`
|
||||
CompiledOperators *operators.Operators `yaml:"-"`
|
||||
|
|
@ -90,6 +101,21 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
|
|||
}
|
||||
}
|
||||
|
||||
// Compile User-Agent
|
||||
switch request.UserAgent.Value {
|
||||
case useragent.Off:
|
||||
request.compiledUserAgent = " "
|
||||
case useragent.Default:
|
||||
request.compiledUserAgent = ""
|
||||
case useragent.Custom:
|
||||
if request.CustomUserAgent == "" {
|
||||
return errors.New("please set custom_user_agent in the template")
|
||||
}
|
||||
request.compiledUserAgent = request.CustomUserAgent
|
||||
case useragent.Random:
|
||||
request.compiledUserAgent = uarand.GetRandom()
|
||||
}
|
||||
|
||||
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
|
||||
compiled := &request.Operators
|
||||
if err := compiled.Compile(); err != nil {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@ func (request *Request) Type() templateTypes.ProtocolType {
|
|||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
|
||||
if request.options.Browser.UserAgent() == "" {
|
||||
request.options.Browser.SetUserAgent(request.compiledUserAgent)
|
||||
}
|
||||
payloads := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
if request.generator != nil {
|
||||
iterator := request.generator.NewIterator()
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
|
|||
actualAddress := replacer.Replace(kv.address, variables)
|
||||
|
||||
if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
|
||||
gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
|
||||
gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
|
@ -223,7 +223,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
|||
default:
|
||||
buf := make([]byte, bufferSize)
|
||||
nBuf, err := conn.Read(buf)
|
||||
if err != nil && !os.IsTimeout(err) {
|
||||
if err != nil && !os.IsTimeout(err) && err != io.EOF {
|
||||
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
|
||||
closeTimer(readInterval)
|
||||
return errors.Wrap(err, "could not read from server")
|
||||
|
|
@ -236,7 +236,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
|||
} else {
|
||||
final = make([]byte, bufferSize)
|
||||
n, err = conn.Read(final)
|
||||
if err != nil && err != io.EOF {
|
||||
if err != nil && !os.IsTimeout(err) && err != io.EOF {
|
||||
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
|
||||
return errors.Wrap(err, "could not read from server")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ var (
|
|||
HEADLESSRequestDoc encoder.Doc
|
||||
ENGINEActionDoc encoder.Doc
|
||||
ActionTypeHolderDoc encoder.Doc
|
||||
USERAGENTUserAgentHolderDoc encoder.Doc
|
||||
SSLRequestDoc encoder.Doc
|
||||
WEBSOCKETRequestDoc encoder.Doc
|
||||
WEBSOCKETInputDoc encoder.Doc
|
||||
|
|
@ -270,6 +271,7 @@ func init() {
|
|||
"medium",
|
||||
"high",
|
||||
"critical",
|
||||
"unknown",
|
||||
}
|
||||
|
||||
MODELClassificationDoc.Type = "model.Classification"
|
||||
|
|
@ -1441,7 +1443,7 @@ func init() {
|
|||
Value: "Headless response received from client (default)",
|
||||
},
|
||||
}
|
||||
HEADLESSRequestDoc.Fields = make([]encoder.Doc, 7)
|
||||
HEADLESSRequestDoc.Fields = make([]encoder.Doc, 9)
|
||||
HEADLESSRequestDoc.Fields[0].Name = "id"
|
||||
HEADLESSRequestDoc.Fields[0].Type = "string"
|
||||
HEADLESSRequestDoc.Fields[0].Note = ""
|
||||
|
|
@ -1462,22 +1464,32 @@ func init() {
|
|||
HEADLESSRequestDoc.Fields[3].Note = ""
|
||||
HEADLESSRequestDoc.Fields[3].Description = "Steps is the list of actions to run for headless request"
|
||||
HEADLESSRequestDoc.Fields[3].Comments[encoder.LineComment] = "Steps is the list of actions to run for headless request"
|
||||
HEADLESSRequestDoc.Fields[4].Name = "matchers"
|
||||
HEADLESSRequestDoc.Fields[4].Type = "[]matchers.Matcher"
|
||||
HEADLESSRequestDoc.Fields[4].Name = "user_agent"
|
||||
HEADLESSRequestDoc.Fields[4].Type = "userAgent.UserAgentHolder"
|
||||
HEADLESSRequestDoc.Fields[4].Note = ""
|
||||
HEADLESSRequestDoc.Fields[4].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
|
||||
HEADLESSRequestDoc.Fields[4].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
|
||||
HEADLESSRequestDoc.Fields[5].Name = "extractors"
|
||||
HEADLESSRequestDoc.Fields[5].Type = "[]extractors.Extractor"
|
||||
HEADLESSRequestDoc.Fields[4].Description = "descriptions: |\n User-Agent is the type of user-agent to use for the request."
|
||||
HEADLESSRequestDoc.Fields[4].Comments[encoder.LineComment] = " descriptions: |"
|
||||
HEADLESSRequestDoc.Fields[5].Name = "custom_user_agent"
|
||||
HEADLESSRequestDoc.Fields[5].Type = "string"
|
||||
HEADLESSRequestDoc.Fields[5].Note = ""
|
||||
HEADLESSRequestDoc.Fields[5].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
|
||||
HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
|
||||
HEADLESSRequestDoc.Fields[6].Name = "matchers-condition"
|
||||
HEADLESSRequestDoc.Fields[6].Type = "string"
|
||||
HEADLESSRequestDoc.Fields[5].Description = "description: |\n If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request."
|
||||
HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = " description: |"
|
||||
HEADLESSRequestDoc.Fields[6].Name = "matchers"
|
||||
HEADLESSRequestDoc.Fields[6].Type = "[]matchers.Matcher"
|
||||
HEADLESSRequestDoc.Fields[6].Note = ""
|
||||
HEADLESSRequestDoc.Fields[6].Description = "MatchersCondition is the condition between the matchers. Default is OR."
|
||||
HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
|
||||
HEADLESSRequestDoc.Fields[6].Values = []string{
|
||||
HEADLESSRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
|
||||
HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
|
||||
HEADLESSRequestDoc.Fields[7].Name = "extractors"
|
||||
HEADLESSRequestDoc.Fields[7].Type = "[]extractors.Extractor"
|
||||
HEADLESSRequestDoc.Fields[7].Note = ""
|
||||
HEADLESSRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
|
||||
HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
|
||||
HEADLESSRequestDoc.Fields[8].Name = "matchers-condition"
|
||||
HEADLESSRequestDoc.Fields[8].Type = "string"
|
||||
HEADLESSRequestDoc.Fields[8].Note = ""
|
||||
HEADLESSRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
|
||||
HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
|
||||
HEADLESSRequestDoc.Fields[8].Values = []string{
|
||||
"and",
|
||||
"or",
|
||||
}
|
||||
|
|
@ -1553,6 +1565,28 @@ func init() {
|
|||
"waitvisible",
|
||||
}
|
||||
|
||||
USERAGENTUserAgentHolderDoc.Type = "userAgent.UserAgentHolder"
|
||||
USERAGENTUserAgentHolderDoc.Comments[encoder.LineComment] = " UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes"
|
||||
USERAGENTUserAgentHolderDoc.Description = "UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes"
|
||||
USERAGENTUserAgentHolderDoc.AppearsIn = []encoder.Appearance{
|
||||
{
|
||||
TypeName: "headless.Request",
|
||||
FieldName: "user_agent",
|
||||
},
|
||||
}
|
||||
USERAGENTUserAgentHolderDoc.Fields = make([]encoder.Doc, 1)
|
||||
USERAGENTUserAgentHolderDoc.Fields[0].Name = ""
|
||||
USERAGENTUserAgentHolderDoc.Fields[0].Type = "UserAgent"
|
||||
USERAGENTUserAgentHolderDoc.Fields[0].Note = ""
|
||||
USERAGENTUserAgentHolderDoc.Fields[0].Description = ""
|
||||
USERAGENTUserAgentHolderDoc.Fields[0].Comments[encoder.LineComment] = ""
|
||||
USERAGENTUserAgentHolderDoc.Fields[0].EnumFields = []string{
|
||||
"random",
|
||||
"off",
|
||||
"default",
|
||||
"custom",
|
||||
}
|
||||
|
||||
SSLRequestDoc.Type = "ssl.Request"
|
||||
SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol"
|
||||
SSLRequestDoc.Description = "Request is a request for the SSL protocol"
|
||||
|
|
@ -1891,6 +1925,7 @@ func GetTemplateDoc() *encoder.FileDoc {
|
|||
&HEADLESSRequestDoc,
|
||||
&ENGINEActionDoc,
|
||||
&ActionTypeHolderDoc,
|
||||
&USERAGENTUserAgentHolderDoc,
|
||||
&SSLRequestDoc,
|
||||
&WEBSOCKETRequestDoc,
|
||||
&WEBSOCKETInputDoc,
|
||||
|
|
|
|||
|
|
@ -53,6 +53,9 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
|
|||
cmd.Args = append(cmd.Args, "-silent")
|
||||
}
|
||||
data, err := cmd.Output()
|
||||
if debug {
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -76,6 +79,9 @@ func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args
|
|||
fmt.Println(cmd.String())
|
||||
}
|
||||
data, err := cmd.CombinedOutput()
|
||||
if debug {
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue