Merge remote-tracking branch 'upstream/dev'

dev
forgedhallpass 2021-08-17 12:08:56 +03:00
commit 1106262a4b
22 changed files with 322 additions and 183 deletions

View File

@ -1,108 +0,0 @@
linters-settings:
dupl:
threshold: 100
exhaustive:
default-signifies-exhaustive: false
# funlen:
# lines: 100
# statements: 50
#goconst:
# min-len: 2
# min-occurrences: 2
gocritic:
enabled-tags:
- diagnostic
- experimental
- opinionated
- performance
- style
disabled-checks:
- dupImport # https://github.com/go-critic/go-critic/issues/845
- ifElseChain
# gocyclo:
# min-complexity: 15
goimports:
local-prefixes: github.com/golangci/golangci-lint
golint:
min-confidence: 0
gomnd:
settings:
mnd:
# don't include the "operation" and "assign"
checks: argument,case,condition,return
govet:
check-shadowing: true
settings:
printf:
funcs:
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
# lll:
# line-length: 140
maligned:
suggest-new: true
misspell:
locale: US
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
linters:
# please, do not use `enable-all`: it's deprecated and will be removed soon.
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
enable:
#- bodyclose
- deadcode
- dogsled
- errcheck
- exhaustive
- gochecknoinits
#- goconst
- gocritic
- gofmt
- goimports
#- gomnd
- goprintffuncname
- gosimple
- govet
- ineffassign
- interfacer
- maligned
- misspell
- nakedret
- noctx
- nolintlint
- rowserrcheck
- scopelint
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
- revive
# don't enable:
# - depguard
# - asciicheck
# - funlen
# - gochecknoglobals
# - gocognit
# - gocyclo
# - godot
# - godox
# - goerr113
# - gosec
# - lll
# - nestif
# - prealloc
# - testpackage
# - wsl

View File

@ -1,4 +1,4 @@
FROM golang:1.16.6-alpine as build-env
FROM golang:1.16.7-alpine as build-env
RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
FROM alpine:latest

View File

@ -83,7 +83,7 @@ Usage:
Flags:
TARGET:
-u, -target string[] target URLs/hosts to scan
-l, -list string path to file containing a list of target URLs/hosts to scan (one per line)
-l, -list string path to file containing a list of target URLs/hosts to scan (one per line)
TEMPLATES:
-tl list all available templates
@ -118,9 +118,11 @@ CONFIGURATIONS:
-config string path to the nuclei configuration file
-rc, -report-config string nuclei reporting module configuration file
-H, -header string[] custom headers in header:value format
-V, -var value custom vars in var=value format
-r, -resolvers string file containing resolver list for nuclei
-system-resolvers use system DNS resolving as error fallback
-passive enable passive HTTP response processing mode
-env-vars Enable environment variables support
INTERACTSH:
-no-interactsh do not use interactsh server for blind interaction polling
@ -131,9 +133,10 @@ INTERACTSH:
-interactions-cooldown-period int extra time for interaction polling before exiting (default 5)
RATE-LIMIT:
-rl, -rate-limit int maximum number of requests to send per second (default 150)
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
-c, -concurrency int maximum number of templates to be executed in parallel (default 10)
-rl, -rate-limit int maximum number of requests to send per second (default 150)
-rlm, -rate-limit-minute int maximum number of requests to send per minute
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
-c, -concurrency int maximum number of templates to be executed in parallel (default 10)
OPTIMIZATIONS:
-timeout int time to wait in seconds before timeout (default 5)
@ -160,6 +163,7 @@ DEBUG:
UPDATE:
-update update nuclei to the latest released version
-ut, -update-templates update the community templates to latest released version
-nut, -no-update-templates Do not check for nuclei-templates updates
-ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
STATISTICS:

View File

@ -98,9 +98,12 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"),
flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"),
flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"),
flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "Enable environment variables support"),
)
createGroup(flagSet, "interactsh", "interactsh",

View File

@ -5,6 +5,7 @@ go 1.15
require (
github.com/Knetic/govaluate v3.0.0+incompatible
github.com/andygrunwald/go-jira v1.13.0
github.com/antchfx/htmlquery v1.2.3
github.com/apex/log v1.9.0
github.com/blang/semver v3.5.1+incompatible
github.com/c4milo/unpackit v0.1.0 // indirect
@ -18,29 +19,29 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
github.com/itchyny/gojq v0.12.4
github.com/json-iterator/go v1.1.10
github.com/json-iterator/go v1.1.11
github.com/julienschmidt/httprouter v1.3.0
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karrick/godirwalk v1.16.1
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/mattn/go-runewidth v0.0.10 // indirect
github.com/miekg/dns v1.1.38
github.com/miekg/dns v1.1.43
github.com/olekukonko/tablewriter v0.0.5
github.com/owenrumney/go-sarif v1.0.4
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/fastdialer v0.0.8
github.com/projectdiscovery/goflags v0.0.6
github.com/projectdiscovery/goflags v0.0.7
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/interactsh v0.0.3
github.com/projectdiscovery/interactsh v0.0.4
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.10
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rs/xid v1.2.1
github.com/rs/xid v1.3.0
github.com/segmentio/ksuid v1.0.3
github.com/shirou/gopsutil/v3 v3.21.5
github.com/spaolacci/murmur3 v1.1.0
@ -54,10 +55,8 @@ require (
go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.6.0
go.uber.org/ratelimit v0.2.0
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/yaml.v2 v2.4.0

View File

@ -31,6 +31,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
@ -44,6 +46,10 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9or
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
@ -104,6 +110,7 @@ github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsH
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -190,16 +197,17 @@ github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
@ -220,7 +228,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@ -234,9 +241,9 @@ github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRR
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@ -267,15 +274,14 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS
github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew=
github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8=
github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE=
github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
@ -284,10 +290,8 @@ github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQY
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e h1:ZT3wZ92sp/EHEE/HcFCWCsYS3ROLjHb6EqSX8qYrgXw=
github.com/prologic/smtpd v0.0.0-20210126001904-0893ad18168e/go.mod h1:GkLsdH1RZj6RDKeI9A05NGZYmEZQ/PbQcZPnZoSZuYI=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
@ -296,8 +300,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY=
github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@ -314,7 +318,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -373,8 +376,6 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -383,11 +384,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw=
golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -440,6 +438,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@ -448,8 +447,10 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -466,8 +467,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -498,11 +499,11 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk=
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

View File

@ -28,7 +28,7 @@ type Config struct {
const nucleiConfigFilename = ".templates-config.json"
// Version is the current version of nuclei
const Version = `2.4.3-dev`
const Version = `2.4.4-dev`
func getConfigDetails() (string, error) {
homeDir, err := os.UserHomeDir()

View File

@ -1,8 +1,12 @@
package extractors
import (
"strings"
"encoding/json"
"github.com/antchfx/htmlquery"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -45,6 +49,35 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
return results
}
// ExtractHTML extracts items from text using XPath selectors
func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} {
results := make(map[string]struct{})
doc, err := htmlquery.Parse(strings.NewReader(corpus))
if err != nil {
return results
}
for _, k := range e.XPath {
nodes, err := htmlquery.QueryAll(doc, k)
if err != nil {
continue
}
for _, node := range nodes {
var value string
if e.Attribute != "" {
value = htmlquery.SelectAttr(node, e.Attribute)
} else {
value = htmlquery.InnerText(node)
}
if _, ok := results[value]; !ok {
results[value] = struct{}{}
}
}
}
return results
}
// ExtractJSON extracts text from a corpus using JQ queries and returns it
func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
results := make(map[string]struct{})

View File

@ -25,6 +25,10 @@ type Extractor struct {
// KVal are the kval to be present in the response headers/cookies
KVal []string `yaml:"kval,omitempty"`
// XPath are the Xpath selectors for the extractor
XPath []string `yaml:"xpath"`
// Attribute is an optional attribute to extract from response XPath
Attribute string `yaml:"attribute"`
// JSON are the json pattern required to be present in the response
JSON []string `yaml:"json"`
// jsonCompiled is the compiled variant
@ -46,6 +50,8 @@ const (
RegexExtractor ExtractorType = iota + 1
// KValExtractor extracts responses with key:value
KValExtractor
// XPathExtractor extracts responses with Xpath selectors
XPathExtractor
// JSONExtractor extracts responses with json
JSONExtractor
)
@ -54,6 +60,7 @@ const (
var ExtractorTypes = map[string]ExtractorType{
"regex": RegexExtractor,
"kval": KValExtractor,
"xpath": XPathExtractor,
"json": JSONExtractor,
}

View File

@ -0,0 +1,28 @@
package generators
import (
"os"
"github.com/projectdiscovery/stringsutil"
)
var envVars map[string]interface{}
func parseEnvVars() map[string]interface{} {
sliceEnvVars := os.Environ()
parsedEnvVars := make(map[string]interface{}, len(sliceEnvVars))
for _, envVar := range sliceEnvVars {
key, val := stringsutil.Before(envVar, "="), stringsutil.After(envVar, "=")
parsedEnvVars[key] = val
}
return parsedEnvVars
}
// EnvVars returns a map with all environment variables into a map
func EnvVars() map[string]interface{} {
if envVars == nil {
envVars = parseEnvVars()
}
return envVars
}

View File

@ -49,10 +49,10 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
if payloadType == PitchFork {
var totalLength int
for v := range compiled {
if totalLength != 0 && totalLength != len(v) {
if totalLength != 0 && totalLength != len(compiled[v]) {
return nil, errors.New("pitchfork payloads must be of equal number")
}
totalLength = len(v)
totalLength = len(compiled[v])
}
}
return generator, nil

View File

@ -1,6 +1,8 @@
package generators
import "strings"
import (
"strings"
)
// MergeMaps merges two maps into a new map
func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {

View File

@ -0,0 +1,16 @@
package generators
import "github.com/projectdiscovery/stringsutil"
// SliceToMap converts a slice of strings to map of string splitting each item at sep as "key sep value"
func SliceToMap(s []string, sep string) map[string]interface{} {
m := make(map[string]interface{})
for _, sliceItem := range s {
key := stringsutil.Before(sliceItem, sep)
value := stringsutil.After(sliceItem, sep)
if key != "" {
m[key] = value
}
}
return m
}

View File

@ -2,6 +2,7 @@ package http
import (
"context"
"fmt"
"io"
"io/ioutil"
"net"
@ -15,7 +16,6 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
"github.com/projectdiscovery/rawhttp"
@ -51,23 +51,30 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
}
data, parsed = baseURLWithTemplatePrefs(data, parsed)
values := generators.MergeMaps(dynamicValues, map[string]interface{}{
"Hostname": parsed.Host,
})
trailingSlash := false
isRawRequest := len(r.request.Raw) > 0
if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") {
parsed.Path = strings.TrimSuffix(parsed.Path, "/")
trailingSlash = true
}
values := generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash))
// merge with vars
if !r.options.Options.Vars.IsEmpty() {
values = generators.MergeMaps(values, r.options.Options.Vars.AsMap())
}
// merge with env vars
if r.options.Options.EnvironmentVariables {
values = generators.MergeMaps(generators.EnvVars(), values)
}
parsedString := parsed.String()
values["BaseURL"] = parsedString
// If data contains \n it's a raw request, process it like raw. Else
// continue with the template based request flow.
if isRawRequest {
return r.makeHTTPRequestFromRaw(ctx, parsedString, data, values, payloads, interactURL)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
}
return r.makeHTTPRequestFromModel(ctx, data, values, interactURL)
return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
}
// Total returns the total number of requests for the generator
@ -96,23 +103,38 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}, interactURL string) (*generatedRequest, error) {
final := replacer.Replace(data, values)
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
if interactURL != "" {
final = r.options.Interactsh.ReplaceMarkers(final, interactURL)
data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
}
// Combine the template payloads along with base
// request values.
finalValues := generators.MergeMaps(generatorValues, values)
// Evaulate the expressions for the request if any.
var err error
data, err = expressions.Evaluate(data, finalValues)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
method, err := expressions.Evaluate(r.request.Method, finalValues)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
// Build a request on the specified URL
req, err := http.NewRequestWithContext(ctx, r.request.Method, final, nil)
req, err := http.NewRequestWithContext(ctx, method, data, nil)
if err != nil {
return nil, err
}
request, err := r.fillRequest(req, values, interactURL)
request, err := r.fillRequest(req, finalValues, interactURL)
if err != nil {
return nil, err
}
return &generatedRequest{request: request, original: r.request}, nil
return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil
}
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
@ -168,7 +190,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
req.Host = value
}
}
request, err := r.fillRequest(req, values, "")
request, err := r.fillRequest(req, finalValues, "")
if err != nil {
return nil, err
}
@ -183,9 +205,13 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
if interactURL != "" {
value = r.options.Interactsh.ReplaceMarkers(value, interactURL)
}
req.Header[header] = []string{replacer.Replace(value, values)}
value, err := expressions.Evaluate(value, values)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
req.Header[header] = []string{value}
if header == "Host" {
req.Host = replacer.Replace(value, values)
req.Host = value
}
}
@ -200,6 +226,10 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
if interactURL != "" {
body = r.options.Interactsh.ReplaceMarkers(body, interactURL)
}
body, err := expressions.Evaluate(body, values)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
req.Body = ioutil.NopCloser(strings.NewReader(body))
}
setHeader(req, "User-Agent", uarand.GetRandom())
@ -221,3 +251,34 @@ func setHeader(req *http.Request, name, value string) {
req.Host = value
}
}
// generateVariables will create default variables after parsing a url
func generateVariables(parsed *url.URL, trailingSlash bool) map[string]interface{} {
domain := parsed.Host
if strings.Contains(parsed.Host, ":") {
domain = strings.Split(parsed.Host, ":")[0]
}
port := parsed.Port()
if port == "" {
if parsed.Scheme == "https" {
port = "443"
} else if parsed.Scheme == "http" {
port = "80"
}
}
if trailingSlash {
parsed.Path = strings.TrimSuffix(parsed.Path, "/")
}
return map[string]interface{}{
"BaseURL": parsed.String(),
"RootURL": fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host),
"Hostname": parsed.Host,
"Host": domain,
"Port": port,
"Path": parsed.EscapedPath(),
"Scheme": parsed.Scheme,
}
}

View File

@ -20,6 +20,44 @@ func TestBaseURLWithTemplatePrefs(t *testing.T) {
require.Equal(t, "{{BaseURL}}/newpath", data, "could not get correct data")
}
func TestVariables(t *testing.T) {
baseURL := "http://localhost:9001/test/123"
parsed, _ := url.Parse(baseURL)
values := generateVariables(parsed, true)
require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
require.Equal(t, values["RootURL"], "http://localhost:9001", "incorrect rootURL")
require.Equal(t, values["Host"], "localhost", "incorrect domain name")
require.Equal(t, values["Path"], "/test/123", "incorrect path")
require.Equal(t, values["Port"], "9001", "incorrect port number")
require.Equal(t, values["Scheme"], "http", "incorrect scheme")
require.Equal(t, values["Hostname"], "localhost:9001", "incorrect hostname")
baseURL = "https://example.com"
parsed, _ = url.Parse(baseURL)
values = generateVariables(parsed, false)
require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
require.Equal(t, values["Host"], "example.com", "incorrect domain name")
require.Equal(t, values["RootURL"], "https://example.com", "incorrect rootURL")
require.Equal(t, values["Path"], "", "incorrect path")
require.Equal(t, values["Port"], "443", "incorrect port number")
require.Equal(t, values["Scheme"], "https", "incorrect scheme")
require.Equal(t, values["Hostname"], "example.com", "incorrect hostname")
baseURL = "ftp://foobar.com/"
parsed, _ = url.Parse(baseURL)
values = generateVariables(parsed, true)
require.Equal(t, values["BaseURL"], parsed.String(), "incorrect baseurl")
require.Equal(t, values["Host"], "foobar.com", "incorrect domain name")
require.Equal(t, values["RootURL"], "ftp://foobar.com", "incorrect rootURL")
require.Equal(t, values["Path"], "", "incorrect path")
require.Equal(t, values["Port"], "", "incorrect port number") // Unsupported protocol results in a blank port
require.Equal(t, values["Scheme"], "ftp", "incorrect scheme")
require.Equal(t, values["Hostname"], "foobar.com", "incorrect hostname")
}
func TestMakeRequestFromModal(t *testing.T) {
options := testutils.DefaultOptions

View File

@ -82,12 +82,19 @@ func (r *Request) GetID() string {
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{
connectionConfiguration := &httpclientpool.Configuration{
Threads: r.Threads,
MaxRedirects: r.MaxRedirects,
FollowRedirects: r.Redirects,
CookieReuse: r.CookieReuse,
})
}
// if the headers contain "Connection" we need to disable the automatic keep alive of the standard library
if _, hasConnectionHeader := r.Headers["Connection"]; hasConnectionHeader {
connectionConfiguration.Connection = &httpclientpool.ConnectionConfiguration{DisableKeepAlive: false}
}
client, err := httpclientpool.Get(options.Options, connectionConfiguration)
if err != nil {
return errors.Wrap(err, "could not get dns client")
}

View File

@ -50,6 +50,12 @@ func Init(options *types.Options) error {
return nil
}
// // Configuration contains the custom configuration options for a connection
type ConnectionConfiguration struct {
// DisableKeepAlive of the connection
DisableKeepAlive bool
}
// Configuration contains the custom configuration options for a client
type Configuration struct {
// Threads contains the threads for the client
@ -60,6 +66,8 @@ type Configuration struct {
CookieReuse bool
// FollowRedirects specifies whether to follow redirects
FollowRedirects bool
// Connection defines custom connection configuration
Connection *ConnectionConfiguration
}
// Hash returns the hash of the configuration to allow client pooling
@ -74,10 +82,17 @@ func (c *Configuration) Hash() string {
builder.WriteString(strconv.FormatBool(c.FollowRedirects))
builder.WriteString("r")
builder.WriteString(strconv.FormatBool(c.CookieReuse))
builder.WriteString("c")
builder.WriteString(strconv.FormatBool(c.Connection != nil))
hash := builder.String()
return hash
}
// HasCustomOptions checks whether the configuration requires custom settings
func (c *Configuration) HasStandardOptions() bool {
return c.Threads == 0 && c.MaxRedirects == 0 && !c.FollowRedirects && !c.CookieReuse && c.Connection == nil
}
// GetRawHTTP returns the rawhttp request client
func GetRawHTTP(options *types.Options) *rawhttp.Client {
if rawhttpClient == nil {
@ -90,7 +105,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) {
if configuration.Threads == 0 && configuration.MaxRedirects == 0 && !configuration.FollowRedirects && !configuration.CookieReuse {
if configuration.HasStandardOptions() {
return normalClient, nil
}
return wrappedGet(options, configuration)
@ -140,6 +155,11 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
followRedirects := configuration.FollowRedirects
maxRedirects := configuration.MaxRedirects
// override connection's settings if required
if configuration.Connection != nil {
disableKeepAlives = configuration.Connection.DisableKeepAlive
}
transport := &http.Transport{
DialContext: Dialer.Dial,
MaxIdleConns: maxIdleConns,

View File

@ -55,6 +55,8 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
return extractor.ExtractRegex(item)
case extractors.KValExtractor:
return extractor.ExtractKval(data)
case extractors.XPathExtractor:
return extractor.ExtractHTML(item)
case extractors.JSONExtractor:
return extractor.ExtractJSON(item)
}

View File

@ -354,7 +354,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
// Ignore body read due to server misconfiguration errors
if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", r.options.TemplateID, formedURL, err.Error())
} else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF") { // ignore EOF error
} else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error
return errors.Wrap(err, "could not read http body")
}
}

View File

@ -31,20 +31,39 @@ func (r *Request) newGenerator() *requestGenerator {
// nextValue returns the next path or the next raw request depending on user input
// It returns false if all the inputs have been exhausted by the generator instance.
func (r *requestGenerator) nextValue() (value string, payloads map[string]interface{}, result bool) {
// If we have paths, return the next path.
// For both raw/path requests, start with the request at current index.
// If we are not at the start, then check if the iterator for payloads
// has finished if there are any.
//
// If the iterator has finished for the current request
// then reset it and move on to the next value, otherwise use the last request.
if len(r.request.Path) > 0 && r.currentIndex < len(r.request.Path) {
if r.payloadIterator != nil {
payload, ok := r.payloadIterator.Value()
if !ok {
r.currentIndex++
r.payloadIterator.Reset()
// No more payloads request for us now.
if len(r.request.Path) == r.currentIndex {
return "", nil, false
}
if item := r.request.Path[r.currentIndex]; item != "" {
newPayload, ok := r.payloadIterator.Value()
return item, newPayload, ok
}
return "", nil, false
}
return r.request.Path[r.currentIndex], payload, true
}
if value := r.request.Path[r.currentIndex]; value != "" {
r.currentIndex++
return value, nil, true
}
}
// If we have raw requests, start with the request at current index.
// If we are not at the start, then check if the iterator for payloads
// has finished if there are any.
//
// If the iterator has finished for the current raw request
// then reset it and move on to the next value, otherwise use the last request.
if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) {
if r.payloadIterator != nil {
payload, ok := r.payloadIterator.Value()

View File

@ -8,6 +8,7 @@ import (
"crypto/sha1"
"io/ioutil"
"os"
"reflect"
"unsafe"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -106,5 +107,7 @@ func (s *Storage) Index(result *output.ResultEvent) (bool, error) {
//
// Reference - https://stackoverflow.com/questions/59209493/how-to-use-unsafe-get-a-byte-slice-from-a-string-without-memory-copy
func unsafeToBytes(data string) []byte {
return *(*[]byte)(unsafe.Pointer(&data))
var buf = *(*[]byte)(unsafe.Pointer(&data))
(*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(data)
return buf
}

View File

@ -21,6 +21,8 @@ type Options struct {
ExcludedTemplates goflags.StringSlice
// CustomHeaders is the list of custom global headers to send with each request.
CustomHeaders goflags.StringSlice
// Vars is the list of custom global vars
Vars goflags.RuntimeMap
// Severities filters templates based on their severity and only run the matching ones.
Severities severity.Severities
// Author filters templates based on their author and only run the matching ones.
@ -146,4 +148,6 @@ type Options struct {
UpdateNuclei bool
// NoUpdateTemplates disables checking for nuclei templates updates
NoUpdateTemplates bool
// EnvironmentVariables enables support for environment variables
EnvironmentVariables bool
}