merging dev branch

dev
Mzack9999 2020-11-26 16:21:39 +01:00
commit 0358040d2d
10 changed files with 168 additions and 84 deletions

View File

@ -14,8 +14,8 @@ require (
github.com/miekg/dns v1.1.35
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.5
github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed
github.com/projectdiscovery/fastdialer v0.0.1
github.com/projectdiscovery/collaborator v0.0.2
github.com/projectdiscovery/fastdialer v0.0.2
github.com/projectdiscovery/gologger v1.0.1
github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/rawhttp v0.0.4
@ -24,8 +24,7 @@ require (
github.com/remeh/sizedwaitgroup v1.0.0
github.com/segmentio/ksuid v1.0.3
github.com/spaolacci/murmur3 v1.1.0
github.com/stretchr/testify v1.5.1
go.uber.org/ratelimit v0.1.0
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v2 v2.4.0
)

View File

@ -1,32 +1,34 @@
github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -45,22 +47,19 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F+/fkRs73a0=
github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed h1:0B4ccP68IL6MNHz99dAFeh0E3AY7fU5wVYwZJVX8ZG4=
github.com/projectdiscovery/collaborator v0.0.2-0.20201122173402-7afbb52febed/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/fastdialer v0.0.1 h1:MgBkJ/zkciFu/PcbAz0DYGiZn2aqv6b39NvfXxfN8qg=
github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc=
github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc=
github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
@ -79,7 +78,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@ -103,7 +101,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -113,20 +110,16 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s=
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

View File

@ -178,16 +178,21 @@ func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result {
for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ {
swg.Add()
// base request
result.Lock()
request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
if err != nil {
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
return result
}
go func(httpRequest *requests.HTTPRequest) {
defer swg.Done()
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
}
@ -218,9 +223,15 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
// Workers that keeps enqueuing new requests
maxWorkers := e.bulkHTTPRequest.Threads
swg := sizedwaitgroup.New(maxWorkers)
for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
for e.bulkHTTPRequest.Next(reqURL) {
result.Lock()
request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
if err != nil {
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
p.Drop(remaining)
} else {
@ -231,7 +242,7 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string)
e.ratelimiter.Take()
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
@ -288,9 +299,15 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
maxWorkers = pipeOptions.MaxPendingRequests
}
swg := sizedwaitgroup.New(maxWorkers)
for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
for e.bulkHTTPRequest.Next(reqURL) {
result.Lock()
request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
if err != nil {
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
} else {
swg.Add()
@ -301,7 +318,7 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
// If the request was built correctly then execute it
request.Pipeline = true
request.PipelineClient = pipeclient
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, "")
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "")
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
@ -353,17 +370,23 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
remaining := e.bulkHTTPRequest.GetRequestCount()
e.bulkHTTPRequest.CreateGenerator(reqURL)
for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
for e.bulkHTTPRequest.Next(reqURL) {
requestNumber++
result.Lock()
httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
if err != nil {
payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL)
result.Unlock()
// ignore the error due to the base request having null paylods
if err == requests.ErrNoPayload {
// pass through
} else if err != nil {
result.Error = err
p.Drop(remaining)
} else {
e.ratelimiter.Take()
// If the request was built correctly then execute it
format := "%s_" + strconv.Itoa(requestNumber)
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, format)
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format)
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
@ -388,7 +411,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result
return result
}
func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, format string) error {
func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error {
// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
if e.randomAgent {
// nolint:errcheck // ignoring error
@ -526,6 +549,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
headers := headersToString(resp.Header)
var matchData map[string]interface{}
if payloads != nil {
matchData = generators.MergeMaps(result.historyData, payloads)
}
// store for internal purposes the DSL matcher data
// hardcode stopping storing data after defaultMaxHistorydata items
if len(result.historyData) < defaultMaxHistorydata {
@ -536,13 +564,21 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
result.historyData["Hostname"] = parsed.Host
}
result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
if payloads == nil {
// merge them to history data
result.historyData = generators.MergeMaps(result.historyData, payloads)
}
result.historyData = generators.MergeMaps(result.historyData, dynamicvalues)
// complement match data with new one if necessary
matchData = generators.MergeMaps(matchData, result.historyData)
result.Unlock()
}
matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()
for _, matcher := range e.bulkHTTPRequest.Matchers {
// Check if the matcher matched
if !matcher.Match(resp, body, headers, duration, result.historyData) {
if !matcher.Match(resp, body, headers, duration, matchData) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
return nil
@ -557,7 +593,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
result.Meta = request.Meta
result.GotResults = true
result.Unlock()
e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta, reqURL)
e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL)
}
}
}
@ -588,7 +624,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta, reqURL)
e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL)
result.Lock()
result.GotResults = true
result.Unlock()
@ -716,7 +752,6 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) {
type Result struct {
sync.Mutex
GotResults bool
Done bool
Meta map[string]interface{}
Matches map[string]interface{}
Extractions map[string]interface{}

View File

@ -1,6 +1,7 @@
package executer
import (
"fmt"
"net/http"
"net/http/httputil"
"strings"
@ -119,9 +120,8 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
builder.WriteString(" [")
var metas []string
for name, value := range req.Meta {
metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String())
metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String())
}
builder.WriteString(strings.Join(metas, ","))

View File

@ -35,112 +35,112 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
// strings
functions["len"] = func(args ...interface{}) (interface{}, error) {
length := len(args[0].(string))
length := len(toString(args[0]))
return float64(length), nil
}
functions["toupper"] = func(args ...interface{}) (interface{}, error) {
return strings.ToUpper(args[0].(string)), nil
return strings.ToUpper(toString(args[0])), nil
}
functions["tolower"] = func(args ...interface{}) (interface{}, error) {
return strings.ToLower(args[0].(string)), nil
return strings.ToLower(toString(args[0])), nil
}
functions["replace"] = func(args ...interface{}) (interface{}, error) {
return strings.ReplaceAll(args[0].(string), args[1].(string), args[2].(string)), nil
return strings.ReplaceAll(toString(args[0]), toString(args[1]), toString(args[2])), nil
}
functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(args[1].(string))
compiled, err := regexp.Compile(toString(args[1]))
if err != nil {
return nil, err
}
return compiled.ReplaceAllString(args[0].(string), args[2].(string)), nil
return compiled.ReplaceAllString(toString(args[0]), toString(args[2])), nil
}
functions["trim"] = func(args ...interface{}) (interface{}, error) {
return strings.Trim(args[0].(string), args[2].(string)), nil
return strings.Trim(toString(args[0]), toString(args[2])), nil
}
functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimLeft(args[0].(string), args[1].(string)), nil
return strings.TrimLeft(toString(args[0]), toString(args[1])), nil
}
functions["trimright"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimRight(args[0].(string), args[1].(string)), nil
return strings.TrimRight(toString(args[0]), toString(args[1])), nil
}
functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimSpace(args[0].(string)), nil
return strings.TrimSpace(toString(args[0])), nil
}
functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimPrefix(args[0].(string), args[1].(string)), nil
return strings.TrimPrefix(toString(args[0]), toString(args[1])), nil
}
functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimSuffix(args[0].(string), args[1].(string)), nil
return strings.TrimSuffix(toString(args[0]), toString(args[1])), nil
}
functions["reverse"] = func(args ...interface{}) (interface{}, error) {
return reverseString(args[0].(string)), nil
return reverseString(toString(args[0])), nil
}
// encoding
functions["base64"] = func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0])))
return sEnc, nil
}
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(args[0].(string)))
sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0])))
return insertInto(sEnc, 76, '\n'), nil
}
functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
return base64.StdEncoding.DecodeString(args[0].(string))
return base64.StdEncoding.DecodeString(toString(args[0]))
}
functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
return url.PathEscape(args[0].(string)), nil
return url.PathEscape(toString(args[0])), nil
}
functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
return url.PathUnescape(args[0].(string))
return url.PathUnescape(toString(args[0]))
}
functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
return hex.EncodeToString([]byte(args[0].(string))), nil
return hex.EncodeToString([]byte(toString(args[0]))), nil
}
functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
hx, _ := hex.DecodeString(args[0].(string))
hx, _ := hex.DecodeString(toString(args[0]))
return string(hx), nil
}
functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
return html.EscapeString(args[0].(string)), nil
return html.EscapeString(toString(args[0])), nil
}
functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
return html.UnescapeString(args[0].(string)), nil
return html.UnescapeString(toString(args[0])), nil
}
// hashing
functions["md5"] = func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(args[0].(string)))
hash := md5.Sum([]byte(toString(args[0])))
return hex.EncodeToString(hash[:]), nil
}
functions["sha256"] = func(args ...interface{}) (interface{}, error) {
h := sha256.New()
_, err := h.Write([]byte(args[0].(string)))
_, err := h.Write([]byte(toString(args[0])))
if err != nil {
return nil, err
@ -151,7 +151,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
functions["sha1"] = func(args ...interface{}) (interface{}, error) {
h := sha1.New()
_, err := h.Write([]byte(args[0].(string)))
_, err := h.Write([]byte(toString(args[0])))
if err != nil {
return nil, err
@ -161,21 +161,21 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
}
functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(args[0].(string)), 0))), nil
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(toString(args[0])), 0))), nil
}
// search
functions["contains"] = func(args ...interface{}) (interface{}, error) {
return strings.Contains(args[0].(string), args[1].(string)), nil
return strings.Contains(toString(args[0]), toString(args[1])), nil
}
functions["regex"] = func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(args[0].(string))
compiled, err := regexp.Compile(toString(args[0]))
if err != nil {
return nil, err
}
return compiled.MatchString(args[1].(string)), nil
return compiled.MatchString(toString(args[1])), nil
}
// random generators
@ -183,10 +183,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
chars := letters + numbers
bad := ""
if len(args) >= 1 {
chars = args[0].(string)
chars = toString(args[0])
}
if len(args) >= withCutSetArgsSize {
bad = args[1].(string)
bad = toString(args[1])
}
chars = TrimAll(chars, bad)
@ -203,10 +203,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = args[1].(string)
bad = toString(args[1])
}
if len(args) >= withBaseRandArgsSize {
base = args[2].(string)
base = toString(args[2])
}
base = TrimAll(base, bad)
@ -223,7 +223,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = args[1].(string)
bad = toString(args[1])
}
chars = TrimAll(chars, bad)
@ -240,7 +240,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = args[1].(string)
bad = toString(args[1])
}
chars = TrimAll(chars, bad)
@ -257,7 +257,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = args[1].(string)
bad = toString(args[1])
}
chars = TrimAll(chars, bad)
@ -289,7 +289,7 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
// Collaborator
functions["collab"] = func(args ...interface{}) (interface{}, error) {
// check if collaborator contains a specific pattern
return collaborator.DefaultCollaborator.Has(args[0].(string)), nil
return collaborator.DefaultCollaborator.Has(toString(args[0])), nil
}
return functions

View File

@ -199,3 +199,7 @@ func insertInto(s string, interval int, sep rune) string {
buffer.WriteRune(sep)
return buffer.String()
}
func toString(v interface{}) string {
return fmt.Sprint(v)
}

View File

@ -199,7 +199,7 @@ func (m *Matcher) matchBinary(corpus string) bool {
// matchDSL matches on a generic map result
func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
// Iterate over all the regexes accepted as valid
// Iterate over all the expressions accepted as valid
for i, expression := range m.dslCompiled {
result, err := expression.Evaluate(mp)
if err != nil {

View File

@ -184,7 +184,12 @@ func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, d
r.gsfm.InitOrSkip(baseURL)
r.ReadOne(baseURL)
return r.handleRawWithPaylods(ctx, data, baseURL, values, r.gsfm.Value(baseURL))
payloads, err := r.GetPayloadsValues(baseURL)
if err != nil {
return nil, err
}
return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
}
// otherwise continue with normal flow
@ -202,7 +207,7 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL
dynamicValues := make(map[string]interface{})
// find all potentials tokens between {{}}
var re = regexp.MustCompile(`(?m)\{\{.+}}`)
var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
for _, match := range re.FindAllString(raw, -1) {
// check if the match contains a dynamic variable
expr := generators.TrimDelimiters(match)
@ -400,7 +405,14 @@ func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest,
value = p[1]
}
rawRequest.Headers[key] = value
// in case of unsafe requests multiple headers should be accepted
// therefore use the full line as key
_, found := rawRequest.Headers[key]
if r.Unsafe && found {
rawRequest.Headers[line] = ""
} else {
rawRequest.Headers[key] = value
}
}
// Handle case with the full http url in path. In that case,
@ -482,3 +494,40 @@ func (r *BulkHTTPRequest) Total() int {
func (r *BulkHTTPRequest) Increment(reqURL string) {
r.gsfm.Increment(reqURL)
}
// GetPayloadsValues for the specified URL
func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface{}, error) {
payloadProcessedValues := make(map[string]interface{})
payloadsFromTemplate := r.gsfm.Value(reqURL)
for k, v := range payloadsFromTemplate {
kexp := v.(string)
// if it doesn't containg markups, we just continue
if !hasMarker(kexp) {
payloadProcessedValues[k] = v
continue
}
// attempts to expand expressions
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions())
if err != nil {
// it is a simple literal payload => proceed with literal value
payloadProcessedValues[k] = v
continue
}
// it is an expression - try to solve it
expValue, err := compiled.Evaluate(payloadsFromTemplate)
if err != nil {
// an error occurred => proceed with literal value
payloadProcessedValues[k] = v
continue
}
payloadProcessedValues[k] = fmt.Sprint(expValue)
}
var err error
if len(payloadProcessedValues) == 0 {
err = ErrNoPayload
}
return payloadProcessedValues, err
}
// ErrNoPayload error to avoid the additional base null request
var ErrNoPayload = fmt.Errorf("No payload found")

View File

@ -71,3 +71,7 @@ func ExpandMapValues(m map[string]string) (m1 map[string][]string) {
}
return
}
func hasMarker(s string) bool {
return strings.Contains(s, markerParenthesisOpen) || strings.Contains(s, markerParenthesisClose) || strings.Contains(s, markerGeneral)
}

View File

@ -66,7 +66,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount())
for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
// apply externally supplied payloads if any
// apply externally supplied headers if any
request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
// apply externally supplied payloads if any
request.Payloads = generators.MergeMaps(request.Payloads, externalVars)