daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial update to latest updates

dev
Manuel Bua 2020-07-23 20:19:19 +02:00
parent 04875a55dd
commit 362fee27a4
8 changed files with 141 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/go.mod
View File

@ -11,11 +11,13 @@ require (
github.com/google/go-github/v32 v32.1.0
github.com/json-iterator/go v1.1.10
github.com/karrick/godirwalk v1.15.6
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381
github.com/miekg/dns v1.1.30
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/gologger v1.0.0
github.com/projectdiscovery/retryabledns v1.0.4
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/vbauerster/mpb/v5 v5.2.4
golang.org/x/net v0.0.0-20200707034311-ab3426394381
gopkg.in/yaml.v2 v2.3.0
)

12
v2/go.sum
View File

@ -1,6 +1,10 @@
github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
@ -26,6 +30,8 @@ github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G
github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
@ -45,6 +51,10 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/vbauerster/mpb v1.1.3 h1:IRgic8VFaURXkW0VxDLkNOiNaAgtw0okB2YIaVvJDI4=
github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw=
github.com/vbauerster/mpb/v5 v5.2.4 h1:PLP8vv75RcEgxGoJVtKaRD2FHSxEmIV/u4ZuOrfO8Qg=
github.com/vbauerster/mpb/v5 v5.2.4/go.mod h1:K4iCHQp5sWnmAgEn+uW1sAxSilctb4JPAGXx49jV+Aw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
@ -63,6 +73,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

121
v2/internal/runner/runner.go
View File

@ -16,6 +16,7 @@ import (
"github.com/d5/tengo/v2/stdlib"
"github.com/karrick/godirwalk"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/executer"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@ -24,6 +25,9 @@ import (
// Runner is a client for running the enumeration process.
type Runner struct {
input *os.File
inputCount int64
// output is the output file to write if any
output *os.File
outputMutex *sync.Mutex
@ -32,6 +36,9 @@ type Runner struct {
templatesConfig *nucleiConfig
// options contains configuration options for runner
options *Options
// progress tracking
progress *progress.Progress
}
// New creates a new client for running enumeration process.
@ -71,6 +78,25 @@ func New(options *Options) (*Runner, error) {
tempInput.Close()
}
// Setup input, handle a list of hosts as argument
var err error
if options.Targets != "" {
runner.input, err = os.Open(options.Targets)
} else if options.Stdin || options.Target != "" {
runner.input, err = os.Open(runner.tempFile)
}
if err != nil {
gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
}
// Precompute total number of targets
scanner := bufio.NewScanner(runner.input)
runner.inputCount = 0
for scanner.Scan() {
runner.inputCount++
}
runner.input.Seek(0, 0)
// Create the output file if asked
if options.Output != "" {
output, err := os.Create(options.Output)
@ -79,12 +105,17 @@ func New(options *Options) (*Runner, error) {
}
runner.output = output
}
// Creates the progress tracking object
runner.progress = progress.NewProgress(runner.options.NoColor)
return runner, nil
}
// Close releases all the resources and cleans up
func (r *Runner) Close() {
r.output.Close()
r.input.Close()
os.Remove(r.tempFile)
}
@ -191,21 +222,51 @@ func (r *Runner) RunEnumeration() {
gologger.Fatalf("Error, no templates were found.\n")
}
p := r.progress
templateCount := len(allTemplates)
isSingleTemplate := templateCount == 1
if !isSingleTemplate {
// precompute request count
var totalRequests int64 = 0
for _, match := range allTemplates {
t, err := r.parse(match)
switch t.(type) {
case *templates.Template:
template := t.(*templates.Template)
totalRequests += template.GetHTTPRequestsCount()
default:
gologger.Errorf("Could not parse file '%s': %s\n", match, err)
}
}
// track global progress
p.SetupGlobalProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests)
}
// run with the specified templates
var results bool
for _, match := range allTemplates {
for i, match := range allTemplates {
t, err := r.parse(match)
switch t.(type) {
case *templates.Template:
template := t.(*templates.Template)
if isSingleTemplate {
// track single template progress
p.SetupTemplateProgressbar(-1, -1, template.ID, r.inputCount*template.GetHTTPRequestsCount())
} else {
p.SetupTemplateProgressbar(i, templateCount, template.ID, r.inputCount * template.GetHTTPRequestsCount())
}
for _, request := range template.RequestsDNS {
dnsResults := r.processTemplateRequest(template, request)
dnsResults := r.processTemplateWithList(p, template, request)
if dnsResults {
results = dnsResults
}
}
for _, request := range template.BulkRequestsHTTP {
httpResults := r.processTemplateRequest(template, request)
httpResults := r.processTemplateWithList(p, template, request)
if httpResults {
results = httpResults
}
@ -214,9 +275,15 @@ func (r *Runner) RunEnumeration() {
workflow := t.(*workflows.Workflow)
r.ProcessWorkflowWithList(workflow)
default:
p.StartStdCapture()
gologger.Errorf("Could not parse file '%s': %s\n", match, err)
p.StopStdCapture()
}
}
p.Wait()
p.ShowStdErr()
p.ShowStdOut()
if !results {
if r.output != nil {
outputFile := r.output.Name()
@ -228,33 +295,16 @@ func (r *Runner) RunEnumeration() {
return
}
// processTemplate processes a template and runs the enumeration on all the targets
func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool {
var file *os.File
var err error
// Handle a list of hosts as argument
if r.options.Targets != "" {
file, err = os.Open(r.options.Targets)
} else if r.options.Stdin || r.options.Target != "" {
file, err = os.Open(r.tempFile)
}
if err != nil {
gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
}
results := r.processTemplateWithList(template, request, file)
file.Close()
return results
}
// processDomain processes the list with a template
func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool {
// processTemplateWithList processes a template and runs the enumeration on all the targets
func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
// Display the message for the template
message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author)
if template.Info.Severity != "" {
message += " [" + template.Info.Severity + "]"
}
p.StartStdCapture()
gologger.Infof("%s\n", message)
p.StopStdCapture()
var writer *bufio.Writer
if r.output != nil {
@ -300,7 +350,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
limiter := make(chan struct{}, r.options.Threads)
wg := &sync.WaitGroup{}
scanner := bufio.NewScanner(reader)
r.input.Seek(0, 0)
scanner := bufio.NewScanner(r.input)
for scanner.Scan() {
text := scanner.Text()
if text == "" {
@ -313,13 +364,15 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
var result executer.Result
if httpExecuter != nil {
result = httpExecuter.ExecuteHTTP(URL)
result = httpExecuter.ExecuteHTTP(p, URL)
}
if dnsExecuter != nil {
result = dnsExecuter.ExecuteDNS(URL)
}
if result.Error != nil {
p.StartStdCapture()
gologger.Warningf("Could not execute step: %s\n", result.Error)
p.StopStdCapture()
}
<-limiter
wg.Done()
@ -343,20 +396,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
// ProcessWorkflowWithList coming from stdin or list of targets
func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) {
var file *os.File
var err error
// Handle a list of hosts as argument
if r.options.Targets != "" {
file, err = os.Open(r.options.Targets)
} else if r.options.Stdin {
file, err = os.Open(r.tempFile)
}
if err != nil {
gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
r.input.Seek(0, 0)
scanner := bufio.NewScanner(r.input)
for scanner.Scan() {
text := scanner.Text()
if text == "" {

30
v2/pkg/executer/executer_http.go
View File

@ -17,6 +17,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@ -99,45 +100,54 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
}
// ExecuteHTTP executes the HTTP request on a URL
func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Result) {
result.Matches = make(map[string]interface{})
result.Extractions = make(map[string]interface{})
dynamicvalues := make(map[string]interface{})
e.bulkHttpRequest.Reset()
remaining := e.template.GetHTTPRequestsCount()
for e.bulkHttpRequest.Next() && !result.Done {
httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, dynamicvalues, e.bulkHttpRequest.Current())
if err != nil {
result.Error = errors.Wrap(err, "could not make http request")
result.Error = errors.Wrap(err, "could not build http request")
return
}
err = e.handleHTTP(URL, httpRequest, dynamicvalues, &result)
err = e.handleHTTP(p, URL, httpRequest, dynamicvalues, &result)
if err != nil {
result.Error = errors.Wrap(err, "could not make http request")
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
return
}
e.bulkHttpRequest.Increment()
p.Update()
remaining--
}
p.StartStdCapture()
gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
p.StopStdCapture()
return
}
func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error {
func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error {
e.setCustomHeaders(request)
req := request.Request
if e.debug {
gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
dumpedRequest, err := httputil.DumpRequest(req.Request, true)
if err != nil {
return errors.Wrap(err, "could not make http request")
}
p.StartStdCapture()
gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
p.StopStdCapture()
}
resp, err := e.httpClient.Do(req)
if err != nil {
@ -148,12 +158,14 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn
}
if e.debug {
gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
return errors.Wrap(err, "could not dump http response")
}
p.StartStdCapture()
gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
p.StopStdCapture()
}
data, err := ioutil.ReadAll(resp.Body)
@ -190,7 +202,9 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn
result.Matches[matcher.Name] = nil
// probably redundant but ensures we snapshot current payload values when matchers are valid
result.Meta = request.Meta
p.StartStdCapture()
e.writeOutputHTTP(request, resp, body, matcher, nil)
p.StopStdCapture()
e.Results = true
}
}
@ -214,7 +228,9 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.bulkHttpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
p.StartStdCapture()
e.writeOutputHTTP(request, resp, body, nil, extractorResults)
p.StopStdCapture()
e.Results = true
}

313
v2/pkg/executor/executer_http.go
View File

@ -1,313 +0,0 @@
package executor
import (
"bufio"
"crypto/tls"
"fmt"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"io"
"io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
"os"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/retryablehttp-go"
"golang.org/x/net/proxy"
)
// HTTPExecutor is client for performing HTTP requests
// for a template.
type HTTPExecutor struct {
debug bool
results uint32
jsonOutput bool
httpClient *retryablehttp.Client
template *templates.Template
httpRequest *requests.HTTPRequest
writer *bufio.Writer
outputMutex *sync.Mutex
customHeaders requests.CustomHeaders
}
// HTTPOptions contains configuration options for the HTTP executor.
type HTTPOptions struct {
Template *templates.Template
HTTPRequest *requests.HTTPRequest
Writer *bufio.Writer
Timeout int
Retries int
ProxyURL string
ProxySocksURL string
Debug bool
JSON bool
CustomHeaders requests.CustomHeaders
}
// NewHTTPExecutor creates a new HTTP executor from a template
// and a HTTP request query.
func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
var proxyURL *url.URL
var err error
if options.ProxyURL != "" {
proxyURL, err = url.Parse(options.ProxyURL)
}
if err != nil {
return nil, err
}
// Create the HTTP Client
client := makeHTTPClient(proxyURL, options)
client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
executer := &HTTPExecutor{
debug: options.Debug,
jsonOutput: options.JSON,
results: 0,
httpClient: client,
template: options.Template,
httpRequest: options.HTTPRequest,
outputMutex: &sync.Mutex{},
writer: options.Writer,
customHeaders: options.CustomHeaders,
}
return executer, nil
}
// GotResults returns true if there were any results for the executor
func (e *HTTPExecutor) GotResults() bool {
if atomic.LoadUint32(&e.results) == 0 {
return false
}
return true
}
// ExecuteHTTP executes the HTTP request on a URL
func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error {
// Compile each request for the template based on the URL
compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL)
if err != nil {
return errors.Wrap(err, "could not compile http request")
}
remaining := e.template.GetHTTPRequestsCount()
// Send the request to the target servers
mainLoop:
for compiledRequest := range compiledRequest {
if compiledRequest.Error != nil {
p.Drop(remaining)
return errors.Wrap(err, "error in compiled http request")
}
e.setCustomHeaders(compiledRequest)
req := compiledRequest.Request
if e.debug {
dumpedRequest, err := httputil.DumpRequest(req.Request, true)
if err != nil {
p.Drop(remaining)
return errors.Wrap(err, "could not dump http request")
}
p.StartStdCapture()
gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
p.StopStdCapture()
}
resp, err := e.httpClient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
p.Drop(1)
p.StartStdCapture()
gologger.Warningf("Could not do request: %s\n", err)
p.StopStdCapture()
continue
}
if e.debug {
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
p.Drop(remaining)
return errors.Wrap(err, "could not dump http response")
}
p.StartStdCapture()
gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
p.StopStdCapture()
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
p.Drop(remaining)
return errors.Wrap(err, "could not read http body")
}
resp.Body.Close()
// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
// so in case we have to manually do it
data, err = requests.HandleDecompression(compiledRequest.Request, data)
if err != nil {
p.Drop(remaining)
return errors.Wrap(err, "could not decompress http body")
}
// Convert response body from []byte to string with zero copy
body := unsafeToString(data)
var headers string
matcherCondition := e.httpRequest.GetMatchersCondition()
for _, matcher := range e.httpRequest.Matchers {
headers = headersToString(resp.Header)
// Check if the matcher matched
if !matcher.Match(resp, body, headers) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
p.Update()
remaining--
continue mainLoop
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 {
// capture stdout and emit it via a mpb.BarFiller
p.StartStdCapture()
e.writeOutputHTTP(compiledRequest, matcher, nil)
p.StopStdCapture()
atomic.CompareAndSwapUint32(&e.results, 0, 1)
}
}
}
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults []string
for _, extractor := range e.httpRequest.Extractors {
headers = headersToString(resp.Header)
for match := range extractor.Extract(body, headers) {
extractorResults = append(extractorResults, match)
}
}
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
// capture stdout and emit it via a mpb.BarFiller
p.StartStdCapture()
e.writeOutputHTTP(compiledRequest, nil, extractorResults)
p.StopStdCapture()
atomic.CompareAndSwapUint32(&e.results, 0, 1)
}
p.Update()
remaining--
}
p.StartStdCapture()
gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
p.StopStdCapture()
return nil
}
// Close closes the http executor for a template.
func (e *HTTPExecutor) Close() {
e.outputMutex.Lock()
e.writer.Flush()
e.outputMutex.Unlock()
}
// makeHTTPClient creates a http client
func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client {
retryablehttpOptions := retryablehttp.DefaultOptionsSpraying
retryablehttpOptions.RetryWaitMax = 10 * time.Second
retryablehttpOptions.RetryMax = options.Retries
followRedirects := options.HTTPRequest.Redirects
maxRedirects := options.HTTPRequest.MaxRedirects
transport := &http.Transport{
MaxIdleConnsPerHost: -1,
TLSClientConfig: &tls.Config{
Renegotiation: tls.RenegotiateOnceAsClient,
InsecureSkipVerify: true,
},
DisableKeepAlives: true,
}
// Attempts to overwrite the dial function with the socks proxied version
if options.ProxySocksURL != "" {
var proxyAuth *proxy.Auth
socksURL, err := url.Parse(options.ProxySocksURL)
if err == nil {
proxyAuth = &proxy.Auth{}
proxyAuth.User = socksURL.User.Username()
proxyAuth.Password, _ = socksURL.User.Password()
}
dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
if err == nil {
transport.Dial = dialer.Dial
}
}
if proxyURL != nil {
transport.Proxy = http.ProxyURL(proxyURL)
}
return retryablehttp.NewWithHTTPClient(&http.Client{
Transport: transport,
Timeout: time.Duration(options.Timeout) * time.Second,
CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
}, retryablehttpOptions)
}
type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error
func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
return func(_ *http.Request, requests []*http.Request) error {
if !followRedirects {
return http.ErrUseLastResponse
}
if maxRedirects == 0 {
if len(requests) > 10 {
return http.ErrUseLastResponse
}
return nil
}
if len(requests) > maxRedirects {
return http.ErrUseLastResponse
}
return nil
}
}
func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) {
for _, customHeader := range e.customHeaders {
// This should be pre-computed somewhere and done only once
tokens := strings.Split(customHeader, ":")
// if it's an invalid header skip it
if len(tokens) < 2 {
continue
}
headerName, headerValue := tokens[0], strings.Join(tokens[1:], "")
headerName = strings.TrimSpace(headerName)
headerValue = strings.TrimSpace(headerValue)
r.Request.Header.Set(headerName, headerValue)
}
}

20
v2/pkg/requests/bulk-http-request.go
View File

@ -8,6 +8,7 @@ import (
"net/url"
"regexp"
"strings"
"sync"
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/nuclei/v2/pkg/extractors"
@ -55,6 +56,7 @@ type BulkHTTPRequest struct {
Raw []string `yaml:"raw,omitempty"`
positionPath int
positionRaw int
positionMutex sync.Mutex
generator func(payloads map[string][]string) (out chan map[string]interface{})
currentPayloads map[string]interface{}
basePayloads map[string][]string
@ -83,8 +85,8 @@ func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) {
}
// Returns the total number of requests the YAML rule will perform
func (r *HTTPRequest) GetRequestCount() int64 {
return int64( len(r.Raw) | len(r.Path) )
func (r *BulkHTTPRequest) GetRequestCount() int64 {
return int64(len(r.Raw) | len(r.Path))
}
func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HttpRequest, error) {
@ -369,19 +371,30 @@ func (r *BulkHTTPRequest) parseRawRequest(request string, baseURL string) (*RawR
}
func (r *BulkHTTPRequest) Next() bool {
r.positionMutex.Lock()
defer r.positionMutex.Unlock()
if r.positionPath+r.positionRaw >= len(r.Path)+len(r.Raw) {
return false
}
return true
}
func (r *BulkHTTPRequest) Position() int {
r.positionMutex.Lock()
defer r.positionMutex.Unlock()
return r.positionPath + r.positionRaw
}
func (r *BulkHTTPRequest) Reset() {
r.positionMutex.Lock()
r.positionPath = 0
r.positionRaw = 0
r.positionMutex.Unlock()
}
func (r *BulkHTTPRequest) Current() string {
r.positionMutex.Lock()
defer r.positionMutex.Unlock()
if r.positionPath < len(r.Path) && len(r.Path) != 0 {
return r.Path[r.positionPath]
}
@ -393,6 +406,9 @@ func (r *BulkHTTPRequest) Total() int {
}
func (r *BulkHTTPRequest) Increment() {
r.positionMutex.Lock()
defer r.positionMutex.Unlock()
if len(r.Path) > 0 && r.positionPath < len(r.Path) {
r.positionPath++
return

2
v2/pkg/templates/templates.go
View File

@ -30,7 +30,7 @@ type Info struct {
func (t* Template) GetHTTPRequestsCount() int64 {
var count int64 = 0
for _, request := range t.RequestsHTTP {
for _, request := range t.BulkRequestsHTTP {
count += request.GetRequestCount()
}
return count

5
v2/pkg/workflows/var.go
View File

@ -51,6 +51,9 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
externalVars = iterableToMap(args[1])
}
// track progress
p := progress.NewProgress(false)
var gotResult bool
for _, template := range n.Templates {
if template.HTTPOptions != nil {
@ -65,7 +68,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
continue
}
result := httpExecuter.ExecuteHTTP(n.URL)
result := httpExecuter.ExecuteHTTP(p, n.URL)
if result.Error != nil {
gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
continue