2020-04-03 18:47:57 +00:00
|
|
|
package requests
|
|
|
|
|
|
|
|
import (
|
2020-04-28 21:02:07 +00:00
|
|
|
"bufio"
|
2020-04-29 00:57:18 +00:00
|
|
|
"fmt"
|
2020-04-03 21:20:32 +00:00
|
|
|
"io/ioutil"
|
|
|
|
"net/http"
|
2020-04-03 21:56:11 +00:00
|
|
|
"net/url"
|
2020-05-04 21:24:59 +00:00
|
|
|
"regexp"
|
2020-04-03 21:20:32 +00:00
|
|
|
"strings"
|
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
"github.com/Knetic/govaluate"
|
2020-04-05 19:14:45 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/pkg/extractors"
|
2020-05-04 21:24:59 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/pkg/generators"
|
2020-04-03 18:47:57 +00:00
|
|
|
"github.com/projectdiscovery/nuclei/pkg/matchers"
|
2020-04-04 11:42:29 +00:00
|
|
|
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
|
2020-04-03 18:47:57 +00:00
|
|
|
)
|
|
|
|
|
2020-04-22 20:45:02 +00:00
|
|
|
// HTTPRequest contains a request to be made from a template
|
|
|
|
type HTTPRequest struct {
|
2020-05-04 21:24:59 +00:00
|
|
|
// AttackType is the attack type
|
|
|
|
// Sniper, PitchFork and ClusterBomb. Default is Sniper
|
|
|
|
AttackType string `yaml:"attack,omitempty"`
|
|
|
|
// attackType is internal attack type
|
|
|
|
attackType generators.Type
|
|
|
|
// Path contains the path/s for the request variables
|
|
|
|
Payloads map[string]string `yaml:"payloads,omitempty"`
|
2020-04-03 18:47:57 +00:00
|
|
|
// Method is the request method, whether GET, POST, PUT, etc
|
|
|
|
Method string `yaml:"method"`
|
|
|
|
// Path contains the path/s for the request
|
|
|
|
Path []string `yaml:"path"`
|
|
|
|
// Headers contains headers to send with the request
|
|
|
|
Headers map[string]string `yaml:"headers,omitempty"`
|
|
|
|
// Body is an optional parameter which contains the request body for POST methods, etc
|
|
|
|
Body string `yaml:"body,omitempty"`
|
|
|
|
// Matchers contains the detection mechanism for the request to identify
|
|
|
|
// whether the request was successful
|
2020-04-05 18:28:22 +00:00
|
|
|
Matchers []*matchers.Matcher `yaml:"matchers,omitempty"`
|
2020-04-26 00:20:33 +00:00
|
|
|
// MatchersCondition is the condition of the matchers
|
|
|
|
// whether to use AND or OR. Default is OR.
|
|
|
|
MatchersCondition string `yaml:"matchers-condition,omitempty"`
|
2020-04-26 01:03:59 +00:00
|
|
|
// matchersCondition is internal condition for the matchers.
|
|
|
|
matchersCondition matchers.ConditionType
|
2020-04-05 18:28:22 +00:00
|
|
|
// Extractors contains the extraction mechanism for the request to identify
|
|
|
|
// and extract parts of the response.
|
2020-04-05 19:14:45 +00:00
|
|
|
Extractors []*extractors.Extractor `yaml:"extractors,omitempty"`
|
2020-04-22 22:26:41 +00:00
|
|
|
// Redirects specifies whether redirects should be followed.
|
|
|
|
Redirects bool `yaml:"redirects,omitempty"`
|
|
|
|
// MaxRedirects is the maximum number of redirects that should be followed.
|
|
|
|
MaxRedirects int `yaml:"max-redirects,omitempty"`
|
2020-04-29 19:19:35 +00:00
|
|
|
// Raw contains raw requests
|
|
|
|
Raw []string `yaml:"raw,omitempty"`
|
2020-04-03 18:47:57 +00:00
|
|
|
}
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2020-04-26 01:03:59 +00:00
|
|
|
// GetMatchersCondition returns the condition for the matcher
|
|
|
|
func (r *HTTPRequest) GetMatchersCondition() matchers.ConditionType {
|
|
|
|
return r.matchersCondition
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetMatchersCondition sets the condition for the matcher
|
|
|
|
func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
|
|
|
|
r.matchersCondition = condition
|
|
|
|
}
|
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
// GetAttackType returns the attack
|
|
|
|
func (r *HTTPRequest) GetAttackType() generators.Type {
|
|
|
|
return r.attackType
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetAttackType sets the attack
|
|
|
|
func (r *HTTPRequest) SetAttackType(attack generators.Type) {
|
|
|
|
r.attackType = attack
|
|
|
|
}
|
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
// MakeHTTPRequest creates a *http.Request from a request configuration
|
2020-05-05 19:42:28 +00:00
|
|
|
func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) {
|
2020-04-03 21:56:11 +00:00
|
|
|
parsed, err := url.Parse(baseURL)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
hostname := parsed.Hostname()
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
values := map[string]interface{}{
|
|
|
|
"BaseURL": baseURL,
|
|
|
|
"Hostname": hostname,
|
|
|
|
}
|
|
|
|
|
2020-04-29 19:19:35 +00:00
|
|
|
if len(r.Raw) > 0 {
|
2020-04-29 00:57:18 +00:00
|
|
|
return r.makeHTTPRequestFromRaw(baseURL, values)
|
|
|
|
}
|
|
|
|
|
|
|
|
return r.makeHTTPRequestFromModel(baseURL, values)
|
|
|
|
}
|
|
|
|
|
|
|
|
// MakeHTTPRequestFromModel creates a *http.Request from a request template
|
2020-05-05 19:42:28 +00:00
|
|
|
func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
|
|
|
|
requests = make(chan *CompiledHTTP)
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
// request generator
|
|
|
|
go func() {
|
|
|
|
defer close(requests)
|
|
|
|
for _, path := range r.Path {
|
|
|
|
// process base request
|
|
|
|
replacer := newReplacer(values)
|
2020-04-04 11:42:29 +00:00
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
// Replace the dynamic variables in the URL if any
|
|
|
|
URL := replacer.Replace(path)
|
|
|
|
|
|
|
|
// Build a request on the specified URL
|
|
|
|
req, err := http.NewRequest(r.Method, URL, nil)
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-04 21:24:59 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
request, err := r.fillRequest(req, values)
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
requests <- &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-04 21:24:59 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-05-14 16:09:36 +00:00
|
|
|
requests <- &CompiledHTTP{Request: request, Error: nil, Meta: nil}
|
2020-05-04 21:24:59 +00:00
|
|
|
}
|
|
|
|
}()
|
2020-04-03 21:20:32 +00:00
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
return
|
2020-04-03 21:20:32 +00:00
|
|
|
}
|
2020-04-28 21:02:07 +00:00
|
|
|
|
|
|
|
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
|
2020-05-05 19:42:28 +00:00
|
|
|
func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests chan *CompiledHTTP, err error) {
|
|
|
|
requests = make(chan *CompiledHTTP)
|
2020-05-04 21:24:59 +00:00
|
|
|
// request generator
|
|
|
|
go func() {
|
|
|
|
defer close(requests)
|
2020-04-29 00:57:18 +00:00
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
for _, raw := range r.Raw {
|
|
|
|
// Add trailing line
|
|
|
|
raw += "\n"
|
2020-04-28 21:02:07 +00:00
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
if len(r.Payloads) > 0 {
|
|
|
|
basePayloads := generators.LoadWordlists(r.Payloads)
|
|
|
|
generatorFunc := generators.SniperGenerator
|
|
|
|
switch r.attackType {
|
|
|
|
case generators.PitchFork:
|
|
|
|
generatorFunc = generators.PitchforkGenerator
|
|
|
|
case generators.ClusterBomb:
|
|
|
|
generatorFunc = generators.ClusterbombGenerator
|
|
|
|
}
|
2020-04-29 19:19:35 +00:00
|
|
|
|
2020-05-04 21:24:59 +00:00
|
|
|
for genValues := range generatorFunc(basePayloads) {
|
2020-05-05 19:42:28 +00:00
|
|
|
compiledHTTP := r.handleRawWithPaylods(raw, baseURL, values, genValues)
|
|
|
|
requests <- compiledHTTP
|
|
|
|
if compiledHTTP.Error != nil {
|
2020-05-04 21:24:59 +00:00
|
|
|
return
|
|
|
|
}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// otherwise continue with normal flow
|
|
|
|
compiledHTTP := r.handleSimpleRaw(raw, baseURL, values)
|
|
|
|
requests <- compiledHTTP
|
|
|
|
if compiledHTTP.Error != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
return requests, nil
|
|
|
|
}
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[string]interface{}) *CompiledHTTP {
|
|
|
|
// base request
|
|
|
|
replacer := newReplacer(values)
|
|
|
|
// Replace the dynamic variables in the request if any
|
|
|
|
raw = replacer.Replace(raw)
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
// Build a parsed request from raw
|
|
|
|
parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
// requests generated from http.ReadRequest have incorrect RequestURI, so they
|
|
|
|
// cannot be used to perform another request directly, we need to generate a new one
|
|
|
|
// with the new target url
|
|
|
|
finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
|
|
|
|
req, err := http.NewRequest(parsedReq.Method, finalURL, parsedReq.Body)
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
// copy headers
|
|
|
|
req.Header = parsedReq.Header.Clone()
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
request, err := r.fillRequest(req, values)
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: request, Error: nil, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, genValues map[string]interface{}) *CompiledHTTP {
|
|
|
|
baseValues := generators.CopyMap(values)
|
|
|
|
finValues := generators.MergeMaps(baseValues, genValues)
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
replacer := newReplacer(finValues)
|
|
|
|
|
|
|
|
// Replace the dynamic variables in the URL if any
|
|
|
|
raw = replacer.Replace(raw)
|
2020-05-04 21:24:59 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
dynamicValues := make(map[string]interface{})
|
|
|
|
// find all potentials tokens between {{}}
|
|
|
|
var re = regexp.MustCompile(`(?m)\{\{.+}}`)
|
|
|
|
for _, match := range re.FindAllString(raw, -1) {
|
|
|
|
// check if the match contains a dynamic variable
|
|
|
|
if generators.StringContainsAnyMapItem(finValues, match) {
|
|
|
|
expr := generators.TrimDelimiters(match)
|
|
|
|
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
|
|
|
result, err := compiled.Evaluate(finValues)
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-04 21:24:59 +00:00
|
|
|
}
|
2020-05-05 19:42:28 +00:00
|
|
|
dynamicValues[expr] = result
|
2020-05-04 21:24:59 +00:00
|
|
|
}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
2020-04-28 21:02:07 +00:00
|
|
|
|
2020-05-05 19:42:28 +00:00
|
|
|
// replace dynamic values
|
|
|
|
dynamicReplacer := newReplacer(dynamicValues)
|
|
|
|
raw = dynamicReplacer.Replace(raw)
|
|
|
|
|
|
|
|
// Build a parsed request from raw
|
|
|
|
parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Bug: http.ReadRequest does not process request body, so building it manually
|
|
|
|
// need to read from first \n\n till end
|
|
|
|
body := raw[strings.Index(raw, "\n\n"):]
|
|
|
|
|
|
|
|
// requests generated from http.ReadRequest have incorrect RequestURI, so they
|
|
|
|
// cannot be used to perform another request directly, we need to generate a new one
|
|
|
|
// with the new target url
|
|
|
|
finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
|
|
|
|
req, err := http.NewRequest(parsedReq.Method, finalURL, strings.NewReader(body))
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// copy headers
|
|
|
|
req.Header = parsedReq.Header.Clone()
|
|
|
|
|
|
|
|
request, err := r.fillRequest(req, values)
|
|
|
|
if err != nil {
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|
|
|
|
|
2020-05-14 16:09:36 +00:00
|
|
|
return &CompiledHTTP{Request: request, Error: nil, Meta: genValues}
|
2020-04-29 00:57:18 +00:00
|
|
|
}
|
2020-04-28 21:02:07 +00:00
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
|
2020-05-05 19:42:28 +00:00
|
|
|
req.Header.Set("Connection", "close")
|
|
|
|
req.Close = true
|
|
|
|
|
|
|
|
// raw requests are left untouched
|
|
|
|
if len(r.Raw) > 0 {
|
|
|
|
return retryablehttp.FromRequest(req)
|
|
|
|
}
|
|
|
|
|
2020-04-30 15:39:33 +00:00
|
|
|
replacer := newReplacer(values)
|
2020-04-29 00:57:18 +00:00
|
|
|
// Check if the user requested a request body
|
|
|
|
if r.Body != "" {
|
|
|
|
req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
|
|
|
|
}
|
2020-04-28 21:02:07 +00:00
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
// Set the header values requested
|
|
|
|
for header, value := range r.Headers {
|
2020-04-30 15:39:33 +00:00
|
|
|
req.Header.Set(header, replacer.Replace(value))
|
2020-04-29 00:57:18 +00:00
|
|
|
}
|
2020-04-28 21:02:07 +00:00
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
// Set some headers only if the header wasn't supplied by the user
|
2020-04-29 21:07:19 +00:00
|
|
|
if _, ok := req.Header["User-Agent"]; !ok {
|
2020-04-29 00:57:18 +00:00
|
|
|
req.Header.Set("User-Agent", "Nuclei (@pdiscoveryio)")
|
|
|
|
}
|
2020-04-29 21:07:19 +00:00
|
|
|
|
|
|
|
if _, ok := req.Header["Accept"]; !ok {
|
2020-04-29 00:57:18 +00:00
|
|
|
req.Header.Set("Accept", "*/*")
|
|
|
|
}
|
2020-04-29 21:07:19 +00:00
|
|
|
if _, ok := req.Header["Accept-Language"]; !ok {
|
2020-04-29 00:57:18 +00:00
|
|
|
req.Header.Set("Accept-Language", "en")
|
2020-04-28 21:02:07 +00:00
|
|
|
}
|
|
|
|
|
2020-04-29 00:57:18 +00:00
|
|
|
return retryablehttp.FromRequest(req)
|
2020-04-28 21:02:07 +00:00
|
|
|
}
|
2020-05-05 19:42:28 +00:00
|
|
|
|
|
|
|
// CompiledHTTP contains Generated HTTP Request or error
|
|
|
|
type CompiledHTTP struct {
|
|
|
|
Request *retryablehttp.Request
|
|
|
|
Error error
|
2020-05-14 16:09:36 +00:00
|
|
|
Meta map[string]interface{}
|
2020-05-05 19:42:28 +00:00
|
|
|
}
|