mirror of https://github.com/daffainfo/nuclei.git
Merge pull request #1247 from projectdiscovery/research-aws-signing
Adding support for aws request signingdev
commit
2dc877a99f
|
@ -63,6 +63,8 @@ require (
|
||||||
moul.io/http2curl v1.0.0
|
moul.io/http2curl v1.0.0
|
||||||
)
|
)
|
||||||
|
|
||||||
|
require github.com/aws/aws-sdk-go v1.42.3
|
||||||
|
|
||||||
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
|
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
@ -108,6 +110,7 @@ require (
|
||||||
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
|
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
|
||||||
github.com/itchyny/timefmt-go v0.1.3 // indirect
|
github.com/itchyny/timefmt-go v0.1.3 // indirect
|
||||||
github.com/jasonlvhit/gocron v0.0.1 // indirect
|
github.com/jasonlvhit/gocron v0.0.1 // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
|
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
|
||||||
github.com/klauspost/compress v1.13.6 // indirect
|
github.com/klauspost/compress v1.13.6 // indirect
|
||||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||||
|
|
|
@ -105,6 +105,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
|
||||||
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
|
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
|
||||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||||
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||||
|
github.com/aws/aws-sdk-go v1.42.3 h1:lBKr3tQ06m1uykiychMNKLK1bRfOzaIEQpsI/S3QiNc=
|
||||||
|
github.com/aws/aws-sdk-go v1.42.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||||
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
||||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||||
|
@ -410,6 +412,10 @@ github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT
|
||||||
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
|
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
|
||||||
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
|
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
|
||||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||||
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
|
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
|
||||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||||
|
@ -1243,6 +1249,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
|
||||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||||
|
|
|
@ -16,49 +16,66 @@ func ContainsUnresolvedVariables(items ...string) error {
|
||||||
if len(matches) == 0 {
|
if len(matches) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
errorString := &strings.Builder{}
|
var unresolvedVariables []string
|
||||||
errorString.WriteString("unresolved variables found: ")
|
for _, match := range matches {
|
||||||
|
|
||||||
for i, match := range matches {
|
|
||||||
if len(match) < 2 {
|
if len(match) < 2 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
errorString.WriteString(match[1])
|
unresolvedVariables = append(unresolvedVariables, match[1])
|
||||||
if i != len(matches)-1 {
|
|
||||||
errorString.WriteString(",")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
errorMessage := errorString.String()
|
return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ","))
|
||||||
return errors.New(errorMessage)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ContainsVariablesWithNames returns an error with variable names if the passed
|
||||||
|
// input contains unresolved {{<pattern-here>}} variables within the provided list
|
||||||
func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error {
|
func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error {
|
||||||
for _, data := range items {
|
for _, data := range items {
|
||||||
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
|
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
|
||||||
if len(matches) == 0 {
|
if len(matches) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
errorString := &strings.Builder{}
|
var unresolvedVariables []string
|
||||||
errorString.WriteString("unresolved variables with values found: ")
|
for _, match := range matches {
|
||||||
|
|
||||||
for i, match := range matches {
|
|
||||||
if len(match) < 2 {
|
if len(match) < 2 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
matchName := match[1]
|
matchName := match[1]
|
||||||
if _, ok := names[matchName]; !ok {
|
if _, ok := names[matchName]; !ok {
|
||||||
errorString.WriteString(matchName)
|
unresolvedVariables = append(unresolvedVariables, matchName)
|
||||||
if i != len(matches)-1 {
|
|
||||||
errorString.WriteString(",")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
errorMessage := errorString.String()
|
return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
|
||||||
return errors.New(errorMessage)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ContainsVariablesWithIgnoreList returns an error with variable names if the passed
|
||||||
|
// input contains unresolved {{<pattern-here>}} other than the ones listed in the ignore list
|
||||||
|
func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...string) error {
|
||||||
|
var unresolvedVariables []string
|
||||||
|
for _, data := range items {
|
||||||
|
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
|
||||||
|
if len(matches) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, match := range matches {
|
||||||
|
if len(match) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
matchName := match[1]
|
||||||
|
if _, ok := skipNames[matchName]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
unresolvedVariables = append(unresolvedVariables, matchName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(unresolvedVariables) > 0 {
|
||||||
|
return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ","))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||||
)
|
)
|
||||||
|
@ -23,6 +24,9 @@ func Init(options *types.Options) error {
|
||||||
if err := httpclientpool.Init(options); err != nil {
|
if err := httpclientpool.Init(options); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := signerpool.Init(options); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return networkclientpool.Init(options)
|
return networkclientpool.Init(options)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@ import (
|
||||||
|
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||||
|
@ -121,13 +122,32 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam
|
||||||
if len(parts) < 3 {
|
if len(parts) < 3 {
|
||||||
return nil, fmt.Errorf("malformed request supplied")
|
return nil, fmt.Errorf("malformed request supplied")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
payloads := generators.BuildPayloadFromOptions(r.request.options.Options)
|
||||||
|
// in case cases (eg requests signing, some variables uses default values if missing)
|
||||||
|
if defaultList := GetVariablesDefault(r.request.Signature.Value); defaultList != nil {
|
||||||
|
payloads = generators.MergeMaps(defaultList, payloads)
|
||||||
|
}
|
||||||
|
parts[1] = replacer.Replace(parts[1], payloads)
|
||||||
|
|
||||||
|
// the url might contain placeholders with ignore list
|
||||||
|
if ignoreList := GetVariablesNamesSkipList(r.request.Signature.Value); ignoreList != nil {
|
||||||
|
if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
} else { // the url might contain placeholders
|
||||||
|
if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
parsed, err := url.Parse(parts[1])
|
parsed, err := url.Parse(parts[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("could not parse request URL: %w", err)
|
return nil, fmt.Errorf("could not parse request URL: %w", err)
|
||||||
}
|
}
|
||||||
values := generators.MergeMaps(
|
values := generators.MergeMaps(
|
||||||
generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
|
generators.MergeMaps(dynamicValues, generateVariables(parsed, false)),
|
||||||
generators.BuildPayloadFromOptions(r.request.options.Options),
|
payloads,
|
||||||
)
|
)
|
||||||
|
|
||||||
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
|
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
|
||||||
|
|
|
@ -128,6 +128,12 @@ type Request struct {
|
||||||
// SelfContained specifies if the request is self-contained.
|
// SelfContained specifies if the request is self-contained.
|
||||||
SelfContained bool `yaml:"-" json:"-"`
|
SelfContained bool `yaml:"-" json:"-"`
|
||||||
|
|
||||||
|
// description: |
|
||||||
|
// Signature is the request signature method
|
||||||
|
// values:
|
||||||
|
// - "AWS"
|
||||||
|
Signature SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
|
||||||
|
|
||||||
// description: |
|
// description: |
|
||||||
// CookieReuse is an optional setting that enables cookie reuse for
|
// CookieReuse is an optional setting that enables cookie reuse for
|
||||||
// all requests defined in raw section.
|
// all requests defined in raw section.
|
||||||
|
|
|
@ -29,7 +29,10 @@ import (
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
|
||||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
|
||||||
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||||
"github.com/projectdiscovery/rawhttp"
|
"github.com/projectdiscovery/rawhttp"
|
||||||
"github.com/projectdiscovery/stringsutil"
|
"github.com/projectdiscovery/stringsutil"
|
||||||
)
|
)
|
||||||
|
@ -336,24 +339,27 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
|
||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Dump request for variables checks
|
||||||
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
|
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
|
||||||
if !generatedRequest.original.Race {
|
if !generatedRequest.original.Race {
|
||||||
var dumpError error
|
var dumpError error
|
||||||
|
// TODO: dump is currently not working with post-processors - somehow it alters the signature
|
||||||
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
|
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
|
||||||
if dumpError != nil {
|
if dumpError != nil {
|
||||||
return dumpError
|
return dumpError
|
||||||
}
|
}
|
||||||
dumpedRequestString := string(dumpedRequest)
|
dumpedRequestString := string(dumpedRequest)
|
||||||
|
|
||||||
// Check if are there any unresolved variables. If yes, skip unless overridden by user.
|
if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
|
||||||
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
|
if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
|
||||||
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
|
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
|
||||||
return errStopExecution
|
return errStopExecution
|
||||||
}
|
}
|
||||||
|
} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
|
||||||
if request.options.Options.Debug || request.options.Options.DebugRequests {
|
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
|
||||||
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
|
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
|
||||||
gologger.Print().Msgf("%s", dumpedRequestString)
|
return errStopExecution
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var formedURL string
|
var formedURL string
|
||||||
|
@ -391,9 +397,27 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if resp == nil {
|
if resp == nil {
|
||||||
|
if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
|
||||||
|
return errSignature
|
||||||
|
}
|
||||||
resp, err = request.httpClient.Do(generatedRequest.request)
|
resp, err = request.httpClient.Do(generatedRequest.request)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Dump the requests containing all headers
|
||||||
|
if !generatedRequest.original.Race {
|
||||||
|
var dumpError error
|
||||||
|
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
|
||||||
|
if dumpError != nil {
|
||||||
|
return dumpError
|
||||||
|
}
|
||||||
|
dumpedRequestString := string(dumpedRequest)
|
||||||
|
if request.options.Options.Debug || request.options.Options.DebugRequests {
|
||||||
|
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
|
||||||
|
gologger.Print().Msgf("%s", dumpedRequestString)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// rawhttp doesn't support draining response bodies.
|
// rawhttp doesn't support draining response bodies.
|
||||||
if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil {
|
if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil {
|
||||||
|
@ -533,6 +557,40 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleSignature of the http request
|
||||||
|
func (request *Request) handleSignature(generatedRequest *generatedRequest) error {
|
||||||
|
switch request.Signature.Value {
|
||||||
|
case AWSSignature:
|
||||||
|
var awsSigner signer.Signer
|
||||||
|
payloads := request.options.Options.Vars.AsMap()
|
||||||
|
awsAccessKeyId := types.ToString(payloads["aws-id"])
|
||||||
|
awsSecretAccessKey := types.ToString(payloads["aws-secret"])
|
||||||
|
awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey}
|
||||||
|
service := types.ToString(payloads["service"])
|
||||||
|
region := types.ToString(payloads["region"])
|
||||||
|
// if region is empty use default value
|
||||||
|
if region == "" {
|
||||||
|
region = types.ToString(signer.AwsDefaultVars["region"])
|
||||||
|
}
|
||||||
|
awsSignatureArguments := signer.AwsSignatureArguments{
|
||||||
|
Service: types.ToString(service),
|
||||||
|
Region: types.ToString(region),
|
||||||
|
Time: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
awsSigner, err := signerpool.Get(request.options.Options, &signerpool.Configuration{SignerArgs: awsSignerArgs})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = awsSigner.SignHTTP(generatedRequest.request.Request, awsSignatureArguments)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// setCustomHeaders sets the custom headers for generated request
|
// setCustomHeaders sets the custom headers for generated request
|
||||||
func (request *Request) setCustomHeaders(req *generatedRequest) {
|
func (request *Request) setCustomHeaders(req *generatedRequest) {
|
||||||
for k, v := range request.customHeaders {
|
for k, v := range request.customHeaders {
|
||||||
|
|
|
@ -0,0 +1,107 @@
|
||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"github.com/alecthomas/jsonschema"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SignatureType is the type of signature
|
||||||
|
type SignatureType int
|
||||||
|
|
||||||
|
// Supported values for the SignatureType
|
||||||
|
const (
|
||||||
|
AWSSignature SignatureType = iota + 1
|
||||||
|
signatureLimit
|
||||||
|
)
|
||||||
|
|
||||||
|
// signatureTypeMappings is a table for conversion of signature type from string.
|
||||||
|
var signatureTypeMappings = map[SignatureType]string{
|
||||||
|
AWSSignature: "AWS",
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetSupportedSignaturesTypes() []SignatureType {
|
||||||
|
var result []SignatureType
|
||||||
|
for index := SignatureType(1); index < signatureLimit; index++ {
|
||||||
|
result = append(result, index)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func toSignatureType(valueToMap string) (SignatureType, error) {
|
||||||
|
normalizedValue := normalizeValue(valueToMap)
|
||||||
|
for key, currentValue := range signatureTypeMappings {
|
||||||
|
if normalizedValue == currentValue {
|
||||||
|
return key, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1, errors.New("invalid signature type: " + valueToMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t SignatureType) String() string {
|
||||||
|
return signatureTypeMappings[t]
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignatureTypeHolder is used to hold internal type of the signature
|
||||||
|
type SignatureTypeHolder struct {
|
||||||
|
Value SignatureType
|
||||||
|
}
|
||||||
|
|
||||||
|
func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Type {
|
||||||
|
gotType := &jsonschema.Type{
|
||||||
|
Type: "string",
|
||||||
|
Title: "type of the signature",
|
||||||
|
Description: "Type of the signature",
|
||||||
|
}
|
||||||
|
for _, types := range GetSupportedSignaturesTypes() {
|
||||||
|
gotType.Enum = append(gotType.Enum, types.String())
|
||||||
|
}
|
||||||
|
return gotType
|
||||||
|
}
|
||||||
|
|
||||||
|
func (holder *SignatureTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
var marshalledTypes string
|
||||||
|
if err := unmarshal(&marshalledTypes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
computedType, err := toSignatureType(marshalledTypes)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
holder.Value = computedType
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) {
|
||||||
|
return json.Marshal(holder.Value.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) {
|
||||||
|
return holder.Value.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var ErrNoIgnoreList = errors.New("uknown signature types")
|
||||||
|
|
||||||
|
// GetVariablesNamesSkipList depending on the signature type
|
||||||
|
func GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {
|
||||||
|
switch signature {
|
||||||
|
case AWSSignature:
|
||||||
|
return signer.AwsSkipList
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetVariablesNamesSkipList depending on the signature type
|
||||||
|
func GetVariablesDefault(signature SignatureType) map[string]interface{} {
|
||||||
|
switch signature {
|
||||||
|
case AWSSignature:
|
||||||
|
return signer.AwsDefaultVars
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,140 @@
|
||||||
|
package signer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AwsSigner struct {
|
||||||
|
creds *credentials.Credentials
|
||||||
|
signer *v4.Signer
|
||||||
|
}
|
||||||
|
|
||||||
|
type AwsSignerArgs struct {
|
||||||
|
AwsId string
|
||||||
|
AwsSecretToken string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (awsSignerArgs AwsSignerArgs) Validate() error {
|
||||||
|
if awsSignerArgs.AwsId == "" {
|
||||||
|
return errors.New("empty id")
|
||||||
|
}
|
||||||
|
if awsSignerArgs.AwsSecretToken == "" {
|
||||||
|
return errors.New("empty token")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type AwsSignatureArguments struct {
|
||||||
|
Service string
|
||||||
|
Region string
|
||||||
|
Time time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (awsSignatureArguments AwsSignatureArguments) Validate() error {
|
||||||
|
if awsSignatureArguments.Region == "" {
|
||||||
|
return errors.New("empty region")
|
||||||
|
}
|
||||||
|
if awsSignatureArguments.Service == "" {
|
||||||
|
return errors.New("empty service")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) {
|
||||||
|
if err := args.Validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
creds := credentials.NewStaticCredentials(args.AwsId, args.AwsSecretToken, "")
|
||||||
|
if creds == nil {
|
||||||
|
return nil, errors.New("couldn't create the credentials structure")
|
||||||
|
}
|
||||||
|
signer := v4.NewSigner(creds)
|
||||||
|
return &AwsSigner{creds: creds, signer: signer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewAwsSignerFromEnv() (*AwsSigner, error) {
|
||||||
|
creds := credentials.NewEnvCredentials()
|
||||||
|
if creds == nil {
|
||||||
|
return nil, errors.New("couldn't create the credentials structure")
|
||||||
|
}
|
||||||
|
signer := v4.NewSigner(creds)
|
||||||
|
return &AwsSigner{creds: creds, signer: signer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewAwsSignerFromFile() (*AwsSigner, error) {
|
||||||
|
creds := credentials.NewSharedCredentials("", "")
|
||||||
|
if creds == nil {
|
||||||
|
return nil, errors.New("couldn't create the credentials structure")
|
||||||
|
}
|
||||||
|
signer := v4.NewSigner(creds)
|
||||||
|
return &AwsSigner{creds: creds, signer: signer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) error {
|
||||||
|
signatureArgs, err := awsSigner.checkSignatureArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
awsSigner.prepareRequest(request)
|
||||||
|
var body *bytes.Reader
|
||||||
|
if request.Body != nil {
|
||||||
|
bodyBytes, err := ioutil.ReadAll(request.Body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
request.Body.Close()
|
||||||
|
body = bytes.NewReader(bodyBytes)
|
||||||
|
}
|
||||||
|
if _, err := awsSigner.signer.Sign(request, body, signatureArgs.Service, signatureArgs.Region, signatureArgs.Time); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error) {
|
||||||
|
signatureArgs, err := awsSigner.checkSignatureArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
reqClone := request.Clone(context.Background())
|
||||||
|
awsSigner.prepareRequest(reqClone)
|
||||||
|
err = awsSigner.SignHTTP(reqClone, signatureArgs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
headers := make(map[string]string)
|
||||||
|
headers["X-Amz-Date"] = reqClone.Header.Get("X-Amz-Date")
|
||||||
|
headers["Authorization"] = reqClone.Header.Get("Authorization")
|
||||||
|
return headers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (awsSigner *AwsSigner) checkSignatureArgs(args interface{}) (AwsSignatureArguments, error) {
|
||||||
|
if signatureArgs, ok := args.(AwsSignatureArguments); ok {
|
||||||
|
return signatureArgs, signatureArgs.Validate()
|
||||||
|
}
|
||||||
|
return AwsSignatureArguments{}, errors.New("wrong signature type")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (awsSigner *AwsSigner) prepareRequest(request *http.Request) {
|
||||||
|
request.Header.Del("Host")
|
||||||
|
}
|
||||||
|
|
||||||
|
var AwsSkipList = map[string]interface{}{
|
||||||
|
"region": struct{}{},
|
||||||
|
}
|
||||||
|
|
||||||
|
var AwsDefaultVars = map[string]interface{}{
|
||||||
|
"region": "us-east-2",
|
||||||
|
}
|
|
@ -0,0 +1,40 @@
|
||||||
|
package signer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Signer interface {
|
||||||
|
SignHTTP(request *http.Request, args interface{}) error
|
||||||
|
CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type SignerArgs interface {
|
||||||
|
Validate() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type SignatureArguments interface {
|
||||||
|
Validate() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewSigner(args SignerArgs) (signer Signer, err error) {
|
||||||
|
switch signerArgs := args.(type) {
|
||||||
|
case AwsSignerArgs:
|
||||||
|
awsSigner, err := NewAwsSigner(signerArgs)
|
||||||
|
if err != nil {
|
||||||
|
// $HOME/.aws/credentials
|
||||||
|
awsSigner, err = NewAwsSignerFromFile()
|
||||||
|
if err != nil {
|
||||||
|
// env variables
|
||||||
|
awsSigner, err = NewAwsSignerFromEnv()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return awsSigner, err
|
||||||
|
default:
|
||||||
|
return nil, errors.New("unknown signature arguments type")
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,57 @@
|
||||||
|
package signerpool
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
|
||||||
|
|
||||||
|
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
poolMutex *sync.RWMutex
|
||||||
|
clientPool map[string]signer.Signer
|
||||||
|
)
|
||||||
|
|
||||||
|
// Init initializes the clientpool implementation
|
||||||
|
func Init(options *types.Options) error {
|
||||||
|
poolMutex = &sync.RWMutex{}
|
||||||
|
clientPool = make(map[string]signer.Signer)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Configuration contains the custom configuration options for a client
|
||||||
|
type Configuration struct {
|
||||||
|
SignerArgs signer.SignerArgs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hash returns the hash of the configuration to allow client pooling
|
||||||
|
func (c *Configuration) Hash() string {
|
||||||
|
builder := &strings.Builder{}
|
||||||
|
builder.WriteString(fmt.Sprintf("%v", c.SignerArgs))
|
||||||
|
hash := builder.String()
|
||||||
|
return hash
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get creates or gets a client for the protocol based on custom configuration
|
||||||
|
func Get(options *types.Options, configuration *Configuration) (signer.Signer, error) {
|
||||||
|
hash := configuration.Hash()
|
||||||
|
poolMutex.RLock()
|
||||||
|
if client, ok := clientPool[hash]; ok {
|
||||||
|
poolMutex.RUnlock()
|
||||||
|
return client, nil
|
||||||
|
}
|
||||||
|
poolMutex.RUnlock()
|
||||||
|
|
||||||
|
client, err := signer.NewSigner(configuration.SignerArgs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
poolMutex.Lock()
|
||||||
|
clientPool[hash] = client
|
||||||
|
poolMutex.Unlock()
|
||||||
|
return client, nil
|
||||||
|
}
|
|
@ -108,6 +108,11 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
|
||||||
|
|
||||||
// parseSelfContainedRequests parses the self contained template requests.
|
// parseSelfContainedRequests parses the self contained template requests.
|
||||||
func (template *Template) parseSelfContainedRequests() {
|
func (template *Template) parseSelfContainedRequests() {
|
||||||
|
if template.Signature.Value.String() != "" {
|
||||||
|
for _, request := range template.RequestsHTTP {
|
||||||
|
request.Signature = template.Signature
|
||||||
|
}
|
||||||
|
}
|
||||||
if !template.SelfContained {
|
if !template.SelfContained {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -87,6 +87,12 @@ type Template struct {
|
||||||
// Stop execution once first match is found
|
// Stop execution once first match is found
|
||||||
StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"`
|
StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"`
|
||||||
|
|
||||||
|
// description: |
|
||||||
|
// Signature is the request signature method
|
||||||
|
// values:
|
||||||
|
// - "AWS"
|
||||||
|
Signature http.SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
|
||||||
|
|
||||||
// TotalRequests is the total number of requests for the template.
|
// TotalRequests is the total number of requests for the template.
|
||||||
TotalRequests int `yaml:"-" json:"-"`
|
TotalRequests int `yaml:"-" json:"-"`
|
||||||
// Executer is the actual template executor for running template requests
|
// Executer is the actual template executor for running template requests
|
||||||
|
|
Loading…
Reference in New Issue