mirror of https://github.com/daffainfo/nuclei.git
fix url re-encoding issues (#3294)
* fix double url encoding in urls * remove extra slash * url encode matchedURLdev
Tarun Koyalwar
GitHub
parent
7d1471fbac
commit
e622b989fe
12
v2/go.mod
12
v2/go.mod
|
|
@ -29,7 +29,7 @@ require (
|
|||
github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
|
||||
github.com/projectdiscovery/rawhttp v0.1.7
|
||||
github.com/projectdiscovery/retryabledns v1.0.20
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.11
|
||||
github.com/projectdiscovery/stringsutil v0.0.2
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
|
|
@ -44,9 +44,9 @@ require (
|
|||
github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37
|
||||
github.com/xanzy/go-gitlab v0.79.0
|
||||
go.uber.org/multierr v1.9.0
|
||||
golang.org/x/net v0.5.0
|
||||
golang.org/x/net v0.6.0
|
||||
golang.org/x/oauth2 v0.4.0
|
||||
golang.org/x/text v0.6.0
|
||||
golang.org/x/text v0.7.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
moul.io/http2curl v1.0.0
|
||||
)
|
||||
|
|
@ -80,7 +80,7 @@ require (
|
|||
github.com/projectdiscovery/sarif v0.0.1
|
||||
github.com/projectdiscovery/tlsx v1.0.2
|
||||
github.com/projectdiscovery/uncover v1.0.2
|
||||
github.com/projectdiscovery/utils v0.0.8
|
||||
github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.79
|
||||
github.com/stretchr/testify v1.8.1
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1
|
||||
|
|
@ -185,7 +185,7 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mholt/acmez v1.0.4 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.21 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.22 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
|
|
@ -215,7 +215,7 @@ require (
|
|||
golang.org/x/crypto v0.5.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/sys v0.4.0 // indirect
|
||||
golang.org/x/sys v0.5.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.5.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
|
|
|
|||
22
v2/go.sum
22
v2/go.sum
|
|
@ -485,8 +485,9 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA
|
|||
github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU=
|
||||
github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.22 h1:p2tT7RNzRdCi0qmwxG+HbqD6ILkmwter1ZwVZn1oTxA=
|
||||
github.com/microcosm-cc/bluemonday v1.0.22/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
|
||||
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
|
||||
|
|
@ -592,8 +593,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
|
|||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
|
||||
github.com/projectdiscovery/retryabledns v1.0.20 h1:grRyh4EzuyqsaK07iNkJKgrGLu/qDJwfDJ+83SBo6yo=
|
||||
github.com/projectdiscovery/retryabledns v1.0.20/go.mod h1:97Et22Kw2iPyvz/Vn41/i3dSbhLMHfeWP/S7EaLgmtg=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a h1:KUHx4Yxx7S+qX94TtCegLj/01obmohdVDeiG86FCHjM=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE=
|
||||
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
|
||||
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
|
||||
github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg=
|
||||
|
|
@ -603,8 +604,8 @@ github.com/projectdiscovery/tlsx v1.0.2 h1:2bbfPQLuMIhs6FPmGsIcAo3uJaB2E+9ssJtZ8
|
|||
github.com/projectdiscovery/tlsx v1.0.2/go.mod h1:WW+PdBImrqnMl18v4Brp3OsbnO4A1tqYPUcfiVtjNLM=
|
||||
github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
|
||||
github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
|
||||
github.com/projectdiscovery/utils v0.0.8 h1:yPl/DwhW0IGnWNjapcw03g97ria8ZM8fH5PbcX4QFUo=
|
||||
github.com/projectdiscovery/utils v0.0.8/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw=
|
||||
github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b h1:7a4pnoEny9vrn9mmoBxo1yRP1RPMKCgFWkUaGKyGdAM=
|
||||
github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.79 h1:hWMxNysxC/P6fxnu6c+opqf5L27hHQ9wD1QzPRCb+I8=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.79/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
|
||||
|
|
@ -883,8 +884,9 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS
|
|||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
|
||||
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
|
||||
|
|
@ -954,8 +956,9 @@ golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
@ -963,8 +966,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
|
|||
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
|
@ -972,8 +975,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
|
|||
|
|
@ -133,8 +133,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
|||
finalparams := parsed.Params
|
||||
finalparams.Merge(reqURL.Params)
|
||||
reqURL.Params = finalparams
|
||||
|
||||
return r.generateHttpRequest(ctx, reqURL.String(), finalVars, payloads)
|
||||
return r.generateHttpRequest(ctx, reqURL, finalVars, payloads)
|
||||
}
|
||||
|
||||
// selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
|
||||
|
|
@ -205,19 +204,23 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
|
|||
if err != nil {
|
||||
return nil, ErrEvalExpression.Wrap(err).WithTag("self-contained")
|
||||
}
|
||||
return r.generateHttpRequest(ctx, data, values, payloads)
|
||||
urlx, err := urlutil.ParseURL(data, true)
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithErr(err).Msgf("failed to parse %v in self contained request", data).WithTag("self-contained")
|
||||
}
|
||||
return r.generateHttpRequest(ctx, urlx, values, payloads)
|
||||
}
|
||||
|
||||
// generateHttpRequest generates http request from request data from template and variables
|
||||
// finalVars = contains all variables including generator and protocol specific variables
|
||||
// generatorValues = contains variables used in fuzzing or other generator specific values
|
||||
func (r *requestGenerator) generateHttpRequest(ctx context.Context, data string, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
|
||||
func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
|
||||
method, err := expressions.Evaluate(r.request.Method.String(), finalVars)
|
||||
if err != nil {
|
||||
return nil, ErrEvalExpression.Wrap(err).Msgf("failed to evaluate while generating http request")
|
||||
}
|
||||
// Build a request on the specified URL
|
||||
req, err := retryablehttp.NewRequestWithContext(ctx, method, data, nil)
|
||||
req, err := retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -254,8 +257,11 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
|
|||
// Todo: sync internally upon writing latest request byte
|
||||
body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second)
|
||||
}
|
||||
|
||||
req, err := retryablehttp.NewRequestWithContext(ctx, rawRequestData.Method, rawRequestData.FullURL, body)
|
||||
urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
|
||||
}
|
||||
req, err := retryablehttp.NewRequestFromURLWithContext(ctx, rawRequestData.Method, urlx, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
|
|||
var req *retryablehttp.Request
|
||||
var err error
|
||||
if input.BaseRequest == nil {
|
||||
req, err = retryablehttp.NewRequest(http.MethodGet, parsed.String(), nil)
|
||||
req, err = retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -512,6 +512,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
}
|
||||
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
|
||||
} else if generatedRequest.request != nil {
|
||||
// hot fix to avoid double url encoding (should only be called once)
|
||||
generatedRequest.request.Prepare()
|
||||
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
|
||||
}
|
||||
} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
|
||||
|
|
@ -562,6 +564,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
}
|
||||
httpclient = client
|
||||
}
|
||||
generatedRequest.request.Prepare()
|
||||
resp, err = httpclient.Do(generatedRequest.request)
|
||||
}
|
||||
}
|
||||
|
|
@ -570,6 +573,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
formedURL = input.MetaInput.Input
|
||||
}
|
||||
|
||||
// converts whitespace and other chars that cannot be printed to url encoded values
|
||||
formedURL = urlutil.URLEncodeWithEscapes(formedURL)
|
||||
|
||||
// Dump the requests containing all headers
|
||||
if !generatedRequest.original.Race {
|
||||
var dumpError error
|
||||
|
|
|
|||
Loading…
Reference in New Issue