daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge from dev

dev
Ice3man543 2021-11-08 15:40:18 +05:30
parent e8197f127f 3c53e262e3
commit 213853c45d
10 changed files with 168 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/cmd/nuclei/main.go
View File

@ -109,7 +109,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"),
flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"),
flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"),
flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
flagSet.IntVar(&options.InteractionsCooldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"),
flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"),
)

2
v2/internal/runner/runner.go
View File

@ -151,7 +151,7 @@ func New(options *types.Options) (*Runner, error) {
opts.Authorization = options.InteractshToken
opts.CacheSize = int64(options.InteractionsCacheSize)
opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second
opts.ColldownPeriod = time.Duration(options.InteractionsColldownPeriod) * time.Second
opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second
opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
interactshClient, err := interactsh.New(opts)

51
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -220,12 +220,14 @@ func (c *Client) Close() bool {
//
// It accepts data to replace as well as the URL to replace placeholders
// with generated uniquely for each request.
func (c *Client) ReplaceMarkers(data, interactshURL string) string {
if !strings.Contains(data, interactshURLMarker) {
return data
func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) {
for strings.Contains(data, interactshURLMarker) {
url := c.URL()
interactshURLs = append(interactshURLs, url)
data = strings.Replace(data, interactshURLMarker, url, 1)
}
replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data)
return replaced
return data, interactshURLs
}
// MakeResultEventFunc is a result making function for nuclei
@ -241,30 +243,29 @@ type RequestData struct {
}
// RequestEvent is the event for a network request sent by nuclei.
func (c *Client) RequestEvent(interactshURL string, data *RequestData) {
id := strings.TrimSuffix(interactshURL, c.dotHostname)
func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) {
for _, interactshURL := range interactshURLs {
id := strings.TrimSuffix(interactshURL, c.dotHostname)
interaction := c.interactions.Get(id)
if interaction != nil {
// If we have previous interactions, get them and process them.
interactions, ok := interaction.Value().([]*server.Interaction)
if !ok {
c.requests.Set(id, data, c.eviction)
return
}
matched := false
for _, interaction := range interactions {
if c.processInteractionForRequest(interaction, data) {
matched = true
break
interaction := c.interactions.Get(id)
if interaction != nil {
// If we have previous interactions, get them and process them.
interactions, ok := interaction.Value().([]*server.Interaction)
if !ok {
c.requests.Set(id, data, c.eviction)
return
}
for _, interaction := range interactions {
if c.processInteractionForRequest(interaction, data) {
c.interactions.Delete(id)
break
}
}
} else {
c.requests.Set(id, data, c.eviction)
}
if matched {
c.interactions.Delete(id)
}
} else {
c.requests.Set(id, data, c.eviction)
}
}
// HasMatchers returns true if an operator has interactsh part

55
v2/pkg/protocols/http/build_request.go
View File

@ -38,6 +38,7 @@ type generatedRequest struct {
pipelinedClient *rawhttp.PipelineClient
request *retryablehttp.Request
dynamicValues map[string]interface{}
interactshURLs []string
}
func (g *generatedRequest) URL() string {
@ -52,9 +53,9 @@ func (g *generatedRequest) URL() string {
// Make creates a http request for the provided input.
// It returns io.EOF as error when all the requests have been exhausted.
func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
if r.request.SelfContained {
return r.makeSelfContainedRequest(dynamicValues, interactURL)
return r.makeSelfContainedRequest(dynamicValues)
}
// We get the next payload for the request.
data, payloads, ok := r.nextValue()
@ -63,12 +64,10 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
}
ctx := context.Background()
if interactURL != "" {
data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
for payloadName, payloadValue := range payloads {
payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL)
}
for payloadName, payloadValue := range payloads {
payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs)
}
parsed, err := url.Parse(baseURL)
@ -98,12 +97,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
// If data contains \n it's a raw request, process it like raw. Else
// continue with the template based request flow.
if isRawRequest {
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
}
return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
return r.makeHTTPRequestFromModel(ctx, data, values, payloads)
}
func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}) (*generatedRequest, error) {
// We get the next payload for the request.
data, payloads, ok := r.nextValue()
if !ok {
@ -136,13 +135,13 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int
generators.BuildPayloadFromOptions(r.request.options.Options),
)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL)
return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads)
}
values := generators.MergeMaps(
dynamicValues,
generators.BuildPayloadFromOptions(r.request.options.Options),
)
return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL)
return r.makeHTTPRequestFromModel(ctx, data, values, payloads)
}
// Total returns the total number of requests for the generator
@ -171,10 +170,8 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) {
if interactURL != "" {
data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
}
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
// Combine the template payloads along with base
// request values.
@ -198,18 +195,16 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
return nil, err
}
request, err := r.fillRequest(req, finalValues, interactURL)
request, err := r.fillRequest(req, finalValues)
if err != nil {
return nil, err
}
return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil
return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil
}
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}, interactURL string) (*generatedRequest, error) {
if interactURL != "" {
data = r.options.Interactsh.ReplaceMarkers(data, interactURL)
}
func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) {
data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs)
return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads)
}
@ -258,21 +253,19 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest
req.Host = value
}
}
request, err := r.fillRequest(req, finalValues, "")
request, err := r.fillRequest(req, finalValues)
if err != nil {
return nil, err
}
return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil
return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil
}
// fillRequest fills various headers in the request with values
func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}, interactURL string) (*retryablehttp.Request, error) {
func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
// Set the header values requested
for header, value := range r.request.Headers {
if interactURL != "" {
value = r.options.Interactsh.ReplaceMarkers(value, interactURL)
}
value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs)
value, err := expressions.Evaluate(value, values)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
@ -290,10 +283,8 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
// Check if the user requested a request body
if r.request.Body != "" {
body := r.request.Body
if interactURL != "" {
body = r.options.Interactsh.ReplaceMarkers(body, interactURL)
}
var body string
body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs)
body, err := expressions.Evaluate(body, values)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")

72
v2/pkg/protocols/http/build_request_test.go
View File

@ -3,13 +3,14 @@ package http
import (
"net/url"
"testing"
"github.com/stretchr/testify/require"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/stretchr/testify/require"
)
func TestBaseURLWithTemplatePrefs(t *testing.T) {
@ -85,7 +86,7 @@ func TestMakeRequestFromModal(t *testing.T) {
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
req, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
bodyBytes, _ := req.request.BodyBytes()
@ -112,12 +113,12 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}, "")
req, err := generator.Make("https://example.com/test.php", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
generator = request.newGenerator()
req, err = generator.Make("https://example.com/test/", map[string]interface{}{}, "")
req, err = generator.Make("https://example.com/test/", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
}
@ -150,12 +151,12 @@ Accept-Encoding: gzip`},
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
req, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization := req.request.Header.Get("Authorization")
require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
req, err = generator.Make("https://example.com", map[string]interface{}{}, "")
req, err = generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization = req.request.Header.Get("Authorization")
require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@ -189,13 +190,66 @@ Accept-Encoding: gzip`},
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
req, err := generator.Make("https://example.com", map[string]interface{}{}, "")
req, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization := req.request.Header.Get("Authorization")
require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
req, err = generator.Make("https://example.com", map[string]interface{}{}, "")
req, err = generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization = req.request.Header.Get("Authorization")
require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
}
func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-unique-interactsh"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"},
Method: "GET",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
generator.options.Interactsh, err = interactsh.New(&interactsh.Options{
ServerURL: options.InteractshURL,
CacheSize: int64(options.InteractionsCacheSize),
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second,
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
})
require.Nil(t, err, "could not create interactsh client")
got, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
// check if all the interactsh markers are replaced with unique urls
require.NotContains(t, got.request.URL.String(), "{{interactsh-url}}", "could not get correct interactsh url")
// check the length of returned urls
require.Equal(t, len(got.interactshURLs), 4, "could not get correct interactsh url")
// check if the interactsh urls are unique
require.True(t, areUnique(got.interactshURLs), "interactsh urls are not unique")
}
// areUnique checks if the elements of string slice are unique
func areUnique(elements []string) bool {
encountered := map[string]bool{}
for v := range elements {
if encountered[elements[v]] {
return false
}
encountered[elements[v]] = true
}
return true
}

16
v2/pkg/protocols/http/request.go
View File

@ -47,7 +47,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
// This will introduce a delay and will populate in hacky way the field "request" of outputEvent
generator := request.newGenerator()
requestForDump, err := generator.Make(reqURL, nil, "")
requestForDump, err := generator.Make(reqURL, nil)
if err != nil {
return err
}
@ -65,7 +65,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
// Pre-Generate requests
for i := 0; i < request.RaceNumberRequests; i++ {
generator := request.newGenerator()
generatedRequest, err := generator.Make(reqURL, nil, "")
generatedRequest, err := generator.Make(reqURL, nil)
if err != nil {
return err
}
@ -104,7 +104,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
var requestErr error
mutex := &sync.Mutex{}
for {
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "")
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
if err == io.EOF {
break
}
@ -167,7 +167,7 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
var requestErr error
mutex := &sync.Mutex{}
for {
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "")
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
if err == io.EOF {
break
}
@ -221,11 +221,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
for {
hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
var interactURL string
if request.options.Interactsh != nil && hasInteractMarkers {
interactURL = request.options.Interactsh.URL()
}
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL)
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues)
if err == io.EOF {
break
}
@ -251,7 +247,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues)
}
if hasInteractMarkers && request.options.Interactsh != nil {
request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,

1
v2/pkg/protocols/http/request_generator.go
View File

@ -16,6 +16,7 @@ type requestGenerator struct {
request *Request
options *protocols.ExecuterOptions
payloadIterator *generators.Iterator
interactshURLs []string
}
// newGenerator creates a new request generator instance

14
v2/pkg/protocols/network/request.go
View File

@ -126,11 +126,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
defer conn.Close()
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second))
hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
var interactURL string
if request.options.Interactsh != nil && hasInteractMarkers {
interactURL = request.options.Interactsh.URL()
}
var interactshURLs []string
responseBuilder := &strings.Builder{}
reqBuilder := &strings.Builder{}
@ -143,9 +139,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
case "hex":
data, err = hex.DecodeString(input.Data)
default:
if interactURL != "" {
input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL)
}
input.Data, interactshURLs = request.options.Interactsh.ReplaceMarkers(input.Data, []string{})
data = []byte(input.Data)
}
if err != nil {
@ -267,14 +261,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
}
var event *output.InternalWrappedEvent
if interactURL == "" {
if len(interactshURLs) == 0 {
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
wrappedEvent.OperatorsResult.PayloadValues = payloads
})
callback(event)
} else if request.options.Interactsh != nil {
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: request.CompiledOperators,

77
v2/pkg/testutils/testutils.go
View File

@ -22,42 +22,47 @@ func Init(options *types.Options) {
// DefaultOptions is the default options structure for nuclei during mocking.
var DefaultOptions = &types.Options{
Metrics: false,
Debug: false,
DebugRequests: false,
DebugResponse: false,
Silent: false,
Version: false,
Verbose: false,
NoColor: true,
UpdateTemplates: false,
JSON: false,
JSONRequests: false,
EnableProgressBar: false,
TemplatesVersion: false,
TemplateList: false,
Stdin: false,
StopAtFirstMatch: false,
NoMeta: false,
Project: false,
MetricsPort: 0,
BulkSize: 25,
TemplateThreads: 10,
Timeout: 5,
Retries: 1,
RateLimit: 150,
ProjectPath: "",
Severities: severity.Severities{},
Targets: []string{},
TargetsFilePath: "",
Output: "",
ProxyURL: "",
ProxySocksURL: "",
TemplatesDirectory: "",
TraceLogFile: "",
Templates: []string{},
ExcludedTemplates: []string{},
CustomHeaders: []string{},
Metrics: false,
Debug: false,
DebugRequests: false,
DebugResponse: false,
Silent: false,
Version: false,
Verbose: false,
NoColor: true,
UpdateTemplates: false,
JSON: false,
JSONRequests: false,
EnableProgressBar: false,
TemplatesVersion: false,
TemplateList: false,
Stdin: false,
StopAtFirstMatch: false,
NoMeta: false,
Project: false,
MetricsPort: 0,
BulkSize: 25,
TemplateThreads: 10,
Timeout: 5,
Retries: 1,
RateLimit: 150,
ProjectPath: "",
Severities: severity.Severities{},
Targets: []string{},
TargetsFilePath: "",
Output: "",
ProxyURL: "",
ProxySocksURL: "",
TemplatesDirectory: "",
TraceLogFile: "",
Templates: []string{},
ExcludedTemplates: []string{},
CustomHeaders: []string{},
InteractshURL: "https://interactsh.com",
InteractionsCacheSize: 5000,
InteractionsEviction: 60,
InteractionsCooldownPeriod: 5,
InteractionsPollDuration: 5,
}
// TemplateInfo contains info for a mock executed template.

4
v2/pkg/types/types.go
View File

@ -109,9 +109,9 @@ type Options struct {
// Eviction is the number of seconds after which to automatically discard
// interaction requests.
InteractionsEviction int
// InteractionsColldownPeriod is additional seconds to wait for interactions after closing
// InteractionsCooldownPeriod is additional seconds to wait for interactions after closing
// of the poller.
InteractionsColldownPeriod int
InteractionsCooldownPeriod int
// OfflineHTTP is a flag that specific offline processing of http response
// using same matchers/extractors from http protocol without the need
// to send a new request, reading responses from a file.