daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Made code changes as per review comments

dev
Ice3man543 2021-11-05 03:01:41 +05:30
parent f3675d547a
commit 8ad3ebcd05
47 changed files with 203 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/cmd/functional-test/main.go
View File

@ -11,7 +11,7 @@ import (
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var (

2
v2/cmd/integration-test/dns.go
View File

@ -1,7 +1,7 @@
package main
import (
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var dnsTestCases = map[string]testutils.TestCase{

2
v2/cmd/integration-test/http.go
View File

@ -11,7 +11,7 @@ import (
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var httpTestcases = map[string]testutils.TestCase{

2
v2/cmd/integration-test/integration-test.go
View File

@ -6,7 +6,7 @@ import (
"strings"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var (

5
v2/cmd/integration-test/loader.go
View File

@ -2,12 +2,13 @@ package main
import (
"fmt"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"net/http"
"net/http/httptest"
"os"
"strings"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var loaderTestcases = map[string]testutils.TestCase{

2
v2/cmd/integration-test/network.go
View File

@ -3,7 +3,7 @@ package main
import (
"net"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var networkTestcases = map[string]testutils.TestCase{

2
v2/cmd/integration-test/websocket.go
View File

@ -5,7 +5,7 @@ import (
"strings"
"github.com/gobwas/ws/wsutil"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var websocketTestCases = map[string]testutils.TestCase{

2
v2/cmd/integration-test/workflow.go
View File

@ -7,7 +7,7 @@ import (
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
var workflowTestcases = map[string]testutils.TestCase{

22
v2/internal/runner/runner.go
View File

@ -145,18 +145,16 @@ func New(options *types.Options) (*Runner, error) {
}
if !options.NoInteractsh {
interactshClient, err := interactsh.New(&interactsh.Options{
ServerURL: options.InteractshURL,
Authorization: options.InteractshToken,
CacheSize: int64(options.InteractionsCacheSize),
Eviction: time.Duration(options.InteractionsEviction) * time.Second,
ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second,
PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second,
Output: runner.output,
IssuesClient: runner.issuesClient,
Progress: runner.progress,
Debug: runner.options.Debug,
})
opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress)
opts.Debug = runner.options.Debug
opts.ServerURL = options.InteractshURL
opts.Authorization = options.InteractshToken
opts.CacheSize = int64(options.InteractionsCacheSize)
opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second
opts.ColldownPeriod = time.Duration(options.InteractionsColldownPeriod) * time.Second
opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
interactshClient, err := interactsh.New(opts)
if err != nil {
gologger.Error().Msgf("Could not create interactsh client: %s", err)
} else {

2
v2/internal/runner/update_test.go
View File

@ -13,8 +13,8 @@ import (
"testing"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/stretchr/testify/require"
)

4
v2/pkg/core/engine.go
View File

@ -27,8 +27,8 @@ type Engine struct {
type InputProvider interface {
// Count returns the number of items for input provider
Count() int64
// Scan iterates the input and for each found item calls the callback
// until the input provider has been exhausted.
// Scan iterates the input and each found item is passed to the
// callback consumer.
Scan(callback func(value string))
}

8
v2/pkg/core/execute.go
View File

@ -17,12 +17,12 @@ import (
//
// All the execution logic for the templates/workflows happens in this part
// of the engine.
func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool {
return e.ExecuteWithOpts(templates, input, false)
func (e *Engine) Execute(templates []*templates.Template, target InputProvider) *atomic.Bool {
return e.ExecuteWithOpts(templates, target, false)
}
// ExecuteWithOpts is execute with the full options
func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool {
func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool {
var finalTemplates []*templates.Template
if !noCluster {
finalTemplates, _ = e.ClusterTemplates(templatesList)
@ -48,7 +48,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input Inpu
e.executeSelfContainedTemplateWithInput(template, results)
default:
// All other request types are executed here
e.executeModelWithInput(templateType, template, input, results)
e.executeModelWithInput(templateType, template, target, results)
}
wg.Done()
}

3
v2/pkg/core/inputs/hybrid/hmap.go
View File

@ -119,7 +119,8 @@ func (i *Input) Count() int64 {
return i.inputCount
}
// Scan calls an input provider till the callback is exhausted
// Scan iterates the input and each found item is passed to the
// callback consumer.
func (i *Input) Scan(callback func(value string)) {
callbackFunc := func(k, _ []byte) error {
callback(string(k))

7
v2/pkg/operators/common/dsl/dsl.go
View File

@ -332,7 +332,7 @@ var functions = map[string]govaluate.ExpressionFunction{
}
now := time.Now()
offset := now.Add(time.Duration(seconds) * time.Second)
return offset.Unix(), nil
return float64(offset.Unix()), nil
},
// Time Functions
"waitfor": func(args ...interface{}) (interface{}, error) {
@ -363,11 +363,6 @@ var functions = map[string]govaluate.ExpressionFunction{
gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args))
return true, nil
},
// is_before_now compares a timestamp and returns true if the first
// passed argument is a time.Time that has already passed.
"time_now": func(args ...interface{}) (interface{}, error) {
return float64(time.Now().Unix()), nil
},
}
// HelperFunctions returns the dsl helper functions

35
v2/pkg/output/output.go
View File

@ -224,38 +224,3 @@ func (w *StandardWriter) Close() {
w.errorFile.Close()
}
}
// MockOutputWriter is a mocked output writer.
type MockOutputWriter struct {
aurora aurora.Aurora
RequestCallback func(templateID, url, requestType string, err error)
WriteCallback func(o *ResultEvent)
}
// NewMockOutputWriter creates a new mock output writer
func NewMockOutputWriter() *MockOutputWriter {
return &MockOutputWriter{aurora: aurora.NewAurora(false)}
}
// Close closes the output writer interface
func (m *MockOutputWriter) Close() {}
// Colorizer returns the colorizer instance for writer
func (m *MockOutputWriter) Colorizer() aurora.Aurora {
return m.aurora
}
// Write writes the event to file and/or screen.
func (m *MockOutputWriter) Write(result *ResultEvent) error {
if m.WriteCallback != nil {
m.WriteCallback(result)
}
return nil
}
// Request writes a log the requests trace log
func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
if m.RequestCallback != nil {
m.RequestCallback(templateID, url, requestType, err)
}
}

24
v2/pkg/progress/progress.go
View File

@ -247,27 +247,3 @@ func (p *StatsTicker) Stop() {
_ = p.server.Shutdown(context.Background())
}
}
type MockProgressClient struct{}
// Stop stops the progress recorder.
func (m *MockProgressClient) Stop() {}
// Init inits the progress bar with initial details for scan
func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {}
// AddToTotal adds a value to the total request count
func (m *MockProgressClient) AddToTotal(delta int64) {}
// IncrementRequests increments the requests counter by 1.
func (m *MockProgressClient) IncrementRequests() {}
// IncrementMatched increments the matched counter by 1.
func (m *MockProgressClient) IncrementMatched() {}
// IncrementErrorsBy increments the error counter by count.
func (m *MockProgressClient) IncrementErrorsBy(count int64) {}
// IncrementFailedRequestsBy increments the number of requests counter by count
// along with errors.
func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {}

10
v2/pkg/protocols/common/generators/generators.go
View File

@ -7,14 +7,14 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
)
// Generator is the generator struct for generating payloads
type Generator struct {
// PayloadGenerator is the generator struct for generating payloads
type PayloadGenerator struct {
Type AttackType
payloads map[string][]string
}
// New creates a new generator structure for payload generation
func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) {
func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*PayloadGenerator, error) {
if attackType.String() == "" {
attackType = BatteringRamAttack
}
@ -35,7 +35,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
}
}
generator := &Generator{}
generator := &PayloadGenerator{}
if err := generator.validate(payloads, templatePath); err != nil {
return nil, err
}
@ -66,7 +66,7 @@ type Iterator struct {
}
// NewIterator creates a new iterator for the payloads generator
func (g *Generator) NewIterator() *Iterator {
func (g *PayloadGenerator) NewIterator() *Iterator {
var payloads []*payloadIterator
for name, values := range g.payloads {

2
v2/pkg/protocols/common/generators/validate.go
View File

@ -11,7 +11,7 @@ import (
)
// validate validates the payloads if any.
func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error {
func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePath string) error {
for name, payload := range payloads {
switch pt := payload.(type) {
case string:

2
v2/pkg/protocols/dns/dns_test.go
View File

@ -5,9 +5,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestGenerateDNSVariables(t *testing.T) {

2
v2/pkg/protocols/dns/operators_test.go
View File

@ -8,13 +8,13 @@ import (
"github.com/miekg/dns"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestResponseToDSLMap(t *testing.T) {

6
v2/pkg/protocols/dns/request.go
View File

@ -35,7 +35,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
// Compile each request for the template based on the URL
compiledRequest, err := request.Make(domain)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not build request")
}
@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
// Send the request to the target servers
response, err := request.dnsClient.Do(compiledRequest)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
}
if response == nil {
@ -61,7 +61,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
}
request.options.Progress.IncrementRequests()
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain)
outputEvent := request.responseToDSLMap(compiledRequest, response, input, input)

2
v2/pkg/protocols/dns/request_test.go
View File

@ -5,13 +5,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestDNSExecuteWithResults(t *testing.T) {

2
v2/pkg/protocols/file/file_test.go
View File

@ -5,9 +5,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestFileCompile(t *testing.T) {

2
v2/pkg/protocols/file/find_test.go
View File

@ -8,9 +8,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestFindInputPaths(t *testing.T) {

2
v2/pkg/protocols/file/operators_test.go
View File

@ -5,13 +5,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestResponseToDSLMap(t *testing.T) {

2
v2/pkg/protocols/file/request.go
View File

@ -74,7 +74,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
})
wg.Wait()
if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, "file", err)
request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request")
}

2
v2/pkg/protocols/file/request_test.go
View File

@ -8,13 +8,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestFileExecuteWithResults(t *testing.T) {

8
v2/pkg/protocols/headless/request.go
View File

@ -26,7 +26,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
instance, err := request.options.Browser.NewInstance()
if err != nil {
request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err)
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
@ -34,19 +34,19 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o
parsedURL, err := url.Parse(inputURL)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err)
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err)
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
defer page.Close()
request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", nil)
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), nil)
request.options.Progress.IncrementRequests()
gologger.Verbose().Msgf("Sent Headless request to %s", inputURL)

2
v2/pkg/protocols/http/build_request_test.go
View File

@ -6,10 +6,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestBaseURLWithTemplatePrefs(t *testing.T) {

2
v2/pkg/protocols/http/http.go
View File

@ -131,7 +131,7 @@ type Request struct {
options *protocols.ExecuterOptions
totalRequests int
customHeaders map[string]string
generator *generators.Generator // optional, only enabled when using payloads
generator *generators.PayloadGenerator // optional, only enabled when using payloads
httpClient *retryablehttp.Client
rawhttpClient *rawhttp.Client
dynamicValues map[string]interface{}

2
v2/pkg/protocols/http/http_test.go
View File

@ -5,10 +5,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestHTTPCompile(t *testing.T) {

2
v2/pkg/protocols/http/operators_test.go
View File

@ -7,13 +7,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestResponseToDSLMap(t *testing.T) {

4
v2/pkg/protocols/http/request.go
View File

@ -363,7 +363,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
_, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize)
resp.Body.Close()
}
request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err)
request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err)
request.options.Progress.IncrementErrorsBy(1)
// If we have interactsh markers and request times out, still send
@ -401,7 +401,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
}
gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL)
request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err)
request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err)
duration := time.Since(timeStart)

2
v2/pkg/protocols/network/network.go
View File

@ -75,7 +75,7 @@ type Request struct {
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
generator *generators.Generator
generator *generators.PayloadGenerator
// cache any variables that may be needed for operation.
dialer *fastdialer.Dialer
options *protocols.ExecuterOptions

2
v2/pkg/protocols/network/network_test.go
View File

@ -5,9 +5,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestNetworkCompileMake(t *testing.T) {

2
v2/pkg/protocols/network/operators_test.go
View File

@ -5,13 +5,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestResponseToDSLMap(t *testing.T) {

18
v2/pkg/protocols/network/request.go
View File

@ -42,7 +42,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
address, err = getAddress(input)
}
if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, "network", err)
request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not get address from url")
}
@ -71,7 +71,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
if !strings.Contains(actualAddress, ":") {
err := errors.New("no port provided in network protocol request")
request.options.Output.Request(request.options.TemplatePath, address, "network", err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return err
}
@ -119,7 +119,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress)
}
if err != nil {
request.options.Output.Request(request.options.TemplatePath, address, "network", err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server request")
}
@ -149,7 +149,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
data = []byte(input.Data)
}
if err != nil {
request.options.Output.Request(request.options.TemplatePath, address, "network", err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not write request to server")
}
@ -157,7 +157,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
finalData, dataErr := expressions.EvaluateByte(data, payloads)
if dataErr != nil {
request.options.Output.Request(request.options.TemplatePath, address, "network", dataErr)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions")
}
@ -168,7 +168,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
return nil
}
if _, err := conn.Write(finalData); err != nil {
request.options.Output.Request(request.options.TemplatePath, address, "network", err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not write request to server")
}
@ -203,7 +203,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
}
}
request.options.Output.Request(request.options.TemplatePath, actualAddress, "network", err)
request.options.Output.Request(request.options.TemplatePath, actualAddress, request.Type().String(), err)
gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress)
bufferSize := 1024
@ -234,7 +234,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
buf := make([]byte, bufferSize)
nBuf, err := conn.Read(buf)
if err != nil && !os.IsTimeout(err) {
request.options.Output.Request(request.options.TemplatePath, address, "network", err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
closeTimer(readInterval)
return errors.Wrap(err, "could not read from server")
}
@ -247,7 +247,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
final = make([]byte, bufferSize)
n, err = conn.Read(final)
if err != nil && err != io.EOF {
request.options.Output.Request(request.options.TemplatePath, address, "network", err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
return errors.Wrap(err, "could not read from server")
}
responseBuilder.Write(final[:n])

2
v2/pkg/protocols/network/request_test.go
View File

@ -10,13 +10,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestNetworkExecuteWithResults(t *testing.T) {

2
v2/pkg/protocols/offlinehttp/find_test.go
View File

@ -8,10 +8,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestFindResponses(t *testing.T) {

2
v2/pkg/protocols/offlinehttp/operators_test.go
View File

@ -7,13 +7,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
)
func TestResponseToDSLMap(t *testing.T) {

3
v2/pkg/protocols/protocols.go
View File

@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -96,6 +97,8 @@ type Request interface {
MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
// GetCompiledOperators returns a list of the compiled operators
GetCompiledOperators() []*operators.Operators
// Type returns the type of the protocol request
Type() templateTypes.ProtocolType
}
// OutputEventCallback is a callback event for any results found during scanning.

46
v2/pkg/protocols/ssl/ssl.go
View File

@ -18,6 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
@ -30,6 +31,9 @@ type Request struct {
// Operators for the current request go here.
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
// description: |
// Address contains address for the request
Address string `yaml:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"`
// cache any variables that may be needed for operation.
dialer *fastdialer.Dialer
@ -72,28 +76,45 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
if err != nil {
return nil
}
hostname, _, _ := net.SplitHostPort(address)
hostname, port, _ := net.SplitHostPort(address)
requestOptions := request.options
payloadValues := make(map[string]interface{})
for k, v := range dynamicValues {
payloadValues[k] = v
}
payloadValues["Hostname"] = address
payloadValues["Host"] = hostname
payloadValues["Port"] = port
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
if dataErr != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions")
}
addressToDial := string(finalAddress)
config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname}
conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config)
conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", addressToDial, config)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "ssl", err)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server")
}
defer conn.Close()
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second))
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(requestOptions.Options.Timeout) * time.Second))
connTLS, ok := conn.(*tls.Conn)
if !ok {
return nil
}
request.options.Output.Request(request.options.TemplateID, address, "ssl", err)
requestOptions.Output.Request(requestOptions.TemplateID, address, request.Type().String(), err)
gologger.Verbose().Msgf("Sent SSL request to %s", address)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", request.options.TemplateID, input)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input)
}
state := connTLS.ConnectionState()
@ -110,13 +131,14 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
data["response"] = jsonDataString
data["host"] = input
data["matched"] = addressToDial
data["not_after"] = float64(cert.NotAfter.Unix())
data["ip"] = request.dialer.GetDialedIP(hostname)
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false))
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
}
callback(event)
return nil

2
v2/pkg/protocols/ssl/ssl_test.go
View File

@ -3,10 +3,10 @@ package ssl
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/stretchr/testify/require"
)

61
v2/pkg/protocols/websocket/websocket.go
View File

@ -65,7 +65,7 @@ type Request struct {
// be provided as payload which will be read on run-time.
Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"`
generator *generators.Generator
generator *generators.PayloadGenerator
// cache any variables that may be needed for operation.
dialer *fastdialer.Dialer
@ -174,46 +174,44 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
payloadValues["Scheme"] = parsed.Scheme
payloadValues["Path"] = parsed.Path
requestOptions := request.options
for key, value := range request.Headers {
finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues)
if dataErr != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions")
}
header.Set(key, string(finalData))
}
websocketDialer := ws.Dialer{
Header: ws.HandshakeHeaderHTTP(header),
Timeout: time.Duration(request.options.Options.Timeout) * time.Second,
Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second,
NetDial: request.dialer.Dial,
TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname},
}
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
if dataErr != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions")
}
addressToDial := string(finalAddress)
parsedAddress, err := url.Parse(addressToDial)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not parse input url")
}
if parsed.Path != "" && parsed.Path != "/" {
parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)
addressToDial = parsedAddress.String()
}
parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)
addressToDial = parsedAddress.String()
conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", err)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server")
}
defer conn.Close()
@ -225,18 +223,18 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
events, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", err)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not read write response")
}
request.options.Progress.IncrementRequests()
requestOptions.Progress.IncrementRequests()
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", requestOptions.TemplateID, input)
gologger.Print().Msgf("%s", requestOutput)
}
request.options.Output.Request(request.options.TemplateID, input, "websocket", err)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
gologger.Verbose().Msgf("Sent Websocket request to %s", input)
data := make(map[string]interface{})
@ -253,13 +251,13 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
data["matched"] = addressToDial
data["ip"] = request.dialer.GetDialedIP(hostname)
event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
event := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues
})
if request.options.Options.Debug || request.options.Options.DebugResponse {
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
responseOutput := responseBuilder.String()
gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", request.options.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor, false))
gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", requestOptions.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, requestOptions.Options.NoColor, false))
}
callback(event)
@ -270,28 +268,29 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map
reqBuilder := &strings.Builder{}
inputEvents := make(map[string]interface{})
requestOptions := request.options
for _, req := range request.Inputs {
reqBuilder.Grow(len(req.Data))
finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues)
if dataErr != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return nil, "", errors.Wrap(dataErr, "could not evaluate template expressions")
}
reqBuilder.WriteString(string(finalData))
err = wsutil.WriteClientMessage(conn, ws.OpText, finalData)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", err)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return nil, "", errors.Wrap(err, "could not write request to server")
}
msg, opCode, err := wsutil.ReadServerData(conn)
if err != nil {
request.options.Output.Request(request.options.TemplateID, input, "websocket", err)
request.options.Progress.IncrementFailedRequestsBy(1)
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
return nil, "", errors.Wrap(err, "could not write request to server")
}
// Only perform matching and writes in case we recieve

4
v2/pkg/templates/templates.go
View File

@ -83,8 +83,8 @@ type Template struct {
Path string `yaml:"-" json:"-"`
}
// TemplateTypes is a list of accepted template types
var TemplateTypes = []string{
// TemplateProtocols is a list of accepted template protocols
var TemplateProtocols = []string{
"dns",
"file",
"http",

0
v2/internal/testutils/integration.go → v2/pkg/testutils/integration.go
View File

62
v2/internal/testutils/testutils.go → v2/pkg/testutils/testutils.go
View File

@ -3,6 +3,7 @@ package testutils
import (
"go.uber.org/ratelimit"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
@ -73,7 +74,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
TemplateID: info.ID,
TemplateInfo: info.Info,
TemplatePath: info.Path,
Output: output.NewMockOutputWriter(),
Output: NewMockOutputWriter(),
Options: options,
Progress: progressImpl,
ProjectFile: nil,
@ -90,3 +91,62 @@ type NoopWriter struct{}
// Write writes the data to an output writer.
func (n *NoopWriter) Write(data []byte, level levels.Level) {}
// MockOutputWriter is a mocked output writer.
type MockOutputWriter struct {
aurora aurora.Aurora
RequestCallback func(templateID, url, requestType string, err error)
WriteCallback func(o *output.ResultEvent)
}
// NewMockOutputWriter creates a new mock output writer
func NewMockOutputWriter() *MockOutputWriter {
return &MockOutputWriter{aurora: aurora.NewAurora(false)}
}
// Close closes the output writer interface
func (m *MockOutputWriter) Close() {}
// Colorizer returns the colorizer instance for writer
func (m *MockOutputWriter) Colorizer() aurora.Aurora {
return m.aurora
}
// Write writes the event to file and/or screen.
func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
if m.WriteCallback != nil {
m.WriteCallback(result)
}
return nil
}
// Request writes a log the requests trace log
func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
if m.RequestCallback != nil {
m.RequestCallback(templateID, url, requestType, err)
}
}
type MockProgressClient struct{}
// Stop stops the progress recorder.
func (m *MockProgressClient) Stop() {}
// Init inits the progress bar with initial details for scan
func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {}
// AddToTotal adds a value to the total request count
func (m *MockProgressClient) AddToTotal(delta int64) {}
// IncrementRequests increments the requests counter by 1.
func (m *MockProgressClient) IncrementRequests() {}
// IncrementMatched increments the matched counter by 1.
func (m *MockProgressClient) IncrementMatched() {}
// IncrementErrorsBy increments the error counter by count.
func (m *MockProgressClient) IncrementErrorsBy(count int64) {}
// IncrementFailedRequestsBy increments the number of requests counter by count
// along with errors.
func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {}