daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extend headless contextargs (#3850) 

* extend headless contextargs

* using darwin-latest

* grouping page options

* temp commenting code out

* fixing test

* adding more checks

* more checks

* fixing first navigation metadata

* adding integration test

* proto update

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
dev
Mzack9999 2023-06-26 19:25:51 +02:00 committed by GitHub
parent fa199ed3b3
commit c9d0942bc1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 333 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build-test.yml vendored
View File

@ -13,7 +13,7 @@ jobs:
strategy:
matrix:
go-version: [1.20.x]
os: [ubuntu-latest, windows-latest, macOS-13]
os: [ubuntu-latest, windows-latest, macOS-latest]
runs-on: ${{ matrix.os }}
steps:

2
.github/workflows/functional-test.yml vendored
View File

@ -13,7 +13,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-13]
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
uses: actions/setup-go@v4

16
integration_tests/workflow/headless-1.yaml Normal file
View File

@ -0,0 +1,16 @@
id: headless-1
info:
name: Headless 1
author: pdteam
severity: info
tags: headless
headless:
- cookie-reuse: true
steps:
- action: navigate
args:
url: "{{BaseURL}}/headless1"
- action: waitload

12
integration_tests/workflow/http-1.yaml Normal file
View File

@ -0,0 +1,12 @@
id: http1
info:
name: http1
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}/http1"
cookie-reuse: true

12
integration_tests/workflow/http-2.yaml Normal file
View File

@ -0,0 +1,12 @@
id: http2
info:
name: http2
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}/http2"
cookie-reuse: true

12
integration_tests/workflow/http-3.yaml Normal file
View File

@ -0,0 +1,12 @@
id: http3
info:
name: http3
author: pdteam
severity: info
http:
- method: GET
path:
- "{{BaseURL}}/http3"
cookie-reuse: true

2
integration_tests/workflow/http-value-share-template-1.yaml
View File

@ -5,7 +5,7 @@ info:
author: pdteam
severity: info
requests:
http:
- path:
- "{{BaseURL}}/path1"
extractors:

2
integration_tests/workflow/http-value-share-template-2.yaml
View File

@ -5,7 +5,7 @@ info:
author: pdteam
severity: info
requests:
http:
- raw:
- |
GET /path2 HTTP/1.1

2
integration_tests/workflow/match-1.yaml
View File

@ -5,7 +5,7 @@ info:
author: pdteam
severity: info
requests:
http:
- method: GET
path:
- "{{BaseURL}}"

2
integration_tests/workflow/match-2.yaml
View File

@ -5,7 +5,7 @@ info:
author: pdteam
severity: info
requests:
http:
- method: GET
path:
- "{{BaseURL}}"

2
integration_tests/workflow/nomatch-1.yaml
View File

@ -5,7 +5,7 @@ info:
author: pdteam
severity: info
requests:
http:
- method: GET
path:
- "{{BaseURL}}"

15
integration_tests/workflow/shared-cookie.yaml Normal file
View File

@ -0,0 +1,15 @@
id: workflow-shared-cookies
info:
name: Test Workflow Shared Cookies
author: pdteam
severity: info
workflows:
# store cookies to standard http client cookie-jar
- template: workflow/http-1.yaml
- template: workflow/http-2.yaml
# store cookie in native browser context
- template: workflow/headless-1.yaml
# retrive 2 standard library cookies + headless cookie
- template: workflow/http-3.yaml

37
v2/cmd/integration-test/workflow.go
View File

@ -18,6 +18,7 @@ var workflowTestcases = map[string]testutils.TestCase{
"workflow/matcher-name.yaml": &workflowMatcherName{},
"workflow/http-value-share-workflow.yaml": &workflowHttpKeyValueShare{},
"workflow/dns-value-share-workflow.yaml": &workflowDnsKeyValueShare{},
"workflow/shared-cookie.yaml": &workflowSharedCookies{},
}
type workflowBasic struct{}
@ -131,3 +132,39 @@ func (h *workflowDnsKeyValueShare) Execute(filePath string) error {
// no results - ensure that the variable sharing works
return expectResultsCount(results, 1)
}
type workflowSharedCookies struct{}
// Execute executes a test case and returns an error if occurred
func (h *workflowSharedCookies) Execute(filePath string) error {
handleFunc := func(name string, w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cookie := &http.Cookie{Name: name, Value: name}
http.SetCookie(w, cookie)
}
var gotCookies []string
router := httprouter.New()
router.GET("/http1", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
handleFunc("http1", w, r, p)
})
router.GET("/http2", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
handleFunc("http2", w, r, p)
})
router.GET("/headless1", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
handleFunc("headless1", w, r, p)
})
router.GET("/http3", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
for _, cookie := range r.Cookies() {
gotCookies = append(gotCookies, cookie.Name)
}
})
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug, "-headless")
if err != nil {
return err
}
return expectResultsCount(gotCookies, 3)
}

13
v2/pkg/protocols/common/fuzz/execute.go
View File

@ -6,6 +6,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
@ -13,8 +14,8 @@ import (
// ExecuteRuleInput is the input for rule Execute function
type ExecuteRuleInput struct {
// URL is the URL for the request
URL *urlutil.URL
// Input is the context args input
Input *contextargs.Context
// Callback is the callback for generated rule requests
Callback func(GeneratedRequest) bool
// InteractURLs contains interact urls for execute call
@ -41,7 +42,7 @@ type GeneratedRequest struct {
// Input is not thread safe and should not be shared between concurrent
// goroutines.
func (rule *Rule) Execute(input *ExecuteRuleInput) error {
if !rule.isExecutable(input.URL) {
if !rule.isExecutable(input.Input) {
return nil
}
baseValues := input.Values
@ -69,7 +70,11 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) error {
}
// isExecutable returns true if the rule can be executed based on provided input
func (rule *Rule) isExecutable(parsed *urlutil.URL) bool {
func (rule *Rule) isExecutable(input *contextargs.Context) bool {
parsed, err := urlutil.Parse(input.MetaInput.Input)
if err != nil {
return false
}
if len(parsed.Query()) > 0 && rule.partType == queryPartType {
return true
}

10
v2/pkg/protocols/common/fuzz/execute_test.go
View File

@ -3,7 +3,7 @@ package fuzz
import (
"testing"
urlutil "github.com/projectdiscovery/utils/url"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/stretchr/testify/require"
)
@ -12,11 +12,11 @@ func TestRuleIsExecutable(t *testing.T) {
err := rule.Compile(nil, nil)
require.NoError(t, err, "could not compile rule")
parsed, _ := urlutil.Parse("https://example.com/?url=localhost")
result := rule.isExecutable(parsed)
input := contextargs.NewWithInput("https://example.com/?url=localhost")
result := rule.isExecutable(input)
require.True(t, result, "could not get correct result")
parsed, _ = urlutil.Parse("https://example.com/")
result = rule.isExecutable(parsed)
input = contextargs.NewWithInput("https://example.com/")
result = rule.isExecutable(input)
require.False(t, result, "could not get correct result")
}

10
v2/pkg/protocols/common/fuzz/parts.go
View File

@ -24,16 +24,20 @@ func (rule *Rule) executePartRule(input *ExecuteRuleInput, payload string) error
// executeQueryPartRule executes query part rules
func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) error {
requestURL := input.URL.Clone()
requestURL, err := urlutil.Parse(input.Input.MetaInput.Input)
if err != nil {
return err
}
origRequestURL := requestURL.Clone()
temp := urlutil.Params{}
for k, v := range input.URL.Query() {
for k, v := range origRequestURL.Query() {
// this has to be a deep copy
x := []string{}
x = append(x, v...)
temp[k] = x
}
for key, values := range input.URL.Query() {
for key, values := range origRequestURL.Query() {
for i, value := range values {
if !rule.matchKeyOrValue(key, value) {
continue

10
v2/pkg/protocols/common/fuzz/parts_test.go
View File

@ -4,13 +4,13 @@ import (
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
urlutil "github.com/projectdiscovery/utils/url"
"github.com/stretchr/testify/require"
)
func TestExecuteQueryPartRule(t *testing.T) {
parsed, _ := urlutil.Parse("http://localhost:8080/?url=localhost&mode=multiple&file=passwdfile")
URL := "http://localhost:8080/?url=localhost&mode=multiple&file=passwdfile"
options := &protocols.ExecutorOptions{
Interactsh: &interactsh.Client{},
}
@ -22,8 +22,9 @@ func TestExecuteQueryPartRule(t *testing.T) {
options: options,
}
var generatedURL []string
input := contextargs.NewWithInput(URL)
err := rule.executeQueryPartRule(&ExecuteRuleInput{
URL: parsed,
Input: input,
Callback: func(gr GeneratedRequest) bool {
generatedURL = append(generatedURL, gr.Request.URL.String())
return true
@ -44,8 +45,9 @@ func TestExecuteQueryPartRule(t *testing.T) {
options: options,
}
var generatedURL string
input := contextargs.NewWithInput(URL)
err := rule.executeQueryPartRule(&ExecuteRuleInput{
URL: parsed,
Input: input,
Callback: func(gr GeneratedRequest) bool {
generatedURL = gr.Request.URL.String()
return true

120
v2/pkg/protocols/headless/engine/page.go
View File

@ -1,7 +1,9 @@
package engine
import (
"bufio"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
@ -9,10 +11,14 @@ import (
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
)
// Page is a single page in an isolated browser instance
type Page struct {
input *contextargs.Context
options *Options
page *rod.Page
rules []rule
instance *Instance
@ -30,13 +36,19 @@ type HistoryData struct {
RawResponse string
}
// Options contains additional configuration options for the browser instance
type Options struct {
Timeout time.Duration
CookieReuse bool
}
// Run runs a list of actions by creating a new page in the browser.
func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]interface{}, timeout time.Duration) (map[string]string, *Page, error) {
func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (map[string]string, *Page, error) {
page, err := i.engine.Page(proto.TargetCreateTarget{})
if err != nil {
return nil, nil, err
}
page = page.Timeout(timeout)
page = page.Timeout(options.Timeout)
if i.browser.customAgent != "" {
if userAgentErr := page.SetUserAgent(&proto.NetworkSetUserAgentOverride{UserAgent: i.browser.customAgent}); userAgentErr != nil {
@ -44,7 +56,14 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]
}
}
createdPage := &Page{page: page, instance: i, mutex: &sync.RWMutex{}, payloads: payloads}
createdPage := &Page{
options: options,
page: page,
input: input,
instance: i,
mutex: &sync.RWMutex{},
payloads: payloads,
}
// in case the page has request/response modification rules - enable global hijacking
if createdPage.hasModificationRules() || containsModificationActions(actions...) {
@ -79,18 +98,76 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]
return nil, nil, err
}
//FIXME: this is a hack, make sure to fix this in the future. See: https://github.com/go-rod/rod/issues/188
var e proto.NetworkResponseReceived
wait := page.WaitEvent(&e)
data, err := createdPage.ExecuteActions(baseURL, actions)
// inject cookies
// each http request is performed via the native go http client
// we first inject the shared cookies
URL, err := url.Parse(input.MetaInput.Input)
if err != nil {
return nil, nil, err
}
wait()
data["header"] = headersToString(e.Response.Headers)
data["status_code"] = fmt.Sprint(e.Response.Status)
if options.CookieReuse {
if cookies := input.CookieJar.Cookies(URL); len(cookies) > 0 {
var NetworkCookies []*proto.NetworkCookie
for _, cookie := range cookies {
networkCookie := &proto.NetworkCookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
HTTPOnly: cookie.HttpOnly,
Secure: cookie.Secure,
Expires: proto.TimeSinceEpoch(cookie.Expires.Unix()),
SameSite: proto.NetworkCookieSameSite(GetSameSite(cookie)),
Priority: proto.NetworkCookiePriorityLow,
}
NetworkCookies = append(NetworkCookies, networkCookie)
}
params := proto.CookiesToParams(NetworkCookies)
for _, param := range params {
param.URL = input.MetaInput.Input
}
err := page.SetCookies(params)
if err != nil {
return nil, nil, err
}
}
}
data, err := createdPage.ExecuteActions(input, actions)
if err != nil {
return nil, nil, err
}
if options.CookieReuse {
// at the end of actions pull out updated cookies from the browser and inject them into the shared cookie jar
if cookies, err := page.Cookies([]string{URL.String()}); options.CookieReuse && err == nil && len(cookies) > 0 {
var httpCookies []*http.Cookie
for _, cookie := range cookies {
httpCookie := &http.Cookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
HttpOnly: cookie.HTTPOnly,
Secure: cookie.Secure,
}
httpCookies = append(httpCookies, httpCookie)
}
input.CookieJar.SetCookies(URL, httpCookies)
}
}
// The first item of history data will contain the very first request from the browser
// we assume it's the one matching the initial URL
if len(createdPage.History) > 0 {
firstItem := createdPage.History[0]
if resp, err := http.ReadResponse(bufio.NewReader(strings.NewReader(firstItem.RawResponse)), nil); err == nil {
data["header"] = utils.HeadersToString(resp.Header)
data["status_code"] = fmt.Sprint(resp.StatusCode)
resp.Body.Close()
}
}
return data, createdPage, nil
}
@ -189,14 +266,17 @@ func containsAnyModificationActionType(actionTypes ...ActionType) bool {
return false
}
// headersToString converts network headers to string
func headersToString(headers proto.NetworkHeaders) string {
builder := &strings.Builder{}
for header, value := range headers {
builder.WriteString(header)
builder.WriteString(": ")
builder.WriteString(value.String())
builder.WriteRune('\n')
func GetSameSite(cookie *http.Cookie) string {
switch cookie.SameSite {
case http.SameSiteNoneMode:
return "none"
case http.SameSiteLaxMode:
return "lax"
case http.SameSiteStrictMode:
return "strict"
case http.SameSiteDefaultMode:
fallthrough
default:
return ""
}
return builder.String()
}

12
v2/pkg/protocols/headless/engine/page_actions.go
View File

@ -18,6 +18,7 @@ import (
"github.com/go-rod/rod/lib/utils"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
@ -38,8 +39,11 @@ const (
)
// ExecuteActions executes a list of actions on a page.
func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]string, error) {
var err error
func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (map[string]string, error) {
baseURL, err := url.Parse(input.MetaInput.Input)
if err != nil {
return nil, err
}
outData := make(map[string]string)
for _, act := range actions {
@ -213,7 +217,7 @@ func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO revi
}
// ActionSetBody executes a SetBody action.
func (p *Page) ActionSetBody(act *Action, out map[string]string /*TODO review unused parameter*/) error {
func (p *Page) ActionSetBody(act *Action, out map[string]string) error {
in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string)
@ -233,7 +237,7 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error {
}
// NavigateURL executes an ActionLoadURL actions loading a URL for the page.
func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL /*TODO review unused parameter*/) error {
func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error {
URL := p.getActionArgWithDefaultValues(action, "url")
if URL == "" {
return errinvalidArguments

11
v2/pkg/protocols/headless/engine/page_actions_test.go
View File

@ -5,8 +5,8 @@ import (
"io"
"math/rand"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strconv"
@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -564,9 +565,11 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
ts := httptest.NewServer(http.HandlerFunc(handler))
defer ts.Close()
parsed, err := url.Parse(ts.URL)
require.Nil(t, err, "could not parse URL")
extractedData, page, err := instance.Run(parsed, actions, nil, timeout)
input := contextargs.NewWithInput(ts.URL)
input.CookieJar, err = cookiejar.New(nil)
require.Nil(t, err)
extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout})
assert(page, err, extractedData)
if page != nil {

18
v2/pkg/protocols/headless/engine/rules.go
View File

@ -35,8 +35,26 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
ctx.Request.SetBody(body)
}
}
if p.options.CookieReuse {
// each http request is performed via the native go http client
// we first inject the shared cookies
if cookies := p.input.CookieJar.Cookies(ctx.Request.URL()); len(cookies) > 0 {
p.instance.browser.httpclient.Jar.SetCookies(ctx.Request.URL(), cookies)
}
}
// perform the request
_ = ctx.LoadResponse(p.instance.browser.httpclient, true)
if p.options.CookieReuse {
// retrieve the updated cookies from the native http client and inject them into the shared cookie jar
// keeps existing one if not present
if cookies := p.instance.browser.httpclient.Jar.Cookies(ctx.Request.URL()); len(cookies) > 0 {
p.input.CookieJar.SetCookies(ctx.Request.URL(), cookies)
}
}
for _, rule := range p.rules {
if rule.Part != "response" {
continue

4
v2/pkg/protocols/headless/headless.go
View File

@ -58,6 +58,10 @@ type Request struct {
// Fuzzing describes schema to fuzz headless requests
Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" json:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz headless requests"`
// description: |
// CookieReuse is an optional setting that enables cookie reuse
CookieReuse bool `yaml:"cookie-reuse,omitempty" json:"cookie-reuse,omitempty" jsonschema:"title=optional cookie reuse enable,description=Optional setting that enables cookie reuse"`
}
// RequestPartDefinitions contains a mapping of request part definitions and their

54
v2/pkg/protocols/headless/request.go
View File

@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
urlutil "github.com/projectdiscovery/utils/url"
@ -35,7 +36,6 @@ func (request *Request) Type() templateTypes.ProtocolType {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
inputURL := input.MetaInput.Input
if request.options.Browser.UserAgent() == "" {
request.options.Browser.SetUserAgent(request.compiledUserAgent)
}
@ -56,7 +56,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
}
// verify if fuzz elaboration was requested
if len(request.Fuzzing) > 0 {
return request.executeFuzzingRule(inputURL, payloads, previous, wrappedCallback)
return request.executeFuzzingRule(input, payloads, previous, wrappedCallback)
}
if request.generator != nil {
iterator := request.generator.NewIterator()
@ -69,23 +69,23 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
return nil
}
value = generators.MergeMaps(value, payloads)
if err := request.executeRequestWithPayloads(inputURL, value, previous, wrappedCallback); err != nil {
if err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil {
return err
}
}
} else {
value := maps.Clone(payloads)
if err := request.executeRequestWithPayloads(inputURL, value, previous, wrappedCallback); err != nil {
if err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil {
return err
}
}
return nil
}
func (request *Request) executeRequestWithPayloads(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
instance, err := request.options.Browser.NewInstance()
if err != nil {
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err)
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, errCouldGetHtmlElement)
}
@ -97,32 +97,39 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
instance.SetInteractsh(request.options.Interactsh)
parsedURL, err := url.Parse(inputURL)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err)
if _, err := url.Parse(input.MetaInput.Input); err != nil {
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, errCouldGetHtmlElement)
}
timeout := time.Duration(request.options.Options.PageTimeout) * time.Second
out, page, err := instance.Run(parsedURL, request.Steps, payloads, timeout)
options := &engine.Options{
Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second,
CookieReuse: request.CookieReuse,
}
if options.CookieReuse && input.CookieJar == nil {
return errors.New("cookie-reuse set but cookie-jar is nil")
}
out, page, err := instance.Run(input, request.Steps, payloads, options)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err)
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, errCouldGetHtmlElement)
}
defer page.Close()
request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), nil)
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil)
request.options.Progress.IncrementRequests()
gologger.Verbose().Msgf("Sent Headless request to %s", inputURL)
gologger.Verbose().Msgf("Sent Headless request to %s", input.MetaInput.Input)
reqBuilder := &strings.Builder{}
if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, inputURL)
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input.MetaInput.Input)
for _, act := range request.Steps {
actStepStr := act.String()
actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", inputURL)
actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", input.MetaInput.Input)
reqBuilder.WriteString("\t" + actStepStr + "\n")
}
gologger.Debug().Msgf(reqBuilder.String())
@ -135,7 +142,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
responseBody, _ = html.HTML()
}
outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory())
for k, v := range out {
outputEvent[k] = v
}
@ -161,7 +168,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
event.UsesInteractsh = true
}
dumpResponse(event, request.options, responseBody, inputURL)
dumpResponse(event, request.options, responseBody, input.MetaInput.Input)
return nil
}
@ -174,26 +181,27 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
}
// executeFuzzingRule executes a fuzzing rule in the template request
func (request *Request) executeFuzzingRule(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// check for operator matches by wrapping callback
gotmatches := false
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
if gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) {
return true
}
if err := request.executeRequestWithPayloads(gr.Request.URL.String(), gr.DynamicValues, previous, callback); err != nil {
newInput := input.Clone()
newInput.MetaInput.Input = gr.Request.URL.String()
if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, callback); err != nil {
return false
}
return true
}
parsedURL, err := urlutil.Parse(inputURL)
if err != nil {
if _, err := urlutil.Parse(input.MetaInput.Input); err != nil {
return errors.Wrap(err, "could not parse url")
}
for _, rule := range request.Fuzzing {
err := rule.Execute(&fuzz.ExecuteRuleInput{
URL: parsedURL,
Input: input,
Callback: fuzzRequestCallback,
Values: payloads,
BaseRequest: nil,

5
v2/pkg/protocols/http/request.go
View File

@ -228,8 +228,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
// executeFuzzingRule executes fuzzing request for a URL
func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
parsed, err := urlutil.Parse(input.MetaInput.Input)
if err != nil {
if _, err := urlutil.Parse(input.MetaInput.Input); err != nil {
return errors.Wrap(err, "could not parse url")
}
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
@ -297,7 +296,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
}
for _, rule := range request.Fuzzing {
err = rule.Execute(&fuzz.ExecuteRuleInput{
URL: parsed,
Input: input,
Callback: fuzzRequestCallback,
Values: generated.dynamicValues,
BaseRequest: generated.request,

27
v2/pkg/protocols/offlinehttp/request.go
View File

@ -2,10 +2,8 @@ package offlinehttp
import (
"io"
"net/http"
"net/http/httputil"
"os"
"strings"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
@ -16,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
@ -86,7 +85,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
return
}
outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil)
outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil)
outputEvent["ip"] = ""
for k, v := range previous {
outputEvent[k] = v
@ -105,25 +104,3 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
request.options.Progress.IncrementRequests()
return nil
}
// headersToString converts http headers to string
func headersToString(headers http.Header) string {
builder := &strings.Builder{}
for header, values := range headers {
builder.WriteString(header)
builder.WriteString(": ")
for i, value := range values {
builder.WriteString(value)
if i != len(values)-1 {
builder.WriteRune('\n')
builder.WriteString(header)
builder.WriteString(": ")
}
}
builder.WriteRune('\n')
}
return builder.String()
}

23
v2/pkg/protocols/utils/utils.go
View File

@ -3,6 +3,7 @@ package utils
import (
"crypto/tls"
"crypto/x509"
"net/http"
"os"
"strings"
@ -46,3 +47,25 @@ func CalculateContentLength(contentLength, bodyLength int64) int64 {
}
return bodyLength
}
// headersToString converts http headers to string
func HeadersToString(headers http.Header) string {
builder := &strings.Builder{}
for header, values := range headers {
builder.WriteString(header)
builder.WriteString(": ")
for i, value := range values {
builder.WriteString(value)
if i != len(values)-1 {
builder.WriteRune('\n')
builder.WriteString(header)
builder.WriteString(": ")
}
}
builder.WriteRune('\n')
}
return builder.String()
}