From c9d0942bc1e0f66a241c82b97e0424e7b53eaedb Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 26 Jun 2023 19:25:51 +0200 Subject: [PATCH] Extend headless contextargs (#3850) * extend headless contextargs * using darwin-latest * grouping page options * temp commenting code out * fixing test * adding more checks * more checks * fixing first navigation metadata * adding integration test * proto update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .github/workflows/build-test.yml | 2 +- .github/workflows/functional-test.yml | 2 +- integration_tests/workflow/headless-1.yaml | 16 +++ integration_tests/workflow/http-1.yaml | 12 ++ integration_tests/workflow/http-2.yaml | 12 ++ integration_tests/workflow/http-3.yaml | 12 ++ .../workflow/http-value-share-template-1.yaml | 2 +- .../workflow/http-value-share-template-2.yaml | 2 +- integration_tests/workflow/match-1.yaml | 2 +- integration_tests/workflow/match-2.yaml | 2 +- integration_tests/workflow/nomatch-1.yaml | 2 +- integration_tests/workflow/shared-cookie.yaml | 15 +++ v2/cmd/integration-test/workflow.go | 37 ++++++ v2/pkg/protocols/common/fuzz/execute.go | 13 +- v2/pkg/protocols/common/fuzz/execute_test.go | 10 +- v2/pkg/protocols/common/fuzz/parts.go | 10 +- v2/pkg/protocols/common/fuzz/parts_test.go | 10 +- v2/pkg/protocols/headless/engine/page.go | 120 +++++++++++++++--- .../protocols/headless/engine/page_actions.go | 12 +- .../headless/engine/page_actions_test.go | 11 +- v2/pkg/protocols/headless/engine/rules.go | 18 +++ v2/pkg/protocols/headless/headless.go | 4 + v2/pkg/protocols/headless/request.go | 54 ++++---- v2/pkg/protocols/http/request.go | 5 +- v2/pkg/protocols/offlinehttp/request.go | 27 +--- v2/pkg/protocols/utils/utils.go | 23 ++++ 26 files changed, 333 insertions(+), 102 deletions(-) create mode 100644 integration_tests/workflow/headless-1.yaml create mode 100644 integration_tests/workflow/http-1.yaml create mode 100644 integration_tests/workflow/http-2.yaml create mode 100644 integration_tests/workflow/http-3.yaml create mode 100644 integration_tests/workflow/shared-cookie.yaml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 57ff74e6..599940d7 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: go-version: [1.20.x] - os: [ubuntu-latest, windows-latest, macOS-13] + os: [ubuntu-latest, windows-latest, macOS-latest] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index c029803f..79b154af 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-13] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go uses: actions/setup-go@v4 diff --git a/integration_tests/workflow/headless-1.yaml b/integration_tests/workflow/headless-1.yaml new file mode 100644 index 00000000..2a589531 --- /dev/null +++ b/integration_tests/workflow/headless-1.yaml @@ -0,0 +1,16 @@ +id: headless-1 +info: + name: Headless 1 + author: pdteam + severity: info + tags: headless + +headless: + - cookie-reuse: true + steps: + - action: navigate + args: + url: "{{BaseURL}}/headless1" + + - action: waitload + \ No newline at end of file diff --git a/integration_tests/workflow/http-1.yaml b/integration_tests/workflow/http-1.yaml new file mode 100644 index 00000000..cdebddf3 --- /dev/null +++ b/integration_tests/workflow/http-1.yaml @@ -0,0 +1,12 @@ +id: http1 + +info: + name: http1 + author: pdteam + severity: info + +http: + - method: GET + path: + - "{{BaseURL}}/http1" + cookie-reuse: true \ No newline at end of file diff --git a/integration_tests/workflow/http-2.yaml b/integration_tests/workflow/http-2.yaml new file mode 100644 index 00000000..e5000e05 --- /dev/null +++ b/integration_tests/workflow/http-2.yaml @@ -0,0 +1,12 @@ +id: http2 + +info: + name: http2 + author: pdteam + severity: info + +http: + - method: GET + path: + - "{{BaseURL}}/http2" + cookie-reuse: true \ No newline at end of file diff --git a/integration_tests/workflow/http-3.yaml b/integration_tests/workflow/http-3.yaml new file mode 100644 index 00000000..230408dd --- /dev/null +++ b/integration_tests/workflow/http-3.yaml @@ -0,0 +1,12 @@ +id: http3 + +info: + name: http3 + author: pdteam + severity: info + +http: + - method: GET + path: + - "{{BaseURL}}/http3" + cookie-reuse: true \ No newline at end of file diff --git a/integration_tests/workflow/http-value-share-template-1.yaml b/integration_tests/workflow/http-value-share-template-1.yaml index a273e113..c2f6123d 100644 --- a/integration_tests/workflow/http-value-share-template-1.yaml +++ b/integration_tests/workflow/http-value-share-template-1.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - path: - "{{BaseURL}}/path1" extractors: diff --git a/integration_tests/workflow/http-value-share-template-2.yaml b/integration_tests/workflow/http-value-share-template-2.yaml index 7bac99f3..f391aae1 100644 --- a/integration_tests/workflow/http-value-share-template-2.yaml +++ b/integration_tests/workflow/http-value-share-template-2.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET /path2 HTTP/1.1 diff --git a/integration_tests/workflow/match-1.yaml b/integration_tests/workflow/match-1.yaml index c7e07e8c..854652c9 100644 --- a/integration_tests/workflow/match-1.yaml +++ b/integration_tests/workflow/match-1.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/workflow/match-2.yaml b/integration_tests/workflow/match-2.yaml index 01e9d4b6..2431ad10 100644 --- a/integration_tests/workflow/match-2.yaml +++ b/integration_tests/workflow/match-2.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/workflow/nomatch-1.yaml b/integration_tests/workflow/nomatch-1.yaml index 71f12642..ec71d9be 100644 --- a/integration_tests/workflow/nomatch-1.yaml +++ b/integration_tests/workflow/nomatch-1.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/workflow/shared-cookie.yaml b/integration_tests/workflow/shared-cookie.yaml new file mode 100644 index 00000000..f2d68be2 --- /dev/null +++ b/integration_tests/workflow/shared-cookie.yaml @@ -0,0 +1,15 @@ +id: workflow-shared-cookies + +info: + name: Test Workflow Shared Cookies + author: pdteam + severity: info + +workflows: + # store cookies to standard http client cookie-jar + - template: workflow/http-1.yaml + - template: workflow/http-2.yaml + # store cookie in native browser context + - template: workflow/headless-1.yaml + # retrive 2 standard library cookies + headless cookie + - template: workflow/http-3.yaml \ No newline at end of file diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 3fa08523..fc22d8d8 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -18,6 +18,7 @@ var workflowTestcases = map[string]testutils.TestCase{ "workflow/matcher-name.yaml": &workflowMatcherName{}, "workflow/http-value-share-workflow.yaml": &workflowHttpKeyValueShare{}, "workflow/dns-value-share-workflow.yaml": &workflowDnsKeyValueShare{}, + "workflow/shared-cookie.yaml": &workflowSharedCookies{}, } type workflowBasic struct{} @@ -131,3 +132,39 @@ func (h *workflowDnsKeyValueShare) Execute(filePath string) error { // no results - ensure that the variable sharing works return expectResultsCount(results, 1) } + +type workflowSharedCookies struct{} + +// Execute executes a test case and returns an error if occurred +func (h *workflowSharedCookies) Execute(filePath string) error { + handleFunc := func(name string, w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + cookie := &http.Cookie{Name: name, Value: name} + http.SetCookie(w, cookie) + } + + var gotCookies []string + router := httprouter.New() + router.GET("/http1", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + handleFunc("http1", w, r, p) + }) + router.GET("/http2", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + handleFunc("http2", w, r, p) + }) + router.GET("/headless1", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + handleFunc("headless1", w, r, p) + }) + router.GET("/http3", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + for _, cookie := range r.Cookies() { + gotCookies = append(gotCookies, cookie.Name) + } + }) + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + + return expectResultsCount(gotCookies, 3) +} diff --git a/v2/pkg/protocols/common/fuzz/execute.go b/v2/pkg/protocols/common/fuzz/execute.go index 3c7173b9..a16c7107 100644 --- a/v2/pkg/protocols/common/fuzz/execute.go +++ b/v2/pkg/protocols/common/fuzz/execute.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/retryablehttp-go" urlutil "github.com/projectdiscovery/utils/url" @@ -13,8 +14,8 @@ import ( // ExecuteRuleInput is the input for rule Execute function type ExecuteRuleInput struct { - // URL is the URL for the request - URL *urlutil.URL + // Input is the context args input + Input *contextargs.Context // Callback is the callback for generated rule requests Callback func(GeneratedRequest) bool // InteractURLs contains interact urls for execute call @@ -41,7 +42,7 @@ type GeneratedRequest struct { // Input is not thread safe and should not be shared between concurrent // goroutines. func (rule *Rule) Execute(input *ExecuteRuleInput) error { - if !rule.isExecutable(input.URL) { + if !rule.isExecutable(input.Input) { return nil } baseValues := input.Values @@ -69,7 +70,11 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) error { } // isExecutable returns true if the rule can be executed based on provided input -func (rule *Rule) isExecutable(parsed *urlutil.URL) bool { +func (rule *Rule) isExecutable(input *contextargs.Context) bool { + parsed, err := urlutil.Parse(input.MetaInput.Input) + if err != nil { + return false + } if len(parsed.Query()) > 0 && rule.partType == queryPartType { return true } diff --git a/v2/pkg/protocols/common/fuzz/execute_test.go b/v2/pkg/protocols/common/fuzz/execute_test.go index 7f44f940..88582e1b 100644 --- a/v2/pkg/protocols/common/fuzz/execute_test.go +++ b/v2/pkg/protocols/common/fuzz/execute_test.go @@ -3,7 +3,7 @@ package fuzz import ( "testing" - urlutil "github.com/projectdiscovery/utils/url" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/stretchr/testify/require" ) @@ -12,11 +12,11 @@ func TestRuleIsExecutable(t *testing.T) { err := rule.Compile(nil, nil) require.NoError(t, err, "could not compile rule") - parsed, _ := urlutil.Parse("https://example.com/?url=localhost") - result := rule.isExecutable(parsed) + input := contextargs.NewWithInput("https://example.com/?url=localhost") + result := rule.isExecutable(input) require.True(t, result, "could not get correct result") - parsed, _ = urlutil.Parse("https://example.com/") - result = rule.isExecutable(parsed) + input = contextargs.NewWithInput("https://example.com/") + result = rule.isExecutable(input) require.False(t, result, "could not get correct result") } diff --git a/v2/pkg/protocols/common/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go index 9c0c4302..0576f302 100644 --- a/v2/pkg/protocols/common/fuzz/parts.go +++ b/v2/pkg/protocols/common/fuzz/parts.go @@ -24,16 +24,20 @@ func (rule *Rule) executePartRule(input *ExecuteRuleInput, payload string) error // executeQueryPartRule executes query part rules func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string) error { - requestURL := input.URL.Clone() + requestURL, err := urlutil.Parse(input.Input.MetaInput.Input) + if err != nil { + return err + } + origRequestURL := requestURL.Clone() temp := urlutil.Params{} - for k, v := range input.URL.Query() { + for k, v := range origRequestURL.Query() { // this has to be a deep copy x := []string{} x = append(x, v...) temp[k] = x } - for key, values := range input.URL.Query() { + for key, values := range origRequestURL.Query() { for i, value := range values { if !rule.matchKeyOrValue(key, value) { continue diff --git a/v2/pkg/protocols/common/fuzz/parts_test.go b/v2/pkg/protocols/common/fuzz/parts_test.go index e4402c94..e1855a3c 100644 --- a/v2/pkg/protocols/common/fuzz/parts_test.go +++ b/v2/pkg/protocols/common/fuzz/parts_test.go @@ -4,13 +4,13 @@ import ( "testing" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" - urlutil "github.com/projectdiscovery/utils/url" "github.com/stretchr/testify/require" ) func TestExecuteQueryPartRule(t *testing.T) { - parsed, _ := urlutil.Parse("http://localhost:8080/?url=localhost&mode=multiple&file=passwdfile") + URL := "http://localhost:8080/?url=localhost&mode=multiple&file=passwdfile" options := &protocols.ExecutorOptions{ Interactsh: &interactsh.Client{}, } @@ -22,8 +22,9 @@ func TestExecuteQueryPartRule(t *testing.T) { options: options, } var generatedURL []string + input := contextargs.NewWithInput(URL) err := rule.executeQueryPartRule(&ExecuteRuleInput{ - URL: parsed, + Input: input, Callback: func(gr GeneratedRequest) bool { generatedURL = append(generatedURL, gr.Request.URL.String()) return true @@ -44,8 +45,9 @@ func TestExecuteQueryPartRule(t *testing.T) { options: options, } var generatedURL string + input := contextargs.NewWithInput(URL) err := rule.executeQueryPartRule(&ExecuteRuleInput{ - URL: parsed, + Input: input, Callback: func(gr GeneratedRequest) bool { generatedURL = gr.Request.URL.String() return true diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 96bbb15a..e23d2eb2 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -1,7 +1,9 @@ package engine import ( + "bufio" "fmt" + "net/http" "net/url" "strings" "sync" @@ -9,10 +11,14 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" ) // Page is a single page in an isolated browser instance type Page struct { + input *contextargs.Context + options *Options page *rod.Page rules []rule instance *Instance @@ -30,13 +36,19 @@ type HistoryData struct { RawResponse string } +// Options contains additional configuration options for the browser instance +type Options struct { + Timeout time.Duration + CookieReuse bool +} + // Run runs a list of actions by creating a new page in the browser. -func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]interface{}, timeout time.Duration) (map[string]string, *Page, error) { +func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (map[string]string, *Page, error) { page, err := i.engine.Page(proto.TargetCreateTarget{}) if err != nil { return nil, nil, err } - page = page.Timeout(timeout) + page = page.Timeout(options.Timeout) if i.browser.customAgent != "" { if userAgentErr := page.SetUserAgent(&proto.NetworkSetUserAgentOverride{UserAgent: i.browser.customAgent}); userAgentErr != nil { @@ -44,7 +56,14 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string] } } - createdPage := &Page{page: page, instance: i, mutex: &sync.RWMutex{}, payloads: payloads} + createdPage := &Page{ + options: options, + page: page, + input: input, + instance: i, + mutex: &sync.RWMutex{}, + payloads: payloads, + } // in case the page has request/response modification rules - enable global hijacking if createdPage.hasModificationRules() || containsModificationActions(actions...) { @@ -79,18 +98,76 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string] return nil, nil, err } - //FIXME: this is a hack, make sure to fix this in the future. See: https://github.com/go-rod/rod/issues/188 - var e proto.NetworkResponseReceived - wait := page.WaitEvent(&e) - - data, err := createdPage.ExecuteActions(baseURL, actions) + // inject cookies + // each http request is performed via the native go http client + // we first inject the shared cookies + URL, err := url.Parse(input.MetaInput.Input) if err != nil { return nil, nil, err } - wait() - data["header"] = headersToString(e.Response.Headers) - data["status_code"] = fmt.Sprint(e.Response.Status) + if options.CookieReuse { + if cookies := input.CookieJar.Cookies(URL); len(cookies) > 0 { + var NetworkCookies []*proto.NetworkCookie + for _, cookie := range cookies { + networkCookie := &proto.NetworkCookie{ + Name: cookie.Name, + Value: cookie.Value, + Domain: cookie.Domain, + Path: cookie.Path, + HTTPOnly: cookie.HttpOnly, + Secure: cookie.Secure, + Expires: proto.TimeSinceEpoch(cookie.Expires.Unix()), + SameSite: proto.NetworkCookieSameSite(GetSameSite(cookie)), + Priority: proto.NetworkCookiePriorityLow, + } + NetworkCookies = append(NetworkCookies, networkCookie) + } + params := proto.CookiesToParams(NetworkCookies) + for _, param := range params { + param.URL = input.MetaInput.Input + } + err := page.SetCookies(params) + if err != nil { + return nil, nil, err + } + } + } + + data, err := createdPage.ExecuteActions(input, actions) + if err != nil { + return nil, nil, err + } + + if options.CookieReuse { + // at the end of actions pull out updated cookies from the browser and inject them into the shared cookie jar + if cookies, err := page.Cookies([]string{URL.String()}); options.CookieReuse && err == nil && len(cookies) > 0 { + var httpCookies []*http.Cookie + for _, cookie := range cookies { + httpCookie := &http.Cookie{ + Name: cookie.Name, + Value: cookie.Value, + Domain: cookie.Domain, + Path: cookie.Path, + HttpOnly: cookie.HTTPOnly, + Secure: cookie.Secure, + } + httpCookies = append(httpCookies, httpCookie) + } + input.CookieJar.SetCookies(URL, httpCookies) + } + } + + // The first item of history data will contain the very first request from the browser + // we assume it's the one matching the initial URL + if len(createdPage.History) > 0 { + firstItem := createdPage.History[0] + if resp, err := http.ReadResponse(bufio.NewReader(strings.NewReader(firstItem.RawResponse)), nil); err == nil { + data["header"] = utils.HeadersToString(resp.Header) + data["status_code"] = fmt.Sprint(resp.StatusCode) + resp.Body.Close() + } + } return data, createdPage, nil } @@ -189,14 +266,17 @@ func containsAnyModificationActionType(actionTypes ...ActionType) bool { return false } -// headersToString converts network headers to string -func headersToString(headers proto.NetworkHeaders) string { - builder := &strings.Builder{} - for header, value := range headers { - builder.WriteString(header) - builder.WriteString(": ") - builder.WriteString(value.String()) - builder.WriteRune('\n') +func GetSameSite(cookie *http.Cookie) string { + switch cookie.SameSite { + case http.SameSiteNoneMode: + return "none" + case http.SameSiteLaxMode: + return "lax" + case http.SameSiteStrictMode: + return "strict" + case http.SameSiteDefaultMode: + fallthrough + default: + return "" } - return builder.String() } diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 294abbe6..b96d4f26 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -18,6 +18,7 @@ import ( "github.com/go-rod/rod/lib/utils" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" @@ -38,8 +39,11 @@ const ( ) // ExecuteActions executes a list of actions on a page. -func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]string, error) { - var err error +func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (map[string]string, error) { + baseURL, err := url.Parse(input.MetaInput.Input) + if err != nil { + return nil, err + } outData := make(map[string]string) for _, act := range actions { @@ -213,7 +217,7 @@ func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO revi } // ActionSetBody executes a SetBody action. -func (p *Page) ActionSetBody(act *Action, out map[string]string /*TODO review unused parameter*/) error { +func (p *Page) ActionSetBody(act *Action, out map[string]string) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -233,7 +237,7 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { } // NavigateURL executes an ActionLoadURL actions loading a URL for the page. -func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL /*TODO review unused parameter*/) error { +func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error { URL := p.getActionArgWithDefaultValues(action, "url") if URL == "" { return errinvalidArguments diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 78602b4a..3524b47d 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -5,8 +5,8 @@ import ( "io" "math/rand" "net/http" + "net/http/cookiejar" "net/http/httptest" - "net/url" "os" "path/filepath" "strconv" @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -564,9 +565,11 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle ts := httptest.NewServer(http.HandlerFunc(handler)) defer ts.Close() - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - extractedData, page, err := instance.Run(parsed, actions, nil, timeout) + input := contextargs.NewWithInput(ts.URL) + input.CookieJar, err = cookiejar.New(nil) + require.Nil(t, err) + + extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout}) assert(page, err, extractedData) if page != nil { diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 2768cc99..22c0057f 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -35,8 +35,26 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { ctx.Request.SetBody(body) } } + + if p.options.CookieReuse { + // each http request is performed via the native go http client + // we first inject the shared cookies + if cookies := p.input.CookieJar.Cookies(ctx.Request.URL()); len(cookies) > 0 { + p.instance.browser.httpclient.Jar.SetCookies(ctx.Request.URL(), cookies) + } + } + + // perform the request _ = ctx.LoadResponse(p.instance.browser.httpclient, true) + if p.options.CookieReuse { + // retrieve the updated cookies from the native http client and inject them into the shared cookie jar + // keeps existing one if not present + if cookies := p.instance.browser.httpclient.Jar.Cookies(ctx.Request.URL()); len(cookies) > 0 { + p.input.CookieJar.SetCookies(ctx.Request.URL(), cookies) + } + } + for _, rule := range p.rules { if rule.Part != "response" { continue diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index d4ba60b4..06203d3e 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -58,6 +58,10 @@ type Request struct { // Fuzzing describes schema to fuzz headless requests Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" json:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz headless requests"` + + // description: | + // CookieReuse is an optional setting that enables cookie reuse + CookieReuse bool `yaml:"cookie-reuse,omitempty" json:"cookie-reuse,omitempty" jsonschema:"title=optional cookie reuse enable,description=Optional setting that enables cookie reuse"` } // RequestPartDefinitions contains a mapping of request part definitions and their diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 814769d1..9365d142 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" urlutil "github.com/projectdiscovery/utils/url" @@ -35,7 +36,6 @@ func (request *Request) Type() templateTypes.ProtocolType { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - inputURL := input.MetaInput.Input if request.options.Browser.UserAgent() == "" { request.options.Browser.SetUserAgent(request.compiledUserAgent) } @@ -56,7 +56,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } // verify if fuzz elaboration was requested if len(request.Fuzzing) > 0 { - return request.executeFuzzingRule(inputURL, payloads, previous, wrappedCallback) + return request.executeFuzzingRule(input, payloads, previous, wrappedCallback) } if request.generator != nil { iterator := request.generator.NewIterator() @@ -69,23 +69,23 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, return nil } value = generators.MergeMaps(value, payloads) - if err := request.executeRequestWithPayloads(inputURL, value, previous, wrappedCallback); err != nil { + if err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil { return err } } } else { value := maps.Clone(payloads) - if err := request.executeRequestWithPayloads(inputURL, value, previous, wrappedCallback); err != nil { + if err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil { return err } } return nil } -func (request *Request) executeRequestWithPayloads(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, errCouldGetHtmlElement) } @@ -97,32 +97,39 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map instance.SetInteractsh(request.options.Interactsh) - parsedURL, err := url.Parse(inputURL) - if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) + if _, err := url.Parse(input.MetaInput.Input); err != nil { + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, errCouldGetHtmlElement) } - timeout := time.Duration(request.options.Options.PageTimeout) * time.Second - out, page, err := instance.Run(parsedURL, request.Steps, payloads, timeout) + options := &engine.Options{ + Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second, + CookieReuse: request.CookieReuse, + } + + if options.CookieReuse && input.CookieJar == nil { + return errors.New("cookie-reuse set but cookie-jar is nil") + } + + out, page, err := instance.Run(input, request.Steps, payloads, options) if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, errCouldGetHtmlElement) } defer page.Close() - request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), nil) + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil) request.options.Progress.IncrementRequests() - gologger.Verbose().Msgf("Sent Headless request to %s", inputURL) + gologger.Verbose().Msgf("Sent Headless request to %s", input.MetaInput.Input) reqBuilder := &strings.Builder{} if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, inputURL) + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input.MetaInput.Input) for _, act := range request.Steps { actStepStr := act.String() - actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", inputURL) + actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", input.MetaInput.Input) reqBuilder.WriteString("\t" + actStepStr + "\n") } gologger.Debug().Msgf(reqBuilder.String()) @@ -135,7 +142,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), inputURL, inputURL, page.DumpHistory()) + outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory()) for k, v := range out { outputEvent[k] = v } @@ -161,7 +168,7 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map event.UsesInteractsh = true } - dumpResponse(event, request.options, responseBody, inputURL) + dumpResponse(event, request.options, responseBody, input.MetaInput.Input) return nil } @@ -174,26 +181,27 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols. } // executeFuzzingRule executes a fuzzing rule in the template request -func (request *Request) executeFuzzingRule(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // check for operator matches by wrapping callback gotmatches := false fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool { if gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) { return true } - if err := request.executeRequestWithPayloads(gr.Request.URL.String(), gr.DynamicValues, previous, callback); err != nil { + newInput := input.Clone() + newInput.MetaInput.Input = gr.Request.URL.String() + if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, callback); err != nil { return false } return true } - parsedURL, err := urlutil.Parse(inputURL) - if err != nil { + if _, err := urlutil.Parse(input.MetaInput.Input); err != nil { return errors.Wrap(err, "could not parse url") } for _, rule := range request.Fuzzing { err := rule.Execute(&fuzz.ExecuteRuleInput{ - URL: parsedURL, + Input: input, Callback: fuzzRequestCallback, Values: payloads, BaseRequest: nil, diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index bbadc187..304eead2 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -228,8 +228,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu // executeFuzzingRule executes fuzzing request for a URL func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - parsed, err := urlutil.Parse(input.MetaInput.Input) - if err != nil { + if _, err := urlutil.Parse(input.MetaInput.Input); err != nil { return errors.Wrap(err, "could not parse url") } fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool { @@ -297,7 +296,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous } for _, rule := range request.Fuzzing { err = rule.Execute(&fuzz.ExecuteRuleInput{ - URL: parsed, + Input: input, Callback: fuzzRequestCallback, Values: generated.dynamicValues, BaseRequest: generated.request, diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 7bafa3a7..2bcbeacc 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -2,10 +2,8 @@ package offlinehttp import ( "io" - "net/http" "net/http/httputil" "os" - "strings" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" @@ -16,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) @@ -86,7 +85,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata return } - outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) + outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil) outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v @@ -105,25 +104,3 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata request.options.Progress.IncrementRequests() return nil } - -// headersToString converts http headers to string -func headersToString(headers http.Header) string { - builder := &strings.Builder{} - - for header, values := range headers { - builder.WriteString(header) - builder.WriteString(": ") - - for i, value := range values { - builder.WriteString(value) - - if i != len(values)-1 { - builder.WriteRune('\n') - builder.WriteString(header) - builder.WriteString(": ") - } - } - builder.WriteRune('\n') - } - return builder.String() -} diff --git a/v2/pkg/protocols/utils/utils.go b/v2/pkg/protocols/utils/utils.go index cdc0b367..505a0a3b 100644 --- a/v2/pkg/protocols/utils/utils.go +++ b/v2/pkg/protocols/utils/utils.go @@ -3,6 +3,7 @@ package utils import ( "crypto/tls" "crypto/x509" + "net/http" "os" "strings" @@ -46,3 +47,25 @@ func CalculateContentLength(contentLength, bodyLength int64) int64 { } return bodyLength } + +// headersToString converts http headers to string +func HeadersToString(headers http.Header) string { + builder := &strings.Builder{} + + for header, values := range headers { + builder.WriteString(header) + builder.WriteString(": ") + + for i, value := range values { + builder.WriteString(value) + + if i != len(values)-1 { + builder.WriteRune('\n') + builder.WriteString(header) + builder.WriteString(": ") + } + } + builder.WriteRune('\n') + } + return builder.String() +}