daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

optional file read in headless protocol (#4055) 

* use -lfa and -lna in headless

* fix lna in headless

* misc update

* fix nil pointer dereference in test

* fix lint & unit test

* use urlutil

* headless protocol scheme improvements

* add unit and integration tests

* run unit test from binary

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
dev
Sandeep Singh 2023-08-25 18:30:46 +05:30 committed by GitHub
parent e146c89930
commit d3928e080d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 347 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
integration_tests/headless/file-upload-negative.yaml Normal file
View File

@ -0,0 +1,29 @@
id: file-upload
# template for testing when file upload is disabled
info:
name: Basic File Upload
author: pdteam
severity: info
headless:
- steps:
- action: navigate
args:
url: "{{BaseURL}}"
- action: waitload
- action: files
args:
by: xpath
xpath: /html/body/form/input[1]
value: headless/file-upload.yaml
- action: sleep
args:
duration: 2
- action: click
args:
by: x
xpath: /html/body/form/input[2]
matchers:
- type: word
words:
- "Basic File Upload"

15
integration_tests/headless/headless-local.yaml Normal file
View File

@ -0,0 +1,15 @@
id: nuclei-headless-local
info:
name: Nuclei Headless Local
author: pdteam
severity: high
headless:
- steps:
- action: navigate
args:
url: "{{BaseURL}}"
- action: waitload

68
v2/cmd/integration-test/headless.go
View File

@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{
{Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}}, {Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},
{Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}}, {Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}},
{Path: "headless/variables.yaml", TestCase: &headlessVariables{}}, {Path: "headless/variables.yaml", TestCase: &headlessVariables{}},
{Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}},
{Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}}, {Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}},
{Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}},
{Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}}, {Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}},
} }
@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error {
return expectResultsCount(results, 1) return expectResultsCount(results, 1)
} }
type headlessLocal struct{}
// Execute executes a test case and returns an error if occurred
// in this testcases local network access is disabled
func (h *headlessLocal) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = w.Write([]byte("<html><body></body></html>"))
})
ts := httptest.NewServer(router)
defer ts.Close()
args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"}
results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}
type headlessHeaderActions struct{} type headlessHeaderActions struct{}
// Execute executes a test case and returns an error if occurred // Execute executes a test case and returns an error if occurred
@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error {
return expectResultsCount(results, 1) return expectResultsCount(results, 1)
} }
type headlessFileUploadNegative struct{}
// Execute executes a test case and returns an error if occurred
func (h *headlessFileUploadNegative) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = w.Write([]byte(`
<!doctype html>
<body>
<form method=post enctype=multipart/form-data>
<input type=file name=file>
<input type=submit value=Upload>
</form>
</body>
</html>
`))
})
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, _, err := r.FormFile("file")
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer file.Close()
content, err := io.ReadAll(file)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
_, _ = w.Write(content)
})
ts := httptest.NewServer(router)
defer ts.Close()
args := []string{"-t", filePath, "-u", ts.URL, "-headless"}
results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}

90
v2/pkg/protocols/common/protocolstate/headless.go Normal file
View File

@ -0,0 +1,90 @@
package protocolstate
import (
"strings"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/networkpolicy"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
"go.uber.org/multierr"
)
// initalize state of headless protocol
var (
ErrURLDenied = errorutil.NewWithFmt("headless: url %v dropped by rule: %v")
networkPolicy *networkpolicy.NetworkPolicy
allowLocalFileAccess bool
)
// ValidateNFailRequest validates and fails request
// if the request does not respect the rules, it will be canceled with reason
func ValidateNFailRequest(page *rod.Page, e *proto.FetchRequestPaused) error {
reqURL := e.Request.URL
normalized := strings.ToLower(reqURL) // normalize url to lowercase
normalized = strings.TrimSpace(normalized) // trim leading & trailing whitespaces
if !allowLocalFileAccess && stringsutil.HasPrefixI(normalized, "file:") {
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "use of file:// protocol disabled use '-lfa' to enable"))
}
// validate potential invalid schemes
// javascript protocol is allowed for xss fuzzing
if HasPrefixAnyI(normalized, "ftp:", "externalfile:", "chrome:", "chrome-extension:") {
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "protocol blocked by network policy"))
}
if !isValidHost(reqURL) {
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "address blocked by network policy"))
}
return nil
}
// FailWithReason fails request with AccessDenied reason
func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error {
m := proto.FetchFailRequest{
RequestID: e.RequestID,
ErrorReason: proto.NetworkErrorReasonAccessDenied,
}
return m.Call(page)
}
// InitHeadless initializes headless protocol state
func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) {
allowLocalFileAccess = localFileAccess
if !RestrictLocalNetworkAccess {
return
}
networkPolicy, _ = networkpolicy.New(networkpolicy.Options{
DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...),
})
}
// isValidHost checks if the host is valid (only limited to http/https protocols)
func isValidHost(targetUrl string) bool {
if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") {
return true
}
if networkPolicy == nil {
return true
}
urlx, err := urlutil.Parse(targetUrl)
if err != nil {
// not a valid url
return false
}
targetUrl = urlx.Hostname()
_, ok := networkPolicy.ValidateHost(targetUrl)
return ok
}
// HasPrefixAnyI checks if the string has any of the prefixes
// TODO: replace with stringsutil.HasPrefixAnyI after implementation
func HasPrefixAnyI(s string, prefixes ...string) bool {
for _, prefix := range prefixes {
if stringsutil.HasPrefixI(s, prefix) {
return true
}
}
return false
}

1
v2/pkg/protocols/common/protocolstate/state.go
View File

@ -22,6 +22,7 @@ func Init(options *types.Options) error {
return nil return nil
} }
opts := fastdialer.DefaultOptions opts := fastdialer.DefaultOptions
InitHeadless(options.RestrictLocalNetworkAccess, options.AllowLocalFileAccess)
switch { switch {
case options.SourceIP != "" && options.Interface != "": case options.SourceIP != "" && options.Interface != "":

2
v2/pkg/protocols/headless/engine/page.go
View File

@ -13,6 +13,7 @@ import (
"github.com/go-rod/rod/lib/proto" "github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
) )
// Page is a single page in an isolated browser instance // Page is a single page in an isolated browser instance
@ -40,6 +41,7 @@ type HistoryData struct {
type Options struct { type Options struct {
Timeout time.Duration Timeout time.Duration
CookieReuse bool CookieReuse bool
Options *types.Options
} }
// Run runs a list of actions by creating a new page in the browser. // Run runs a list of actions by creating a new page in the browser.

7
v2/pkg/protocols/headless/engine/page_actions.go
View File

@ -30,7 +30,8 @@ import (
) )
var ( var (
errinvalidArguments = errors.New("invalid arguments provided") errinvalidArguments = errorutil.New("invalid arguments provided")
ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access")
) )
const ( const (
@ -70,7 +71,11 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
case ActionWaitEvent: case ActionWaitEvent:
err = p.WaitEvent(act, outData) err = p.WaitEvent(act, outData)
case ActionFilesInput: case ActionFilesInput:
if p.options.Options.AllowLocalFileAccess {
err = p.FilesInput(act, outData) err = p.FilesInput(act, outData)
} else {
err = ErrLFAccessDenied
}
case ActionAddHeader: case ActionAddHeader:
err = p.ActionAddHeader(act, outData) err = p.ActionAddHeader(act, outData)
case ActionSetHeader: case ActionSetHeader:

102
v2/pkg/protocols/headless/engine/page_actions_test.go
View File

@ -8,6 +8,7 @@ import (
"net/http/cookiejar" "net/http/cookiejar"
"net/http/httptest" "net/http/httptest"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
@ -20,6 +21,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless" "github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
stringsutil "github.com/projectdiscovery/utils/strings"
) )
func TestActionNavigate(t *testing.T) { func TestActionNavigate(t *testing.T) {
@ -36,7 +38,7 @@ func TestActionNavigate(t *testing.T) {
actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}} actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions") require.Nilf(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
}) })
} }
@ -316,6 +318,29 @@ func TestActionFilesInput(t *testing.T) {
}) })
} }
// Negative testcase for files input where it should fail
func TestActionFilesInputNegative(t *testing.T) {
response := `
<html>
<head>
<title>Nuclei Test Page</title>
</head>
<body>Nuclei Test Page</body>
<input type="file">
</html>`
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
{ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
}
t.Setenv("LOCAL_FILE_ACCESS", "false")
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false")
})
}
func TestActionWaitLoad(t *testing.T) { func TestActionWaitLoad(t *testing.T) {
response := ` response := `
<html> <html>
@ -569,7 +594,10 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
input.CookieJar, err = cookiejar.New(nil) input.CookieJar, err = cookiejar.New(nil)
require.Nil(t, err) require.Nil(t, err)
extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout}) lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true)
rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false)
extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna}}) // allow file access in test
assert(page, err, extractedData) assert(page, err, extractedData)
if page != nil { if page != nil {
@ -591,3 +619,73 @@ func TestContainsAnyModificationActionType(t *testing.T) {
t.Error("Expected true, got false") t.Error("Expected true, got false")
} }
} }
func TestBlockedHeadlessURLS(t *testing.T) {
// run this test from binary since we are changing values
// of global variables
if os.Getenv("TEST_BLOCK_HEADLESS_URLS") != "1" {
cmd := exec.Command(os.Args[0], "-test.run=TestBlockedHeadlessURLS", "-test.v")
cmd.Env = append(cmd.Env, "TEST_BLOCK_HEADLESS_URLS=1")
out, err := cmd.CombinedOutput()
if !strings.Contains(string(out), "PASS\n") || err != nil {
t.Fatalf("%s\n(exit status %v)", string(out), err)
}
return
}
opts := &types.Options{
AllowLocalFileAccess: false,
RestrictLocalNetworkAccess: true,
}
err := protocolstate.Init(opts)
require.Nil(t, err, "could not init protocol state")
browser, err := New(&types.Options{ShowBrowser: false, UseInstalledChrome: testheadless.HeadlessLocal})
require.Nil(t, err, "could not create browser")
defer browser.Close()
instance, err := browser.NewInstance()
require.Nil(t, err, "could not create browser instance")
defer instance.Close()
ts := httptest.NewServer(nil)
defer ts.Close()
testcases := []string{
"file:/etc/hosts",
" file:///etc/hosts\r\n",
" fILe:/../../../../etc/hosts",
ts.URL, // local test server
"fTP://example.com:21\r\n",
"ftp://example.com:21",
"chrome://settings",
" chROme://version",
"chrome-extension://version\r",
" chrOme-EXTension://settings",
"view-source:file:/etc/hosts",
}
for _, testcase := range testcases {
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": testcase}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
}
data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
require.Error(t, err, "expected error for url %s got %v", testcase, data)
require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase)
require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data)
if page != nil {
page.Close()
}
}
}
func getBoolFromEnv(key string, defaultValue bool) bool {
val := os.Getenv(key)
if val == "" {
return defaultValue
}
return strings.EqualFold(val, "true")
}

7
v2/pkg/protocols/headless/engine/rules.go
View File

@ -7,6 +7,7 @@ import (
"github.com/go-rod/rod" "github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto" "github.com/go-rod/rod/lib/proto"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
) )
// routingRuleHandler handles proxy rule for actions related to request/response modification // routingRuleHandler handles proxy rule for actions related to request/response modification
@ -103,6 +104,12 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
// routingRuleHandlerNative handles native proxy rule // routingRuleHandlerNative handles native proxy rule
func (p *Page) routingRuleHandlerNative(e *proto.FetchRequestPaused) error { func (p *Page) routingRuleHandlerNative(e *proto.FetchRequestPaused) error {
// ValidateNFailRequest validates if Local file access is enabled
// and local network access is enables if not it will fail the request
// that don't match the rules
if err := protocolstate.ValidateNFailRequest(p.page, e); err != nil {
return err
}
body, _ := FetchGetResponseBody(p.page, e) body, _ := FetchGetResponseBody(p.page, e)
headers := make(map[string][]string) headers := make(map[string][]string)
for _, h := range e.ResponseHeaders { for _, h := range e.ResponseHeaders {

1
v2/pkg/protocols/headless/request.go
View File

@ -106,6 +106,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
options := &engine.Options{ options := &engine.Options{
Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second, Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second,
CookieReuse: request.CookieReuse, CookieReuse: request.CookieReuse,
Options: request.options.Options,
} }
if options.CookieReuse && input.CookieJar == nil { if options.CookieReuse && input.CookieJar == nil {

27
v2/pkg/testutils/integration.go
View File

@ -77,6 +77,33 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
return parts, nil return parts, nil
} }
// RunNucleiArgsAndGetResults returns result,and runtime errors
func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) {
cmd := exec.Command("./nuclei", args...)
if debug {
cmd.Args = append(cmd.Args, "-debug")
cmd.Stderr = os.Stderr
fmt.Println(cmd.String())
} else {
cmd.Args = append(cmd.Args, "-silent")
}
data, err := cmd.Output()
if debug {
fmt.Println(string(data))
}
if len(data) < 1 && err != nil {
return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
}
var parts []string
items := strings.Split(string(data), "\n")
for _, i := range items {
if i != "" {
parts = append(parts, i)
}
}
return parts, nil
}
// RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) // RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL)
func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei") cmd := exec.Command("./nuclei")