mirror of https://github.com/daffainfo/nuclei.git
optional file read in headless protocol (#4055)
* use -lfa and -lna in headless * fix lna in headless * misc update * fix nil pointer dereference in test * fix lint & unit test * use urlutil * headless protocol scheme improvements * add unit and integration tests * run unit test from binary --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>dev
parent
e146c89930
commit
d3928e080d
|
@ -0,0 +1,29 @@
|
|||
id: file-upload
|
||||
# template for testing when file upload is disabled
|
||||
info:
|
||||
name: Basic File Upload
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
headless:
|
||||
- steps:
|
||||
- action: navigate
|
||||
args:
|
||||
url: "{{BaseURL}}"
|
||||
- action: waitload
|
||||
- action: files
|
||||
args:
|
||||
by: xpath
|
||||
xpath: /html/body/form/input[1]
|
||||
value: headless/file-upload.yaml
|
||||
- action: sleep
|
||||
args:
|
||||
duration: 2
|
||||
- action: click
|
||||
args:
|
||||
by: x
|
||||
xpath: /html/body/form/input[2]
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "Basic File Upload"
|
|
@ -0,0 +1,15 @@
|
|||
id: nuclei-headless-local
|
||||
|
||||
info:
|
||||
name: Nuclei Headless Local
|
||||
author: pdteam
|
||||
severity: high
|
||||
|
||||
headless:
|
||||
- steps:
|
||||
- action: navigate
|
||||
args:
|
||||
url: "{{BaseURL}}"
|
||||
|
||||
- action: waitload
|
||||
|
|
@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{
|
|||
{Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},
|
||||
{Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}},
|
||||
{Path: "headless/variables.yaml", TestCase: &headlessVariables{}},
|
||||
{Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}},
|
||||
{Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}},
|
||||
{Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}},
|
||||
{Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}},
|
||||
}
|
||||
|
||||
|
@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error {
|
|||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type headlessLocal struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
// in this testcases local network access is disabled
|
||||
func (h *headlessLocal) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
_, _ = w.Write([]byte("<html><body></body></html>"))
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"}
|
||||
|
||||
results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(results, 0)
|
||||
}
|
||||
|
||||
type headlessHeaderActions struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
|
@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error {
|
|||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type headlessFileUploadNegative struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *headlessFileUploadNegative) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
_, _ = w.Write([]byte(`
|
||||
<!doctype html>
|
||||
<body>
|
||||
<form method=post enctype=multipart/form-data>
|
||||
<input type=file name=file>
|
||||
<input type=submit value=Upload>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
})
|
||||
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
|
||||
content, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
_, _ = w.Write(content)
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
args := []string{"-t", filePath, "-u", ts.URL, "-headless"}
|
||||
|
||||
results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(results, 0)
|
||||
}
|
||||
|
|
|
@ -0,0 +1,90 @@
|
|||
package protocolstate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
"go.uber.org/multierr"
|
||||
)
|
||||
|
||||
// initalize state of headless protocol
|
||||
|
||||
var (
|
||||
ErrURLDenied = errorutil.NewWithFmt("headless: url %v dropped by rule: %v")
|
||||
networkPolicy *networkpolicy.NetworkPolicy
|
||||
allowLocalFileAccess bool
|
||||
)
|
||||
|
||||
// ValidateNFailRequest validates and fails request
|
||||
// if the request does not respect the rules, it will be canceled with reason
|
||||
func ValidateNFailRequest(page *rod.Page, e *proto.FetchRequestPaused) error {
|
||||
reqURL := e.Request.URL
|
||||
normalized := strings.ToLower(reqURL) // normalize url to lowercase
|
||||
normalized = strings.TrimSpace(normalized) // trim leading & trailing whitespaces
|
||||
if !allowLocalFileAccess && stringsutil.HasPrefixI(normalized, "file:") {
|
||||
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "use of file:// protocol disabled use '-lfa' to enable"))
|
||||
}
|
||||
// validate potential invalid schemes
|
||||
// javascript protocol is allowed for xss fuzzing
|
||||
if HasPrefixAnyI(normalized, "ftp:", "externalfile:", "chrome:", "chrome-extension:") {
|
||||
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "protocol blocked by network policy"))
|
||||
}
|
||||
if !isValidHost(reqURL) {
|
||||
return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "address blocked by network policy"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FailWithReason fails request with AccessDenied reason
|
||||
func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error {
|
||||
m := proto.FetchFailRequest{
|
||||
RequestID: e.RequestID,
|
||||
ErrorReason: proto.NetworkErrorReasonAccessDenied,
|
||||
}
|
||||
return m.Call(page)
|
||||
}
|
||||
|
||||
// InitHeadless initializes headless protocol state
|
||||
func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) {
|
||||
allowLocalFileAccess = localFileAccess
|
||||
if !RestrictLocalNetworkAccess {
|
||||
return
|
||||
}
|
||||
networkPolicy, _ = networkpolicy.New(networkpolicy.Options{
|
||||
DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...),
|
||||
})
|
||||
}
|
||||
|
||||
// isValidHost checks if the host is valid (only limited to http/https protocols)
|
||||
func isValidHost(targetUrl string) bool {
|
||||
if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") {
|
||||
return true
|
||||
}
|
||||
if networkPolicy == nil {
|
||||
return true
|
||||
}
|
||||
urlx, err := urlutil.Parse(targetUrl)
|
||||
if err != nil {
|
||||
// not a valid url
|
||||
return false
|
||||
}
|
||||
targetUrl = urlx.Hostname()
|
||||
_, ok := networkPolicy.ValidateHost(targetUrl)
|
||||
return ok
|
||||
}
|
||||
|
||||
// HasPrefixAnyI checks if the string has any of the prefixes
|
||||
// TODO: replace with stringsutil.HasPrefixAnyI after implementation
|
||||
func HasPrefixAnyI(s string, prefixes ...string) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if stringsutil.HasPrefixI(s, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
|
@ -22,6 +22,7 @@ func Init(options *types.Options) error {
|
|||
return nil
|
||||
}
|
||||
opts := fastdialer.DefaultOptions
|
||||
InitHeadless(options.RestrictLocalNetworkAccess, options.AllowLocalFileAccess)
|
||||
|
||||
switch {
|
||||
case options.SourceIP != "" && options.Interface != "":
|
||||
|
|
|
@ -13,6 +13,7 @@ import (
|
|||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
)
|
||||
|
||||
// Page is a single page in an isolated browser instance
|
||||
|
@ -40,6 +41,7 @@ type HistoryData struct {
|
|||
type Options struct {
|
||||
Timeout time.Duration
|
||||
CookieReuse bool
|
||||
Options *types.Options
|
||||
}
|
||||
|
||||
// Run runs a list of actions by creating a new page in the browser.
|
||||
|
|
|
@ -30,7 +30,8 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
errinvalidArguments = errors.New("invalid arguments provided")
|
||||
errinvalidArguments = errorutil.New("invalid arguments provided")
|
||||
ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access")
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -70,7 +71,11 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
|
|||
case ActionWaitEvent:
|
||||
err = p.WaitEvent(act, outData)
|
||||
case ActionFilesInput:
|
||||
err = p.FilesInput(act, outData)
|
||||
if p.options.Options.AllowLocalFileAccess {
|
||||
err = p.FilesInput(act, outData)
|
||||
} else {
|
||||
err = ErrLFAccessDenied
|
||||
}
|
||||
case ActionAddHeader:
|
||||
err = p.ActionAddHeader(act, outData)
|
||||
case ActionSetHeader:
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
"net/http/cookiejar"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
@ -20,6 +21,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
func TestActionNavigate(t *testing.T) {
|
||||
|
@ -36,7 +38,7 @@ func TestActionNavigate(t *testing.T) {
|
|||
actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}}
|
||||
|
||||
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
|
||||
require.Nil(t, err, "could not run page actions")
|
||||
require.Nilf(t, err, "could not run page actions")
|
||||
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
|
||||
})
|
||||
}
|
||||
|
@ -316,6 +318,29 @@ func TestActionFilesInput(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// Negative testcase for files input where it should fail
|
||||
func TestActionFilesInputNegative(t *testing.T) {
|
||||
response := `
|
||||
<html>
|
||||
<head>
|
||||
<title>Nuclei Test Page</title>
|
||||
</head>
|
||||
<body>Nuclei Test Page</body>
|
||||
<input type="file">
|
||||
</html>`
|
||||
|
||||
actions := []*Action{
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
|
||||
}
|
||||
t.Setenv("LOCAL_FILE_ACCESS", "false")
|
||||
|
||||
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
|
||||
require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false")
|
||||
})
|
||||
}
|
||||
|
||||
func TestActionWaitLoad(t *testing.T) {
|
||||
response := `
|
||||
<html>
|
||||
|
@ -569,7 +594,10 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
|
|||
input.CookieJar, err = cookiejar.New(nil)
|
||||
require.Nil(t, err)
|
||||
|
||||
extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout})
|
||||
lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true)
|
||||
rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false)
|
||||
|
||||
extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna}}) // allow file access in test
|
||||
assert(page, err, extractedData)
|
||||
|
||||
if page != nil {
|
||||
|
@ -591,3 +619,73 @@ func TestContainsAnyModificationActionType(t *testing.T) {
|
|||
t.Error("Expected true, got false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockedHeadlessURLS(t *testing.T) {
|
||||
|
||||
// run this test from binary since we are changing values
|
||||
// of global variables
|
||||
if os.Getenv("TEST_BLOCK_HEADLESS_URLS") != "1" {
|
||||
cmd := exec.Command(os.Args[0], "-test.run=TestBlockedHeadlessURLS", "-test.v")
|
||||
cmd.Env = append(cmd.Env, "TEST_BLOCK_HEADLESS_URLS=1")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if !strings.Contains(string(out), "PASS\n") || err != nil {
|
||||
t.Fatalf("%s\n(exit status %v)", string(out), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
opts := &types.Options{
|
||||
AllowLocalFileAccess: false,
|
||||
RestrictLocalNetworkAccess: true,
|
||||
}
|
||||
err := protocolstate.Init(opts)
|
||||
require.Nil(t, err, "could not init protocol state")
|
||||
|
||||
browser, err := New(&types.Options{ShowBrowser: false, UseInstalledChrome: testheadless.HeadlessLocal})
|
||||
require.Nil(t, err, "could not create browser")
|
||||
defer browser.Close()
|
||||
|
||||
instance, err := browser.NewInstance()
|
||||
require.Nil(t, err, "could not create browser instance")
|
||||
defer instance.Close()
|
||||
|
||||
ts := httptest.NewServer(nil)
|
||||
defer ts.Close()
|
||||
|
||||
testcases := []string{
|
||||
"file:/etc/hosts",
|
||||
" file:///etc/hosts\r\n",
|
||||
" fILe:/../../../../etc/hosts",
|
||||
ts.URL, // local test server
|
||||
"fTP://example.com:21\r\n",
|
||||
"ftp://example.com:21",
|
||||
"chrome://settings",
|
||||
" chROme://version",
|
||||
"chrome-extension://version\r",
|
||||
" chrOme-EXTension://settings",
|
||||
"view-source:file:/etc/hosts",
|
||||
}
|
||||
|
||||
for _, testcase := range testcases {
|
||||
actions := []*Action{
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": testcase}},
|
||||
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
|
||||
}
|
||||
|
||||
data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
|
||||
require.Error(t, err, "expected error for url %s got %v", testcase, data)
|
||||
require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase)
|
||||
require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data)
|
||||
if page != nil {
|
||||
page.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getBoolFromEnv(key string, defaultValue bool) bool {
|
||||
val := os.Getenv(key)
|
||||
if val == "" {
|
||||
return defaultValue
|
||||
}
|
||||
return strings.EqualFold(val, "true")
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
|
||||
// routingRuleHandler handles proxy rule for actions related to request/response modification
|
||||
|
@ -103,6 +104,12 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
|
|||
|
||||
// routingRuleHandlerNative handles native proxy rule
|
||||
func (p *Page) routingRuleHandlerNative(e *proto.FetchRequestPaused) error {
|
||||
// ValidateNFailRequest validates if Local file access is enabled
|
||||
// and local network access is enables if not it will fail the request
|
||||
// that don't match the rules
|
||||
if err := protocolstate.ValidateNFailRequest(p.page, e); err != nil {
|
||||
return err
|
||||
}
|
||||
body, _ := FetchGetResponseBody(p.page, e)
|
||||
headers := make(map[string][]string)
|
||||
for _, h := range e.ResponseHeaders {
|
||||
|
|
|
@ -106,6 +106,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
|
|||
options := &engine.Options{
|
||||
Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second,
|
||||
CookieReuse: request.CookieReuse,
|
||||
Options: request.options.Options,
|
||||
}
|
||||
|
||||
if options.CookieReuse && input.CookieJar == nil {
|
||||
|
|
|
@ -77,6 +77,33 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
|
|||
return parts, nil
|
||||
}
|
||||
|
||||
// RunNucleiArgsAndGetResults returns result,and runtime errors
|
||||
func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) {
|
||||
cmd := exec.Command("./nuclei", args...)
|
||||
if debug {
|
||||
cmd.Args = append(cmd.Args, "-debug")
|
||||
cmd.Stderr = os.Stderr
|
||||
fmt.Println(cmd.String())
|
||||
} else {
|
||||
cmd.Args = append(cmd.Args, "-silent")
|
||||
}
|
||||
data, err := cmd.Output()
|
||||
if debug {
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
if len(data) < 1 && err != nil {
|
||||
return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
|
||||
}
|
||||
var parts []string
|
||||
items := strings.Split(string(data), "\n")
|
||||
for _, i := range items {
|
||||
if i != "" {
|
||||
parts = append(parts, i)
|
||||
}
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL)
|
||||
func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) {
|
||||
cmd := exec.Command("./nuclei")
|
||||
|
|
Loading…
Reference in New Issue