Adding same host redirect support (#2655)

* simplifying test syntax

* adding same host redirect + refactoring redirect handling

* adding missing file

* adding support for template syntax

* adding integration test

* updating options

* fixing issue on same host redirect

integration_tests/http/get-host-redirects.yaml

@@ -0,0 +1,17 @@
+id: basic-get-host-redirects
+
+info:
+  name: Basic GET Host Redirects Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    host-redirects: true
+    max-redirects: 3
+    matchers:
+      - type: dsl
+        dsl:
+          - "status_code==307"

v2/cmd/integration-test/http.go

@@ -22,6 +22,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/get-headers.yaml":                         &httpGetHeaders{},
 	"http/get-query-string.yaml":                    &httpGetQueryString{},
 	"http/get-redirects.yaml":                       &httpGetRedirects{},
+	"http/get-host-redirects.yaml":                  &httpGetHostRedirects{},
 	"http/disable-redirects.yaml":                   &httpDisableRedirects{},
 	"http/get.yaml":                                 &httpGet{},
 	"http/post-body.yaml":                           &httpPostBody{},
@@ -167,6 +168,34 @@ func (h *httpGetRedirects) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type httpGetHostRedirects struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpGetHostRedirects) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		http.Redirect(w, r, "/redirected1", http.StatusFound)
+	})
+	router.GET("/redirected1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		http.Redirect(w, r, "redirected2", http.StatusFound)
+	})
+	router.GET("/redirected2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		http.Redirect(w, r, "/redirected3", http.StatusFound)
+	})
+	router.GET("/redirected3", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		http.Redirect(w, r, "https://scanme.sh", http.StatusTemporaryRedirect)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
 type httpDisableRedirects struct{}
 
 // Execute executes a test case and returns an error if occurred

v2/cmd/nuclei/main.go

@@ -175,6 +175,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("configs", "Configurations",
 		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
 		flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"),
+		flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
 		flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
 		flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable redirects for http templates"),
 		flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking

v2/internal/runner/options.go

@@ -109,7 +109,11 @@ func validateOptions(options *types.Options) error {
 	if options.Verbose && options.Silent {
 		return errors.New("both verbose and silent mode specified")
 	}
-	if options.FollowRedirects && options.DisableRedirects {
+
+	if options.FollowHostRedirects && options.FollowRedirects {
+		return errors.New("both follow host redirects and follow redirects specified")
+	}
+	if options.ShouldFollowHTTPRedirects() && options.DisableRedirects {
 		return errors.New("both follow redirects and disable redirects specified")
 	}
 	// loading the proxy server list from file or cli and test the connectivity

v2/pkg/operators/extractors/extract_test.go

@@ -12,7 +12,7 @@ func TestExtractor_ExtractRegex(t *testing.T) {
 	require.Nil(t, err)
 
 	got := e.ExtractRegex("RegEx")
-	require.Equal(t, map[string]struct{}{"RegEx": struct{}{}}, got)
+	require.Equal(t, map[string]struct{}{"RegEx": {}}, got)
 
 	got = e.ExtractRegex("regex")
 	require.Equal(t, map[string]struct{}{}, got)
@@ -24,7 +24,7 @@ func TestExtractor_ExtractKval(t *testing.T) {
 	require.Nil(t, err)
 
 	got := e.ExtractKval(map[string]interface{}{"content_type": "text/html"})
-	require.Equal(t, map[string]struct{}{"text/html": struct{}{}}, got)
+	require.Equal(t, map[string]struct{}{"text/html": {}}, got)
 
 	got = e.ExtractKval(map[string]interface{}{"authorization": "Basic YWxhZGRpbjpvcGVuc2VzYW1l"})
 	require.Equal(t, map[string]struct{}{}, got)
@@ -58,7 +58,7 @@ func TestExtractor_ExtractXPath(t *testing.T) {
 	require.Nil(t, err)
 
 	got := e.ExtractXPath(body)
-	require.Equal(t, map[string]struct{}{"More information...": struct{}{}}, got)
+	require.Equal(t, map[string]struct{}{"More information...": {}}, got)
 
 	e = &Extractor{Type: ExtractorTypeHolder{ExtractorType: XPathExtractor}, XPath: []string{"/html/body/div/p[3]/a"}}
 	got = e.ExtractXPath(body)
@@ -71,7 +71,7 @@ func TestExtractor_ExtractJSON(t *testing.T) {
 	require.Nil(t, err)
 
 	got := e.ExtractJSON(`[{"id": 1}]`)
-	require.Equal(t, map[string]struct{}{"1": struct{}{}}, got)
+	require.Equal(t, map[string]struct{}{"1": {}}, got)
 
 	got = e.ExtractJSON(`{"id": 1}`)
 	require.Equal(t, map[string]struct{}{}, got)
@@ -83,7 +83,7 @@ func TestExtractor_ExtractDSL(t *testing.T) {
 	require.Nil(t, err)
 
 	got := e.ExtractDSL(map[string]interface{}{"hello": "hi"})
-	require.Equal(t, map[string]struct{}{"HI": struct{}{}}, got)
+	require.Equal(t, map[string]struct{}{"HI": {}}, got)
 
 	got = e.ExtractDSL(map[string]interface{}{"hi": "hello"})
 	require.Equal(t, map[string]struct{}{}, got)

v2/pkg/protocols/http/http.go

@@ -152,6 +152,11 @@ type Request struct {
 	//   This can be used in conjunction with `max-redirects` to control the HTTP request redirects.
 	Redirects bool `yaml:"redirects,omitempty" jsonschema:"title=follow http redirects,description=Specifies whether redirects should be followed by the HTTP Client"`
 	// description: |
+	//   Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.
+	//
+	//   This can be used in conjunction with `max-redirects` to control the HTTP request redirects.
+	HostRedirects bool `yaml:"host-redirects,omitempty" jsonschema:"title=follow same host http redirects,description=Specifies whether redirects to the same host should be followed by the HTTP Client"`
+	// description: |
 	//   Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining
 	//
 	//   All requests must be idempotent (GET/POST). This can be used for race conditions/billions requests.
@@ -232,13 +237,21 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	}
 
 	connectionConfiguration := &httpclientpool.Configuration{
-		Threads:         request.Threads,
-		MaxRedirects:    request.MaxRedirects,
-		NoTimeout:       false,
-		FollowRedirects: request.Redirects,
-		CookieReuse:     request.CookieReuse,
-		Connection:      &httpclientpool.ConnectionConfiguration{},
+		Threads:      request.Threads,
+		MaxRedirects: request.MaxRedirects,
+		NoTimeout:    false,
+		CookieReuse:  request.CookieReuse,
+		Connection:   &httpclientpool.ConnectionConfiguration{},
+		RedirectFlow: httpclientpool.DontFollowRedirect,
 	}
+
+	if request.Redirects || options.Options.FollowRedirects {
+		connectionConfiguration.RedirectFlow = httpclientpool.FollowAllRedirect
+	}
+	if request.HostRedirects || options.Options.FollowHostRedirects {
+		connectionConfiguration.RedirectFlow = httpclientpool.FollowSameHostRedirect
+	}
+
 	// If we have request level timeout, ignore http client timeouts
 	for _, req := range request.Raw {
 		if reTimeoutAnnotation.MatchString(req) {

v2/pkg/protocols/http/httpclientpool/clientpool.go

@@ -41,7 +41,7 @@ func Init(options *types.Options) error {
 	if normalClient != nil {
 		return nil
 	}
-	if options.FollowRedirects {
+	if options.ShouldFollowHTTPRedirects() {
 		forceMaxRedirects = options.MaxRedirects
 	}
 	poolMutex = &sync.RWMutex{}
@@ -71,8 +71,8 @@ type Configuration struct {
 	NoTimeout bool
 	// CookieReuse enables cookie reuse for the http client (cookiejar impl)
 	CookieReuse bool
-	// FollowRedirects specifies whether to follow redirects
-	FollowRedirects bool
+	// FollowRedirects specifies the redirects flow
+	RedirectFlow RedirectFlow
 	// Connection defines custom connection configuration
 	Connection *ConnectionConfiguration
 }
@@ -88,7 +88,7 @@ func (c *Configuration) Hash() string {
 	builder.WriteString("n")
 	builder.WriteString(strconv.FormatBool(c.NoTimeout))
 	builder.WriteString("f")
-	builder.WriteString(strconv.FormatBool(c.FollowRedirects))
+	builder.WriteString(strconv.Itoa(int(c.RedirectFlow)))
 	builder.WriteString("r")
 	builder.WriteString(strconv.FormatBool(c.CookieReuse))
 	builder.WriteString("c")
@@ -99,7 +99,7 @@ func (c *Configuration) Hash() string {
 
 // HasStandardOptions checks whether the configuration requires custom settings
 func (c *Configuration) HasStandardOptions() bool {
-	return c.Threads == 0 && c.MaxRedirects == 0 && !c.FollowRedirects && !c.CookieReuse && c.Connection == nil && !c.NoTimeout
+	return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && !c.CookieReuse && c.Connection == nil && !c.NoTimeout
 }
 
 // GetRawHTTP returns the rawhttp request client
@@ -160,16 +160,23 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 
 	retryableHttpOptions.RetryWaitMax = 10 * time.Second
 	retryableHttpOptions.RetryMax = options.Retries
-	followRedirects := configuration.FollowRedirects
+	redirectFlow := configuration.RedirectFlow
 	maxRedirects := configuration.MaxRedirects
 
 	if forceMaxRedirects > 0 {
-		followRedirects = true
+		// by default we enable general redirects following
+		switch {
+		case options.FollowHostRedirects:
+			redirectFlow = FollowSameHostRedirect
+		default:
+			redirectFlow = FollowAllRedirect
+		}
 		maxRedirects = forceMaxRedirects
 	}
 	if options.DisableRedirects {
 		options.FollowRedirects = false
-		followRedirects = false
+		options.FollowHostRedirects = false
+		redirectFlow = DontFollowRedirect
 		maxRedirects = 0
 	}
 	// override connection's settings if required
@@ -242,7 +249,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 
 	httpclient := &http.Client{
 		Transport:     transport,
-		CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects),
+		CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects),
 	}
 	if !configuration.NoTimeout {
 		httpclient.Timeout = time.Duration(options.Timeout) * time.Second
@@ -262,26 +269,51 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	return client, nil
 }
 
+type RedirectFlow uint8
+
+const (
+	DontFollowRedirect RedirectFlow = iota
+	FollowSameHostRedirect
+	FollowAllRedirect
+)
+
 const defaultMaxRedirects = 10
 
 type checkRedirectFunc func(req *http.Request, via []*http.Request) error
 
-func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc {
+func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRedirectFunc {
 	return func(req *http.Request, via []*http.Request) error {
-		if !followRedirects {
+		switch redirectType {
+		case DontFollowRedirect:
 			return http.ErrUseLastResponse
-		}
-
-		if maxRedirects == 0 {
-			if len(via) > defaultMaxRedirects {
+		case FollowSameHostRedirect:
+			var newHost = req.URL.Host
+			var oldHost = via[0].Host
+			if oldHost == "" {
+				oldHost = via[0].URL.Host
+			}
+			if newHost != oldHost {
+				// Tell the http client to not follow redirect
 				return http.ErrUseLastResponse
 			}
-			return nil
-		}
-
-		if len(via) > maxRedirects {
-			return http.ErrUseLastResponse
+			return checkMaxRedirects(req, via, maxRedirects)
+		case FollowAllRedirect:
+			return checkMaxRedirects(req, via, maxRedirects)
 		}
 		return nil
 	}
 }
+
+func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error {
+	if maxRedirects == 0 {
+		if len(via) > defaultMaxRedirects {
+			return http.ErrUseLastResponse
+		}
+		return nil
+	}
+
+	if len(via) > maxRedirects {
+		return http.ErrUseLastResponse
+	}
+	return nil
+}

v2/pkg/protocols/http/validate.go

@@ -7,5 +7,9 @@ func (request *Request) validate() error {
 		return errors.New("'race' and 'req-condition' can't be used together")
 	}
 
+	if request.Redirects && request.HostRedirects {
+		return errors.New("'redirects' and 'host-redirects' can't be used together")
+	}
+
 	return nil
 }

v2/pkg/types/types.go

@@ -135,6 +135,8 @@ type Options struct {
 	MaxRedirects int
 	// FollowRedirects enables following redirects for http request module
 	FollowRedirects bool
+	// FollowRedirects enables following redirects for http request module only on the same host
+	FollowHostRedirects bool
 	// OfflineHTTP is a flag that specific offline processing of http response
 	// using same matchers/extractors from http protocol without the need
 	// to send a new request, reading responses from a file.
@@ -276,6 +278,11 @@ func (options *Options) ShouldSaveResume() bool {
 	return true
 }
 
+// ShouldFollowHTTPRedirects determines if http redirects should be followed
+func (options *Options) ShouldFollowHTTPRedirects() bool {
+	return options.FollowRedirects || options.FollowHostRedirects
+}
+
 // DefaultOptions returns default options for nuclei
 func DefaultOptions() *Options {
 	return &Options{