Merge branch 'dev' into issue-2188-reporting-client

README.md

@@ -203,7 +203,7 @@ INTERACTSH:
 UNCOVER:
    -uc, -uncover                  enable uncover engine
    -uq, -uncover-query string[]   uncover search query
-   -ue, -uncover-engine string[]  uncover search engine (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas) (default shodan)
+   -ue, -uncover-engine string[]  uncover search engine (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan)
    -uf, -uncover-field string     uncover fields to return (ip,port,host) (default "ip:port")
    -ul, -uncover-limit int        uncover results to return (default 100)
    -ucd, -uncover-delay int       delay between uncover query requests in seconds (0 to disable) (default 1)

integration_tests/http/http-paths.yaml

@@ -0,0 +1,84 @@
+id: http-paths
+
+info:
+  name: Test Http Path Edgecases
+  author: pd-team
+  severity: info
+  description: >
+    - https://github.com/projectdiscovery/nuclei/pull/3211
+    - https://github.com/projectdiscovery/nuclei/pull/3127
+  reference:
+    # adding expected results here for context and debugging
+    - "/1337?with=param"
+    - "/some%0A/%0D"
+    - "/%73%6f%6d%65%0A/%0D"
+    - "/%00test%20"
+    - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d"
+    - "/test/..;/..;/"
+    - "/xyz/%25u2s/%25invalid"
+    # duplicating here because same results are expected even if http request is written in different format
+    - "/1337?with=param"
+    - "/some%0A/%0D"
+    - "/%73%6f%6d%65%0A/%0D"
+    - "/%00test%20"
+    - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d"
+    - "/test/..;/..;/"
+    - "/xyz/%25u2s/%25invalid"
+
+# Test all templates with FullURLs
+requests:
+  - raw:
+    # relative path without leading slash with param
+    # If relative path does not have `/` prefix it is autocorrected 
+      - |+
+        GET 1337?with=param HTTP/1.1 
+        Host: scanme.sh
+    # url encoded characters in path
+      - |+
+        GET /some%0A/%0D HTTP/1.1
+        Host: scanme.sh
+    # percent encoded characters in path
+    # In URL encoding only key characters are encoded 
+    # while in percent encoding all characters are url encoded (similar to burp decoder)
+      - |+
+        GET /%73%6f%6d%65%0A/%0D HTTP/1.1
+        Host: scanme.sh
+    # test null and % chars in path
+      - |+
+        GET /%00test%20 HTTP/1.1
+        Host: scanme.sh
+    # test payload integrity in parameter
+      - |+
+        GET /text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d HTTP/1.1
+        Host: scanme.sh
+    # test for missing trailing slash
+      - |+
+        GET /test/..;/..;/ HTTP/1.1
+        Host: scanme.sh
+        Origin: {{BaseURL}}
+    # test relative path with invalid/corrupted characters
+    # In such case instead of error or panic nuclei escaped unsupported character (i.e /xyz/%25u2s/%25invalid)
+    # if template requires this condition to not escape unsupported characters. It can only be done in unsafe raw requests
+      - |+
+        GET /xyz/%u2s/%invalid HTTP/1.1
+        Host: scanme.sh
+
+    matchers:
+      - type: status
+        status:
+          - 200
+  # Same testcases as mentioned above but in path based request format
+  - method: GET
+    path:
+     - "{{BaseURL}}/1337?with=param"
+     - "{{BaseURL}}/some%0A/%0D"
+     - "{{BaseURL}}/%73%6f%6d%65%0A/%0D"
+     - "{{BaseURL}}/%00test%20"
+     - "{{BaseURL}}/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d"
+     - "{{BaseURL}}/test/..;/..;/"
+     - "{{BaseURL}}/xyz/%u2s/%invalid"
+
+    matchers:
+    - type: status
+      status:
+        - 200

integration_tests/http/raw-unsafe-path.yaml

@@ -0,0 +1,58 @@
+id: raw-unsafe-path
+
+info:
+  name: Test RAW Unsafe Paths
+  author: pd-team
+  severity: info
+  description: >
+    - https://github.com/projectdiscovery/nuclei/pull/3211
+    - https://github.com/projectdiscovery/nuclei/pull/3127
+  reference:
+    # adding expected results here for context and debugging
+    - "1337"
+    - "1337?with=param"
+    - "/some%0A/%0D"
+    - "/%20test%0a"
+    - "/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d"
+    - "/test/..;/..;/"
+    - "/xyz/%u2s/%invalid"
+
+
+# Test all unsafe URL Handling Edgecases
+requests:
+  - raw:
+    # relative path without leading slash
+      - |+
+        GET 1337 HTTP/1.1 
+        Host: scanme.sh
+    # same but with param
+      - |+
+        GET 1337?with=param HTTP/1.1 
+        Host: scanme.sh
+    # url encoded characters in path
+      - |+
+        GET /some%0A/%0D HTTP/1.1
+        Host: scanme.sh
+    # test unsupported  chars in path
+      - |+
+        GET /%20test%0a HTTP/1.1
+        Host: scanme.sh
+    # test payload integrity params
+      - |+
+        GET /text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{}.getparam')%7d HTTP/1.1
+        Host: scanme.sh
+    # test for missing trailing slash
+      - |+
+        GET /test/..;/..;/ HTTP/1.1
+        Host: scanme.sh
+        Origin: {{BaseURL}}
+    # test relative path with invalid/corrupted characters
+      - |+
+        GET /xyz/%u2s/%invalid HTTP/1.1
+        Host: scanme.sh
+
+    unsafe: true
+    matchers:
+      - type: status
+        status:
+          - 200

v2/cmd/integration-test/http.go

@@ -7,11 +7,14 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
+	"os"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/julienschmidt/httprouter"
+	"gopkg.in/yaml.v2"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
 	logutil "github.com/projectdiscovery/utils/log"
@@ -37,6 +40,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/raw-path-trailing-slash.yaml":             &httpRawPathTrailingSlash{},
 	"http/raw-payload.yaml":                         &httpRawPayload{},
 	"http/raw-post-body.yaml":                       &httpRawPostBody{},
+	"http/raw-unsafe-path.yaml":                     &httpRawUnsafePath{},
+	"http/http-paths.yaml":                          &httpPaths{},
 	"http/request-condition.yaml":                   &httpRequestCondition{},
 	"http/request-condition-new.yaml":               &httpRequestCondition{},
 	"http/interactsh.yaml":                          &httpInteractshRequest{},
@@ -587,6 +592,104 @@ func (h *httpRawPostBody) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type httpRawUnsafePath struct{}
+
+func (h *httpRawUnsafePath) Execute(filepath string) error {
+	// testing unsafe paths using router feedback is not possible cause they are `unsafe urls`
+	// hence it is done by parsing and matching paths from nuclei output with `-debug-req` flag
+	// read template files
+	bin, err := os.ReadFile(filepath)
+	if err != nil {
+		return err
+	}
+
+	// Instead of storing expected `paths` in code it is stored in
+	// `reference` section of template
+	type template struct {
+		Info struct {
+			Reference []string `yaml:"reference"`
+		}
+	}
+	var tpl template
+	if err = yaml.Unmarshal(bin, &tpl); err != nil {
+		return err
+	}
+	// expected relative paths
+	expected := []string{}
+	expected = append(expected, tpl.Info.Reference...)
+	if len(expected) == 0 {
+		return fmt.Errorf("something went wrong with %v template", filepath)
+	}
+
+	results, err := testutils.RunNucleiBinaryAndGetCombinedOutput(debug, []string{"-t", filepath, "-u", "scanme.sh", "-debug-req"})
+	if err != nil {
+		return err
+	}
+
+	actual := []string{}
+	for _, v := range strings.Split(results, "\n") {
+		if strings.Contains(v, "GET") {
+			parts := strings.Fields(v)
+			if len(parts) == 3 {
+				actual = append(actual, parts[1])
+			}
+		}
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		return fmt.Errorf("%8v: %v\n%-8v: %v", "expected", expected, "actual", actual)
+	}
+	return nil
+}
+
+type httpPaths struct{}
+
+func (h *httpPaths) Execute(filepath string) error {
+	// covers testcases similar to httpRawUnsafePath but when `unsafe:false`
+	bin, err := os.ReadFile(filepath)
+	if err != nil {
+		return err
+	}
+
+	// Instead of storing expected `paths` in code it is stored in
+	// `reference` section of template
+	type template struct {
+		Info struct {
+			Reference []string `yaml:"reference"`
+		}
+	}
+	var tpl template
+	if err = yaml.Unmarshal(bin, &tpl); err != nil {
+		return err
+	}
+	// expected relative paths
+	expected := []string{}
+	expected = append(expected, tpl.Info.Reference...)
+	if len(expected) == 0 {
+		return fmt.Errorf("something went wrong with %v template", filepath)
+	}
+
+	results, err := testutils.RunNucleiBinaryAndGetCombinedOutput(debug, []string{"-t", filepath, "-u", "scanme.sh", "-debug-req"})
+	if err != nil {
+		return err
+	}
+
+	actual := []string{}
+	for _, v := range strings.Split(results, "\n") {
+		if strings.Contains(v, "GET") {
+			parts := strings.Fields(v)
+			if len(parts) == 3 {
+				actual = append(actual, parts[1])
+			}
+		}
+	}
+
+	if !reflect.DeepEqual(expected, actual) {
+		return fmt.Errorf("%8v: %v\n%-8v: %v", "expected", expected, "actual", actual)
+	}
+	return nil
+}
+
 type httpRawCookieReuse struct{}
 
 // Execute executes a test case and returns an error if occurred

v2/go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
 	github.com/projectdiscovery/rawhttp v0.1.9
 	github.com/projectdiscovery/retryabledns v1.0.21
-	github.com/projectdiscovery/retryablehttp-go v1.0.11
+	github.com/projectdiscovery/retryablehttp-go v1.0.12-0.20230220094538-f406add578ab
 	github.com/projectdiscovery/stringsutil v0.0.2
 	github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
 	github.com/remeh/sizedwaitgroup v1.0.0
@@ -44,8 +44,8 @@ require (
 	github.com/weppos/publicsuffix-go v0.20.0
 	github.com/xanzy/go-gitlab v0.80.2
 	go.uber.org/multierr v1.9.0
-	golang.org/x/net v0.6.0
-	golang.org/x/oauth2 v0.5.0
+	golang.org/x/net v0.7.0
+	golang.org/x/oauth2 v0.4.0
 	golang.org/x/text v0.7.0
 	gopkg.in/yaml.v2 v2.4.0
 	moul.io/http2curl v1.0.0
@@ -80,7 +80,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.0.5
 	github.com/projectdiscovery/uncover v1.0.2
-	github.com/projectdiscovery/utils v0.0.9
+	github.com/projectdiscovery/utils v0.0.10-0.20230217185600-008d111dd1c1
 	github.com/projectdiscovery/wappalyzergo v0.0.81
 	github.com/stretchr/testify v1.8.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
@@ -94,37 +94,16 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/bits-and-blooms/bitset v1.3.1 // indirect
 	github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
-	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
-	github.com/cloudflare/cfssl v1.6.3 // indirect
+	github.com/cloudflare/cfssl v1.6.4-0.20221208165709-c5e40da60306 // indirect
 	github.com/cloudflare/circl v1.1.0 // indirect
-	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
-	github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect
-	github.com/envoyproxy/protoc-gen-validate v0.6.1 // indirect
-	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
-	github.com/fullstorydev/grpcurl v1.8.1 // indirect
 	github.com/getsentry/sentry-go v0.18.0 // indirect
-	github.com/golang/mock v1.5.0 // indirect
-	github.com/google/btree v1.0.1 // indirect
-	github.com/google/certificate-transparency-go v1.1.2-0.20210511102531-373a877eec92 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/google/certificate-transparency-go v1.1.4 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jhump/protoreflect v1.8.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/karlseguin/expect v1.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pjbgf/sha1cd v0.2.3 // indirect
@@ -135,12 +114,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
-	github.com/soheilhy/cmux v0.1.5 // indirect
-	github.com/spf13/cobra v1.1.3 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/buntdb v1.2.10 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
@@ -149,24 +123,8 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
-	github.com/urfave/cli v1.22.5 // indirect
-	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/client/v2 v2.305.0-alpha.0 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/etcdctl/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/server/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 // indirect
-	go.etcd.io/etcd/v3 v3.5.0-alpha.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	google.golang.org/genproto v0.0.0-20220804142021-4e6b2dfa6612 // indirect
-	google.golang.org/grpc v1.51.0 // indirect
-	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
 )
 
 require (

v2/pkg/protocols/common/uncover/uncover.go

@@ -15,6 +15,7 @@ import (
 	ucRunner "github.com/projectdiscovery/uncover/runner"
 	"github.com/projectdiscovery/uncover/uncover"
 	"github.com/projectdiscovery/uncover/uncover/agent/censys"
+	"github.com/projectdiscovery/uncover/uncover/agent/criminalip"
 	"github.com/projectdiscovery/uncover/uncover/agent/fofa"
 	"github.com/projectdiscovery/uncover/uncover/agent/hunter"
 	"github.com/projectdiscovery/uncover/uncover/agent/netlas"
@@ -29,7 +30,7 @@ import (
 const maxConcurrentAgents = 50
 
 func GetUncoverSupportedAgents() string {
-	uncoverSupportedAgents := []string{"shodan", "shodan-idb", "fofa", "censys", "quake", "hunter", "zoomeye", "netlas"}
+	uncoverSupportedAgents := []string{"shodan", "shodan-idb", "fofa", "censys", "quake", "hunter", "zoomeye", "netlas", "criminalip"}
 	return strings.Join(uncoverSupportedAgents, ",")
 }
 
@@ -72,6 +73,8 @@ func GetUncoverTargetsFromMetadata(templates []*templates.Template, delay, limit
 				eng = "zoomeye"
 			case "netlas-query":
 				eng = "netlas"
+			case "criminalip-query":
+				eng = "criminalip"
 			default:
 				continue
 			}
@@ -131,6 +134,8 @@ func getTargets(uncoverOptions *ucRunner.Options, field string) (chan string, er
 			agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
 		case "netlas":
 			agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
+		case "criminalip":
+			agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
 		default:
 			err = errors.Errorf("%s unknown uncover agent type", engine)
 		}
@@ -231,6 +236,12 @@ func loadKeys(engine string, options *ucRunner.Options) error {
 		} else {
 			return errors.Errorf("NETLAS_API_KEY env variable is not configured")
 		}
+	case "criminalip":
+		if key, exists := os.LookupEnv("CRIMINALIP_API_KEY"); exists {
+			options.Provider.CriminalIP = append(options.Provider.CriminalIP, key)
+		} else {
+			return errors.Errorf("CRIMINALIP_API_KEY env variable is not configured")
+		}
 	default:
 		return errors.Errorf("unknown uncover agent")
 	}

v2/pkg/protocols/http/build_request.go

@@ -115,6 +115,9 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 		gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(finalVars))
 	}
 
+	// Note: If possible any changes to current logic (i.e evaluate -> then parse URL)
+	// should be avoided since it is dependent on `urlutil` core logic
+
 	// Evaluate (replace) variable with final values
 	reqData, err = expressions.Evaluate(reqData, finalVars)
 	if err != nil {

v2/pkg/protocols/http/raw/raw.go

@@ -54,12 +54,29 @@ func Parse(request string, inputURL *urlutil.URL, unsafe bool) (*Request, error)
 	case unsafe:
 		prevPath := rawrequest.Path
 		cloned := inputURL.Clone()
-		err := cloned.MergePath(rawrequest.Path, true)
-		unsafeRelativePath := cloned.GetRelativePath()
-		if err != nil {
-			return nil, errorutil.NewWithErr(err).WithTag("raw").Msgf("failed to automerge %v from unsafe template", rawrequest.Path)
+		unsafeRelativePath := ""
+		if (cloned.Path == "" || cloned.Path == "/") && !strings.HasPrefix(prevPath, "/") {
+			// Edgecase if raw unsafe request is
+			// GET 1337?with=param HTTP/1.1
+			if tmpurl, err := urlutil.ParseRelativePath(prevPath, true); err == nil && len(tmpurl.Params) > 0 {
+				// if raw request contains parameters
+				cloned.Params.Merge(tmpurl.Params)
+				unsafeRelativePath = strings.TrimPrefix(tmpurl.Path, "/") + "?" + cloned.Params.Encode()
+			} else {
+				// if raw request does not contain param
+				if len(cloned.Params) > 0 {
+					unsafeRelativePath = prevPath + "?" + cloned.Params.Encode()
+				} else {
+					unsafeRelativePath = prevPath
+				}
+			}
+		} else {
+			err = cloned.MergePath(rawrequest.Path, true)
+			if err != nil {
+				return nil, errorutil.NewWithErr(err).WithTag("raw").Msgf("failed to automerge %v from unsafe template", rawrequest.Path)
+			}
+			unsafeRelativePath = cloned.GetRelativePath()
 		}
-		// replace itself
 		rawrequest.UnsafeRawBytes = bytes.Replace(rawrequest.UnsafeRawBytes, []byte(prevPath), []byte(unsafeRelativePath), 1)
 
 	default:

v2/pkg/protocols/http/request.go

@@ -512,8 +512,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			}
 			resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
 		} else if generatedRequest.request != nil {
-			// hot fix to avoid double url encoding (should only be called once)
-			generatedRequest.request.Prepare()
 			resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
 		}
 	} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
@@ -564,7 +562,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 				}
 				httpclient = client
 			}
-			generatedRequest.request.Prepare()
 			resp, err = httpclient.Do(generatedRequest.request)
 		}
 	}