Merge branch 'master' into enumeration-progressbar

README.md

@@ -20,6 +20,7 @@ We have also [open-sourced a dedicated repository](https://github.com/projectdis
 - [Installation Instructions](#installation-instructions)
     - [From Binary](#from-binary)
     - [From Source](#from-source)
+    - [From Github](#from-github)
     - [Running in a Docker Container](#running-in-a-docker-container)
 - [Nuclei templates](#nuclei-templates)
 - [Running nuclei](#running-nuclei)
@@ -94,6 +95,17 @@ nuclei requires go1.13+ to install successfully. Run the following command to ge
 
 In order to update the tool, you can use -u flag with `go get` command.
 
+
+### From Github
+
+```bash
+git clone https://github.com/projectdiscovery/nuclei.git
+cd v2/cmd/nuclei/
+go build .
+mv nuclei /usr/local/bin/
+nuclei -h
+```
+
 ### Running in a Docker Container
 
 You can use the [nuclei dockerhub image](https://hub.docker.com/r/projectdiscovery/nuclei). Simply run -

v2/pkg/requests/http-request.go

@@ -179,21 +179,12 @@ func (r *HTTPRequest) handleSimpleRaw(raw string, baseURL string, values map[str
 	// Replace the dynamic variables in the request if any
 	raw = replacer.Replace(raw)
 
-	compiledRequest, err := r.parseRawRequest(raw)
+	compiledRequest, err := r.parseRawRequest(raw, baseURL)
 	if err != nil {
 		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
 	}
 
-	// requests generated from http.ReadRequest have incorrect RequestURI, so they
-	// cannot be used to perform another request directly, we need to generate a new one
-	// with the new target url
-	var finalURL string
-	if compiledRequest.Path != "?" {
-		finalURL = fmt.Sprintf("%s%s", baseURL, compiledRequest.Path)
-	} else {
-		finalURL = baseURL
-	}
-	req, err := http.NewRequest(compiledRequest.Method, finalURL, strings.NewReader(compiledRequest.Data))
+	req, err := http.NewRequest(compiledRequest.Method, compiledRequest.FullURL, strings.NewReader(compiledRequest.Data))
 	if err != nil {
 		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
 	}
@@ -243,21 +234,12 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
 	dynamicReplacer := newReplacer(dynamicValues)
 	raw = dynamicReplacer.Replace(raw)
 
-	compiledRequest, err := r.parseRawRequest(raw)
+	compiledRequest, err := r.parseRawRequest(raw, baseURL)
 	if err != nil {
 		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
 	}
 
-	// requests generated from http.ReadRequest have incorrect RequestURI, so they
-	// cannot be used to perform another request directly, we need to generate a new one
-	// with the new target url
-	var finalURL string
-	if compiledRequest.Path != "?" {
-		finalURL = fmt.Sprintf("%s%s", baseURL, compiledRequest.Path)
-	} else {
-		finalURL = baseURL
-	}
-	req, err := http.NewRequest(compiledRequest.Method, finalURL, strings.NewReader(compiledRequest.Data))
+	req, err := http.NewRequest(compiledRequest.Method, compiledRequest.FullURL, strings.NewReader(compiledRequest.Data))
 	if err != nil {
 		return &CompiledHTTP{Request: nil, Error: err, Meta: nil}
 	}
@@ -278,13 +260,8 @@ func (r *HTTPRequest) handleRawWithPaylods(raw string, baseURL string, values, g
 func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
 	req.Header.Set("Connection", "close")
 	req.Close = true
-
-	// raw requests are left untouched
-	if len(r.Raw) > 0 {
-		return retryablehttp.FromRequest(req)
-	}
-
 	replacer := newReplacer(values)
+
 	// Check if the user requested a request body
 	if r.Body != "" {
 		req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
@@ -300,6 +277,11 @@ func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface
 		req.Header.Set("User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
 	}
 
+	// raw requests are left untouched
+	if len(r.Raw) > 0 {
+		return retryablehttp.FromRequest(req)
+	}
+
 	if _, ok := req.Header["Accept"]; !ok {
 		req.Header.Set("Accept", "*/*")
 	}
@@ -332,6 +314,7 @@ func (c *CustomHeaders) Set(value string) error {
 }
 
 type compiledRawRequest struct {
+	FullURL string
 	Method  string
 	Path    string
 	Data    string
@@ -339,7 +322,7 @@ type compiledRawRequest struct {
 }
 
 // parseRawRequest parses the raw request as supplied by the user
-func (r *HTTPRequest) parseRawRequest(request string) (*compiledRawRequest, error) {
+func (r *HTTPRequest) parseRawRequest(request string, baseURL string) (*compiledRawRequest, error) {
 	reader := bufio.NewReader(strings.NewReader(request))
 
 	rawRequest := compiledRawRequest{
@@ -390,6 +373,33 @@ func (r *HTTPRequest) parseRawRequest(request string) (*compiledRawRequest, erro
 		rawRequest.Path = parts[1]
 	}
 
+	// If raw request doesn't have a Host header and/ path,
+	// this will be generated from the parsed baseURL
+	parsedURL, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse request URL: %s", err)
+	}
+
+	var hostURL string
+	if len(rawRequest.Headers["Host"]) == 0 {
+		hostURL = parsedURL.Host
+	} else {
+		hostURL = rawRequest.Headers["Host"]
+	}
+
+	if len(rawRequest.Path) == 0 {
+		rawRequest.Path = parsedURL.Path
+	} else {
+		// requests generated from http.ReadRequest have incorrect RequestURI, so they
+		// cannot be used to perform another request directly, we need to generate a new one
+		// with the new target url
+		if strings.HasPrefix(rawRequest.Path, "?") {
+			rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path)
+		}
+	}
+
+	rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, hostURL, rawRequest.Path)
+
 	// Set the request body
 	b, err := ioutil.ReadAll(reader)
 	if err != nil {