nuclei/v2/pkg/protocols/http/utils.go

package http

import (
	"bytes"
	"compress/gzip"
	"compress/zlib"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"strings"

	"github.com/pkg/errors"
	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
	"github.com/projectdiscovery/rawhttp"
	"github.com/projectdiscovery/stringsutil"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
)

type redirectedResponse struct {
	headers      []byte
	body         []byte
	fullResponse []byte
	resp         *http.Response
}

// dumpResponseWithRedirectChain dumps a http response with the
// complete http redirect chain.
//
// It preserves the order in which responses were given to requests
// and returns the data to the user for matching and viewing in that order.
//
// Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923
func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) {
	var response []redirectedResponse

	respData, err := httputil.DumpResponse(resp, false)
	if err != nil {
		return nil, err
	}
	respObj := redirectedResponse{
		headers:      respData,
		body:         body,
		resp:         resp,
		fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
	}
	if err := normalizeResponseBody(resp, &respObj); err != nil {
		return nil, err
	}
	response = append(response, respObj)

	var redirectResp *http.Response
	if resp != nil && resp.Request != nil {
		redirectResp = resp.Request.Response
	}
	for redirectResp != nil {
		var body []byte

		respData, err := httputil.DumpResponse(redirectResp, false)
		if err != nil {
			break
		}
		if redirectResp.Body != nil {
			body, _ = ioutil.ReadAll(redirectResp.Body)
		}
		respObj := redirectedResponse{
			headers:      respData,
			body:         body,
			resp:         redirectResp,
			fullResponse: bytes.Join([][]byte{respData, body}, []byte{}),
		}
		if err := normalizeResponseBody(redirectResp, &respObj); err != nil {
			return nil, err
		}
		response = append(response, respObj)
		redirectResp = redirectResp.Request.Response
	}
	return response, nil
}

// normalizeResponseBody performs normalization on the http response object.
func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error {
	var err error
	// net/http doesn't automatically decompress the response body if an
	// encoding has been specified by the user in the request so in case we have to
	// manually do it.
	dataOrig := response.body
	response.body, err = handleDecompression(resp, response.body)
	// in case of error use original data
	if err != nil {
		response.body = dataOrig
	}
	response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body)

	// Decode gbk response content-types
	// gb18030 supersedes gb2312
	responseContentType := resp.Header.Get("Content-Type")
	if isContentTypeGbk(responseContentType) {
		response.fullResponse, err = decodegbk(response.fullResponse)
		if err != nil {
			return errors.Wrap(err, "could not gbk decode")
		}

		// the uncompressed body needs to be decoded to standard utf8
		response.body, err = decodegbk(response.body)
		if err != nil {
			return errors.Wrap(err, "could not gbk decode")
		}
	}
	return nil
}

// dump creates a dump of the http request in form of a byte slice
func dump(req *generatedRequest, reqURL string) ([]byte, error) {
	if req.request != nil {
		// Create a copy on the fly of the request body - ignore errors
		bodyBytes, _ := req.request.BodyBytes()
		req.request.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
		return httputil.DumpRequestOut(req.request.Request, true)
	}
	return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(req.rawRequest.Data)), rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes})
}

// handleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically)
func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte, err error) {
	if resp == nil {
		return bodyOrig, nil
	}

	var reader io.ReadCloser
	switch resp.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err = gzip.NewReader(bytes.NewReader(bodyOrig))
	case "deflate":
		reader, err = zlib.NewReader(bytes.NewReader(bodyOrig))
	default:
		return bodyOrig, nil
	}
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	bodyDec, err = ioutil.ReadAll(reader)
	if err != nil {
		return bodyOrig, err
	}
	return bodyDec, nil
}

// decodegbk converts GBK to UTF-8
func decodegbk(s []byte) ([]byte, error) {
	I := bytes.NewReader(s)
	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(O)
	if e != nil {
		return nil, e
	}
	return d, nil
}

// isContentTypeGbk checks if the content-type header is gbk
func isContentTypeGbk(contentType string) bool {
	contentType = strings.ToLower(contentType)
	return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030")
}