Misc work on extractors + compat

v2/pkg/executer/executer_dns.go

@@ -3,61 +3,31 @@ package executer
 import (
 	"fmt"
 	"os"
-	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
 	"github.com/projectdiscovery/nuclei/v2/internal/progress"
-	"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	retryabledns "github.com/projectdiscovery/retryabledns"
-	"go.uber.org/ratelimit"
 )
 
 // DNSExecuter is a client for performing a DNS request
 // for a template.
 type DNSExecuter struct {
-	dnsClient   *retryabledns.Client
 	template *templates.Template
-	dnsRequest  *requests.DNSRequest
-	ratelimiter ratelimit.Limiter
-}
-
-// DefaultResolvers contains the list of resolvers known to be trusted.
-var DefaultResolvers = []string{
-	"1.1.1.1:53", // Cloudflare
-	"1.0.0.1:53", // Cloudflare
-	"8.8.8.8:53", // Google
-	"8.8.4.4:53", // Google
 }
 
 // DNSOptions contains configuration options for the DNS executer.
 type DNSOptions struct {
-	ColoredOutput bool
-	Debug         bool
-	JSON          bool
-	JSONRequests  bool
-	NoMeta        bool
-	VHost         bool
-	TraceLog      tracelog.Log
 	Template   *templates.Template
 	DNSRequest *requests.DNSRequest
-	Writer        *bufwriter.Writer
-
-	Colorizer   colorizer.NucleiColorizer
-	Decolorizer *regexp.Regexp
-	RateLimiter ratelimit.Limiter
 }
 
 // NewDNSExecuter creates a new DNS executer from a template
 // and a DNS request query.
 func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
-	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
 
 	executer := &DNSExecuter{
 		debug:         options.Debug,

v2/pkg/executer/executer_http.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
-	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
 	"github.com/projectdiscovery/nuclei/v2/internal/progress"
@@ -74,8 +73,6 @@ type HTTPOptions struct {
 	BulkHTTPRequest *requests.BulkHTTPRequest
 	CookieJar       *cookiejar.Jar
 	PF              *projetctfile.ProjectFile
-	RateLimiter     ratelimit.Limiter
-	Dialer          *fastdialer.Dialer
 }
 
 // NewHTTPExecuter creates a new HTTP executer from a template
@@ -93,7 +90,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 	// Create the HTTP Client
 	client := makeHTTPClient(proxyURL, options)
 	// nolint:bodyclose // false positive there is no body to close yet
-	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
 
 	if options.CookieJar != nil {
 		client.HTTPClient.Jar = options.CookieJar
@@ -105,9 +101,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 		client.HTTPClient.Jar = jar
 	}
 
-	// initiate raw http client
-	rawClient := rawhttp.NewClient(rawhttp.DefaultOptions)
-
 	executer := &HTTPExecuter{
 		debug:            options.Debug,
 		jsonOutput:       options.JSON,
@@ -257,17 +250,6 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
 		return result
 	}
 
-	pipeOptions := rawhttp.DefaultPipelineOptions
-	pipeOptions.Host = URL.Host
-	pipeOptions.MaxConnections = 1
-	if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 {
-		pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections
-	}
-	if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 {
-		pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection
-	}
-	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
-
 	// defaultMaxWorkers should be a sufficient value to keep queues always full
 	maxWorkers := defaultMaxWorkers
 	// in case the queue is bigger increase the workers

v2/pkg/operators/common/dsl/dsl.go

@@ -18,8 +18,8 @@ import (
 
 	"github.com/Knetic/govaluate"
 	"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spaolacci/murmur3"
-	"github.com/spf13/cast"
 )
 
 const (
@@ -35,110 +35,110 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	functions := make(map[string]govaluate.ExpressionFunction)
 
 	functions["len"] = func(args ...interface{}) (interface{}, error) {
-		length := len(cast.ToString(args[0]))
+		length := len(types.ToString(args[0]))
 		return float64(length), nil
 	}
 
 	functions["toupper"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToUpper(cast.ToString(args[0])), nil
+		return strings.ToUpper(types.ToString(args[0])), nil
 	}
 
 	functions["tolower"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ToLower(cast.ToString(args[0])), nil
+		return strings.ToLower(types.ToString(args[0])), nil
 	}
 
 	functions["replace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.ReplaceAll(cast.ToString(args[0]), cast.ToString(args[1]), cast.ToString(args[2])), nil
+		return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
 	}
 
 	functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(cast.ToString(args[1]))
+		compiled, err := regexp.Compile(types.ToString(args[1]))
 		if err != nil {
 			return nil, err
 		}
-		return compiled.ReplaceAllString(cast.ToString(args[0]), cast.ToString(args[2])), nil
+		return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
 	}
 
 	functions["trim"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Trim(cast.ToString(args[0]), cast.ToString(args[2])), nil
+		return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
 	}
 
 	functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimLeft(cast.ToString(args[0]), cast.ToString(args[1])), nil
+		return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["trimright"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimRight(cast.ToString(args[0]), cast.ToString(args[1])), nil
+		return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSpace(cast.ToString(args[0])), nil
+		return strings.TrimSpace(types.ToString(args[0])), nil
 	}
 
 	functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimPrefix(cast.ToString(args[0]), cast.ToString(args[1])), nil
+		return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
-		return strings.TrimSuffix(cast.ToString(args[0]), cast.ToString(args[1])), nil
+		return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["reverse"] = func(args ...interface{}) (interface{}, error) {
-		return reverseString(cast.ToString(args[0])), nil
+		return reverseString(types.ToString(args[0])), nil
 	}
 
 	// encoding
 	functions["base64"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0])))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 
 		return sEnc, nil
 	}
 
 	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
 	functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
-		sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0])))
+		sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
 		return insertInto(sEnc, 76, '\n'), nil
 	}
 
 	functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
-		return base64.StdEncoding.DecodeString(cast.ToString(args[0]))
+		return base64.StdEncoding.DecodeString(types.ToString(args[0]))
 	}
 
 	functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathEscape(cast.ToString(args[0])), nil
+		return url.PathEscape(types.ToString(args[0])), nil
 	}
 
 	functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
-		return url.PathUnescape(cast.ToString(args[0]))
+		return url.PathUnescape(types.ToString(args[0]))
 	}
 
 	functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
-		return hex.EncodeToString([]byte(cast.ToString(args[0]))), nil
+		return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
 	}
 
 	functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
-		hx, _ := hex.DecodeString(cast.ToString(args[0]))
+		hx, _ := hex.DecodeString(types.ToString(args[0]))
 		return string(hx), nil
 	}
 
 	functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
-		return html.EscapeString(cast.ToString(args[0])), nil
+		return html.EscapeString(types.ToString(args[0])), nil
 	}
 
 	functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
-		return html.UnescapeString(cast.ToString(args[0])), nil
+		return html.UnescapeString(types.ToString(args[0])), nil
 	}
 
 	// hashing
 	functions["md5"] = func(args ...interface{}) (interface{}, error) {
-		hash := md5.Sum([]byte(cast.ToString(args[0])))
+		hash := md5.Sum([]byte(types.ToString(args[0])))
 
 		return hex.EncodeToString(hash[:]), nil
 	}
 
 	functions["sha256"] = func(args ...interface{}) (interface{}, error) {
 		h := sha256.New()
-		_, err := h.Write([]byte(cast.ToString(args[0])))
+		_, err := h.Write([]byte(types.ToString(args[0])))
 
 		if err != nil {
 			return nil, err
@@ -148,7 +148,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 
 	functions["sha1"] = func(args ...interface{}) (interface{}, error) {
 		h := sha1.New()
-		_, err := h.Write([]byte(cast.ToString(args[0])))
+		_, err := h.Write([]byte(types.ToString(args[0])))
 
 		if err != nil {
 			return nil, err
@@ -157,20 +157,20 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	}
 
 	functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
-		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(cast.ToString(args[0])), 0))), nil
+		return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
 	}
 
 	// search
 	functions["contains"] = func(args ...interface{}) (interface{}, error) {
-		return strings.Contains(cast.ToString(args[0]), cast.ToString(args[1])), nil
+		return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
 	}
 
 	functions["regex"] = func(args ...interface{}) (interface{}, error) {
-		compiled, err := regexp.Compile(cast.ToString(args[0]))
+		compiled, err := regexp.Compile(types.ToString(args[0]))
 		if err != nil {
 			return nil, err
 		}
-		return compiled.MatchString(cast.ToString(args[1])), nil
+		return compiled.MatchString(types.ToString(args[1])), nil
 	}
 
 	// random generators
@@ -178,10 +178,10 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 		chars := letters + numbers
 		bad := ""
 		if len(args) >= 1 {
-			chars = cast.ToString(args[0])
+			chars = types.ToString(args[0])
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = cast.ToString(args[1])
+			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return chars[rand.Intn(len(chars))], nil
@@ -196,10 +196,10 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = cast.ToString(args[1])
+			bad = types.ToString(args[1])
 		}
 		if len(args) >= withBaseRandArgsSize {
-			base = cast.ToString(args[2])
+			base = types.ToString(args[2])
 		}
 		base = trimAll(base, bad)
 		return randSeq(base, l), nil
@@ -214,7 +214,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = cast.ToString(args[1])
+			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
@@ -229,7 +229,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = cast.ToString(args[1])
+			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
@@ -244,7 +244,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 			l = args[0].(int)
 		}
 		if len(args) >= withCutSetArgsSize {
-			bad = cast.ToString(args[1])
+			bad = types.ToString(args[1])
 		}
 		chars = trimAll(chars, bad)
 		return randSeq(chars, l), nil
@@ -273,7 +273,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
 	// Collaborator
 	functions["collab"] = func(args ...interface{}) (interface{}, error) {
 		// check if collaborator contains a specific pattern
-		return collaborator.DefaultCollaborator.Has(cast.ToString(args[0])), nil
+		return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil
 	}
 	return functions
 }

v2/pkg/operators/extractors/compile.go

@@ -25,14 +25,8 @@ func (e *Extractor) CompileExtractors() error {
 	}
 
 	// Setup the part of the request to match, if any.
-	if e.Part != "" {
+	if e.Part == "" {
-		e.part, ok = PartTypes[e.Part]
+		e.Part = "body"
-		if !ok {
-			return fmt.Errorf("unknown matcher part specified: %s", e.Part)
 	}
-	} else {
-		e.part = BodyPart
-	}
-
 	return nil
 }

v2/pkg/operators/extractors/extract.go

@@ -1,52 +1,21 @@
 package extractors
 
-import (
+import "github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"net/http"
-
-	"github.com/miekg/dns"
-)
-
-// Extract extracts response from the parts of request using a regex
-func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} {
-	switch e.extractorType {
-	case RegexExtractor:
-		if e.part == BodyPart {
-			return e.extractRegex(body)
-		} else if e.part == HeaderPart {
-			return e.extractRegex(headers)
-		} else {
-			matches := e.extractRegex(headers)
-			if len(matches) > 0 {
-				return matches
-			}
-			return e.extractRegex(body)
-		}
-	case KValExtractor:
-		if e.part == HeaderPart {
-			return e.extractKVal(resp)
-		}
-
-		matches := e.extractKVal(resp)
-
-		if len(matches) > 0 {
-			return matches
-		}
-
-		return e.extractCookieKVal(resp)
-	}
 
+// Extract extracts data from an output structure based on user options
+func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} {
+	part, ok := data[e.Part]
+	if !ok {
 		return nil
 	}
+	partString := types.ToString(part)
 
-// ExtractDNS extracts response from dns message using a regex
-// nolint:interfacer // dns.Msg is out of current scope
-func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} {
 	switch e.extractorType {
 	case RegexExtractor:
-		return e.extractRegex(msg.String())
+		return e.extractRegex(partString)
 	case KValExtractor:
+		return e.extractKVal(data)
 	}
-
 	return nil
 }
 
@@ -57,39 +26,34 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
 	groupPlusOne := e.RegexGroup + 1
 	for _, regex := range e.regexCompiled {
 		matches := regex.FindAllStringSubmatch(corpus, -1)
+
 		for _, match := range matches {
-			if len(match) >= groupPlusOne {
+			if len(match) < groupPlusOne {
-				results[match[e.RegexGroup]] = struct{}{}
+				continue
+			}
+			matchString := match[e.RegexGroup]
+
+			if _, ok := results[matchString]; !ok {
+				results[matchString] = struct{}{}
 			}
 		}
 	}
 	return results
 }
 
-// extractKVal extracts text from http response
+// extractKVal extracts key value pairs from a data map
-func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} {
+func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} {
 	results := make(map[string]struct{})
 
 	for _, k := range e.KVal {
-		for _, v := range r.Header.Values(k) {
+		item, ok := data[k]
-			results[v] = struct{}{}
+		if !ok {
+			continue
+		}
+		itemString := types.ToString(item)
+		if _, ok := results[itemString]; !ok {
+			results[itemString] = struct{}{}
 		}
 	}
-
-	return results
-}
-
-// extractCookieKVal extracts text from cookies
-func (e *Extractor) extractCookieKVal(r *http.Response) map[string]struct{} {
-	results := make(map[string]struct{})
-
-	for _, k := range e.KVal {
-		for _, cookie := range r.Cookies() {
-			if cookie.Name == k {
-				results[cookie.Value] = struct{}{}
-			}
-		}
-	}
-
 	return results
 }

v2/pkg/operators/extractors/extractors.go

@@ -25,8 +25,6 @@ type Extractor struct {
 	//
 	// By default, matching is performed in request body.
 	Part string `yaml:"part,omitempty"`
-	// part is the part of the request to match
-	part Part
 	// Internal defines if this is used internally
 	Internal bool `yaml:"internal,omitempty"`
 }
@@ -46,27 +44,3 @@ var ExtractorTypes = map[string]ExtractorType{
 	"regex": RegexExtractor,
 	"kval":  KValExtractor,
 }
-
-// Part is the part of the request to match
-type Part int
-
-const (
-	// BodyPart matches body of the response.
-	BodyPart Part = iota + 1
-	// HeaderPart matches headers of the response.
-	HeaderPart
-	// AllPart matches both response body and headers of the response.
-	AllPart
-)
-
-// PartTypes is an table for conversion of part type from string.
-var PartTypes = map[string]Part{
-	"body":   BodyPart,
-	"header": HeaderPart,
-	"all":    AllPart,
-}
-
-// GetPart returns the part of the matcher
-func (e *Extractor) GetPart() Part {
-	return e.part
-}

v2/pkg/operators/matchers/compile.go

@@ -17,9 +17,9 @@ func (m *Matcher) CompileMatchers() error {
 	if !ok {
 		return fmt.Errorf("unknown matcher type specified: %s", m.Type)
 	}
-	// By default, match on all if user hasn't provided any specific items
+	// By default, match on body if user hasn't provided any specific items
 	if m.Part == "" {
-		m.Part = "all"
+		m.Part = "body"
 	}
 
 	// Compile the regexes

v2/pkg/output/format_screen.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"errors"
 
-	"github.com/spf13/cast"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
 // formatScreen formats the output for showing on screen.
@@ -52,7 +52,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) {
 	if ok {
 		builder.WriteString(" [")
 
-		extractorResults := cast.ToStringSlice(extractedResults)
+		extractorResults := types.ToStringSlice(extractedResults)
 		for i, item := range extractorResults {
 			builder.WriteString(w.aurora.BrightCyan(item).String())
 
@@ -68,7 +68,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) {
 	if ok {
 		builder.WriteString(" [")
 
-		metaResults := cast.ToStringMap(metaResults)
+		metaResults := types.ToStringMap(metaResults)
 
 		var first = true
 		for name, value := range metaResults {
@@ -79,7 +79,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) {
 
 			builder.WriteString(w.aurora.BrightYellow(name).String())
 			builder.WriteRune('=')
-			builder.WriteString(w.aurora.BrightYellow(cast.ToString(value)).String())
+			builder.WriteString(w.aurora.BrightYellow(types.ToString(value)).String())
 		}
 		builder.WriteString("]")
 	}

v2/pkg/protocols/dns/matchers.go

@@ -35,7 +35,9 @@ func responseToDSLMap(msg *dns.Msg) map[string]interface{} {
 	data["ns"] = buffer.String()
 	buffer.Reset()
 
-	data["raw"] = msg.String()
+	rawData := msg.String()
+	data["raw"] = rawData
+	data["body"] = rawData // Use rawdata as body for dns responses matching
 	data["status_code"] = msg.Rcode
 	return data
 }

v2/pkg/protocols/http/matchers.go

@@ -9,7 +9,7 @@ import (
 
 // responseToDSLMap converts a HTTP response to a map for use in DSL matching
 func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
-	data := make(map[string]interface{}, len(extra)+6+len(resp.Header))
+	data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies()))
 	for k, v := range extra {
 		data[k] = v
 	}
@@ -18,14 +18,20 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D
 	data["status_code"] = resp.StatusCode
 
 	data["body"] = body
+	for _, cookie := range resp.Cookies() {
+		data[cookie.Name] = cookie.Value
+	}
 	for k, v := range resp.Header {
 		k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
 		data[k] = strings.Join(v, " ")
 	}
-	data["headers"] = headers
+	data["header"] = headers
+	data["all_headers"] = headers
 
 	if r, err := httputil.DumpResponse(resp, true); err == nil {
-		data["raw"] = string(r)
+		rawString := string(r)
+		data["raw"] = rawString
+		data["all"] = rawString
 	}
 	data["duration"] = duration.Seconds()
 	return data

v2/pkg/requests/bulk-http-request.go

@@ -279,24 +279,9 @@ func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string {
 			parsedURL.Host = hostname
 		}
 	}
-
 	return parsedURL.String()
 }
 
-// CustomHeaders valid for all requests
-type CustomHeaders []string
-
-// String returns just a label
-func (c *CustomHeaders) String() string {
-	return "Custom Global Headers"
-}
-
-// Set a new global header
-func (c *CustomHeaders) Set(value string) error {
-	*c = append(*c, value)
-	return nil
-}
-
 // Next returns the next generator by URL
 func (r *BulkHTTPRequest) Next(reqURL string) bool {
 	return r.gsfm.Next(reqURL)

v2/pkg/types/interfaces.go

@@ -0,0 +1,89 @@
+// Taken from https://github.com/spf13/cast.
+
+package types
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// ToString converts an interface to string in a quick way
+func ToString(data interface{}) string {
+	switch s := data.(type) {
+	case string:
+		return s
+	case bool:
+		return strconv.FormatBool(s)
+	case float64:
+		return strconv.FormatFloat(s, 'f', -1, 64)
+	case float32:
+		return strconv.FormatFloat(float64(s), 'f', -1, 32)
+	case int:
+		return strconv.Itoa(s)
+	case int64:
+		return strconv.FormatInt(s, 10)
+	case int32:
+		return strconv.Itoa(int(s))
+	case int16:
+		return strconv.FormatInt(int64(s), 10)
+	case int8:
+		return strconv.FormatInt(int64(s), 10)
+	case uint:
+		return strconv.FormatUint(uint64(s), 10)
+	case uint64:
+		return strconv.FormatUint(uint64(s), 10)
+	case uint32:
+		return strconv.FormatUint(uint64(s), 10)
+	case uint16:
+		return strconv.FormatUint(uint64(s), 10)
+	case uint8:
+		return strconv.FormatUint(uint64(s), 10)
+	case []byte:
+		return string(s)
+	case fmt.Stringer:
+		return s.String()
+	case error:
+		return s.Error()
+	default:
+		return fmt.Sprintf("%v", data)
+	}
+}
+
+// ToStringSlice casts an interface to a []string type.
+func ToStringSlice(i interface{}) []string {
+	var a []string
+
+	switch v := i.(type) {
+	case []interface{}:
+		for _, u := range v {
+			a = append(a, ToString(u))
+		}
+		return a
+	case []string:
+		return v
+	case string:
+		return strings.Fields(v)
+	case interface{}:
+		return []string{ToString(v)}
+	default:
+		return nil
+	}
+}
+
+// ToStringMap casts an interface to a map[string]interface{} type.
+func ToStringMap(i interface{}) map[string]interface{} {
+	var m = map[string]interface{}{}
+
+	switch v := i.(type) {
+	case map[interface{}]interface{}:
+		for k, val := range v {
+			m[ToString(k)] = val
+		}
+		return m
+	case map[string]interface{}:
+		return v
+	default:
+		return nil
+	}
+}