Merge pull request #418 from projectdiscovery/bugfix-collab-match

adding metadata for exact collab match

v2/go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/miekg/dns v1.1.35
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.5
-	github.com/projectdiscovery/collaborator v0.0.1
+	github.com/projectdiscovery/collaborator v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.2
 	github.com/projectdiscovery/gologger v1.0.1
 	github.com/projectdiscovery/hmap v0.0.1

v2/go.sum

@@ -56,6 +56,8 @@ github.com/projectdiscovery/clistats v0.0.5 h1:vcvOR9PrFRawO/7FWD6pER9nYVSoSTD2F
 github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
 github.com/projectdiscovery/collaborator v0.0.1 h1:dbQ5BCL/a3c+BB9cGtrGgiLs23+EfSzoaTzX/pxqiTI=
 github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
+github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s=
+github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
 github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc=
 github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
 github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=

v2/pkg/collaborator/collaborator.go

@@ -2,6 +2,7 @@ package collaborator
 
 import (
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/projectdiscovery/collaborator"
@@ -17,6 +18,7 @@ var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds)
 var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()}
 
 type BurpCollaborator struct {
+	sync.RWMutex
 	options *Options // unused
 	Collab  *collaborator.BurpCollaborator
 }
@@ -41,19 +43,23 @@ func (b *BurpCollaborator) Poll() {
 	}
 }
 
-func (b *BurpCollaborator) Has(s string) bool {
+func (b *BurpCollaborator) Has(s string) (found bool) {
+	foundAt := 0
 	for _, r := range b.Collab.RespBuffer {
 		for i := 0; i < len(r.Responses); i++ {
-			// search in dns
+			// search in dns - http - smtp
-			if strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) {
+			b.RLock()
-				return true
+			found = strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) || strings.Contains(r.Responses[i].Data.RequestDecoded, s) || strings.Contains(r.Responses[i].Data.MessageDecoded, s)
-			}
+			b.RUnlock()
-			// search in http
+			if found {
-			if strings.Contains(r.Responses[i].Data.RequestDecoded, s) {
+				b.Lock()
-				return true
+				r.Responses = removeMatch(r.Responses, foundAt)
+				b.Unlock()
+				break
 			}
+
 		}
 	}
 
-	return false
+	return
 }

v2/pkg/collaborator/util.go

@@ -0,0 +1,9 @@
+package collaborator
+
+import (
+	"github.com/projectdiscovery/collaborator"
+)
+
+func removeMatch(responses []collaborator.BurpResponse, index int) []collaborator.BurpResponse {
+	return append(responses[:index], responses[index+1:]...)
+}

v2/pkg/executer/executer_http.go

@@ -558,6 +558,11 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 	// hardcode stopping storing data after defaultMaxHistorydata items
 	if len(result.historyData) < defaultMaxHistorydata {
 		result.Lock()
+		// update history data with current reqURL and hostname
+		result.historyData["reqURL"] = reqURL
+		if parsed, err := url.Parse(reqURL); err == nil {
+			result.historyData["Hostname"] = parsed.Host
+		}
 		result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format))
 		if payloads == nil {
 			// merge them to history data