Added Go Concurrency and Redesigned Response Format

libsubfinder/engines/passive/passive.go

@@ -28,70 +28,35 @@ import (
 
 func PassiveDiscovery(state *helper.State) (finalPassiveSubdomains []string) {
 
-	// TODO : Add Go Concurrency to requests for data sources :-)
+	// TODO : Add Selection for search sources
 	fmt.Printf("\n\n[-] Searching For Subdomains in Crt.sh")
 	fmt.Printf("\n[-] Searching For Subdomains in Certspotter")
 	fmt.Printf("\n[-] Searching For Subdomains in Threatcrowd")
 	fmt.Printf("\n[-] Searching For Subdomains in Hackertarget")
-	fmt.Printf("\n[-] Searching For Subdomains in Certspotter")
 	fmt.Printf("\n[-] Searching For Subdomains in Virustotal")
 	fmt.Printf("\n[-] Searching For Subdomains in Netcraft\n")
 
-	crtSh, err := crtsh.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range crtSh {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
-	}
+	ch := make(chan helper.Result, 6)
 
-	certspotterResults, err := certspotter.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range certspotterResults {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
-	}
+	// Create goroutines for added speed and recieve data via channels
+	go crtsh.Query(state, ch)
+	go certspotter.Query(state, ch)
+	go hackertarget.Query(state, ch)
+	go threatcrowd.Query(state, ch)
+	go virustotal.Query(state, ch)
+	go netcraft.Query(state, ch)
 
-	threatcrowdResults, err := threatcrowd.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range threatcrowdResults {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
-	}
+	// recieve data from all goroutines running
+	for i := 1; i <= 6; i++ {
+		result := <-ch
 
-	hackertargetResults, err := hackertarget.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range hackertargetResults {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
-	}
-
-	/*fmt.Printf("\n\n[-] Trying DNSDB Domain Search")
-	dnsdbResults, err := dnsdb.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range dnsdbResults {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
-	}*/
-
-	virustotalResults, err := virustotal.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range virustotalResults {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
-	}
-
-	netcraftResults, err := netcraft.Query(state)
-	if err != nil {
-		fmt.Println(err)
-	}
-	for _, subdomain := range netcraftResults {
-		finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
+		if result.Error != nil {
+			// some error occured
+			fmt.Println(result.Error)
+		}
+		for _, subdomain := range result.Subdomains {
+			finalPassiveSubdomains = append(finalPassiveSubdomains, subdomain)
+		}
 	}
 
 	// Now remove duplicate items from the slice

libsubfinder/helper/misc.go

@@ -12,6 +12,13 @@ import (
 	"io/ioutil"
 	"encoding/json"
 )
+
+// Current result structure
+type Result struct {
+    Subdomains []string     // Subdomains found
+    Error      error        // Any error that has occured
+}
+
 //
 // ReadConfigFile : Reads a config file from disk
 // 

libsubfinder/sources/certspotter/certspotter.go

@@ -25,31 +25,39 @@ type certspotter_object struct {
 // array of all results returned
 var certspotter_data []certspotter_object
 
+// all subdomains found
+var subdomains []string
+
 // 
 // Query : Queries awesome Certspotter service for subdomains
 // @param state : current application state, holds all information found
-// 
-// @return subdomain : String array containing subdomains found
-// @return err : nil if successfull and error if failed
 //
-func Query(state *helper.State) (subdomains []string, err error) {
+func Query(state *helper.State, ch chan helper.Result) {
+
+	// Create a result object 
+	var result helper.Result
+	result.Subdomains = subdomains
 
 	// Make a http request to Certspotter
 	resp, err := helper.GetHTTPResponse("https://certspotter.com/api/v0/certs?domain="+state.Domain, 3000)
 	if err != nil {
-		return subdomains, err
+		// Set values and return
+		result.Error = err
+		ch <- result
 	}
 
 	// Get the response body
 	resp_body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Decode the json format
 	err = json.Unmarshal([]byte(resp_body), &certspotter_data)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Append each subdomain found to subdomains array
@@ -73,5 +81,7 @@ func Query(state *helper.State) (subdomains []string, err error) {
 		}	
 	}
 
-	return subdomains, nil
+	result.Subdomains = subdomains
+	result.Error = nil
+	ch <-result
 }

libsubfinder/sources/crtsh/crtsh.go

@@ -26,27 +26,39 @@ type crtsh_object struct {
 // array of all results returned
 var crtsh_data []crtsh_object
 
+// all subdomains found
+var subdomains []string 
+
 // 
 // Query : Queries awesome crt.sh service by comodo
 // @param state : current application state, holds all information found
-// 
-// @return subdomain : String array containing subdomains found
-// @return err : nil if successfull and error if failed
 //
-func Query(state *helper.State) (subdomains []string, err error) {
+func Query(state *helper.State, ch chan helper.Result) {
+
+	var result helper.Result
+	result.Subdomains = subdomains
 
 	// Make a http request to CRT.SH server and request output in JSON
 	// format.
 	// I Think 5 minutes would be more than enough for CRT.SH :-)
 	resp, err := helper.GetHTTPResponse("https://crt.sh/?q=%25."+state.Domain+"&output=json", 3000)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Get the response body
 	resp_body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
+	}
+
+	if strings.Contains(string(resp_body), "The requested URL / was not found on this server.") {
+		// crt.sh is not showing subdomains for some reason
+		// move back
+		result.Error = nil
+		ch <- result
 	}
 
 	// Convert Response Body to string and then replace }{ to },{
@@ -64,7 +76,8 @@ func Query(state *helper.State) (subdomains []string, err error) {
 	// Decode the json format
 	err = json.Unmarshal([]byte(json_output), &crtsh_data)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Append each subdomain found to subdomains array
@@ -86,5 +99,7 @@ func Query(state *helper.State) (subdomains []string, err error) {
 		subdomains = append(subdomains, subdomain.Name_value)
 	}
 
-	return subdomains, nil
+	result.Subdomains = subdomains
+	result.Error = nil
+	ch <-result
 }

libsubfinder/sources/hackertarget/hackertarget.go

@@ -17,24 +17,29 @@ import (
 	"subfinder/libsubfinder/helper"
 )
 
+// all subdomains found
+var subdomains []string 
+
 // 
 // Query : Queries awesome Hackertarget subdomain search service
 // @param state : current application state, holds all information found
-// 
-// @return subdomain : String array containing subdomains found
-// @return err : nil if successfull and error if failed
 //
-func Query(state *helper.State) (subdomains []string, err error) {
+func Query(state *helper.State, ch chan helper.Result) {
+
+	var result helper.Result
+	result.Subdomains = subdomains
 
 	resp, err := helper.GetHTTPResponse("https://api.hackertarget.com/hostsearch/?q="+state.Domain, 3000)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Get the response body
 	resp_body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	scanner := bufio.NewScanner(strings.NewReader(string(resp_body)))
@@ -51,5 +56,7 @@ func Query(state *helper.State) (subdomains []string, err error) {
 		}
 	}
 
-	return subdomains, nil
+	result.Subdomains = subdomains
+	result.Error = nil
+	ch <-result
 }

libsubfinder/sources/netcraft/netcraft.go

@@ -111,16 +111,22 @@ func enumerate(state *helper.State, baseUrl string) (err error) {
 // @return subdomain : String array containing subdomains found
 // @return err : nil if successfull and error if failed
 //
-func Query(state *helper.State) (subdomains []string, err error) {
+func Query(state *helper.State, ch chan helper.Result) {
 
+	var result helper.Result
+	
 	// Initialize global cookie holder
 	gCookies = nil
 
 	// Query using first page. Everything from there would be recursive
-	err = enumerate(state, "https://searchdns.netcraft.com/?restriction=site+ends+with&host="+state.Domain+"&lookup=wait..&position=limited")
+	err := enumerate(state, "https://searchdns.netcraft.com/?restriction=site+ends+with&host="+state.Domain+"&lookup=wait..&position=limited")
 	if err != nil {
-		return subdomains, err
+		result.Subdomains = globalSubdomains
+		result.Error = err
+		ch <- result
 	}
 
-  	return globalSubdomains, nil
+  	result.Subdomains = globalSubdomains
+	result.Error = nil
+	ch <- result
 }

libsubfinder/sources/threatcrowd/threatcrowd.go

@@ -25,31 +25,37 @@ type threatcrowd_object struct {
 // array of all results returned
 var threatcrowd_data threatcrowd_object
 
+// all subdomains found
+var subdomains []string 
+
 // 
 // Query : Queries awesome ThreatCrowd service for subdomains
 // @param state : current application state, holds all information found
-// 
-// @return subdomain : String array containing subdomains found
-// @return err : nil if successfull and error if failed
 //
-func Query(state *helper.State) (subdomains []string, err error) {
+func Query(state *helper.State, ch chan helper.Result) {
 
+	var result helper.Result
+	result.Subdomains = subdomains
+	
 	// Make a http request to Threatcrowd
 	resp, err := helper.GetHTTPResponse("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain="+state.Domain, 3000)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Get the response body
 	resp_body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Decode the json format
 	err = json.Unmarshal([]byte(resp_body), &threatcrowd_data)
 	if err != nil {
-		return subdomains, err
+		result.Error = err
+		ch <- result
 	}
 
 	// Append each subdomain found to subdomains array
@@ -71,5 +77,7 @@ func Query(state *helper.State) (subdomains []string, err error) {
 		subdomains = append(subdomains, subdomain)
 	}	
 
-	return subdomains, nil
+	result.Subdomains = subdomains
+	result.Error = nil
+	ch <-result
 }

libsubfinder/sources/virustotal/virustotal.go

@@ -79,23 +79,26 @@ func queryVirustotalApi(state *helper.State) (subdomains []string, err error) {
 // 
 // Query : Queries awesome Virustotal Service for Subdomains
 // @param state : Current application state
-// 
-// @return subdomain : String array containing subdomains found
-// @return err : nil if successfull and error if failed
 //
-func Query(state *helper.State) (subdomains []string, err error) {
+func Query(state *helper.State, ch chan helper.Result) {
+
+	var result helper.Result
 
 	// We have recieved an API Key
 	// Now, we will use Virustotal API key to fetch subdomain info
 	if state.ConfigState.VirustotalAPIKey != "" {
 
 		// Get subdomains via API
-		subdomains, err = queryVirustotalApi(state)
+		subdomains, err := queryVirustotalApi(state)
 
 		if err != nil {
-			return subdomains, err
+			result.Subdomains = subdomains
+			result.Error = err
+			ch <- result
 		}
-	}
 
-	return subdomains, nil
+		result.Subdomains = subdomains
+		result.Error = nil
+		ch <-result
+	}
 }

main.go

@@ -49,8 +49,8 @@ func ParseCmdLine() (state *helper.State, err error) {
 func main() {
 
 	fmt.Println(banner)
-	fmt.Printf("\nSubFinder v0.1.0 			Made with ❤ by @Ice3man")
-	fmt.Printf("\n===============================================================")
+	fmt.Printf("\nSubFinder v0.1.0 	  Made with ❤ by @Ice3man")
+	fmt.Printf("\n==================================================")
 
 	state, err := ParseCmdLine()
 	if err != nil {