Added Project Files

libsubfinder/helper/misc.go

@@ -0,0 +1,53 @@
+// 
+// misc.go : contains misc helper function
+// Written By : @ice3man (Nizamul Rana)
+//
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package helper
+
+import (
+	"io/ioutil"
+	"encoding/json"
+)
+//
+// ReadConfigFile : Reads a config file from disk
+// 
+// @return config : configuration structure
+// @return err : if no error nil, else error
+//
+func ReadConfigFile() (configuration *Config, err error) {
+
+	var config Config
+
+	// Read the file
+	raw, err := ioutil.ReadFile("./config.json")
+    if err != nil {
+        return &config, err
+    }
+
+    err = json.Unmarshal(raw, &config)
+    if (err != nil) {
+    	return &config, err
+    }
+
+    return &config, nil
+}
+
+// 
+// Returns unique items in a slice
+// Adapted from http://www.golangprograms.com/remove-duplicate-values-from-slice.html
+//
+func Unique(strSlice []string) []string {
+    keys := make(map[string]bool)
+    list := []string{} 
+    for _, entry := range strSlice {
+        if _, value := keys[entry]; !value {
+            keys[entry] = true
+            list = append(list, entry)
+        }
+    }    
+    return list
+}

libsubfinder/helper/sources.go

@@ -0,0 +1,53 @@
+// 
+// helper.go : Main sources driver. Contains helper functions for other sources.
+// Written By : @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package helper
+
+import (
+	"net/http"
+	"crypto/tls"
+	"time"
+)
+
+// 
+// GetHTTPResponse : Returns a HTTP Response object
+// @param url : URL To Visit (Note, It needs full url with scheme)
+// @param timeout : Seconds to wait for response until timeout
+// 
+// @return resp : HTTP Response object
+// @return err : nil if successfull else error
+//
+func GetHTTPResponse(url string, timeout int) (resp *http.Response, err error) {
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   time.Duration(timeout) * time.Second,
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return resp, err
+	}
+
+	// TODO : Figure out a way to handle user agents as per user intention
+	// @codingo, I don't think it's correct to spam services by making requests with fake user agent
+	// 	What do you think
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1")
+	req.Header.Add("Connection", "close")
+
+	resp, err = client.Do(req)
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}

libsubfinder/helper/state.go

@@ -0,0 +1,36 @@
+// 
+// state.go : Contains current program state
+// Written By :  @codingo (Michael)
+//				 @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package helper
+
+// Holds the State read in from the CLI
+type State struct {
+	Color   		bool		// Whether to use color or not
+	Threads 		int 		// Number of threads to use
+	Verbose 		bool 		// Show verbose information
+	Domain  		string		// Domain name to find subdomains for
+	Recursive 		bool		// Whether perform recursive subdomain discovery or not
+
+	ConfigState  	Config		// Current configuration file state
+}
+
+type Config struct {
+	VirustotalAPIKey	string	`json:"virustotalapikey"`		// Virustotal API Key
+}
+
+func InitState() (state State, err error) {
+
+	// Read the configuration file
+	config, err := ReadConfigFile()
+	if err != nil {
+		return state, err
+	}
+
+	return State{true, 10, false, "", false, *config}, nil
+}

libsubfinder/sources/certspotter/certspotter.go

@@ -0,0 +1,77 @@
+// 
+// certspotter.go : A Golang based client for Certspotter Parsing
+// Written By : @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package certspotter
+
+import (
+	"io/ioutil"
+	"encoding/json"
+	"strings"
+	"fmt"
+
+	"subfinder/libsubfinder/helper"
+)
+
+// Structure of a single dictionary of output by crt.sh
+type certspotter_object struct {
+	Dns_names	[]string `json:"dns_names"`
+}
+
+// array of all results returned
+var certspotter_data []certspotter_object
+
+// 
+// Query : Queries awesome Certspotter service for subdomains
+// @param state : current application state, holds all information found
+// 
+// @return subdomain : String array containing subdomains found
+// @return err : nil if successfull and error if failed
+//
+func Query(state *helper.State) (subdomains []string, err error) {
+
+	// Make a http request to Certspotter
+	resp, err := helper.GetHTTPResponse("https://certspotter.com/api/v0/certs?domain="+state.Domain, 3000)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Get the response body
+	resp_body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Decode the json format
+	err = json.Unmarshal([]byte(resp_body), &certspotter_data)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Append each subdomain found to subdomains array
+	for _, block := range certspotter_data {
+		for _, dns_name := range block.Dns_names {
+
+			// Fix Wildcard subdomains containg asterisk before them
+			if strings.Contains(dns_name, "*.") {
+				dns_name = strings.Split(dns_name, "*.")[1]
+			}
+
+			if state.Verbose == true {
+				if state.Color == true {
+					fmt.Printf("\n[\033[31;1;4mCERTSPOTTER\033[0m] %s", dns_name)
+				} else {
+					fmt.Printf("\n[CERTSPOTTER] %s", dns_name)
+				}
+			}
+
+			subdomains = append(subdomains, dns_name)
+		}	
+	}
+
+	return subdomains, nil
+}

libsubfinder/sources/crtsh/crtsh.go

@@ -0,0 +1,90 @@
+// 
+// crtsh.go : A Golang based client for CRT.SH Parsing
+// Written By : @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package crtsh
+
+import (
+	"io/ioutil"
+	"strings"
+	"encoding/json"
+	"fmt"
+
+	"subfinder/libsubfinder/helper"
+)
+
+// Structure of a single dictionary of output by crt.sh
+// We only need name_value object hence this :-)
+type crtsh_object struct {
+	Name_value		string  `json:"name_value"`
+}
+
+// array of all results returned
+var crtsh_data []crtsh_object
+
+// 
+// Query : Queries awesome crt.sh service by comodo
+// @param state : current application state, holds all information found
+// 
+// @return subdomain : String array containing subdomains found
+// @return err : nil if successfull and error if failed
+//
+func Query(state *helper.State) (subdomains []string, err error) {
+
+	// Make a http request to CRT.SH server and request output in JSON
+	// format.
+	// I Think 5 minutes would be more than enough for CRT.SH :-)
+	resp, err := helper.GetHTTPResponse("https://crt.sh/?q=%25."+state.Domain+"&output=json", 3000)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Get the response body
+	resp_body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Convert Response Body to string and then replace }{ to },{
+	// This is done in order to enable parsing of JSON format employed by
+	// crt.sh
+	correct_format := strings.Replace(string(resp_body), "}{", "},{", -1)
+
+	// Now convert it into a json array like this
+	// [
+	// 		{abc},
+	//		{abc}
+	// ]
+	json_output := "["+correct_format+"]"
+
+	// Decode the json format
+	err = json.Unmarshal([]byte(json_output), &crtsh_data)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Append each subdomain found to subdomains array
+	for _, subdomain := range crtsh_data {
+
+		// Fix Wildcard subdomains containg asterisk before them
+		if strings.Contains(subdomain.Name_value, "*.") {
+			subdomain.Name_value = strings.Split(subdomain.Name_value, "*.")[1]
+		}
+
+		if state.Verbose == true {
+			if state.Color == true {
+				fmt.Printf("\n[\033[31;1;4mCRT.SH\033[0m] %s", subdomain.Name_value)
+			} else {
+				fmt.Printf("\n[CRT.SH] %s", subdomain.Name_value)
+			}
+		}
+
+		subdomains = append(subdomains, subdomain.Name_value)
+	}
+
+	return subdomains, nil
+}

libsubfinder/sources/netcraft/netcraft.go

@@ -0,0 +1,75 @@
+// 
+// netcraft.go : Netcraft Scraping Engine in Golang
+// Written By : @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package threatcrowd
+
+import (
+	"io/ioutil"
+	"encoding/json"
+	"strings"
+	"fmt"
+
+	"subfinder/libsubfinder/helper"
+)
+
+// Struct containing json data we actually need
+type netcraft_object struct {
+	Subdomains	[]string `json:"subdomains"`
+}
+
+// array of all results returned
+var netcraft_data netcraft_object
+
+// 
+// Query : Queries awesome Netcraft service for subdomains
+// @param state : current application state, holds all information found
+// 
+// @return subdomain : String array containing subdomains found
+// @return err : nil if successfull and error if failed
+//
+func Query(state *helper.State) (subdomains []string, err error) {
+
+	// Make a http request to Netcraft
+	resp, err := helper.GetHTTPResponse("https://searchdns.netcraft.com/?restriction=site+ends+with&host="+state.Domain+"&lookup=wait..&position=limited", 3000)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Get the response body
+	resp_body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Decode the json format
+	err = json.Unmarshal([]byte(resp_body), &threatcrowd_data)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Append each subdomain found to subdomains array
+	for _, subdomain := range threatcrowd_data.Subdomains {
+
+		// Fix Wildcard subdomains containg asterisk before them
+		if strings.Contains(subdomain, "*.") {
+			subdomain = strings.Split(subdomain, "*.")[1]
+		}
+
+		if state.Verbose == true {
+			if state.Color == true {
+				fmt.Printf("\n[\033[31;1;4mTHREATCROWD\033[0m] %s", subdomain)
+			} else {
+				fmt.Printf("\n[THREATCROWD] %s", subdomain)
+			}
+		}
+
+		subdomains = append(subdomains, subdomain)
+	}	
+
+	return subdomains, nil
+}

libsubfinder/sources/threatcrowd/threatcrowd.go

@@ -0,0 +1,75 @@
+// 
+// threatcrowd.go : A Golang based client for Threatcrowd API
+// Written By : @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+package threatcrowd
+
+import (
+	"io/ioutil"
+	"encoding/json"
+	"strings"
+	"fmt"
+
+	"subfinder/libsubfinder/helper"
+)
+
+// Struct containing json data we actually need
+type threatcrowd_object struct {
+	Subdomains	[]string `json:"subdomains"`
+}
+
+// array of all results returned
+var threatcrowd_data threatcrowd_object
+
+// 
+// Query : Queries awesome ThreatCrowd service for subdomains
+// @param state : current application state, holds all information found
+// 
+// @return subdomain : String array containing subdomains found
+// @return err : nil if successfull and error if failed
+//
+func Query(state *helper.State) (subdomains []string, err error) {
+
+	// Make a http request to Threatcrowd
+	resp, err := helper.GetHTTPResponse("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain="+state.Domain, 3000)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Get the response body
+	resp_body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Decode the json format
+	err = json.Unmarshal([]byte(resp_body), &threatcrowd_data)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Append each subdomain found to subdomains array
+	for _, subdomain := range threatcrowd_data.Subdomains {
+
+		// Fix Wildcard subdomains containg asterisk before them
+		if strings.Contains(subdomain, "*.") {
+			subdomain = strings.Split(subdomain, "*.")[1]
+		}
+
+		if state.Verbose == true {
+			if state.Color == true {
+				fmt.Printf("\n[\033[31;1;4mTHREATCROWD\033[0m] %s", subdomain)
+			} else {
+				fmt.Printf("\n[THREATCROWD] %s", subdomain)
+			}
+		}
+
+		subdomains = append(subdomains, subdomain)
+	}	
+
+	return subdomains, nil
+}

libsubfinder/sources/virustotal/virustotal.go

@@ -0,0 +1,101 @@
+// 
+// virustotal.go : A Virustotal Client for Subdomain Enumeration
+// Written By : @ice3man (Nizamul Rana)
+// 
+// Distributed Under MIT License
+// Copyrights (C) 2018 Ice3man
+//
+
+// NOTE : We are using Virustotal API here Since we wanted to eliminate the 
+// rate limiting performed by Virustotal on scraping.
+// Direct queries and parsing can be also done :-)
+
+package virustotal
+
+import (
+	"io/ioutil"
+	"fmt"
+	"encoding/json"
+	"strings"
+
+	"subfinder/libsubfinder/helper"
+)
+
+type virustotalapi_object struct {
+	Subdomains	[]string `json:"subdomains"`
+}
+
+var virustotalapi_data virustotalapi_object
+
+// 
+// Local function to query virustotal API
+// Requires an API key
+//
+// @note : If the user specifies an API key in config.json, we use API
+//	If not, we try to scrape pages though it is highly discouraged
+//
+func queryVirustotalApi(state *helper.State) (subdomains []string, err error) {
+
+	// Make a search for a domain name and get HTTP Response
+	resp, err := helper.GetHTTPResponse("https://www.virustotal.com/vtapi/v2/domain/report?apikey="+state.ConfigState.VirustotalAPIKey+"&domain="+state.Domain, 3000)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Get the response body
+	resp_body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Decode the json format
+	err = json.Unmarshal([]byte(resp_body), &virustotalapi_data)
+	if err != nil {
+		return subdomains, err
+	}
+
+	// Append each subdomain found to subdomains array
+	for _, subdomain := range virustotalapi_data.Subdomains {
+
+		// Fix Wildcard subdomains containg asterisk before them
+		if strings.Contains(subdomain, "*.") {
+			subdomain = strings.Split(subdomain, "*.")[1]
+		}
+
+		if state.Verbose == true {
+			if state.Color == true {
+				fmt.Printf("\n[\033[31;1;4mVIRUSTOTAL\033[0m] %s", subdomain)
+			} else {
+				fmt.Printf("\n[VIRUSTOTAL] %s", subdomain)
+			}
+		}
+
+		subdomains = append(subdomains, subdomain)
+	}	
+
+	return subdomains, nil
+}
+
+// 
+// Query : Queries awesome Virustotal Service for Subdomains
+// @param state : Current application state
+// 
+// @return subdomain : String array containing subdomains found
+// @return err : nil if successfull and error if failed
+//
+func Query(state *helper.State) (subdomains []string, err error) {
+
+	// We have recieved an API Key
+	// Now, we will use Virustotal API key to fetch subdomain info
+	if state.ConfigState.VirustotalAPIKey != "" {
+
+		// Get subdomains via API
+		subdomains, err = queryVirustotalApi(state)
+
+		if err != nil {
+			return subdomains, err
+		}
+	}
+
+	return subdomains, nil
+}