Merge pull request #314 from vzamanillo/threatbook-source

Implemented ThreatBook source

pkg/passive/sources.go

@@ -29,6 +29,7 @@ import (
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/spyse"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sublist3r"
+	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatbook"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatcrowd"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatminer"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/virustotal"
@@ -59,6 +60,7 @@ var DefaultSources = []string{
 	"shodan",
 	"spyse",
 	"sublist3r",
+	"threatbook",
 	"threatcrowd",
 	"threatminer",
 	"virustotal",
@@ -110,6 +112,7 @@ var DefaultAllSources = []string{
 	"sitedossier",
 	"spyse",
 	"sublist3r",
+	"threatbook",
 	"threatcrowd",
 	"threatminer",
 	"virustotal",
@@ -194,6 +197,8 @@ func (a *Agent) addSources(sources []string) {
 			a.sources[source] = &spyse.Source{}
 		case "sublist3r":
 			a.sources[source] = &sublist3r.Source{}
+		case "threatbook":
+			a.sources[source] = &threatbook.Source{}
 		case "threatcrowd":
 			a.sources[source] = &threatcrowd.Source{}
 		case "threatminer":

pkg/runner/config.go

@@ -41,6 +41,7 @@ type ConfigFile struct {
 	SecurityTrails []string `yaml:"securitytrails"`
 	Shodan         []string `yaml:"shodan"`
 	Spyse          []string `yaml:"spyse"`
+	ThreatBook     []string `yaml:"threatbook"`
 	URLScan        []string `yaml:"urlscan"`
 	Virustotal     []string `yaml:"virustotal"`
 	ZoomEye        []string `yaml:"zoomeye"`
@@ -171,6 +172,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
 	if len(c.Spyse) > 0 {
 		keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))]
 	}
+	if len(c.ThreatBook) > 0 {
+		keys.ThreatBook = c.ThreatBook[rand.Intn(len(c.ThreatBook))]
+	}
 	if len(c.URLScan) > 0 {
 		keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))]
 	}

pkg/subscraping/sources/threatbook/threatbook.go

@@ -0,0 +1,78 @@
+package threatbook
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/projectdiscovery/subfinder/pkg/subscraping"
+)
+
+type threatBookResponse struct {
+	ResponseCode int64  `json:"response_code"`
+	VerboseMsg   string `json:"verbose_msg"`
+	Data         struct {
+		Domain     string `json:"domain"`
+		SubDomains struct {
+			Total string   `json:"total"`
+			Data  []string `json:"data"`
+		} `json:"sub_domains"`
+	} `json:"data"`
+}
+
+// Source is the passive scraping agent
+type Source struct{}
+
+// Run function returns all subdomains found with the service
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+
+	go func() {
+		defer close(results)
+
+		if session.Keys.ThreatBook == "" {
+			return
+		}
+
+		resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", session.Keys.ThreatBook, domain))
+		if err != nil && resp == nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+
+		var response threatBookResponse
+		err = jsoniter.NewDecoder(resp.Body).Decode(&response)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
+			return
+		}
+		resp.Body.Close()
+
+		if response.ResponseCode != 0 {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg)}
+			return
+		}
+
+		total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			return
+		}
+
+		if total > 0 {
+			for _, subdomain := range response.Data.SubDomains.Data {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
+			}
+		}
+	}()
+
+	return results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "threatbook"
+}

pkg/subscraping/types.go

@@ -50,6 +50,7 @@ type Keys struct {
 	Securitytrails       string   `json:"securitytrails"`
 	Shodan               string   `json:"shodan"`
 	Spyse                string   `json:"spyse"`
+	ThreatBook           string   `json:"threatbook"`
 	URLScan              string   `json:"urlscan"`
 	Virustotal           string   `json:"virustotal"`
 	ZoomEyeUsername      string   `json:"zoomeye_username"`