Merge pull request #326 from vzamanillo/robtex-pro

Move from Robtex free API to PRO API

README.md

@@ -138,6 +138,7 @@ Subfinder will work after using the installation instructions however to configu
 - [Github](https://github.com)
 - [Intelx](https://intelx.io)
 - [Recon](https://recon.dev)
+- [Robtex](https://www.robtex.com/api/)
 
 Theses values are stored in the `$HOME/.config/subfinder/config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.
 

pkg/runner/config.go

@@ -38,6 +38,7 @@ type ConfigFile struct {
 	IntelX         []string `yaml:"intelx"`
 	PassiveTotal   []string `yaml:"passivetotal"`
 	Recon          []string `yaml:"recon"`
+	Robtex         []string `yaml:"robtex"`
 	SecurityTrails []string `yaml:"securitytrails"`
 	Shodan         []string `yaml:"shodan"`
 	Spyse          []string `yaml:"spyse"`
@@ -163,6 +164,10 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
 		keys.Recon = c.Recon[rand.Intn(len(c.Recon))]
 	}
 
+	if len(c.Robtex) > 0 {
+		keys.Robtex = c.Robtex[rand.Intn(len(c.Robtex))]
+	}
+
 	if len(c.SecurityTrails) > 0 {
 		keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))]
 	}

pkg/subscraping/sources/robtex/robtext.go

@@ -13,7 +13,7 @@ import (
 const (
 	addrRecord     = "A"
 	iPv6AddrRecord = "AAAA"
-	baseURL        = "https://freeapi.robtex.com/pdns"
+	baseURL        = "https://proapi.robtex.com/pdns"
 )
 
 // Source is the passive scraping agent
@@ -32,9 +32,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 	go func() {
 		defer close(results)
 
+		if session.Keys.Robtex == "" {
+			return
+		}
+
 		headers := map[string]string{"Content-Type": "application/x-ndjson"}
 
-		ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s", baseURL, domain), headers)
+		ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, session.Keys.Robtex), headers)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 			return
@@ -42,7 +46,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 
 		for _, result := range ips {
 			if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord {
-				domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s", baseURL, result.Rrdata), headers)
+				domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, session.Keys.Robtex), headers)
 				if err != nil {
 					results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 					return

pkg/subscraping/types.go

@@ -47,6 +47,7 @@ type Keys struct {
 	PassiveTotalUsername string   `json:"passivetotal_username"`
 	PassiveTotalPassword string   `json:"passivetotal_password"`
 	Recon                string   `json:"recon"`
+	Robtex               string   `json:"robtex"`
 	Securitytrails       string   `json:"securitytrails"`
 	Shodan               string   `json:"shodan"`
 	Spyse                string   `json:"spyse"`