Merge branch 'lc/add-chaos-upload' of github.com:lc/subfinder into lc-lc/add-chaos-upload

config.yaml

@@ -41,10 +41,11 @@ sources:
 binaryedge: []
 censys: []
 certspotter: []
+chaos: []
 dnsdb: []
 passivetotal: []
 securitytrails: []
 shodan: []
 urlscan: []
 virustotal: []
-zoomeye: []
+zoomeye: []

pkg/runner/config.go

@@ -22,6 +22,7 @@ type ConfigFile struct {
 	Binaryedge     []string `yaml:"binaryedge"`
 	Censys         []string `yaml:"censys"`
 	Certspotter    []string `yaml:"certspotter"`
+	Chaos          []string `yaml:"chaos"`
 	DNSDB          []string `yaml:"dnsdb"`
 	PassiveTotal   []string `yaml:"passivetotal"`
 	SecurityTrails []string `yaml:"securitytrails"`
@@ -109,7 +110,9 @@ func (c ConfigFile) GetKeys() subscraping.Keys {
 	if len(c.Certspotter) > 0 {
 		keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))]
 	}
-
+	if len(c.Chaos) > 0 {
+		keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
+	}
 	if (len(c.DNSDB)) > 0 {
 		keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
 	}

pkg/runner/enumerate.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"bytes"
 	"os"
 	"strings"
 	"sync"
@@ -113,7 +114,25 @@ func (r *Runner) EnumerateSingleDomain(domain, output string, append bool) error
 			}
 		}
 	}
-
+	// In case the user has specified to upload to chaos, write everything to a temporary buffer and upload
+	if r.options.ChaosUpload {
+		var buf = &bytes.Buffer{}
+		err := WriteHostOutput(uniqueMap, buf)
+		// If an error occurs, do not interrupt, continue to check if user specifed an output file
+		if err != nil {
+			gologger.Errorf("Could not prepare results for chaos %s\n", err)
+		} else {
+			// no error in writing host output, upload to chaos
+			err = r.UploadToChaos(buf)
+			if err != nil {
+				gologger.Errorf("Could not upload results to chaos %s\n", err)
+			} else {
+				gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
+			}
+			// clear buffer
+			buf = nil
+		}
+	}
 	// In case the user has given an output file, write all the found
 	// subdomains to the output file.
 	if output != "" {

pkg/runner/options.go

@@ -18,6 +18,7 @@ type Options struct {
 	MaxEnumerationTime int    // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
 	Domain             string // Domain is the domain to find subdomains for
 	DomainsFile        string // DomainsFile is the file containing list of domains to find subdomains for
+	ChaosUpload        bool   // ChaosUpload indicates whether to upload results to the Chaos API
 	Output             string // Output is the file to write found subdomains to.
 	OutputDirectory    string // OutputDirectory is the directory to write results to in case list of domains is given
 	JSON               bool   // JSON specifies whether to use json for output format or text file
@@ -52,6 +53,7 @@ func ParseOptions() *Options {
 	flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results")
 	flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for")
 	flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate")
+	flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)")
 	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
 	flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)")
 	flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format")

pkg/runner/utils.go

@@ -2,10 +2,16 @@ package runner
 
 import (
 	"bufio"
+	"crypto/tls"
+	"fmt"
 	"io"
+	"io/ioutil"
+	"net/http"
 	"strings"
+	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
 )
 
 // JSONResult contains the result for a host in JSON format
@@ -14,6 +20,39 @@ type JSONResult struct {
 	IP   string `json:"ip"`
 }
 
+func (r *Runner) UploadToChaos(reader io.Reader) error {
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			MaxIdleConnsPerHost: 100,
+			MaxIdleConns:        100,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+		Timeout: time.Duration(600) * time.Second, // 10 minutes - uploads may take long
+	}
+
+	request, err := http.NewRequest("POST", "https://dns.projectdiscovery.io/dns/add", reader)
+	if err != nil {
+		return errors.Wrap(err, "could not create request")
+	}
+	request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos)
+
+	resp, err := httpClient.Do(request)
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("invalid status code received: %d", resp.StatusCode)
+	}
+	return nil
+}
+
 // WriteHostOutput writes the output list of subdomain to an io.Writer
 func WriteHostOutput(results map[string]struct{}, writer io.Writer) error {
 	bufwriter := bufio.NewWriter(writer)

pkg/subscraping/types.go

@@ -33,6 +33,7 @@ type Keys struct {
 	CensysToken          string `json:"censysUsername"`
 	CensysSecret         string `json:"censysPassword"`
 	Certspotter          string `json:"certspotter"`
+	Chaos                string `json:"chaos"`
 	DNSDB                string `json:"dnsdb"`
 	PassiveTotalUsername string `json:"passivetotal_username"`
 	PassiveTotalPassword string `json:"passivetotal_password"`