From 6867824a6bf631e7cfdaae032eb3650c94f9835a Mon Sep 17 00:00:00 2001 From: lc Date: Sun, 17 May 2020 16:18:09 -0500 Subject: [PATCH 1/4] Add Chaos upload functionality, -upload flag --- config.yaml | 3 ++- pkg/runner/config.go | 5 ++++- pkg/runner/enumerate.go | 16 +++++++++++++++- pkg/runner/options.go | 2 ++ pkg/runner/utils.go | 40 ++++++++++++++++++++++++++++++++++++++++ pkg/subscraping/types.go | 1 + 6 files changed, 64 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index d0af439..d25a889 100644 --- a/config.yaml +++ b/config.yaml @@ -41,10 +41,11 @@ sources: binaryedge: [] censys: [] certspotter: [] +chaos: [] dnsdb: [] passivetotal: [] securitytrails: [] shodan: [] urlscan: [] virustotal: [] -zoomeye: [] \ No newline at end of file +zoomeye: [] diff --git a/pkg/runner/config.go b/pkg/runner/config.go index f4f6feb..7328831 100644 --- a/pkg/runner/config.go +++ b/pkg/runner/config.go @@ -22,6 +22,7 @@ type ConfigFile struct { Binaryedge []string `yaml:"binaryedge"` Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` + Chaos []string `yaml:"chaos"` DNSDB []string `yaml:"dnsdb"` PassiveTotal []string `yaml:"passivetotal"` SecurityTrails []string `yaml:"securitytrails"` @@ -109,7 +110,9 @@ func (c ConfigFile) GetKeys() subscraping.Keys { if len(c.Certspotter) > 0 { keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))] } - + if len(c.Chaos) > 0 { + keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] + } if (len(c.DNSDB)) > 0 { keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] } diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index ae1680a..fc70478 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -1,6 +1,7 @@ package runner import ( + "bytes" "os" "strings" "sync" @@ -113,7 +114,20 @@ func (r *Runner) EnumerateSingleDomain(domain, output string, append bool) error } } } - + // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload + if r.options.ChaosUpload { + var buf = &bytes.Buffer{} + err := WriteHostOutput(uniqueMap, buf) + if err != nil { + gologger.Errorf("Could not prepare results for chaos %s\n", err) + } + err = r.UploadToChaos(buf) + if err != nil { + gologger.Errorf("Could not upload results to chaos %s\n", err) + } else { + gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") + } + } // In case the user has given an output file, write all the found // subdomains to the output file. if output != "" { diff --git a/pkg/runner/options.go b/pkg/runner/options.go index 4c90c52..82f7453 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -18,6 +18,7 @@ type Options struct { MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration Domain string // Domain is the domain to find subdomains for DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for + ChaosUpload bool // ChaosUpload indicates whether to upload results to the Chaos API Output string // Output is the file to write found subdomains to. OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given JSON bool // JSON specifies whether to use json for output format or text file @@ -52,6 +53,7 @@ func ParseOptions() *Options { flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results") flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for") flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate") + flag.BoolVar(&options.ChaosUpload, "upload", false, "Upload results to the Chaos API (api-key required)") flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)") flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format") diff --git a/pkg/runner/utils.go b/pkg/runner/utils.go index e2aaddf..e770ba5 100644 --- a/pkg/runner/utils.go +++ b/pkg/runner/utils.go @@ -2,10 +2,16 @@ package runner import ( "bufio" + "crypto/tls" + "fmt" "io" + "io/ioutil" + "net/http" "strings" + "time" jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" ) // JSONResult contains the result for a host in JSON format @@ -14,6 +20,40 @@ type JSONResult struct { IP string `json:"ip"` } +func (r *Runner) UploadToChaos(reader io.Reader) error { + httpClient := &http.Client{ + Transport: &http.Transport{ + MaxIdleConnsPerHost: 100, + MaxIdleConns: 100, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: time.Duration(600) * time.Second, // 10 minutes - uploads may take long + } + + request, err := http.NewRequest("POST", "https://dns.projectdiscovery.io/dns/add", reader) + if err != nil { + return errors.Wrap(err, "could not create request") + } + request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos) + + resp, err := httpClient.Do(request) + if err != nil { + return errors.Wrap(err, "could not make request") + } + defer func() { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != 200 { + return fmt.Errorf("invalid status code received: %d", resp.StatusCode) + } + return nil + +} + // WriteHostOutput writes the output list of subdomain to an io.Writer func WriteHostOutput(results map[string]struct{}, writer io.Writer) error { bufwriter := bufio.NewWriter(writer) diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index d5d6b2c..e87e8e7 100755 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -33,6 +33,7 @@ type Keys struct { CensysToken string `json:"censysUsername"` CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` + Chaos string `json:"chaos"` DNSDB string `json:"dnsdb"` PassiveTotalUsername string `json:"passivetotal_username"` PassiveTotalPassword string `json:"passivetotal_password"` From 8dc5f68f6c4c678accfbcada9a01e3a90f7540a1 Mon Sep 17 00:00:00 2001 From: lc Date: Sun, 17 May 2020 16:19:59 -0500 Subject: [PATCH 2/4] Change -upload flag to -cd per issue #7 --- pkg/runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/options.go b/pkg/runner/options.go index 82f7453..9eb74a6 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -53,7 +53,7 @@ func ParseOptions() *Options { flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results") flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for") flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate") - flag.BoolVar(&options.ChaosUpload, "upload", false, "Upload results to the Chaos API (api-key required)") + flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)") flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)") flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format") From 092e40d61b913b005bbaf9f11f3320766cbab2fd Mon Sep 17 00:00:00 2001 From: lc Date: Sun, 17 May 2020 16:29:15 -0500 Subject: [PATCH 3/4] Fix error handling, clear buffer after use --- pkg/runner/enumerate.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index fc70478..91794c6 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -118,14 +118,19 @@ func (r *Runner) EnumerateSingleDomain(domain, output string, append bool) error if r.options.ChaosUpload { var buf = &bytes.Buffer{} err := WriteHostOutput(uniqueMap, buf) + // If an error occurs, do not interrupt, continue to check if user specifed an output file if err != nil { gologger.Errorf("Could not prepare results for chaos %s\n", err) - } - err = r.UploadToChaos(buf) - if err != nil { - gologger.Errorf("Could not upload results to chaos %s\n", err) } else { - gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") + // no error in writing host output, upload to chaos + err = r.UploadToChaos(buf) + if err != nil { + gologger.Errorf("Could not upload results to chaos %s\n", err) + } else { + gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") + } + // clear buffer + buf = nil } } // In case the user has given an output file, write all the found From 9e1d2abde179aa62c0a93092e27c4d53ebeccede Mon Sep 17 00:00:00 2001 From: lc Date: Sun, 17 May 2020 16:38:28 -0500 Subject: [PATCH 4/4] Remove newline --- pkg/runner/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/runner/utils.go b/pkg/runner/utils.go index e770ba5..0009ad3 100644 --- a/pkg/runner/utils.go +++ b/pkg/runner/utils.go @@ -51,7 +51,6 @@ func (r *Runner) UploadToChaos(reader io.Reader) error { return fmt.Errorf("invalid status code received: %d", resp.StatusCode) } return nil - } // WriteHostOutput writes the output list of subdomain to an io.Writer