From d7d7dccf0906ba079b8f13dc46ae2ca8fb690c06 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 1 May 2018 22:58:20 +0200 Subject: [PATCH 1/2] added baidu passive subdomains search - Added better fix to Dockerfile - Added Baidu passive search --- Dockerfile | 4 +- libsubfinder/engines/passive/passive.go | 18 ++++++-- libsubfinder/sources/baidu/baidu.go | 60 +++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 libsubfinder/sources/baidu/baidu.go diff --git a/Dockerfile b/Dockerfile index d0b5590..c38cd3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,13 @@ FROM iron/go:dev WORKDIR /app # Set an env var that matches your github repo name -ENV SRC_DIR=/go/src/github.com/ice3man543/subfinder/ +ENV SRC_DIR=/go/src/github.com/Ice3man543/subfinder/ # Add the source code: ADD . $SRC_DIR # Build it: -RUN cd $SRC_DIR; go get; go build -o main; cp main /app/ +RUN cd $SRC_DIR; go build -o main; cp main /app/ ENTRYPOINT ["./main"] CMD ["-h"] diff --git a/libsubfinder/engines/passive/passive.go b/libsubfinder/engines/passive/passive.go index 0b5e304..530c5ce 100644 --- a/libsubfinder/engines/passive/passive.go +++ b/libsubfinder/engines/passive/passive.go @@ -34,6 +34,7 @@ import ( "github.com/Ice3man543/subfinder/libsubfinder/sources/threatminer" "github.com/Ice3man543/subfinder/libsubfinder/sources/virustotal" "github.com/Ice3man543/subfinder/libsubfinder/sources/waybackarchive" + "github.com/Ice3man543/subfinder/libsubfinder/sources/baidu" ) // Sources configuration structure specifying what should we use @@ -56,12 +57,13 @@ type Source struct { Threatminer bool Riddler bool Dnsdb bool + Baidu bool NoOfSources int } func PassiveDiscovery(state *helper.State) (finalPassiveSubdomains []string) { - sourceConfig := Source{false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, 0} + sourceConfig := Source{false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, 0} fmt.Printf("\n") if state.Sources == "all" { @@ -84,10 +86,11 @@ func PassiveDiscovery(state *helper.State) (finalPassiveSubdomains []string) { fmt.Printf("\n[-] Searching For Subdomains in ThreatMiner") fmt.Printf("\n[-] Searching For Subdomains in Riddler") fmt.Printf("\n[-] Searching For Subdomains in Netcraft") - fmt.Printf("\n[-] Searching For Subdomains in Dnsdb\n") + fmt.Printf("\n[-] Searching For Subdomains in Dnsdb") + fmt.Printf("\n[-] Searching For Subdomains in Baidu\n") } - sourceConfig = Source{true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, 17} + sourceConfig = Source{true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, 18} } else { // Check data sources and create a source configuration structure @@ -195,6 +198,12 @@ func PassiveDiscovery(state *helper.State) (finalPassiveSubdomains []string) { } sourceConfig.Dnsdb = true sourceConfig.NoOfSources = sourceConfig.NoOfSources + 1 + } else if source == "baidu" { + if state.Silent != true { + fmt.Printf("\n[-] Searching For Subdomains in Baidu") + } + sourceConfig.Baidu = true + sourceConfig.NoOfSources = sourceConfig.NoOfSources + 1 } } } @@ -255,6 +264,9 @@ func PassiveDiscovery(state *helper.State) (finalPassiveSubdomains []string) { if sourceConfig.Dnsdb == true { go dnsdb.Query(state, ch) } + if sourceConfig.Baidu == true { + go baidu.Query(state, ch) + } // Recieve data from all goroutines running for i := 0; i < sourceConfig.NoOfSources; i++ { diff --git a/libsubfinder/sources/baidu/baidu.go b/libsubfinder/sources/baidu/baidu.go new file mode 100644 index 0000000..3e5a851 --- /dev/null +++ b/libsubfinder/sources/baidu/baidu.go @@ -0,0 +1,60 @@ +// +// Written By : @Mzack9999 (Marco Rivoli) +// +// Distributed Under MIT License +// Copyrights (C) 2018 Ice3man +// + +// A golang client for Baidu Subdomain Discovery +package baidu + +import ( + "fmt" + "io/ioutil" + "regexp" + + "github.com/Ice3man543/subfinder/libsubfinder/helper" +) + +// all subdomains found +var subdomains []string + +// Query function returns all subdomains found using the service. +func Query(state *helper.State, ch chan helper.Result) { + + var result helper.Result + result.Subdomains = subdomains + resp, err := helper.GetHTTPResponse("https://www.baidu.com/s?rn=100&pn=0&wd=site:" + state.Domain +"&oq=site:" + state.Domain, state.Timeout) + if err != nil { + result.Error = err + ch <- result + return + } + + // Get the response body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + result.Error = err + ch <- result + return + } + src := string(body) + + re := regexp.MustCompile(`(?m)(\w+\.)+` + state.Domain) + match := re.FindAllString(src, -1) + for _, subdomain := range match { + + if state.Verbose == true { + if state.Color == true { + fmt.Printf("\n[%sBaidu%s] %s", helper.Red, helper.Reset, subdomain) + } else { + fmt.Printf("\n[Baidu] %s", subdomain) + } + } + + subdomains = append(subdomains, subdomain) + } + result.Subdomains = subdomains + result.Error = nil + ch <- result +} From 1bbc572f4a18e1dd456c40aa0abf6cc333391357 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 2 May 2018 13:50:56 +0200 Subject: [PATCH 2/2] added search recursion --- libsubfinder/sources/baidu/baidu.go | 95 ++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 28 deletions(-) diff --git a/libsubfinder/sources/baidu/baidu.go b/libsubfinder/sources/baidu/baidu.go index 3e5a851..beecffa 100644 --- a/libsubfinder/sources/baidu/baidu.go +++ b/libsubfinder/sources/baidu/baidu.go @@ -12,6 +12,11 @@ import ( "fmt" "io/ioutil" "regexp" + "time" + "math/rand" + "strconv" + "sort" + "net/url" "github.com/Ice3man543/subfinder/libsubfinder/helper" ) @@ -24,37 +29,71 @@ func Query(state *helper.State, ch chan helper.Result) { var result helper.Result result.Subdomains = subdomains - resp, err := helper.GetHTTPResponse("https://www.baidu.com/s?rn=100&pn=0&wd=site:" + state.Domain +"&oq=site:" + state.Domain, state.Timeout) - if err != nil { - result.Error = err - ch <- result - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - result.Error = err - ch <- result - return - } - src := string(body) - - re := regexp.MustCompile(`(?m)(\w+\.)+` + state.Domain) - match := re.FindAllString(src, -1) - for _, subdomain := range match { - - if state.Verbose == true { - if state.Color == true { - fmt.Printf("\n[%sBaidu%s] %s", helper.Red, helper.Reset, subdomain) - } else { - fmt.Printf("\n[Baidu] %s", subdomain) - } + min_iterations := 5 + max_iterations := 760 + search_query := "" + current_page := 0 + for current_iteration := 0; current_iteration <= max_iterations; current_iteration++ { + new_search_query := "site:" + state.Domain + if len(subdomains) > 0 { + new_search_query += " -site:www." + state.Domain + } + new_search_query = url.QueryEscape(new_search_query) + if search_query != new_search_query { + current_page = 0 + search_query = new_search_query } - subdomains = append(subdomains, subdomain) + resp, err := helper.GetHTTPResponse("https://www.baidu.com/s?rn=100&pn=" + strconv.Itoa(current_page) + "&wd=" + search_query +"&oq=" + search_query, state.Timeout) + if err != nil { + result.Error = err + ch <- result + return + } + + // Get the response body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + result.Error = err + ch <- result + return + } + src := string(body) + + re := regexp.MustCompile(`([a-z0-9]+\.)+` + state.Domain) + match := re.FindAllString(src, -1) + + new_subdomains_found := 0 + for _, subdomain := range match { + if sort.StringsAreSorted(subdomains) == false { + sort.Strings(subdomains) + } + + insert_index := sort.SearchStrings(subdomains, subdomain) + if insert_index < len(subdomains) && subdomains[insert_index] == subdomain { + continue + } + + if state.Verbose == true { + if state.Color == true { + fmt.Printf("\n[%sBaidu%s] %s", helper.Red, helper.Reset, subdomain) + } else { + fmt.Printf("\n[Baidu] %s", subdomain) + } + } + + subdomains = append(subdomains, subdomain) + new_subdomains_found++ + } + // If no new subdomains are found exits after min_iterations + if new_subdomains_found == 0 && current_iteration > min_iterations { + break + } + current_page++ + time.Sleep(time.Duration((3 +rand.Intn(5))) * time.Second) } + result.Subdomains = subdomains result.Error = nil ch <- result -} +} \ No newline at end of file