new source dnsrepo

v2/pkg/passive/sources.go

@@ -16,7 +16,9 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
@@ -38,7 +40,6 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
-	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
 )
 
 // DefaultSources contains the list of fast sources used by default.
@@ -67,6 +68,7 @@ var DefaultSources = []string{
 	"virustotal",
 	"fofa",
 	"fullhunt",
+	"dnsrepo",
 }
 
 // DefaultRecursiveSources contains list of default recursive sources
@@ -123,6 +125,7 @@ var DefaultAllSources = []string{
 	"zoomeyeapi",
 	"fofa",
 	"fullhunt",
+	"dnsrepo",
 }
 
 // Agent is a struct for running passive subdomain enumeration
@@ -221,6 +224,8 @@ func (a *Agent) addSources(sources []string) {
 			a.sources[source] = &fofa.Source{}
 		case "fullhunt":
 			a.sources[source] = &fullhunt.Source{}
+		case "dnsrepo":
+			a.sources[source] = &dnsrepo.Source{}
 		}
 	}
 }

v2/pkg/runner/config.go

@@ -51,6 +51,7 @@ type ConfigFile struct {
 	ZoomEyeApi     []string `yaml:"zoomeyeapi"`
 	Fofa           []string `yaml:"fofa"`
 	FullHunt       []string `json:"fullhunt"`
+	DnsRepo        []string `json:"dnsrepo"`
 	// Version indicates the version of subfinder installed.
 	Version string `yaml:"subfinder-version"`
 }
@@ -219,5 +220,8 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
 	if len(c.FullHunt) > 0 {
 		keys.FullHunt = c.FullHunt[rand.Intn(len(c.FullHunt))]
 	}
+	if len(c.DnsRepo) > 0 {
+		keys.DnsRepo = c.DnsRepo[rand.Intn(len(c.DnsRepo))]
+	}
 	return keys
 }

v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go

@@ -0,0 +1,53 @@
+package dnsrepo
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strings"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+)
+
+// Source is the passive scraping agent
+type Source struct{}
+
+var reNext = regexp.MustCompile(`<a href=[\"\'](\/\?domain=)[\"\']*.*>(<]+|.*?)?<\/a>`)
+var reSubNext = regexp.MustCompile(`[\"\'](\/\?domain=)\w+.*[\"\']`)
+
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+
+	go func() {
+		defer close(results)
+		resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/?search=%s", domain))
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+		responseData, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+		resp.Body.Close()
+		src := string(responseData)
+		for _, match := range reNext.FindAllStringSubmatch(src, len(src)) {
+			for _, subMatch := range reSubNext.FindAllStringSubmatch(match[0], len(match[0])) {
+				splt := strings.Split(subMatch[0], "=")[1]
+				splt = strings.Trim(splt, `".`)
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: splt}
+			}
+		}
+
+	}()
+	return results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "dnsrepo"
+}

v2/pkg/subscraping/types.go

@@ -66,6 +66,7 @@ type Keys struct {
 	FofaUsername         string   `json:"fofa_username"`
 	FofaSecret           string   `json:"fofa_secret"`
 	FullHunt             string   `json:"fullhunt"`
+	DnsRepo              string   `json:"dnsrepo"`
 }
 
 // Result is a result structure returned by a source