added intelx source which also supports https://phonebook.cz/

.gitignore

@@ -1,3 +1,4 @@
 .DS_Store
 cmd/subfinder/subfinder
-vendor/
+vendor/
+.idea

config.yaml

@@ -25,6 +25,7 @@ sources:
   - entrust
   - googleter
   - hackertarget
+  - intelx
   - ipv4info
   - passivetotal
   - rapiddns
@@ -41,19 +42,21 @@ sources:
   - zoomeye
 censys:
   - <key-here>
-binaryedge: 
+binaryedge:
   - <key-here>
-certspotter: 
+certspotter:
   - <key-here>
 facebook: []
-passivetotal: 
+intelx:
+  - <hostname:key-here>
+passivetotal:
   - <email:key-here>
-securitytrails: 
+securitytrails:
   - <key-here>
 urlscan: []
-virustotal: 
+virustotal:
   - <key-here>
-chaos: 
+chaos:
   - <key-here>
 spyse:
   - <key-here>

go.sum

@@ -1,3 +1,4 @@
+github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 h1:P5U+E4x5OkVEKQDklVPmzs71WM56RTTRqV4OrDC//Y4=
 github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -22,6 +23,7 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc=
 github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g=

pkg/passive/sources.go

@@ -16,6 +16,7 @@ import (
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget"
+	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/intelx"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/ipv4info"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns"
@@ -49,6 +50,7 @@ var DefaultSources = []string{
 	"entrust",
 	"hackertarget",
 	"ipv4info",
+	"intelx",
 	"passivetotal",
 	"rapiddns",
 	"securitytrails",
@@ -116,6 +118,8 @@ func (a *Agent) addSources(sources []string) {
 			a.sources[source] = &hackertarget.Source{}
 		case "ipv4info":
 			a.sources[source] = &ipv4info.Source{}
+		case "intelx":
+			a.sources[source] = &intelx.Source{}
 		case "passivetotal":
 			a.sources[source] = &passivetotal.Source{}
 		case "rapiddns":

pkg/runner/config.go

@@ -24,10 +24,11 @@ type ConfigFile struct {
 	Certspotter    []string `yaml:"certspotter"`
 	Chaos          []string `yaml:"chaos"`
 	DNSDB          []string `yaml:"dnsdb"`
+	IntelX         []string `yaml:"intelx"`
 	PassiveTotal   []string `yaml:"passivetotal"`
 	SecurityTrails []string `yaml:"securitytrails"`
 	Shodan         []string `yaml:"shodan"`
-	Spyse        	 []string `yaml:"spyse"`
+	Spyse          []string `yaml:"spyse"`
 	URLScan        []string `yaml:"urlscan"`
 	Virustotal     []string `yaml:"virustotal"`
 	ZoomEye        []string `yaml:"zoomeye"`
@@ -118,6 +119,15 @@ func (c ConfigFile) GetKeys() subscraping.Keys {
 		keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
 	}
 
+	if len(c.IntelX) > 0 {
+		intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))]
+		parts := strings.Split(intelxKeys, ":")
+		if len(parts) == 2 {
+			keys.IntelXHost = parts[0]
+			keys.IntelXKey = parts[1]
+		}
+	}
+
 	if len(c.PassiveTotal) > 0 {
 		passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))]
 		parts := strings.Split(passiveTotalKeys, ":")

pkg/subscraping/sources/intelx/intelx.go

@@ -0,0 +1,121 @@
+package intelx
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/projectdiscovery/subfinder/pkg/subscraping"
+	"io/ioutil"
+	"net/http"
+)
+
+type searchResponseType struct {
+	Id     string `json:"id"`
+	Status int    `json:"status"`
+}
+
+type selectorType struct {
+	Selectvalue string `json:"selectorvalue"`
+}
+
+type searchResultType struct {
+	Selectors []selectorType `json:"selectors"`
+	Status    int            `json:"status"`
+}
+
+type requestBody struct {
+	Term       string
+	Maxresults int
+	Media      int
+	Target     int
+	Terminate  []int
+	Timeout    int
+}
+
+type Source struct{}
+
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+
+	results := make(chan subscraping.Result)
+
+	go func() {
+		if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" {
+			fmt.Println(session.Keys)
+			close(results)
+			return
+		}
+
+		search_url := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey)
+
+		reqBody := requestBody{
+			Term:       domain,
+			Maxresults: 100000,
+			Media:      0,
+			Target:     1,
+			Timeout:    20,
+		}
+
+		body, err := json.Marshal(reqBody)
+
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			close(results)
+			return
+		}
+
+		resp, err := http.Post(search_url, "application/json", bytes.NewBuffer(body))
+
+		var response searchResponseType
+
+		err = jsoniter.NewDecoder(resp.Body).Decode(&response)
+
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			close(results)
+			return
+		}
+
+		results_url := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.Id)
+
+		var status = 0
+
+		for status == 0 {
+
+			resp, err = session.Get(ctx, results_url, "", map[string]string{})
+			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
+				close(results)
+				return
+			}
+			var response searchResultType
+			err = jsoniter.NewDecoder(resp.Body).Decode(&response)
+
+			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				close(results)
+				return
+			}
+
+			body, err = ioutil.ReadAll(resp.Body)
+
+			status = response.Status
+
+			for _, hostname := range response.Selectors {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}
+
+			}
+
+		}
+
+		close(results)
+	}()
+
+	return results
+}
+
+func (s *Source) Name() string {
+	return "intelx"
+}

pkg/subscraping/types.go

@@ -35,6 +35,8 @@ type Keys struct {
 	Certspotter          string `json:"certspotter"`
 	Chaos                string `json:"chaos"`
 	DNSDB                string `json:"dnsdb"`
+	IntelXHost           string `json:"intelXHost"`
+	IntelXKey            string `json:"intelXKey"`
 	PassiveTotalUsername string `json:"passivetotal_username"`
 	PassiveTotalPassword string `json:"passivetotal_password"`
 	Securitytrails       string `json:"securitytrails"`