mirror of https://github.com/daffainfo/nuclei.git
uncover logic refactor to v0.0.4 (#3663)
* uncover logic refactor to v0.0.4 * remove deprecated import: stringsutildev
Tarun Koyalwar
GitHub
parent
abc1d595ac
commit
c62dc01f9f
|
|
@ -230,7 +230,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
|||
flagSet.StringSliceVarP(&options.UncoverEngine, "uncover-engine", "ue", nil, fmt.Sprintf("uncover search engine (%s) (default shodan)", uncover.GetUncoverSupportedAgents()), goflags.FileStringSliceOptions),
|
||||
flagSet.StringVarP(&options.UncoverField, "uncover-field", "uf", "ip:port", "uncover fields to return (ip,port,host)"),
|
||||
flagSet.IntVarP(&options.UncoverLimit, "uncover-limit", "ul", 100, "uncover results to return"),
|
||||
flagSet.IntVarP(&options.UncoverDelay, "uncover-delay", "ucd", 1, "delay between uncover query requests in seconds (0 to disable)"),
|
||||
flagSet.IntVarP(&options.UncoverRateLimit, "uncover-ratelimit", "ur", 60, "override ratelimit of engines with unknown ratelimit (default 60 req/min)"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("rate-limit", "Rate-Limit",
|
||||
|
|
|
|||
|
|
@ -26,8 +26,7 @@ require (
|
|||
github.com/projectdiscovery/interactsh v1.1.3
|
||||
github.com/projectdiscovery/rawhttp v0.1.11
|
||||
github.com/projectdiscovery/retryabledns v1.0.24
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.15
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 // indirect
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.16
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.4
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.5.0
|
||||
|
|
@ -78,8 +77,8 @@ require (
|
|||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
|
||||
github.com/projectdiscovery/sarif v0.0.1
|
||||
github.com/projectdiscovery/tlsx v1.0.9
|
||||
github.com/projectdiscovery/uncover v1.0.2
|
||||
github.com/projectdiscovery/utils v0.0.27
|
||||
github.com/projectdiscovery/uncover v1.0.4
|
||||
github.com/projectdiscovery/utils v0.0.28
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.92
|
||||
github.com/stretchr/testify v1.8.2
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1
|
||||
|
|
@ -241,7 +240,6 @@ require (
|
|||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/imdario/mergo v0.3.13 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
|
|
|
|||
15
v2/go.sum
15
v2/go.sum
|
|
@ -255,8 +255,6 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD
|
|||
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
|
||||
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
|
||||
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
|
||||
|
|
@ -440,19 +438,18 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
|
|||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
|
||||
github.com/projectdiscovery/retryabledns v1.0.24 h1:CbC0a1EcyRDBcGFHZDGfW5orkWkOCfa0mAMF060XJpI=
|
||||
github.com/projectdiscovery/retryabledns v1.0.24/go.mod h1:bCmv0neiqgemgmFChevfX2BgCxIp8sn5OnbwL1Gov9M=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhEymvnZXS2Wp2Zs4rWk/c=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.15/go.mod h1:+OzSFUv3sQcPt+MgbNx6X/Q3ESxqPUQSphqG5kxoIgI=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.16 h1:xrEK9VVkLf0eN4bYOTc2Pg0qRqz47g8T823vqpCIsl8=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.16/go.mod h1:9m76To4lNgBtVfqADzLxZg1wWajv6y/uYMWCOs1Olo8=
|
||||
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
|
||||
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
|
||||
github.com/projectdiscovery/tlsx v1.0.9 h1:StUwVpPG0lrXq6FvxbFarPM+fu9Vke1sOPoyJcBVOFA=
|
||||
github.com/projectdiscovery/tlsx v1.0.9/go.mod h1:4vGhl8lxUWjXuyoVvnL2ObMY/FSrb9Mpx5U2KaTV05A=
|
||||
github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
|
||||
github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
|
||||
github.com/projectdiscovery/uncover v1.0.4 h1:d0lUNcnqDMfG4gR/WP2+UoQKLVmLE7opBiwnmI7ir/0=
|
||||
github.com/projectdiscovery/uncover v1.0.4/go.mod h1:fvSHBhj8zZFEXGOhQIpPe5j5LtDXI7adOns1iFJ5TTY=
|
||||
github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0=
|
||||
github.com/projectdiscovery/utils v0.0.27 h1:HfLjy1ZuWWtGfLW5ixHcgWPqybuJhPltpHnHeBID2QQ=
|
||||
github.com/projectdiscovery/utils v0.0.27/go.mod h1:RpMyTA/KACEBK+Bo9mk9MgLH0Hfc+BZtSlZ4T9bjLBE=
|
||||
github.com/projectdiscovery/utils v0.0.28 h1:lFTXbW/SZX3vjmCjReCdvbi5Fj46gCU3egdp098tHzo=
|
||||
github.com/projectdiscovery/utils v0.0.28/go.mod h1:RpMyTA/KACEBK+Bo9mk9MgLH0Hfc+BZtSlZ4T9bjLBE=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.92 h1:IQgYS3xN6J9KKq1FjMVjso+gVchhPBqi/wzChYrr1JA=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.92/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
|
||||
"github.com/projectdiscovery/nuclei/v2/internal/installer"
|
||||
"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
|
||||
uncoverlib "github.com/projectdiscovery/uncover"
|
||||
updateutils "github.com/projectdiscovery/utils/update"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
|
|
@ -489,7 +490,14 @@ func (r *Runner) RunEnumeration() error {
|
|||
|
||||
// add the hosts from the metadata queries of loaded templates into input provider
|
||||
if r.options.Uncover && len(r.options.UncoverQuery) == 0 {
|
||||
ret := uncover.GetUncoverTargetsFromMetadata(store.Templates(), r.options.UncoverDelay, r.options.UncoverLimit, r.options.UncoverField)
|
||||
uncoverOpts := &uncoverlib.Options{
|
||||
Limit: r.options.UncoverLimit,
|
||||
MaxRetry: r.options.Retries,
|
||||
Timeout: r.options.Timeout,
|
||||
RateLimit: uint(r.options.UncoverRateLimit),
|
||||
RateLimitUnit: time.Minute, // default unit is minute
|
||||
}
|
||||
ret := uncover.GetUncoverTargetsFromMetadata(context.TODO(), store.Templates(), r.options.UncoverField, uncoverOpts)
|
||||
for host := range ret {
|
||||
r.hmapInputProvider.Set(host)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ package hybrid
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
|
@ -22,6 +23,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
uncoverlib "github.com/projectdiscovery/uncover"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
readerutil "github.com/projectdiscovery/utils/reader"
|
||||
|
|
@ -137,7 +139,16 @@ func (i *Input) initializeInputSources(opts *Options) error {
|
|||
}
|
||||
if options.Uncover && options.UncoverQuery != nil {
|
||||
gologger.Info().Msgf("Running uncover query against: %s", strings.Join(options.UncoverEngine, ","))
|
||||
ch, err := uncover.GetTargetsFromUncover(options.UncoverDelay, options.UncoverLimit, options.UncoverField, options.UncoverEngine, options.UncoverQuery)
|
||||
uncoverOpts := &uncoverlib.Options{
|
||||
Agents: options.UncoverEngine,
|
||||
Queries: options.UncoverQuery,
|
||||
Limit: options.UncoverLimit,
|
||||
MaxRetry: options.Retries,
|
||||
Timeout: options.Timeout,
|
||||
RateLimit: uint(options.UncoverRateLimit),
|
||||
RateLimitUnit: time.Minute, // default unit is minute
|
||||
}
|
||||
ch, err := uncover.GetTargetsFromUncover(context.TODO(), options.UncoverField, uncoverOpts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,247 +3,133 @@ package uncover
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
ucRunner "github.com/projectdiscovery/uncover/runner"
|
||||
"github.com/projectdiscovery/uncover/uncover"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/censys"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/criminalip"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/fofa"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/hunter"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/netlas"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/quake"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/shodan"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/shodanidb"
|
||||
"github.com/projectdiscovery/uncover/uncover/agent/zoomeye"
|
||||
"github.com/projectdiscovery/uncover"
|
||||
"github.com/projectdiscovery/uncover/sources"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
const maxConcurrentAgents = 50
|
||||
|
||||
// returns csv string of uncover supported agents
|
||||
func GetUncoverSupportedAgents() string {
|
||||
uncoverSupportedAgents := []string{"shodan", "shodan-idb", "fofa", "censys", "quake", "hunter", "zoomeye", "netlas", "criminalip"}
|
||||
return strings.Join(uncoverSupportedAgents, ",")
|
||||
u, _ := uncover.New(&uncover.Options{})
|
||||
return strings.Join(u.AllAgents(), ",")
|
||||
}
|
||||
|
||||
func GetTargetsFromUncover(delay, limit int, field string, engine, query []string) (chan string, error) {
|
||||
uncoverOptions := &ucRunner.Options{
|
||||
Provider: &ucRunner.Provider{},
|
||||
Delay: delay,
|
||||
Limit: limit,
|
||||
Query: query,
|
||||
Engine: engine,
|
||||
// GetTargetsFromUncover returns targets from uncover
|
||||
func GetTargetsFromUncover(ctx context.Context, outputFormat string, opts *uncover.Options) (chan string, error) {
|
||||
u, err := uncover.New(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, eng := range engine {
|
||||
err := loadKeys(eng, uncoverOptions)
|
||||
if err != nil {
|
||||
gologger.Error().Label("WRN").Msgf(err.Error())
|
||||
continue
|
||||
}
|
||||
resChan, err := u.Execute(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return getTargets(uncoverOptions, field)
|
||||
}
|
||||
|
||||
func GetUncoverTargetsFromMetadata(templates []*templates.Template, delay, limit int, field string) chan string {
|
||||
ret := make(chan string)
|
||||
var uqMap = make(map[string][]string)
|
||||
var eng, query string
|
||||
for _, template := range templates {
|
||||
for k, v := range template.Info.Metadata {
|
||||
switch k {
|
||||
case "shodan-query":
|
||||
eng = "shodan"
|
||||
case "fofa-query":
|
||||
eng = "fofa"
|
||||
case "censys-query":
|
||||
eng = "censys"
|
||||
case "quake-query":
|
||||
eng = "quake"
|
||||
case "hunter-query":
|
||||
eng = "hunter"
|
||||
case "zoomeye-query":
|
||||
eng = "zoomeye"
|
||||
case "netlas-query":
|
||||
eng = "netlas"
|
||||
case "criminalip-query":
|
||||
eng = "criminalip"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
query = fmt.Sprintf("%v", v)
|
||||
uqMap[eng] = append(uqMap[eng], query)
|
||||
}
|
||||
}
|
||||
keys := mapsutil.GetKeys(uqMap)
|
||||
gologger.Info().Msgf("Running uncover query against: %s", strings.Join(keys, ","))
|
||||
var wg sync.WaitGroup
|
||||
outputChan := make(chan string) // buffered channel
|
||||
go func() {
|
||||
for k, v := range uqMap {
|
||||
wg.Add(1)
|
||||
go func(engine, query []string) {
|
||||
ch, _ := GetTargetsFromUncover(delay, limit, field, engine, query)
|
||||
for c := range ch {
|
||||
ret <- c
|
||||
defer close(outputChan)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case res, ok := <-resChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
wg.Done()
|
||||
}([]string{k}, v)
|
||||
if res.Error != nil {
|
||||
// only log in verbose mode
|
||||
gologger.Verbose().Msgf("uncover: %v", res.Error)
|
||||
continue
|
||||
}
|
||||
outputChan <- processUncoverOutput(res, outputFormat)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
close(ret)
|
||||
}()
|
||||
return ret
|
||||
return outputChan, nil
|
||||
}
|
||||
|
||||
func getTargets(uncoverOptions *ucRunner.Options, field string) (chan string, error) {
|
||||
var rateLimiter *ratelimit.Limiter
|
||||
// create rateLimiter for uncover delay
|
||||
if uncoverOptions.Delay > 0 {
|
||||
rateLimiter = ratelimit.New(context.Background(), 1, time.Duration(uncoverOptions.Delay))
|
||||
} else {
|
||||
rateLimiter = ratelimit.NewUnlimited(context.Background())
|
||||
// processUncoverOutput returns output strign depending on uncover field
|
||||
func processUncoverOutput(result sources.Result, outputFormat string) string {
|
||||
if (result.IP == "" || result.Port == 0) && stringsutil.ContainsAny(outputFormat, "ip", "port") {
|
||||
// if ip or port is not present, fallback to using host
|
||||
outputFormat = "host"
|
||||
}
|
||||
var agents []uncover.Agent
|
||||
// declare clients
|
||||
for _, engine := range uncoverOptions.Engine {
|
||||
var (
|
||||
agent uncover.Agent
|
||||
err error
|
||||
)
|
||||
switch engine {
|
||||
case "shodan":
|
||||
agent, err = shodan.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "censys":
|
||||
agent, err = censys.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "fofa":
|
||||
agent, err = fofa.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "shodan-idb":
|
||||
agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "quake":
|
||||
agent, err = quake.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "hunter":
|
||||
agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "zoomeye":
|
||||
agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "netlas":
|
||||
agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
case "criminalip":
|
||||
agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: rateLimiter})
|
||||
default:
|
||||
err = errors.Errorf("%s unknown uncover agent type", engine)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
agents = append(agents, agent)
|
||||
}
|
||||
// enumerate
|
||||
swg := sizedwaitgroup.New(maxConcurrentAgents)
|
||||
ret := make(chan string)
|
||||
go func() {
|
||||
for _, q := range uncoverOptions.Query {
|
||||
uncoverQuery := &uncover.Query{
|
||||
Query: q,
|
||||
Limit: uncoverOptions.Limit,
|
||||
replacer := strings.NewReplacer(
|
||||
"ip", result.IP,
|
||||
"host", result.Host,
|
||||
"port", fmt.Sprint(result.Port),
|
||||
"url", result.Url,
|
||||
)
|
||||
return replacer.Replace(outputFormat)
|
||||
}
|
||||
|
||||
// GetUncoverTargetsFromMetadata returns targets from uncover metadata
|
||||
func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.Template, outputFormat string, opts *uncover.Options) chan string {
|
||||
// contains map[engine]queries
|
||||
queriesMap := make(map[string][]string)
|
||||
for _, template := range templates {
|
||||
innerLoop:
|
||||
for k, v := range template.Info.Metadata {
|
||||
if !strings.HasSuffix(k, "-query") {
|
||||
// this is not a query
|
||||
// query keys are like shodan-query, fofa-query, etc
|
||||
continue innerLoop
|
||||
}
|
||||
for _, agent := range agents {
|
||||
swg.Add()
|
||||
go func(agent uncover.Agent, uncoverQuery *uncover.Query) {
|
||||
defer swg.Done()
|
||||
keys := uncoverOptions.Provider.GetKeys()
|
||||
session, err := uncover.NewSession(&keys, uncoverOptions.Retries, uncoverOptions.Timeout)
|
||||
if err != nil {
|
||||
gologger.Error().Label(agent.Name()).Msgf("couldn't create uncover new session: %s", err)
|
||||
}
|
||||
ch, err := agent.Query(session, uncoverQuery)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("%s", err)
|
||||
engine := strings.TrimSuffix(k, "-query")
|
||||
if queriesMap[engine] == nil {
|
||||
queriesMap[engine] = []string{}
|
||||
}
|
||||
queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v))
|
||||
}
|
||||
}
|
||||
keys := mapsutil.GetKeys(queriesMap)
|
||||
gologger.Info().Msgf("Running uncover queries from template against: %s", strings.Join(keys, ","))
|
||||
result := make(chan string, runtime.NumCPU())
|
||||
go func() {
|
||||
defer close(result)
|
||||
// unfortunately uncover doesn't support execution of map[engine]queries
|
||||
// if queries are given they are executed against all engines which is not what we want
|
||||
// TODO: add support for map[engine]queries in uncover
|
||||
// Note below implementation is intentionally sequential to avoid burning all the API keys
|
||||
counter := 0
|
||||
|
||||
for eng, queries := range queriesMap {
|
||||
// create new uncover options for each engine
|
||||
uncoverOpts := &uncover.Options{
|
||||
Agents: []string{eng},
|
||||
Queries: queries,
|
||||
Limit: opts.Limit,
|
||||
MaxRetry: opts.MaxRetry,
|
||||
Timeout: opts.Timeout,
|
||||
RateLimit: opts.RateLimit,
|
||||
RateLimitUnit: opts.RateLimitUnit,
|
||||
}
|
||||
ch, err := GetTargetsFromUncover(ctx, outputFormat, uncoverOpts)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not get targets using %v engine from uncover: %s", eng, err)
|
||||
return
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case res, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for result := range ch {
|
||||
replacer := strings.NewReplacer(
|
||||
"ip", result.IP,
|
||||
"host", result.Host,
|
||||
"port", fmt.Sprint(result.Port),
|
||||
)
|
||||
ret <- replacer.Replace(field)
|
||||
result <- res
|
||||
counter++
|
||||
if opts.Limit > 0 && counter >= opts.Limit {
|
||||
return
|
||||
}
|
||||
}(agent, uncoverQuery)
|
||||
}
|
||||
}
|
||||
}
|
||||
swg.Wait()
|
||||
close(ret)
|
||||
}()
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func loadKeys(engine string, options *ucRunner.Options) error {
|
||||
switch engine {
|
||||
case "fofa":
|
||||
if email, exists := os.LookupEnv("FOFA_EMAIL"); exists {
|
||||
if key, exists := os.LookupEnv("FOFA_KEY"); exists {
|
||||
options.Provider.Fofa = append(options.Provider.Fofa, fmt.Sprintf("%s:%s", email, key))
|
||||
} else {
|
||||
return errors.New("missing FOFA_KEY env variable")
|
||||
}
|
||||
} else {
|
||||
return errors.Errorf("FOFA_EMAIL & FOFA_KEY env variables are not configured")
|
||||
}
|
||||
case "shodan":
|
||||
if key, exists := os.LookupEnv("SHODAN_API_KEY"); exists {
|
||||
options.Provider.Shodan = append(options.Provider.Shodan, key)
|
||||
} else {
|
||||
return errors.Errorf("SHODAN_API_KEY env variable is not configured")
|
||||
}
|
||||
case "censys":
|
||||
if id, exists := os.LookupEnv("CENSYS_API_ID"); exists {
|
||||
if secret, exists := os.LookupEnv("CENSYS_API_SECRET"); exists {
|
||||
options.Provider.Censys = append(options.Provider.Censys, fmt.Sprintf("%s:%s", id, secret))
|
||||
} else {
|
||||
return errors.New("missing CENSYS_API_SECRET env variable")
|
||||
}
|
||||
} else {
|
||||
return errors.Errorf("CENSYS_API_ID & CENSYS_API_SECRET env variable is not configured")
|
||||
}
|
||||
case "hunter":
|
||||
if key, exists := os.LookupEnv("HUNTER_API_KEY"); exists {
|
||||
options.Provider.Hunter = append(options.Provider.Hunter, key)
|
||||
} else {
|
||||
return errors.Errorf("HUNTER_API_KEY env variable is not configured")
|
||||
}
|
||||
case "zoomeye":
|
||||
if key, exists := os.LookupEnv("ZOOMEYE_API_KEY"); exists {
|
||||
options.Provider.ZoomEye = append(options.Provider.ZoomEye, key)
|
||||
} else {
|
||||
return errors.Errorf("ZOOMEYE_API_KEY env variable is not configured")
|
||||
}
|
||||
case "quake":
|
||||
if key, exists := os.LookupEnv("QUAKE_TOKEN"); exists {
|
||||
options.Provider.Quake = append(options.Provider.Quake, key)
|
||||
} else {
|
||||
return errors.Errorf("QUAKE_TOKEN env variable is not configured")
|
||||
}
|
||||
case "netlas":
|
||||
if key, exists := os.LookupEnv("NETLAS_API_KEY"); exists {
|
||||
options.Provider.Netlas = append(options.Provider.Netlas, key)
|
||||
} else {
|
||||
return errors.Errorf("NETLAS_API_KEY env variable is not configured")
|
||||
}
|
||||
case "criminalip":
|
||||
if key, exists := os.LookupEnv("CRIMINALIP_API_KEY"); exists {
|
||||
options.Provider.CriminalIP = append(options.Provider.CriminalIP, key)
|
||||
} else {
|
||||
return errors.Errorf("CRIMINALIP_API_KEY env variable is not configured")
|
||||
}
|
||||
default:
|
||||
return errors.Errorf("unknown uncover agent")
|
||||
}
|
||||
return nil
|
||||
return result
|
||||
}
|
||||
|
|
|
|||
|
|
@ -321,7 +321,7 @@ type Options struct {
|
|||
// Uncover search limit
|
||||
UncoverLimit int
|
||||
// Uncover search delay
|
||||
UncoverDelay int
|
||||
UncoverRateLimit int
|
||||
// ScanAllIPs associated to a dns record
|
||||
ScanAllIPs bool
|
||||
// IPVersion to scan (4,6)
|
||||
|
|
|
|||
Loading…
Reference in New Issue