daffainfo
/
subfinder
mirror of https://github.com/daffainfo/subfinder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More flexibility

master
Ice3man543 2019-12-04 21:06:44 +05:30
parent a4c861c5c0
commit 146b7e4aa7
5 changed files with 47 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/passive/passive.go
View File

@ -30,7 +30,6 @@ func (a *Agent) EnumerateSubdomains(domain string, keys subscraping.Keys, timeou
for resp := range runner.Run(ctx, domain, session) {
results <- resp
}
fmt.Printf("%v source done\n", source)
wg.Done()
}(source, runner)
}

27
pkg/runner/initialize.go
View File

@ -11,13 +11,19 @@ import (
func (r *Runner) initializePassiveEngine() {
var sources, exclusions []string
// Append all the sources from YAML and CLI flag
sources = append(sources, r.options.YAMLConfig.Sources...)
sources = append(sources, strings.Split(r.options.Sources, ",")...)
// If there are any sources from CLI, only use them
// Otherwise, use the yaml file sources
if r.options.Sources != "" {
sources = append(sources, strings.Split(r.options.Sources, ",")...)
} else {
sources = append(sources, r.options.YAMLConfig.Sources...)
}
// Append all excluded sources from YAML and CLI flag
exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...)
exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...)
if r.options.ExcludeSources != "" {
exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...)
} else {
exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...)
}
r.passiveAgent = passive.New(sources, exclusions)
}
@ -35,9 +41,12 @@ func (r *Runner) initializeActiveEngine() error {
}
var resolvers []string
// Append all the resolvers read from the config as well as the CLI
resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...)
resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...)
if r.options.Resolvers != "" {
resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...)
} else {
resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...)
}
r.resolverClient.AppendResolversFromSlice(resolvers)
return nil
}

40
pkg/runner/options.go
View File

@ -11,26 +11,25 @@ import (
// Options contains the configuration options for tuning
// the subdomain enumeration process.
type Options struct {
Verbose bool // Verbose flag indicates whether to show verbose output or not
NoColor bool // No-Color disables the colored output
Threads int // Thread controls the number of threads to use for active enumerations
Timeout int // Timeout is the seconds to wait for sources to respond
MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
Domain string // Domain is the domain to find subdomains for
DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for
Output string // Output is the file to write found subdomains to.
OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given
JSON bool // JSON specifies whether to use json for output format or text file
HostIP bool // HostIP specifies whether to write subdomains in host:ip format
Silent bool // Silent suppresses any extra text and only writes subdomains to screen
Sources string // Sources contains a comma-separated list of sources to use for enumeration
ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process
Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results.
UnauthenticatedOnly bool // UnauthenticatedOnly specifies to run enumeration using only sources that do not require an API key
ConfigFile string // ConfigFile contains the location of the config file
Stdin bool // Stdin specifies whether stdin input was given to the process
Verbose bool // Verbose flag indicates whether to show verbose output or not
NoColor bool // No-Color disables the colored output
Threads int // Thread controls the number of threads to use for active enumerations
Timeout int // Timeout is the seconds to wait for sources to respond
MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
Domain string // Domain is the domain to find subdomains for
DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for
Output string // Output is the file to write found subdomains to.
OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given
JSON bool // JSON specifies whether to use json for output format or text file
HostIP bool // HostIP specifies whether to write subdomains in host:ip format
Silent bool // Silent suppresses any extra text and only writes subdomains to screen
Sources string // Sources contains a comma-separated list of sources to use for enumeration
ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process
Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results.
ConfigFile string // ConfigFile contains the location of the config file
Stdin bool // Stdin specifies whether stdin input was given to the process
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
}
@ -62,7 +61,6 @@ func ParseOptions() *Options {
flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use")
flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use")
flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
flag.BoolVar(&options.UnauthenticatedOnly, "uA", false, "Only use sources requiring no API Keys")
flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
flag.Parse()

4
pkg/runner/runner.go
View File

@ -75,7 +75,7 @@ func (r *Runner) EnumerateMultipleDomains(reader io.Reader) error {
continue
}
outputFile := path.Join(r.options.OutputDirectory, domain)
outputFile := path.Join(r.options.OutputDirectory, domain+".txt")
err := r.EnumerateSingleDomain(domain, outputFile)
if err != nil {
return err
@ -86,6 +86,8 @@ func (r *Runner) EnumerateMultipleDomains(reader io.Reader) error {
// EnumerateSingleDomain performs subdomain enumeration against a single domain
func (r *Runner) EnumerateSingleDomain(domain, output string) error {
log.Infof("Enumerating subdomains for %s\n", domain)
// Get the API keys for sources from the configuration
// and also create the active resolving engine for the domain.
keys := r.options.YAMLConfig.GetKeys()

7
pkg/subscraping/sources/commoncrawl/commoncrawl.go
View File

@ -2,6 +2,7 @@ package commoncrawl
import (
"context"
"errors"
"fmt"
"io/ioutil"
"strings"
@ -34,6 +35,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
if resp.StatusCode == 500 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("internal server error")}
close(results)
return
}
indexes := []indexResponse{}
err = jsoniter.NewDecoder(resp.Body).Decode(&indexes)
if err != nil {