From 06bfd8a087ab66c6a9a26d4692442793f032518f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 3 Dec 2019 16:47:19 +0530 Subject: [PATCH] Added new runner package, configs, yaml, etc --- pkg/runner/banners.go | 50 +++++++++++++++++++ pkg/runner/config.go | 71 +++++++++++++++++++++++++++ pkg/runner/config_test.go | 22 +++++++++ pkg/runner/doc.go | 3 ++ pkg/runner/options.go | 100 ++++++++++++++++++++++++++++++++++++++ pkg/runner/runner.go | 14 ++++++ pkg/runner/validate.go | 49 +++++++++++++++++++ 7 files changed, 309 insertions(+) create mode 100644 pkg/runner/banners.go create mode 100644 pkg/runner/config.go create mode 100644 pkg/runner/config_test.go create mode 100644 pkg/runner/doc.go create mode 100644 pkg/runner/options.go create mode 100644 pkg/runner/runner.go create mode 100644 pkg/runner/validate.go diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go new file mode 100644 index 0000000..f8ed6b6 --- /dev/null +++ b/pkg/runner/banners.go @@ -0,0 +1,50 @@ +package runner + +import "github.com/subfinder/subfinder/pkg/log" + +const banner = ` + _ __ _ _ +____ _| |__ / _(_)_ _ __| |___ _ _ +(_-< || | '_ \ _| | ' \/ _ / -_) '_| +/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2 +` + +// showBanner is used to show the banner to the user +func showBanner() { + log.Printf("%s\n", banner) + log.Printf("\t\tprojectdiscovery.io\n\n") +} + +func showNotice() { + log.Labelf("Use with caution. You are responsible for your actions\n") + log.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") + log.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n") +} + +// normalRunTasks runs the normal startup tasks +func (options *Options) normalRunTasks() { + configFile := ConfigFile{} + err := configFile.UnmarshalRead(options.ConfigFile) + if err != nil { + log.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err) + } + options.YAMLConfig = configFile +} + +// firstRunTasks runs some housekeeping tasks done +// when the program is ran for the first time +func (options *Options) firstRunTasks() { + // Unauthorized use text + showNotice() + + // Create the configuration file and display information + // about it to the user. + config := ConfigFile{} + + err := config.MarshalWrite(options.ConfigFile) + if err != nil { + log.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err) + } + + log.Infof("Configuration file saved to %s\n", options.ConfigFile) +} diff --git a/pkg/runner/config.go b/pkg/runner/config.go new file mode 100644 index 0000000..7899739 --- /dev/null +++ b/pkg/runner/config.go @@ -0,0 +1,71 @@ +package runner + +import ( + "os" + + "gopkg.in/yaml.v2" +) + +// ConfigFile contains the fields stored in the configuration file +type ConfigFile struct { + // Resolvers contains the list of resolvers to use while resolving + Resolvers []string `yaml:"resolvers,omitempty"` + // Sources contains a list of sources to use for enumeration + Sources []string `yaml:"sources,omitempty"` + // ExcludeSources contains the sources to not include in the enumeration process + ExcludeSources []string `yaml:"exclude-sources,omitempty"` + // API keys for different sources + Binaryedge []string `yaml:"binaryedge"` + Certspotter []string `yaml:"certspotter"` + Facebook []string `yaml:"facebook"` + PassiveTotal []string `yaml:"passivetotal"` + SecurityTrails []string `yaml:"securitytrails"` + Virustotal []string `yaml:"virustotal"` +} + +// GetConfigDirectory gets the subfinder config directory for a user +func GetConfigDirectory() (string, error) { + var config string + + directory, err := os.UserHomeDir() + if err != nil { + return config, err + } + config = directory + "/.config/subfinder" + // Create All directory for subfinder even if they exist + os.MkdirAll(config, os.ModePerm) + + return config, nil +} + +// CheckConfigExists checks if the config file exists in the given path +func CheckConfigExists(configPath string) bool { + if _, err := os.Stat(configPath); err == nil { + return true + } else if os.IsNotExist(err) { + return false + } + return false +} + +// MarshalWrite writes the marshalled yaml config to disk +func (c ConfigFile) MarshalWrite(file string) error { + f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + err = yaml.NewEncoder(f).Encode(&c) + f.Close() + return err +} + +// UnmarshalRead reads the unmarshalled config yaml file from disk +func (c ConfigFile) UnmarshalRead(file string) error { + f, err := os.Open(file) + if err != nil { + return err + } + err = yaml.NewDecoder(f).Decode(&c) + f.Close() + return err +} diff --git a/pkg/runner/config_test.go b/pkg/runner/config_test.go new file mode 100644 index 0000000..d086174 --- /dev/null +++ b/pkg/runner/config_test.go @@ -0,0 +1,22 @@ +package runner + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigGetDirectory(t *testing.T) { + directory, err := GetConfigDirectory() + if err != nil { + t.Fatalf("Expected nil got %v while getting home\n", err) + } + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("Expected nil got %v while getting dir\n", err) + } + config := home + "/.config/subfinder" + + assert.Equal(t, directory, config, "Directory and config should be equal") +} diff --git a/pkg/runner/doc.go b/pkg/runner/doc.go new file mode 100644 index 0000000..744872f --- /dev/null +++ b/pkg/runner/doc.go @@ -0,0 +1,3 @@ +// Package runner implements the mechanism to drive the +// subdomain enumeration process +package runner diff --git a/pkg/runner/options.go b/pkg/runner/options.go new file mode 100644 index 0000000..48ceb33 --- /dev/null +++ b/pkg/runner/options.go @@ -0,0 +1,100 @@ +package runner + +import ( + "flag" + "os" + "path" + + "github.com/subfinder/subfinder/pkg/log" +) + +// Options contains the configuration options for tuning +// the subdomain enumeration process. +type Options struct { + Verbose bool // Verbose flag indicates whether to show verbose output or not + NoColor bool // No-Color disables the colored output + Threads int // Thread controls the number of threads to use for active enumerations + Timeout int // Timeout is the seconds to wait for sources to respond + Domain string // Domain is the domain to find subdomains for + DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for + Output string // Output is the file to write found subdomains to. + OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given + JSON bool // JSON specifies whether to use json for output format or text file + HostIP bool // HostIP specifies whether to write subdomains in host:ip format + Silent bool // Silent suppresses any extra text and only writes subdomains to screen + Sources string // Sources contains a comma-separated list of sources to use for enumeration + sourcesSlice []string // unmarshaled list of sources + ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process + excludeSourcesSlice []string // unmarshaled list of excluded sources + Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration + resolversSlice []string // unmarshaled list of resolvers + ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration + RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results. + UnauthenticatedOnly bool // UnauthenticatedOnly specifies to run enumeration using only sources that do not require an API key + ConfigFile string // ConfigFile contains the location of the config file + Stdin bool // Stdin specifies whether stdin input was given to the process + + YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file +} + +// ParseOptions parses the command line flags provided by a user +func ParseOptions() *Options { + options := &Options{} + + config, err := GetConfigDirectory() + if err != nil { + // This should never be reached + log.Fatalf("Could not get user home: %s\n", err) + } + + flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output") + flag.BoolVar(&options.NoColor, "nC", true, "Don't Use colors in output") + flag.IntVar(&options.Threads, "t", 10, "Number of concurrent threads for active enumeration") + flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out") + flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for") + flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate") + flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") + flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)") + flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON Format") + flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host:IP format") + flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output") + flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use") + flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration") + flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use") + flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") + flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") + flag.BoolVar(&options.UnauthenticatedOnly, "uA", false, "Only use sources requiring no API Keys") + flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") + flag.Parse() + + // Check if stdin pipe was given + options.Stdin = hasStdin() + + // Read the inputs and configure the logging + options.configureOutput() + + // Show the user the banner + showBanner() + + // Check if the config file exists. If not, it means this is the + // first run of the program. Show the first run notices and initialize the config file. + // Else show the normal banners and read the yaml fiile to the config + if !CheckConfigExists(options.ConfigFile) { + options.firstRunTasks() + } else { + options.normalRunTasks() + } + + return options +} + +func hasStdin() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } + return true +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go new file mode 100644 index 0000000..d6e67f0 --- /dev/null +++ b/pkg/runner/runner.go @@ -0,0 +1,14 @@ +package runner + +// Runner is an instance of the subdomain enumeration +// client used to orchestrate the whole process. +type Runner struct { +} + +// NewRunner creates a new runner struct instance by parsing +// the configuration options, configuring sources, reading lists +// and setting up loggers, etc. +func NewRunner(options Options) *Runner { + + return &Runner{} +} diff --git a/pkg/runner/validate.go b/pkg/runner/validate.go new file mode 100644 index 0000000..065cece --- /dev/null +++ b/pkg/runner/validate.go @@ -0,0 +1,49 @@ +package runner + +import ( + "errors" + + "github.com/subfinder/subfinder/pkg/log" +) + +// validateOptions validates the configuration options passed +func (options *Options) validateOptions() error { + // Check if domain, list of domains, or stdin info was provided. + // If none was provided, then return. + if options.Domain == "" && options.DomainsFile == "" && !options.Stdin { + return errors.New("no input list provided") + } + + // Both verbose and silent flags were used + if options.Verbose && options.Silent { + return errors.New("both verbose and silent mode specified") + } + + // Validate threads and options + if options.Threads == 0 { + return errors.New("threads cannot be zero") + } + if options.Timeout == 0 { + return errors.New("timeout cannot be zero") + } + + // JSON cannot be used with hostIP + if options.JSON && options.HostIP { + return errors.New("hostip flag cannot be used with json flag") + } + return nil +} + +// configureOutput configures the output on the screen +func (options *Options) configureOutput() { + // If the user desires verbose output, show verbose output + if options.Verbose { + log.MaxLevel = log.Verbose + } + if options.NoColor { + log.UseColors = false + } + if options.Silent { + log.MaxLevel = log.Silent + } +}