Added new runner package, configs, yaml, etc

pkg/runner/banners.go

@@ -0,0 +1,50 @@
+package runner
+
+import "github.com/subfinder/subfinder/pkg/log"
+
+const banner = `
+        _     __ _         _         
+____  _| |__ / _(_)_ _  __| |___ _ _ 
+(_-< || | '_ \  _| | ' \/ _  / -_) '_|
+/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2
+`
+
+// showBanner is used to show the banner to the user
+func showBanner() {
+	log.Printf("%s\n", banner)
+	log.Printf("\t\tprojectdiscovery.io\n\n")
+}
+
+func showNotice() {
+	log.Labelf("Use with caution. You are responsible for your actions\n")
+	log.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	log.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
+}
+
+// normalRunTasks runs the normal startup tasks
+func (options *Options) normalRunTasks() {
+	configFile := ConfigFile{}
+	err := configFile.UnmarshalRead(options.ConfigFile)
+	if err != nil {
+		log.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
+	}
+	options.YAMLConfig = configFile
+}
+
+// firstRunTasks runs some housekeeping tasks done
+// when the program is ran for the first time
+func (options *Options) firstRunTasks() {
+	// Unauthorized use text
+	showNotice()
+
+	// Create the configuration file and display information
+	// about it to the user.
+	config := ConfigFile{}
+
+	err := config.MarshalWrite(options.ConfigFile)
+	if err != nil {
+		log.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
+	}
+
+	log.Infof("Configuration file saved to %s\n", options.ConfigFile)
+}

pkg/runner/config.go

@@ -0,0 +1,71 @@
+package runner
+
+import (
+	"os"
+
+	"gopkg.in/yaml.v2"
+)
+
+// ConfigFile contains the fields stored in the configuration file
+type ConfigFile struct {
+	// Resolvers contains the list of resolvers to use while resolving
+	Resolvers []string `yaml:"resolvers,omitempty"`
+	// Sources contains a list of sources to use for enumeration
+	Sources []string `yaml:"sources,omitempty"`
+	// ExcludeSources contains the sources to not include in the enumeration process
+	ExcludeSources []string `yaml:"exclude-sources,omitempty"`
+	// API keys for different sources
+	Binaryedge     []string `yaml:"binaryedge"`
+	Certspotter    []string `yaml:"certspotter"`
+	Facebook       []string `yaml:"facebook"`
+	PassiveTotal   []string `yaml:"passivetotal"`
+	SecurityTrails []string `yaml:"securitytrails"`
+	Virustotal     []string `yaml:"virustotal"`
+}
+
+// GetConfigDirectory gets the subfinder config directory for a user
+func GetConfigDirectory() (string, error) {
+	var config string
+
+	directory, err := os.UserHomeDir()
+	if err != nil {
+		return config, err
+	}
+	config = directory + "/.config/subfinder"
+	// Create All directory for subfinder even if they exist
+	os.MkdirAll(config, os.ModePerm)
+
+	return config, nil
+}
+
+// CheckConfigExists checks if the config file exists in the given path
+func CheckConfigExists(configPath string) bool {
+	if _, err := os.Stat(configPath); err == nil {
+		return true
+	} else if os.IsNotExist(err) {
+		return false
+	}
+	return false
+}
+
+// MarshalWrite writes the marshalled yaml config to disk
+func (c ConfigFile) MarshalWrite(file string) error {
+	f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	err = yaml.NewEncoder(f).Encode(&c)
+	f.Close()
+	return err
+}
+
+// UnmarshalRead reads the unmarshalled config yaml file from disk
+func (c ConfigFile) UnmarshalRead(file string) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	err = yaml.NewDecoder(f).Decode(&c)
+	f.Close()
+	return err
+}

pkg/runner/config_test.go

@@ -0,0 +1,22 @@
+package runner
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigGetDirectory(t *testing.T) {
+	directory, err := GetConfigDirectory()
+	if err != nil {
+		t.Fatalf("Expected nil got %v while getting home\n", err)
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		t.Fatalf("Expected nil got %v while getting dir\n", err)
+	}
+	config := home + "/.config/subfinder"
+
+	assert.Equal(t, directory, config, "Directory and config should be equal")
+}

pkg/runner/doc.go

@@ -0,0 +1,3 @@
+// Package runner implements the mechanism to drive the
+// subdomain enumeration process
+package runner

pkg/runner/options.go

@@ -0,0 +1,100 @@
+package runner
+
+import (
+	"flag"
+	"os"
+	"path"
+
+	"github.com/subfinder/subfinder/pkg/log"
+)
+
+// Options contains the configuration options for tuning
+// the subdomain enumeration process.
+type Options struct {
+	Verbose             bool     // Verbose flag indicates whether to show verbose output or not
+	NoColor             bool     // No-Color disables the colored output
+	Threads             int      // Thread controls the number of threads to use for active enumerations
+	Timeout             int      // Timeout is the seconds to wait for sources to respond
+	Domain              string   // Domain is the domain to find subdomains for
+	DomainsFile         string   // DomainsFile is the file containing list of domains to find subdomains for
+	Output              string   // Output is the file to write found subdomains to.
+	OutputDirectory     string   // OutputDirectory is the directory to write results to in case list of domains is given
+	JSON                bool     // JSON specifies whether to use json for output format or text file
+	HostIP              bool     // HostIP specifies whether to write subdomains in host:ip format
+	Silent              bool     // Silent suppresses any extra text and only writes subdomains to screen
+	Sources             string   // Sources contains a comma-separated list of sources to use for enumeration
+	sourcesSlice        []string // unmarshaled list of sources
+	ExcludeSources      string   // ExcludeSources contains the comma-separated sources to not include in the enumeration process
+	excludeSourcesSlice []string // unmarshaled list of excluded sources
+	Resolvers           string   // Resolvers is the comma-separated resolvers to use for enumeration
+	resolversSlice      []string // unmarshaled list of resolvers
+	ResolverList        string   // ResolverList is a text file containing list of resolvers to use for enumeration
+	RemoveWildcard      bool     // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results.
+	UnauthenticatedOnly bool     // UnauthenticatedOnly specifies to run enumeration using only sources that do not require an API key
+	ConfigFile          string   // ConfigFile contains the location of the config file
+	Stdin               bool     // Stdin specifies whether stdin input was given to the process
+
+	YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
+}
+
+// ParseOptions parses the command line flags provided by a user
+func ParseOptions() *Options {
+	options := &Options{}
+
+	config, err := GetConfigDirectory()
+	if err != nil {
+		// This should never be reached
+		log.Fatalf("Could not get user home: %s\n", err)
+	}
+
+	flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
+	flag.BoolVar(&options.NoColor, "nC", true, "Don't Use colors in output")
+	flag.IntVar(&options.Threads, "t", 10, "Number of concurrent threads for active enumeration")
+	flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out")
+	flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for")
+	flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate")
+	flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
+	flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)")
+	flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON Format")
+	flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host:IP format")
+	flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output")
+	flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use")
+	flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration")
+	flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use")
+	flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use")
+	flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
+	flag.BoolVar(&options.UnauthenticatedOnly, "uA", false, "Only use sources requiring no API Keys")
+	flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
+	flag.Parse()
+
+	// Check if stdin pipe was given
+	options.Stdin = hasStdin()
+
+	// Read the inputs and configure the logging
+	options.configureOutput()
+
+	// Show the user the banner
+	showBanner()
+
+	// Check if the config file exists. If not, it means this is the
+	// first run of the program. Show the first run notices and initialize the config file.
+	// Else show the normal banners and read the yaml fiile to the config
+	if !CheckConfigExists(options.ConfigFile) {
+		options.firstRunTasks()
+	} else {
+		options.normalRunTasks()
+	}
+
+	return options
+}
+
+func hasStdin() bool {
+	fi, err := os.Stdin.Stat()
+	if err != nil {
+		return false
+	}
+	if fi.Mode()&os.ModeNamedPipe == 0 {
+		return false
+	}
+	return true
+}

pkg/runner/runner.go

@@ -0,0 +1,14 @@
+package runner
+
+// Runner is an instance of the subdomain enumeration
+// client used to orchestrate the whole process.
+type Runner struct {
+}
+
+// NewRunner creates a new runner struct instance by parsing
+// the configuration options, configuring sources, reading lists
+// and setting up loggers, etc.
+func NewRunner(options Options) *Runner {
+
+	return &Runner{}
+}

pkg/runner/validate.go

@@ -0,0 +1,49 @@
+package runner
+
+import (
+	"errors"
+
+	"github.com/subfinder/subfinder/pkg/log"
+)
+
+// validateOptions validates the configuration options passed
+func (options *Options) validateOptions() error {
+	// Check if domain, list of domains, or stdin info was provided.
+	// If none was provided, then return.
+	if options.Domain == "" && options.DomainsFile == "" && !options.Stdin {
+		return errors.New("no input list provided")
+	}
+
+	// Both verbose and silent flags were used
+	if options.Verbose && options.Silent {
+		return errors.New("both verbose and silent mode specified")
+	}
+
+	// Validate threads and options
+	if options.Threads == 0 {
+		return errors.New("threads cannot be zero")
+	}
+	if options.Timeout == 0 {
+		return errors.New("timeout cannot be zero")
+	}
+
+	// JSON cannot be used with hostIP
+	if options.JSON && options.HostIP {
+		return errors.New("hostip flag cannot be used with json flag")
+	}
+	return nil
+}
+
+// configureOutput configures the output on the screen
+func (options *Options) configureOutput() {
+	// If the user desires verbose output, show verbose output
+	if options.Verbose {
+		log.MaxLevel = log.Verbose
+	}
+	if options.NoColor {
+		log.UseColors = false
+	}
+	if options.Silent {
+		log.MaxLevel = log.Silent
+	}
+}