Merge pull request #187 from projectdiscovery/v2.0

Updated README, added dep
master
Ice3man 2019-12-05 03:20:04 -08:00 committed by GitHub
commit 690d7e0974
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 314 additions and 105 deletions

3
.gitignore vendored
View File

@ -1,2 +1,3 @@
.DS_Store
cmd/subfinder/subfinder
cmd/subfinder/subfinder
vendor/

View File

@ -1,5 +1,5 @@
# Build Container
FROM golang:1.11.4-alpine3.7 AS build-env
FROM golang:1.13.4-alpine3.10 AS build-env
RUN apk add --no-cache --upgrade git openssh-client ca-certificates
RUN go get -u github.com/golang/dep/cmd/dep
WORKDIR /go/src/app
@ -9,6 +9,6 @@ COPY Gopkg.toml Gopkg.lock ./
RUN dep ensure -vendor-only -v
# Install
RUN go get -u github.com/projectdiscovery/subfinder
RUN go get -u github.com/projectdiscovery/subfinder/cmd/subfinder
ENTRYPOINT ["subfinder"]

173
Gopkg.lock generated Normal file
View File

@ -0,0 +1,173 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = "UT"
revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
version = "v1.1.1"
[[projects]]
digest = "1:beb5b4f42a25056f0aa291b5eadd21e2f2903a05d15dfe7caf7eaee7e12fa972"
name = "github.com/json-iterator/go"
packages = ["."]
pruneopts = "UT"
revision = "03217c3e97663914aec3faafde50d081f197a0a2"
version = "v1.1.8"
[[projects]]
digest = "1:31e761d97c76151dde79e9d28964a812c46efc5baee4085b86f68f0c654450de"
name = "github.com/konsorten/go-windows-terminal-sequences"
packages = ["."]
pruneopts = "UT"
revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e"
version = "v1.0.2"
[[projects]]
digest = "1:ba852707958e39694e7f64328008287892adf9b1aed0174480e2f50e0c23e521"
name = "github.com/logrusorgru/aurora"
packages = ["."]
pruneopts = "UT"
revision = "21d75270181e0436fee7bd58b991c212cf309068"
version = "v2.0"
[[projects]]
digest = "1:4f6be1cc9a78ba558c6d14f0c8e65ca644666ddfe753f53e54563d427ef39259"
name = "github.com/m-mizutani/urlscan-go"
packages = ["urlscan"]
pruneopts = "UT"
revision = "21d37c8d3d34d514f2ef49db9b59cc94f335e9c3"
version = "v1.0.0"
[[projects]]
digest = "1:311658edb63e7c43bce1fed43495d75f3c0d6ddc801719b428b873b5049c683f"
name = "github.com/miekg/dns"
packages = ["."]
pruneopts = "UT"
revision = "1e224ff5dead8366ed6fcdcb832794be42e73f0e"
version = "v1.1.22"
[[projects]]
digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563"
name = "github.com/modern-go/concurrent"
packages = ["."]
pruneopts = "UT"
revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
version = "1.0.3"
[[projects]]
digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855"
name = "github.com/modern-go/reflect2"
packages = ["."]
pruneopts = "UT"
revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd"
version = "1.0.1"
[[projects]]
digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = "UT"
revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
version = "v0.8.1"
[[projects]]
digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = "UT"
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
digest = "1:2e76a73cb51f42d63a2a1a85b3dc5731fd4faf6821b434bd0ef2c099186031d6"
name = "github.com/rs/xid"
packages = ["."]
pruneopts = "UT"
revision = "15d26544def341f036c5f8dca987a4cbe575032c"
version = "v1.2.1"
[[projects]]
digest = "1:04457f9f6f3ffc5fea48e71d62f2ca256637dee0a04d710288e27e05c8b41976"
name = "github.com/sirupsen/logrus"
packages = ["."]
pruneopts = "UT"
revision = "839c75faf7f98a33d445d181f3018b5c3409a45e"
version = "v1.4.2"
[[projects]]
digest = "1:8548c309c65a85933a625be5e7d52b6ac927ca30c56869fae58123b8a77a75e1"
name = "github.com/stretchr/testify"
packages = ["assert"]
pruneopts = "UT"
revision = "221dbe5ed46703ee255b1da0dec05086f5035f62"
version = "v1.4.0"
[[projects]]
branch = "master"
digest = "1:cd7e85fc3687e062714febdee3e8efeb00a413a2a620d28908fd0258261d2353"
name = "golang.org/x/crypto"
packages = [
"ed25519",
"ed25519/internal/edwards25519",
]
pruneopts = "UT"
revision = "86a70503ff7e82ffc18c7b0de83db35da4791e6a"
[[projects]]
branch = "master"
digest = "1:7182ef5a2af56ca8c788b291e7f9926b85c354eb0a93bc5a57ce19c99e42d74f"
name = "golang.org/x/net"
packages = [
"bpf",
"internal/iana",
"internal/socket",
"ipv4",
"ipv6",
]
pruneopts = "UT"
revision = "5ee1b9f4859acd2e99987ef94ec7a58427c53bef"
[[projects]]
branch = "master"
digest = "1:4dade6126937937ae62a2d345ceef96e1122e49bf3360a567ccd4177c0fb547b"
name = "golang.org/x/sys"
packages = [
"unix",
"windows",
]
pruneopts = "UT"
revision = "ce4227a45e2eb77e5c847278dcc6a626742e2945"
[[projects]]
digest = "1:b75b3deb2bce8bc079e16bb2aecfe01eb80098f5650f9e93e5643ca8b7b73737"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = "UT"
revision = "1f64d6156d11335c3f22d9330b0ad14fc1e789ce"
version = "v2.2.7"
[[projects]]
branch = "v3"
digest = "1:3f3455f22627a44aab35023af8336ffc40df4d6180ef56bfceb7f51ecada3ef2"
name = "gopkg.in/yaml.v3"
packages = ["."]
pruneopts = "UT"
revision = "4206685974f28e3178b35fa198a59899aa4dee3a"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/json-iterator/go",
"github.com/logrusorgru/aurora",
"github.com/m-mizutani/urlscan-go/urlscan",
"github.com/miekg/dns",
"github.com/rs/xid",
"github.com/stretchr/testify/assert",
"gopkg.in/yaml.v3",
]
solver-name = "gps-cdcl"
solver-version = 1

58
Gopkg.toml Normal file
View File

@ -0,0 +1,58 @@
# Gopkg.toml example
#
# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]
name = "github.com/json-iterator/go"
version = "1.1.8"
[[constraint]]
name = "github.com/logrusorgru/aurora"
version = "2.0.0"
[[constraint]]
name = "github.com/m-mizutani/urlscan-go"
version = "1.0.0"
[[constraint]]
name = "github.com/miekg/dns"
version = "1.1.22"
[[constraint]]
name = "github.com/rs/xid"
version = "1.2.1"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.4.0"
[[constraint]]
branch = "v3"
name = "gopkg.in/yaml.v3"
[prune]
go-tests = true
unused-packages = true

176
README.md
View File

@ -1,14 +1,13 @@
# SubFinder
# subfinder
[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/subfinder)](https://goreportcard.com/report/github.com/projectdiscovery/subfinder)
[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/subfinder/issues)
SubFinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and has been aimed as a successor to sublist3r project. SubFinder uses Passive Sources, Search Engines, Pastebins, Internet Archives, etc to find subdomains and then it uses a permutation module inspired by altdns to generate permutations and resolve them quickly using a powerful bruteforcing engine. It can also perform plain bruteforce if needed. The tool is highly customizable, and the code is built with a modular approach in mind making it easy to add functionalities and remove errors.
Subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and is optimized for speed. Subfinder is built for doing one thing only - passive subdomain enumeration, and it does that very well.
We have designed SubFinder to comply with all passive sources licenses, and usage restrictions, as well as maintained a consistently passive model to make it useful to both penetration testers and bug bounty hunters alike.
# Resources
- [Full Documentation](https://github.com/subfinder/documentation)
- [Features](#features)
- [Usage](#usage)
- [Installation Instuctions (direct)](#direct-installation)
@ -17,69 +16,59 @@ We have designed SubFinder to comply with all passive sources licenses, and usag
- [Post Installation Instructions](#post-installation-instructions)
- [Running SubFinder](#running-subfinder)
[![asciicast](https://raw.githubusercontent.com/Ice3man543/ice3man543.github.io/master/assets/asciinema.png)](https://asciinema.org/a/az7rub4RzDMqjI9dcPJpxm7zf)
# Features
- Simple and modular code base making it easy to contribute.
- Fast And Powerful Bruteforcing Module
- Powerful Permutation generation engine. (In Development)
- Many Passive Data Sources (31 At Present)
- Multiple Output formats
- Embeddable Project
- Raspberry Pi Support
> Ask, Archive.is, Baidu, Bing, Censys, CertDB, CertSpotter, Commoncrawl, CrtSH, DnsDB, DNSDumpster, Dnstable, Dogpile, Entrust CT-Search, Exalead, FindSubdomains, GoogleTER, Hackertarget, IPv4Info, Netcraft, PassiveTotal, PTRArchive, Riddler, SecurityTrails, SiteDossier, Shodan, ThreatCrowd, ThreatMiner, Virustotal, WaybackArchive, Yahoo
***We ensure that we abide by the terms and conditions of all sources that we query. For this reason we don't perform scraping on any site that doesn't allow it.***
- Fast And Powerful Resolution and wildcard elimination module
- Curated passive sources to maximize results (26 Sources as of now)
- Multiple Output formats supported (Json, File, Stdout)
- Optimized for speed, very fast and lightweight on resources
- Stdin and stdout support for integrating in workflows
# Usage
```bash
./subfinder -h
subfinder -h
```
This will display help for the tool. Here are all the switches it supports.
| Flag | Description | Example |
|------|-------------|---------|
| -b | Use bruteforcing to find subdomains | ./subfinder -d example.com -b |
| -c | Don't show colored output | ./subfinder -c |
| -d | Domain to find subdomains for | ./subfinder -d example.com |
| -dL | List of domains to find subdomains for | ./subfinder -dL hosts.txt |
| -nW | Remove wildcard subdomains | ./subfinder -nW |
| -o | Name of the output file (Optional) | ./subfinder -o output.txt |
| -oT | Write output in Aquatone style JSON format (Required -nW) | ./subfinder -o output.txt -nW -oT |
| -oJ | Write output in JSON format | ./subfinder -o output.json -oJ |
| -oD | Output to directory (When using multiple hosts) | ./subfinder -oD ~/misc/out/ |
| -r | Comma-separated list of resolvers to use | ./subfinder -r 8.8.8.8,1.1.1.1 |
| -rL | File containing list of resolvers to use | ./subfinder -rL resolvers.txt |
| --recursive | Use recursive subdomain finding (default: false) | ./subfinder --recursive |
| --set-config | Sets a configuration option | ./subfinder --set-config example=something |
| --set-settings | Sets a setting option | ./subfinder --set-settings CensysPages=10 |
| --no-passive | Do not perform passive subdomain enumeration | ./subfinder -d freelancer.com --no-passive |
| --silent | Show only the subdomains found | ./subfinder --silent |
| --sources | Comma separated list of sources to use (optional) | ./subfinder --sources threatcrowd,virustotal |
| --exclude-sources | Comma separated list of sources not to use (optional) | ./subfinder --exclude-sources threatcrowd,virustotal |
| -t | Number of concurrent threads (Bruteforce) | ./subfinder -b -t 10 -w words.txt |
| --timeout | Seconds to wait until quitting connection | ./subfinder --timeout 10 |
| -v | Display verbose output | ./subfinder -v |
| -w | Wordlist for doing bruteforcing and permutation | ./subfinder -w words.txt |
| -config string | Configuration file for API Keys, etc | subfinder -config config.yaml |
| -d | Domain to find subdomains for | subfinder -d uber.com |
| -dL | File containing list of domains to enumerate | subfinder -d hackerone-hosts.txt |
| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
| -nC | Don't Use colors in output | subfinder -nC |
| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
| -o | File to write output to (optional) | subfinder -o output.txt |
| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
| -oI | Write output in Host,IP format | subfinder -oI |
| -oJ | Write output in JSON lines Format | subfinder -oJ |
| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt
| -silent | Show only subdomains in output | subfinder -silent |
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
| -v | Show Verbose output | subfinder -v |
# Installation Instructions
## Direct Installation
#### SubFinder requires go1.10+ to install successfully !
#### SubFinder requires go1.12+ to install successfully !
The installation is easy. Git clone the repo and run go build.
The installation is easy. Go get the repo
```bash
go get github.com/projectdiscovery/subfinder
go get -v github.com/projectdiscovery/subfinder/cmd/subfinder
```
## Upgrading
If you wish to upgrade the package you can use:
```bash
go get -u github.com/projectdiscovery/subfinder
go get -u -v github.com/projectdiscovery/subfinder/cmd/subfinder
```
## Running in a Docker Container
@ -97,8 +86,6 @@ docker run -it subfinder
```
> The above command is the same as running `-h`
***NOTE: Please follow the Post Install steps given after this to correctly configure the tool.***
For example, this runs the tool against uber.com and output the results to your host file system:
```bash
docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it subfinder -d uber.com > uber.com.txt
@ -106,40 +93,49 @@ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it subfinder -d u
## Post Installation Instructions
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. These following services do not work without an API key:
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
- [Virustotal](https://www.virustotal.com/)
- [Passivetotal](http://passivetotal.org/)
- [SecurityTrails](http://securitytrails.com/)
- [Censys](https://censys.io)
- [Riddler](https://riddler.io)
- [Binaryedge](https://binaryedge.io)
- [Shodan](https://shodan.io)
- [URLScan](https://urlscan.io)
These are the configuration options you have to specify via the command line.
```bash
VirustotalAPIKey
PassivetotalUsername
PassivetotalKey
SecurityTrailsKey
RiddlerEmail
RiddlerPassword
CensysUsername
CensysSecret
ShodanAPIKey
```
Theses values are stored in the $HOME/.config/subfinder/config.yaml file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.
Theses values are stored in the $HOME/.config/subfinder/config.json file which will be created when you run the tool for the first time. To configure the services to use an API key, you need to use the tool with --set-config option which will allow you to set a configuration option. For example:
For sources that require multiple keys, namely `Censys`, `Passivetotal`, they can be added by separating them via a colon (:).
```bash
./subfinder --set-config VirustotalAPIKey=0x41414141
./subfinder --set-config PassivetotalUsername=hacker,PassivetotalKey=supersecret
An example config file -
```yaml
resolvers:
- 1.1.1.1
- 1.0.0.1
sources:
- binaryedge
- bufferover
- censys
- passivetotal
- sitedossier
binaryedge:
- 0bf8919b-aab9-42e4-9574-d3b639324597
- ac244e2f-b635-4581-878a-33f4e79a2c13
censys:
- ac244e2f-b635-4581-878a-33f4e79a2c13:dd510d6e-1b6e-4655-83f6-f347b363def9
certspotter: []
passivetotal:
- sampleemail@user.com:sample_password
securitytrails: []
shodan: []
```
If you are using docker, you need to first create your directory structure holding subfinder configuration file. You can either run the binary in your host system and let it create the directory structure of files, after which you can use --set-config flag to set the api values like before. Or you can run:
```bash
mkdir $HOME/.config/subfinder
cp config.json $HOME/.config/subfinder/config.json
nano $HOME/.config/subfinder/config.json
cp config.json $HOME/.config/subfinder/config.yaml
nano $HOME/.config/subfinder/config.yaml
```
After that, you can pass it as a volume using the following sample command.
```bash
@ -173,47 +169,29 @@ The -o command can be used to specify an output file.
./subfinder -d freelancer.com -o output.txt
```
You can also get output in json format using -oJ switch.
You can also get output in json format using -oJ switch. This switch saves the output in the JSON lines format.
If you use the JSON format, or the Host:IP format, then it becomes mandatory for you to use the **-nW** format as resolving is essential for these output format. By default, resolving the found subdomains is disabled.
```bash
> ./subfinder -d hackerone.com -o output.json -oJ -nW
> cat output.json
{"host":"www.hackerone.com","ip":"104.16.99.52"}
{"host":"mta-sts.hackerone.com","ip":"185.199.108.153"}
{"host":"hackerone.com","ip":"104.16.100.52"}
{"host":"mta-sts.managed.hackerone.com","ip":"185.199.110.153"}
```
The --silent switch can be used to show only subdomains found without any other info.
The --set-config switch can be used to set the value of any configuration option as explained above in the readme.
You can also pass some special settings for the tool through the command line by using --set-setting flag.
For example, you can pass the number of Censys pages to check using the following command.
```bash
./subfinder -d freelancer.com --sources censys --set-settings CensysPages=2 -v
```
For checking all pages returned by censys, you can use "all" option. Note, It is a string.
These are the settings currently supported
```bash
CensysPages
AskPages
BaiduPages
BingPages
```
For using bruteforcing capabilities, you can use -b flag with -w option to specify a wordlist.
```bash
./subfinder -d freelancer.com -b -w jhaddix_all.txt -t 100 --sources censys --set-settings CensysPages=2 -v
```
You can also write output in JSON format as used by Aquatone.
```bash
./subfinder -d freelancer.com -o result_aquatone.json -oT -nW -v
```
You can specify custom resolvers too.
```bash
./subfinder -d freelancer.com -o result_aquatone.json -oT -nW -v -r 8.8.8.8,1.1.1.1
./subfinder -d freelancer.com -o result_aquatone.json -oT -nW -v -rL resolvers.txt
```
If you want to do bruteforce only and do not want to run the passive subdomain discovery engine, you can use `--no-passive` flag which will not run passive discovery. You can use this functionality to run plain bruteforce, etc.
```bash
./subfinder -d freelancer.com --no-passive -v -b -w ~/dnslist.txt
./subfinder -d freelancer.com -o result_aquatone.json -nW -v -r 8.8.8.8,1.1.1.1
./subfinder -d freelancer.com -o result_aquatone.json -nW -v -rL resolvers.txt
```
# License
SubFinder is made with 🖤 by the [dev](https://github.com/orgs/subfinder/people) team.
See the **License** file for more details.
subfinder is made with 🖤 by the projectdiscovery team.
See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details.

View File

@ -1,7 +1,6 @@
### Thanks
Many people have contributed to subfinder making it a wonderful tool either by making a pull request fixing some stuff or giving generous
donations to support the furthur development of this tool. Here, we recognize these persons and thank them.
Many people have contributed to subfinder making it a wonderful tool either by making a pull request fixing some stuff or giving generous donations to support the furthur development of this tool. Here, we recognize these persons and thank them.
### Donations
- @infosec-au - Donating to the project

View File

@ -46,7 +46,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
flag.IntVar(&options.Threads, "t", 10, "Number of concurrent threads for active enumeration")
flag.IntVar(&options.Threads, "t", 10, "Number of concurrent goroutines for resolving")
flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out")
flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results")
flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for")