daffainfo
/
subfinder
mirror of https://github.com/daffainfo/subfinder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #446 from projectdiscovery/dev 

v2.4.9 Preparation
master v2.4.9
Sandeep Singh 2021-09-26 00:02:29 +05:30 committed by GitHub
parent 27611ed38f d23471b698
commit 64bc34e76f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 598 additions and 512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/dependabot.yml vendored
View File

@ -11,6 +11,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
@ -20,6 +21,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
@ -29,6 +31,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
include: "scope"

26
.github/workflows/build-test.yml vendored Normal file
View File

@ -0,0 +1,26 @@
name: 🔨 Build Test
on:
push:
pull_request:
workflow_dispatch:
jobs:
build:
name: Test Builds
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Check out code
uses: actions/checkout@v2
- name: Test
run: go test .
working-directory: v2/cmd/subfinder/
- name: Build
run: go build .
working-directory: v2/cmd/subfinder/

38
.github/workflows/build.yaml vendored
View File

@ -1,38 +0,0 @@
name: Build
on:
push:
branches:
- master
pull_request:
jobs:
golangci-lint:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v2.5.2
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.33
args: --timeout 5m
working-directory: v2/
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Test
run: go test ./...
working-directory: v2/
- name: Build
run: go build .
working-directory: v2/cmd/subfinder/

38
.github/workflows/codeql-analysis.yml vendored Normal file
View File

@ -0,0 +1,38 @@
name: 🚨 CodeQL Analysis
on:
workflow_dispatch:
pull_request:
branches:
- dev
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'go' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@v1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1

17
.github/workflows/dockerhub-push-on-release.yml vendored
View File

@ -1,17 +0,0 @@
# dockerhub-push pushes docker build to dockerhub automatically
# on the creation of a new release
name: Publish to Dockerhub on creation of a new release
on:
release:
types: [published]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Publish to Dockerhub Registry
uses: elgohr/Publish-Docker-Github-Action@master
with:
name: projectdiscovery/subfinder
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

39
.github/workflows/dockerhub-push.yml vendored Normal file
View File

@ -0,0 +1,39 @@
name: 🌥 Docker Push
on:
release:
types: [published]
workflow_dispatch:
jobs:
docker:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get Github tag
id: meta
run: |
echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name)"
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
platforms: linux/amd64,linux/arm64,linux/arm
push: true
tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.tag }}

19
.github/workflows/lint-test.yml vendored Normal file
View File

@ -0,0 +1,19 @@
name: 🙏🏻 Lint Test
on:
push:
pull_request:
workflow_dispatch:
jobs:
lint:
name: Lint Test
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v2
with:
version: latest
args: --timeout 5m
working-directory: v2/

0
.github/workflows/release.yml → .github/workflows/release-binary.yml vendored
View File

122
.golangci.yml
View File

@ -1,122 +0,0 @@
linters-settings:
dupl:
threshold: 100
exhaustive:
default-signifies-exhaustive: false
# funlen:
# lines: 100
# statements: 50
goconst:
min-len: 2
min-occurrences: 2
gocritic:
enabled-tags:
- diagnostic
- experimental
- opinionated
- performance
- style
disabled-checks:
- dupImport # https://github.com/go-critic/go-critic/issues/845
- ifElseChain
# gocyclo:
# min-complexity: 15
goimports:
local-prefixes: github.com/golangci/golangci-lint
golint:
min-confidence: 0
gomnd:
settings:
mnd:
# don't include the "operation" and "assign"
checks: argument,case,condition,return
govet:
check-shadowing: true
settings:
printf:
funcs:
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
# lll:
# line-length: 140
maligned:
suggest-new: true
misspell:
locale: US
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
linters:
# please, do not use `enable-all`: it's deprecated and will be removed soon.
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
enable:
- bodyclose
- deadcode
- dogsled
- dupl
# - errcheck
- exhaustive
- gochecknoinits
- goconst
- gocritic
- gofmt
- goimports
- golint
- gomnd
- goprintffuncname
- gosimple
- govet
- ineffassign
- interfacer
- maligned
- misspell
- nakedret
- noctx
# - nolintlint
- rowserrcheck
- scopelint
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
# don't enable:
# - depguard
# - asciicheck
# - funlen
# - gochecknoglobals
# - gocognit
# - gocyclo
# - godot
# - godox
# - goerr113
# - gosec
# - lll
# - nestif
# - prealloc
# - testpackage
# - wsl
issues:
exclude-use-default: false
exclude:
# should have a package comment, unless it's in another file for this package (golint)
- 'in another file for this package'
# golangci.com configuration
# https://github.com/golangci/golangci/wiki/Configuration
service:
golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
prepare:
- echo "here I can run custom commands, but no preparation needed for this repo"

2
Dockerfile
View File

@ -1,6 +1,6 @@
# Build
FROM golang:1.17-alpine AS build-env
RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
# Release
FROM alpine:3.14

206
README.md
View File

@ -53,38 +53,41 @@ subfinder -h
```
This will display help for the tool. Here are all the switches it supports.
| Flag | Description | Example |
| ---------------- | ---------------------------------------------------------- | -------------------------------------- |
| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
| -config | Configuration file for API Keys, etc | subfinder -config config.yaml |
| -d | Domain to find subdomains for | subfinder -d uber.com |
| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt |
| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
| -nC | Don't Use colors in output | subfinder -nC |
| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
| -ls | List all available sources | subfinder -ls |
| -o | File to write output to (optional) | subfinder -o output.txt |
| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
| -oI | Write output in Host,IP format | subfinder -oI |
| -oJ | Write output in JSON lines Format | subfinder -oJ |
| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt |
| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive |
| -silent | Show only subdomains in output | subfinder -silent |
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
| -v | Show Verbose output | subfinder -v |
| -version | Show current program version | subfinder -version |
| Flag | Description | Example |
| ---------------- | ---------------------------------------------------------- | --------------------------------------------|
| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
| -b | IP address to be used as local bind | subfinder -b 172.16.0.1 |
| -config | Configuration file for API Keys, etc | subfinder -config config.yaml |
| -d | Domain to find subdomains for | subfinder -d uber.com |
| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt |
| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
| -nC | Don't Use colors in output | subfinder -nC |
| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
| -ls | List all available sources | subfinder -ls |
| -o | File to write output to (optional) | subfinder -o output.txt |
| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
| -oI | Write output in Host,IP format | subfinder -oI |
| -oJ | Write output in JSON lines Format | subfinder -oJ |
| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt |
| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive |
| -silent | Show only subdomains in output | subfinder -silent |
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
| -proxy | HTTP proxy to use with subfinder | subfinder -proxy http://localhost:3128 |
| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 |
| -v | Show Verbose output | subfinder -v |
| -version | Show current program version | subfinder -version |
# Installation
Subfinder requires **go1.14+** to install successfully. Run the following command to get the repo -
Subfinder requires **go1.17** to install successfully. Run the following command to get the repo -
```sh
▶ GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
```
@ -92,22 +95,7 @@ Subfinder requires **go1.14+** to install successfully. Run the following comman
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
- [Binaryedge](https://binaryedge.io)
- [Certspotter](https://sslmate.com/certspotter/api/)
- [Censys](https://censys.io)
- [Chaos](https://chaos.projectdiscovery.io)
- [DnsDB](https://api.dnsdb.info)
- [Github](https://github.com)
- [Intelx](https://intelx.io)
- [Passivetotal](http://passivetotal.org)
- [Recon.dev](https://recon.dev)
- [Robtex](https://www.robtex.com/api/)
- [SecurityTrails](http://securitytrails.com)
- [Shodan](https://shodan.io)
- [Spyse](https://spyse.com)
- [Threatbook](https://x.threatbook.cn/en)
- [Virustotal](https://www.virustotal.com)
- [Zoomeye](https://www.zoomeye.org)
[Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.so/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Recon.dev](https://recon.dev), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Spyse](https://spyse.com), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [Zoomeye](https://www.zoomeye.org)
Theses values are stored in the `$HOME/.config/subfinder/config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.
@ -144,63 +132,50 @@ github:
# Running Subfinder
To run the tool on a target, just use the following command.
```sh
▶ subfinder -d example.com
```
The verbose flag `v` can be used to display verbose information.
```
[bufferover] lutin.dima.example.com
[bufferover] izosimdima.example.com
[bufferover] glazkovdima.example.com
[bufferover] dengshima.example.com
[bufferover] wwwkima.example.com
[bufferover] proxima.example.com
[bufferover] mma.example.com
[bufferover] damidoma.example.com
[bufferover] nomerdoma.example.com
[bufferover] soti.croma.example.com
```
The `-o` command can be used to specify an output file.
```sh
▶ subfinder -d example.com -o output.txt
```
subfinder -d hackerone.com
To run the tool on a list of domains, `-dL` option can be used. This requires a directory to write the output files. Subdomains for each domain from the list are written in a text file in the directory specified by the `-oD` flag with their name being the domain name.
__ _____ __
_______ __/ /_ / __(_)___ ____/ /__ _____
/ ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
(__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9
```sh
▶ cat domains.txt
projectdiscovery.io
Use with caution. You are responsible for your actions
Developers assume no liability and are not responsible for any misuse or damage.
By using subfinder, you also agree to the terms of the APIs used.
[INF] Enumerating subdomains for hackerone.com
www.hackerone.com
support.hackerone.com
links.hackerone.com
api.hackerone.com
o1.email.hackerone.com
go.hackerone.com
3d.hackerone.com
resources.hackerone.com
a.ns.hackerone.com
b.ns.hackerone.com
mta-sts.hackerone.com
docs.hackerone.com
mta-sts.forwarding.hackerone.com
gslink.hackerone.com
hackerone.com
google.com
info.hackerone.com
mta-sts.managed.hackerone.com
events.hackerone.com
▶ subfinder -dL domains.txt -oD ~/path/to/output
▶ ls ~/path/to/output
hackerone.com.txt
google.com.txt
```
You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format.
If you use the JSON format, or the `Host:IP` format, then it becomes mandatory for you to use the **-nW** format as resolving is essential for these output format. By default, resolving the found subdomains is disabled.
```sh
▶ subfinder -d hackerone.com -o output.json -oJ -nW
▶ cat output.json
{"host":"www.hackerone.com","ip":"104.16.99.52"}
{"host":"mta-sts.hackerone.com","ip":"185.199.108.153"}
{"host":"hackerone.com","ip":"104.16.100.52"}
{"host":"mta-sts.managed.hackerone.com","ip":"185.199.110.153"}
[INF] Found 18 subdomains for hackerone.com in 3 seconds 672 milliseconds
```
The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host.
```sh
▶ echo hackerone.com | subfinder -silent | httpx -silent
echo hackerone.com | subfinder -silent | httpx -silent
http://hackerone.com
http://www.hackerone.com
@ -210,44 +185,53 @@ https://docs.hackerone.com
http://mta-sts.managed.hackerone.com
```
If your enterprise uses source routing to choose network output, or your computer has many public network interfaces (eg: public Wi-Fi + 4G connection + Ethernet Wire + VPN), you might want to choose your output network by binding IP source. In this case, you can use `-b` option.
In the example below, we have 3 network interfaces able to communicate to the Internet through 3 different outputs. Each output is chosen by binding one source IP with `-b` option.
```console
ip addr
[...]
3: wlp3s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether e8:b1:fc:50:90:a0 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.87/24 brd 192.168.1.255 scope global dynamic noprefixroute wlp3s0
valid_lft 62538sec preferred_lft 62538sec
4: tun0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN group default qlen 100
link/none
inet 192.168.254.70 peer 192.168.254.69/32 scope global tun0
valid_lft forever preferred_lft forever
5: enx0c5b8f279a64: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 0c:5b:8f:a5:63:25 brd ff:ff:ff:ff:ff:ff
inet 192.168.8.100/24 brd 192.168.8.255 scope global dynamic noprefixroute enx0c5b8f279a64
valid_lft 86396sec preferred_lft 86396sec
```
```sh
subfinder -d hackerone.com -b 192.168.1.87
```
<table>
<tr>
<td>
## Subfinder with docker
You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run -
Pull the latest tagged [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder) docker image:
```sh
▶ docker pull projectdiscovery/subfinder
docker pull projectdiscovery/subfinder:latest
```
The above command will pull the latest tagged release from the dockerhub repository.
Running subfinder using docker image:
If you want to build the container yourself manually, git clone the repo, then build and run the following commands
- Clone the repo using `git clone https://github.com/projectdiscovery/subfinder.git`
- Build your docker container
```sh
docker build -t projectdiscovery/subfinder .
docker -t projectdiscovery/subfinder:latest -d hackerone.com
```
If you are using docker, you need to first create your directory structure holding subfinder configuration file. After modifying the default config.yaml file, you can run:
Running subfinder using docker image with local config file:
```sh
▶ mkdir -p $HOME/.config/subfinder
▶ cp config.yaml $HOME/.config/subfinder/config.yaml
▶ nano $HOME/.config/subfinder/config.yaml
```
After that, you can pass it as a volume using the following sample command.
```sh
▶ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d example.com
```
For example, this runs the tool against example.com and output the results to your host file system:
```sh
docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d example.com > example.com.txt
docker run -v $HOME/.config/subfinder:/root/.config/subfinder -t projectdiscovery/subfinder -d hackerone.com
```
</td>
@ -262,7 +246,7 @@ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscove
Usage example:
``` go
```go
package main
import (

46
v2/.goreleaser.yml
View File

@ -3,23 +3,31 @@ before:
- go mod tidy
builds:
- binary: subfinder
main: cmd/subfinder/main.go
goos:
- linux
- windows
- darwin
goarch:
- amd64
- 386
- arm
- arm64
- env:
- CGO_ENABLED=0
goos:
- windows
- linux
- darwin
goarch:
- amd64
- 386
- arm
- arm64
ignore:
- goos: darwin
goarch: '386'
- goos: windows
goarch: 'arm'
binary: '{{ .ProjectName }}'
main: cmd/subfinder/main.go
archives:
- id: tgz
format: tar.gz
replacements:
darwin: macOS
format_overrides:
- goos: windows
format: zip
- format: zip
replacements:
darwin: macOS
checksum:
algorithm: sha256

6
v2/go.mod
View File

@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2
go 1.17
require (
github.com/corpix/uarand v0.1.1
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
github.com/json-iterator/go v1.1.10
github.com/lib/pq v1.10.0
@ -11,15 +12,19 @@ require (
github.com/projectdiscovery/fdmax v0.0.3
github.com/projectdiscovery/gologger v1.1.4
github.com/rs/xid v1.3.0
github.com/spyse-com/go-spyse v1.2.3
github.com/stretchr/testify v1.7.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
go.uber.org/ratelimit v0.2.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
require (
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
@ -27,4 +32,5 @@ require (
github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e // indirect
golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d // indirect
golang.org/x/sys v0.0.0-20210419170143-37df388d1f33 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
)

17
v2/go.sum
View File

@ -1,8 +1,15 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -51,12 +58,16 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -87,6 +98,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/spyse-com/go-spyse v1.2.3 h1:0qo0OP5kLv0equyvI7H5pAGuDFTiFON3zXLv4BSw1yY=
github.com/spyse-com/go-spyse v1.2.3/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@ -97,7 +110,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -152,6 +167,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

5
v2/pkg/passive/passive.go
View File

@ -3,6 +3,7 @@ package passive
import (
"context"
"fmt"
"net"
"sync"
"time"
@ -11,11 +12,11 @@ import (
)
// EnumerateSubdomains enumerates all the subdomains for a given domain
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration, localIP net.IP) chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
session, err := subscraping.NewSession(domain, keys, proxy, timeout)
session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout, localIP)
if err != nil {
results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
}

13
v2/pkg/passive/sources.go
View File

@ -7,14 +7,15 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
@ -45,6 +46,7 @@ var DefaultSources = []string{
"certspotter",
"censys",
"chaos",
"chinaz",
"crtsh",
"dnsdumpster",
"hackertarget",
@ -59,6 +61,7 @@ var DefaultSources = []string{
"threatcrowd",
"threatminer",
"virustotal",
"fofa",
}
// DefaultRecursiveSources contains list of default recursive sources
@ -84,7 +87,6 @@ var DefaultAllSources = []string{
"archiveis",
"binaryedge",
"bufferover",
"c99",
"censys",
"certspotter",
"chaos",
@ -112,6 +114,7 @@ var DefaultAllSources = []string{
"virustotal",
"waybackarchive",
"zoomeye",
"fofa",
}
// Agent is a struct for running passive subdomain enumeration
@ -146,14 +149,14 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &binaryedge.Source{}
case "bufferover":
a.sources[source] = &bufferover.Source{}
case "c99":
a.sources[source] = &c99.Source{}
case "censys":
a.sources[source] = &censys.Source{}
case "certspotter":
a.sources[source] = &certspotter.Source{}
case "chaos":
a.sources[source] = &chaos.Source{}
case "chinaz":
a.sources[source] = &chinaz.Source{}
case "commoncrawl":
a.sources[source] = &commoncrawl.Source{}
case "crtsh":
@ -202,6 +205,8 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &waybackarchive.Source{}
case "zoomeye":
a.sources[source] = &zoomeye.Source{}
case "fofa":
a.sources[source] = &fofa.Source{}
}
}
}

4
v2/pkg/runner/banners.go
View File

@ -11,11 +11,11 @@ const banner = `
_______ __/ /_ / __(_)___ ____/ /__ _____
/ ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
(__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.8
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9
`
// Version is the current version of subfinder
const Version = `2.4.8`
const Version = `v2.4.9`
// showBanner is used to show the banner to the user
func showBanner() {

17
v2/pkg/runner/config.go
View File

@ -30,10 +30,10 @@ type ConfigFile struct {
ExcludeSources []string `yaml:"exclude-sources,omitempty"`
// API keys for different sources
Binaryedge []string `yaml:"binaryedge"`
C99 []string `yaml:"c99"`
Censys []string `yaml:"censys"`
Certspotter []string `yaml:"certspotter"`
Chaos []string `yaml:"chaos"`
Chinaz []string `yaml:"chinaz"`
DNSDB []string `yaml:"dnsdb"`
GitHub []string `yaml:"github"`
IntelX []string `yaml:"intelx"`
@ -47,6 +47,7 @@ type ConfigFile struct {
URLScan []string `yaml:"urlscan"`
Virustotal []string `yaml:"virustotal"`
ZoomEye []string `yaml:"zoomeye"`
Fofa []string `yaml:"fofa"`
// Version indicates the version of subfinder installed.
Version string `yaml:"subfinder-version"`
}
@ -120,9 +121,6 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
if len(c.Binaryedge) > 0 {
keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
}
if len(c.C99) > 0 {
keys.C99 = c.C99[rand.Intn(len(c.C99))]
}
if len(c.Censys) > 0 {
censysKeys := c.Censys[rand.Intn(len(c.Censys))]
@ -139,6 +137,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
if len(c.Chaos) > 0 {
keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
}
if len(c.Chinaz) > 0 {
keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))]
}
if (len(c.DNSDB)) > 0 {
keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
}
@ -198,6 +199,14 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
keys.ZoomEyePassword = parts[1]
}
}
if len(c.Fofa) > 0 {
fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))]
parts := strings.Split(fofaKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.FofaUsername = parts[0]
keys.FofaSecret = parts[1]
}
}
return keys
}

2
v2/pkg/runner/enumerate.go
View File

@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
// Run the passive subdomain enumeration
now := time.Now()
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute, r.options.LocalIP)
wg := &sync.WaitGroup{}
wg.Add(1)

27
v2/pkg/runner/options.go
View File

@ -3,6 +3,7 @@ package runner
import (
"flag"
"io"
"net"
"os"
"path"
"reflect"
@ -32,15 +33,19 @@ type Options struct {
Domain string // Domain is the domain to find subdomains for
DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for
Output io.Writer
OutputFile string // Output is the file to write found subdomains to.
OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given
Sources string // Sources contains a comma-separated list of sources to use for enumeration
ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process
Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
ConfigFile string // ConfigFile contains the location of the config file
Proxy string // HTTP proxy
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
OutputFile string // Output is the file to write found subdomains to.
OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given
Sources string // Sources contains a comma-separated list of sources to use for enumeration
ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process
Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
ConfigFile string // ConfigFile contains the location of the config file
Proxy string // HTTP proxy
RateLimit int // Maximum number of HTTP requests to send per second
LocalIP net.IP // LocalIP is the IP address used as local bind
LocalIPString string // LocalIPString is the IP address in string format got from command line
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
}
// ParseOptions parses the command line flags provided by a user
@ -75,8 +80,10 @@ func ParseOptions() *Options {
flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use")
flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use")
flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
flag.StringVar(&options.LocalIPString, "b", "", "IP address to be used as local bind")
flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use")
flag.StringVar(&options.Proxy, "proxy", "", "HTTP proxy to use with subfinder")
flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second")
flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
flag.Parse()

9
v2/pkg/runner/validate.go
View File

@ -2,6 +2,7 @@ package runner
import (
"errors"
"net"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
@ -34,6 +35,14 @@ func (options *Options) validateOptions() error {
return errors.New("hostip flag must be used with RemoveWildcard option")
}
// Check local IP address
if options.LocalIPString != "" {
options.LocalIP = net.ParseIP(options.LocalIPString)
if options.LocalIP == nil {
return errors.New("local bind ip is malformed")
}
}
return nil
}

23
v2/pkg/subscraping/agent.go Executable file → Normal file
View File

@ -6,21 +6,31 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"time"
"github.com/corpix/uarand"
"github.com/projectdiscovery/gologger"
"go.uber.org/ratelimit"
)
// NewSession creates a new session object for a domain
func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) {
func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int, localIP net.IP) (*Session, error) {
dialer := &net.Dialer{
LocalAddr: &net.TCPAddr{
IP: localIP,
},
}
Transport := &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
DialContext: dialer.DialContext,
}
// Add proxy
@ -44,6 +54,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session,
Keys: keys,
}
// Initiate rate limit instance
if rateLimit > 0 {
session.RateLimiter = ratelimit.New(rateLimit)
} else {
session.RateLimiter = ratelimit.NewUnlimited()
}
// Create a new extractor object for the current domain
extractor, err := NewSubdomainExtractor(domain)
session.Extractor = extractor
@ -78,7 +95,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
req.Header.Set("User-Agent", uarand.GetRandom())
req.Header.Set("Accept", "*/*")
req.Header.Set("Accept-Language", "en")
req.Header.Set("Connection", "close")
@ -95,6 +112,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
req.Header.Set(key, value)
}
s.RateLimiter.Take()
return httpRequestWrapper(s.Client, req)
}

94
v2/pkg/subscraping/sources/archiveis/archiveis.go Executable file → Normal file
View File

@ -5,74 +5,48 @@ import (
"context"
"fmt"
"io/ioutil"
"regexp"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type agent struct {
Results chan subscraping.Result
Session *subscraping.Session
}
var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">&rarr;</a>")
func (a *agent) enumerate(ctx context.Context, baseURL string) {
select {
case <-ctx.Done():
return
default:
}
resp, err := a.Session.SimpleGet(ctx, baseURL)
if err != nil {
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
a.Session.DiscardHTTPResponse(resp)
return
}
// Get the response body
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
src := string(body)
for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) {
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
}
match1 := reNext.FindStringSubmatch(src)
if len(match1) > 0 {
a.enumerate(ctx, match1[1])
}
}
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
a := agent{
Session: session,
Results: results,
}
go func() {
a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain))
close(a.Results)
}()
return a.Results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "archiveis"
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain))
if err != nil {
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
src := string(body)
for _, subdomain := range session.Extractor.FindAllString(src, -1) {
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
}
}()
return results
}

0
v2/pkg/subscraping/sources/binaryedge/binaryedge.go Executable file → Normal file
View File

0
v2/pkg/subscraping/sources/bufferover/bufferover.go Executable file → Normal file
View File

71
v2/pkg/subscraping/sources/c99/c99.go
View File

@ -1,71 +0,0 @@
// Package c99 logic
package c99
import (
"context"
"fmt"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct{}
type dnsdbLookupResponse struct {
Success bool `json:"success"`
Subdomains []struct {
Subdomain string `json:"subdomain"`
IP string `json:"ip"`
Cloudflare bool `json:"cloudflare"`
} `json:"subdomains"`
Error string `json:"error"`
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.C99 == "" {
return
}
searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain)
resp, err := session.SimpleGet(ctx, searchURL)
if err != nil {
session.DiscardHTTPResponse(resp)
return
}
defer resp.Body.Close()
var response dnsdbLookupResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if response.Error != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)}
return
}
for _, data := range response.Subdomains {
if !strings.HasPrefix(data.Subdomain, ".") {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "c99"
}

0
v2/pkg/subscraping/sources/certspotter/certspotter.go Executable file → Normal file
View File

56
v2/pkg/subscraping/sources/chinaz/chinaz.go Normal file
View File

@ -0,0 +1,56 @@
package chinaz
// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi
import (
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
"io/ioutil"
)
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.Chinaz == "" {
return
}
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList")
if SubdomainList.ToBool() {
_data := []byte(SubdomainList.ToString())
for i := 0 ; i< SubdomainList.Size() ; i++{
subdomain := jsoniter.Get(_data,i,"DataUrl").ToString()
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
}
} else {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "chinaz"
}

0
v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go Executable file → Normal file
View File

2
v2/pkg/subscraping/sources/crtsh/crtsh.go Executable file → Normal file
View File

@ -44,6 +44,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re
return 0
}
defer db.Close()
pattern := "%." + domain
query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci
WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))

1
v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go Executable file → Normal file
View File

@ -30,6 +30,7 @@ func postForm(ctx context.Context, session *subscraping.Session, token, domain s
params := url.Values{
"csrfmiddlewaretoken": {token},
"targetip": {domain},
"user": {"free"},
}
resp, err := session.HTTPRequest(

74
v2/pkg/subscraping/sources/fofa/fofa.go Normal file
View File

@ -0,0 +1,74 @@
// Package fofa logic
package fofa
import (
"context"
"encoding/base64"
"fmt"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type fofaResponse struct {
Error bool `json:"error"`
ErrMsg string `json:"errmsg"`
Size int `json:"size"`
Results []string `json:"results"`
}
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" {
return
}
// fofa api doc https://fofa.so/static_pages/api_help
qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain)))
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.so/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
var response fofaResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
if response.Error {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)}
return
}
if response.Size > 0 {
for _, subdomain := range response.Results {
if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") {
subdomain = subdomain[strings.Index(subdomain, "//")+2:]
}
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "fofa"
}

0
v2/pkg/subscraping/sources/hackertarget/hackertarget.go Executable file → Normal file
View File

0
v2/pkg/subscraping/sources/passivetotal/passivetotal.go Executable file → Normal file
View File

10
v2/pkg/subscraping/sources/recon/recon.go
View File

@ -10,7 +10,11 @@ import (
)
type subdomain struct {
RawDomain string `json:"rawDomain"`
Domains []string `json:"domains"`
Ip string `json:"ip"`
RawDomains []string `json:"rawDomains"`
RawPort string `json:"rawPort"`
RawIp string `json:"rawIp"`
}
// Source is the passive scraping agent
@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
resp.Body.Close()
for _, subdomain := range subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain}
for _, dmn := range subdomain.RawDomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn}
}
}
}()

0
v2/pkg/subscraping/sources/securitytrails/securitytrails.go Executable file → Normal file
View File

0
v2/pkg/subscraping/sources/sitedossier/sitedossier.go Executable file → Normal file
View File

113
v2/pkg/subscraping/sources/spyse/spyse.go
View File

@ -3,31 +3,12 @@ package spyse
import (
"context"
"fmt"
"strconv"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
spyse "github.com/spyse-com/go-spyse/pkg"
)
type resultObject struct {
Name string `json:"name"`
}
type dataObject struct {
Items []resultObject `json:"items"`
TotalCount int `json:"total_count"`
}
type errorObject struct {
Code string `json:"code"`
Message string `json:"message"`
}
type spyseResult struct {
Data dataObject `json:"data"`
Error []errorObject `json:"error"`
}
const searchMethodResultsLimit = 10000
// Source is the passive scraping agent
type Source struct{}
@ -43,33 +24,77 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
maxCount := 100
client, err := spyse.NewClient(session.Keys.Spyse, nil)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
for offSet := 0; offSet <= maxCount; offSet += 100 {
resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
domainSvc := spyse.NewDomainService(client)
var searchDomain = "." + domain
var subdomainsSearchParams spyse.QueryBuilder
subdomainsSearchParams.AppendParam(spyse.QueryParam{
Name: domainSvc.Params().Name.Name,
Operator: domainSvc.Params().Name.Operator.EndsWith,
Value: searchDomain,
})
totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if totalResults == 0 {
return
}
// The default "Search" method returns only first 10 000 subdomains
// To obtain more than 10 000 subdomains the "Scroll" method should be using
// Note: The "Scroll" method is only available for "PRO" customers, so we need to check
// quota.IsScrollSearchEnabled param
if totalResults > searchMethodResultsLimit && client.Account().IsScrollSearchEnabled {
var scrollID string
var scrollResults *spyse.DomainScrollResponse
for {
scrollResults, err = domainSvc.ScrollSearch(ctx, subdomainsSearchParams.Query, scrollID)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if len(scrollResults.Items) > 0 {
scrollID = scrollResults.SearchID
for i := range scrollResults.Items {
results <- subscraping.Result{
Source: s.Name(),
Type: subscraping.Subdomain,
Value: scrollResults.Items[i].Name,
}
}
}
}
} else {
var limit = 100
var searchResults []spyse.Domain
var response spyseResult
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
for offset := 0; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit {
searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if response.Data.TotalCount == 0 {
return
}
maxCount = response.Data.TotalCount
for _, hostname := range response.Data.Items {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name}
for i := range searchResults {
results <- subscraping.Result{
Source: s.Name(),
Type: subscraping.Subdomain,
Value: searchResults[i].Name,
}
}
}
}
}()

0
v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go Executable file → Normal file
View File

0
v2/pkg/subscraping/sources/threatminer/threatminer.go Executable file → Normal file
View File

0
v2/pkg/subscraping/sources/virustotal/virustotal.go Executable file → Normal file
View File

0
v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go Executable file → Normal file
View File

8
v2/pkg/subscraping/types.go Executable file → Normal file
View File

@ -4,6 +4,8 @@ import (
"context"
"net/http"
"regexp"
"go.uber.org/ratelimit"
)
// BasicAuth request's Authorization header
@ -31,16 +33,18 @@ type Session struct {
Keys *Keys
// Client is the current http client
Client *http.Client
// Rate limit instance
RateLimiter ratelimit.Limiter
}
// Keys contains the current API Keys we have in store
type Keys struct {
Binaryedge string `json:"binaryedge"`
C99 string `json:"c99"`
CensysToken string `json:"censysUsername"`
CensysSecret string `json:"censysPassword"`
Certspotter string `json:"certspotter"`
Chaos string `json:"chaos"`
Chinaz string `json:"chinaz"`
DNSDB string `json:"dnsdb"`
GitHub []string `json:"github"`
IntelXHost string `json:"intelXHost"`
@ -57,6 +61,8 @@ type Keys struct {
Virustotal string `json:"virustotal"`
ZoomEyeUsername string `json:"zoomeye_username"`
ZoomEyePassword string `json:"zoomeye_password"`
FofaUsername string `json:"fofa_username"`
FofaSecret string `json:"fofa_secret"`
}
// Result is a result structure returned by a source

0
v2/pkg/subscraping/utils.go Executable file → Normal file
View File