daffainfo
/
subfinder
mirror of https://github.com/daffainfo/subfinder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #557 from projectdiscovery/dev 

Bugfix release
master v2.5.2
Sandeep Singh 2022-06-08 01:56:21 +05:30 committed by GitHub
parent 9c01de8410 c6fcaa4289
commit 327e4033b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 93 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build-test.yml vendored
View File

@ -13,9 +13,9 @@ jobs:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: 1.17
go-version: 1.18
- name: Check out code
uses: actions/checkout@v3

6
.github/workflows/codeql-analysis.yml vendored
View File

@ -27,12 +27,12 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@v1
uses: github/codeql-action/autobuild@v2
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
uses: github/codeql-action/analyze@v2

8
.github/workflows/dockerhub-push.yml vendored
View File

@ -21,19 +21,19 @@ jobs:
echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name)"
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
context: .
platforms: linux/amd64,linux/arm64,linux/arm

6
.github/workflows/lint-test.yml vendored
View File

@ -10,11 +10,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v2
- uses: actions/setup-go@v3
with:
go-version: 1.17
go-version: 1.18
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v3.1.0
uses: golangci/golangci-lint-action@v3.2.0
with:
version: latest
args: --timeout 5m

6
.github/workflows/release-binary.yml vendored
View File

@ -14,12 +14,12 @@ jobs:
fetch-depth: 0
- name: "Set up Go"
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: 1.17
go-version: 1.18
- name: "Create release on GitHub"
uses: goreleaser/goreleaser-action@v2
uses: goreleaser/goreleaser-action@v3
with:
args: "release --rm-dist"
version: latest

4
.github/workflows/sonarcloud.yml vendored
View File

@ -18,9 +18,9 @@ jobs:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: "Set up Go"
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: 1.17
go-version: 1.18
- name: Run unit Tests
working-directory: v2/

4
Dockerfile
View File

@ -1,9 +1,9 @@
# Build
FROM golang:1.18.0-alpine AS build-env
FROM golang:1.18.3-alpine AS build-env
RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
# Release
FROM alpine:3.15.3
FROM alpine:3.16.0
RUN apk -U upgrade --no-cache \
&& apk add --no-cache bind-tools ca-certificates
COPY --from=build-env /go/bin/subfinder /usr/local/bin/subfinder

10
README.md
View File

@ -28,7 +28,7 @@
Subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and is optimized for speed. subfinder is built for doing one thing only - passive subdomain enumeration, and it does that very well.
We have designed subfinder to comply with all passive sources licenses, and usage restrictions, as well as maintained a consistently passive model to make it useful to both penetration testers and bug bounty hunters alike.
We have designed `subfinder` to comply with all passive sources licenses, and usage restrictions, as well as maintained a consistently passive model to make it useful to both penetration testers and bug bounty hunters alike.
# Features
@ -98,7 +98,7 @@ OPTIMIZATION:
# Installation
Subfinder requires **go1.17** to install successfully. Run the following command to get the repo -
Subfinder requires **go1.17** to install successfully. Run the following command to install the latest version:
```sh
go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
@ -109,9 +109,9 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
[Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.so/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Spyse](https://spyse.com), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [Zoomeye](https://www.zoomeye.org)
[Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.so/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [Zoomeye](https://www.zoomeye.org)
Theses values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.
These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.
For sources that require multiple keys, namely `Censys`, `Passivetotal`, they can be added by separating them via a colon (:).
@ -278,6 +278,6 @@ func main() {
# License
subfinder is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details.
`subfinder` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details.
Read the disclaimer for usage at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal.

13
v2/Makefile
View File

@ -3,12 +3,17 @@ GOCMD=go
GOBUILD=$(GOCMD) build
GOMOD=$(GOCMD) mod
GOTEST=$(GOCMD) test
GOGET=$(GOCMD) get
GOFLAGS := -v
LDFLAGS := -s -w
ifneq ($(shell go env GOOS),darwin)
LDFLAGS := -extldflags "-static"
endif
all: build
build:
$(GOBUILD) -v -ldflags="-extldflags=-static" -o "subfinder" cmd/subfinder/main.go
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "subfinder" cmd/subfinder/main.go
test:
$(GOTEST) -v ./...
$(GOTEST) $(GOFLAGS) ./...
tidy:
$(GOMOD) tidy
$(GOMOD) tidy

4
v2/go.mod
View File

@ -12,8 +12,8 @@ require (
github.com/projectdiscovery/fdmax v0.0.3
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/sliceutil v0.0.0-20220426000009-1d2b7c02f65c
github.com/rs/xid v1.3.0
github.com/spyse-com/go-spyse v1.2.3
github.com/stretchr/testify v1.7.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
go.uber.org/ratelimit v0.2.0
@ -28,7 +28,6 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.9.1
@ -38,6 +37,5 @@ require (
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect
golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d // indirect
golang.org/x/sys v0.0.0-20210419170143-37df388d1f33 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

10
v2/go.sum
View File

@ -63,8 +63,6 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -95,8 +93,6 @@ github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0
github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f h1:FKTkdM1pPIL0gQRRQDoWjd/mZz+4DZ2Bk1l+ZbOJmIQ=
github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/goflags v0.0.8-0.20220328195035-cc76049ee216 h1:Th8GrVWt6LJQAwneHikni4BpoLEPa69DPEnYCxAINmo=
github.com/projectdiscovery/goflags v0.0.8-0.20220328195035-cc76049ee216/go.mod h1:37KhVbVLllyuIAgpXGqcvE/hsFEwJ+ctEUSHawjhsBY=
github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
@ -105,13 +101,13 @@ github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e h1:cxke2L/GKym765W0UnA9RuyaY/LI2u5z+fYqDtoQY0M=
github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/sliceutil v0.0.0-20220426000009-1d2b7c02f65c h1:bdZ81T+C9W5b12RL2ndggctrTaOk1bveFG6UfuOck94=
github.com/projectdiscovery/sliceutil v0.0.0-20220426000009-1d2b7c02f65c/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/spyse-com/go-spyse v1.2.3 h1:0qo0OP5kLv0equyvI7H5pAGuDFTiFON3zXLv4BSw1yY=
github.com/spyse-com/go-spyse v1.2.3/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@ -179,8 +175,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

5
v2/pkg/passive/sources.go
View File

@ -29,7 +29,6 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sonarsearch"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/spyse"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sublist3r"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
@ -59,7 +58,6 @@ var DefaultSources = []string{
"riddler",
"securitytrails",
"shodan",
"spyse",
"sublist3r",
"threatcrowd",
"threatminer",
@ -110,7 +108,6 @@ var DefaultAllSources = []string{
"shodan",
"sitedossier",
"sonarsearch",
"spyse",
"sublist3r",
"threatbook",
"threatcrowd",
@ -195,8 +192,6 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &sitedossier.Source{}
case "sonarsearch":
a.sources[source] = &sonarsearch.Source{}
case "spyse":
a.sources[source] = &spyse.Source{}
case "sublist3r":
a.sources[source] = &sublist3r.Source{}
case "threatbook":

4
v2/pkg/runner/banners.go
View File

@ -14,11 +14,11 @@ const banner = `
_______ __/ /_ / __(_)___ ____/ /__ _____
/ ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
(__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.1
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.2
`
// Version is the current version of subfinder
const Version = `v2.5.1`
const Version = `v2.5.2`
// showBanner is used to show the banner to the user
func showBanner() {

85
v2/pkg/runner/config.go
View File

@ -1,12 +1,12 @@
package runner
import (
"math/rand"
"os"
"strings"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
"gopkg.in/yaml.v3"
"github.com/projectdiscovery/sliceutil"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// MultipleKeyPartsLength is the max length for multiple keys
@ -29,7 +29,6 @@ type Providers struct {
Robtex []string `yaml:"robtex"`
SecurityTrails []string `yaml:"securitytrails"`
Shodan []string `yaml:"shodan"`
Spyse []string `yaml:"spyse"`
ThreatBook []string `yaml:"threatbook"`
URLScan []string `yaml:"urlscan"`
Virustotal []string `yaml:"virustotal"`
@ -87,102 +86,94 @@ func (c *Providers) GetKeys() subscraping.Keys {
keys := subscraping.Keys{}
if len(c.Binaryedge) > 0 {
keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
keys.Binaryedge = sliceutil.PickRandom(c.Binaryedge)
}
if len(c.C99) > 0 {
keys.C99 = c.C99[rand.Intn(len(c.C99))]
keys.C99 = sliceutil.PickRandom(c.C99)
}
if len(c.Bufferover) > 0 {
keys.Bufferover = c.Bufferover[rand.Intn(len(c.Bufferover))]
keys.Bufferover = sliceutil.PickRandom(c.Bufferover)
}
if len(c.Censys) > 0 {
censysKeys := c.Censys[rand.Intn(len(c.Censys))]
parts := strings.Split(censysKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.CensysToken = parts[0]
keys.CensysSecret = parts[1]
censysKeys := sliceutil.PickRandom(c.Censys)
if keyPartA, keyPartB, ok := multipartKey(censysKeys); ok {
keys.CensysToken = keyPartA
keys.CensysSecret = keyPartB
}
}
if len(c.Certspotter) > 0 {
keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))]
keys.Certspotter = sliceutil.PickRandom(c.Certspotter)
}
if len(c.Chaos) > 0 {
keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
keys.Chaos = sliceutil.PickRandom(c.Chaos)
}
if len(c.Chinaz) > 0 {
keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))]
keys.Chinaz = sliceutil.PickRandom(c.Chinaz)
}
if (len(c.DNSDB)) > 0 {
keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
keys.DNSDB = sliceutil.PickRandom(c.DNSDB)
}
if (len(c.GitHub)) > 0 {
keys.GitHub = c.GitHub
}
if len(c.IntelX) > 0 {
intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))]
parts := strings.Split(intelxKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.IntelXHost = parts[0]
keys.IntelXKey = parts[1]
intelxKeys := sliceutil.PickRandom(c.IntelX)
if keyPartA, keyPartB, ok := multipartKey(intelxKeys); ok {
keys.IntelXHost = keyPartA
keys.IntelXKey = keyPartB
}
}
if len(c.PassiveTotal) > 0 {
passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))]
parts := strings.Split(passiveTotalKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.PassiveTotalUsername = parts[0]
keys.PassiveTotalPassword = parts[1]
passiveTotalKeys := sliceutil.PickRandom(c.PassiveTotal)
if keyPartA, keyPartB, ok := multipartKey(passiveTotalKeys); ok {
keys.PassiveTotalUsername = keyPartA
keys.PassiveTotalPassword = keyPartB
}
}
if len(c.Robtex) > 0 {
keys.Robtex = c.Robtex[rand.Intn(len(c.Robtex))]
keys.Robtex = sliceutil.PickRandom(c.Robtex)
}
if len(c.SecurityTrails) > 0 {
keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))]
keys.Securitytrails = sliceutil.PickRandom(c.SecurityTrails)
}
if len(c.Shodan) > 0 {
keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))]
}
if len(c.Spyse) > 0 {
keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))]
keys.Shodan = sliceutil.PickRandom(c.Shodan)
}
if len(c.ThreatBook) > 0 {
keys.ThreatBook = c.ThreatBook[rand.Intn(len(c.ThreatBook))]
keys.ThreatBook = sliceutil.PickRandom(c.ThreatBook)
}
if len(c.URLScan) > 0 {
keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))]
keys.URLScan = sliceutil.PickRandom(c.URLScan)
}
if len(c.Virustotal) > 0 {
keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))]
keys.Virustotal = sliceutil.PickRandom(c.Virustotal)
}
if len(c.ZoomEye) > 0 {
zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))]
parts := strings.Split(zoomEyeKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.ZoomEyeUsername = parts[0]
keys.ZoomEyePassword = parts[1]
zoomEyeKeys := sliceutil.PickRandom(c.ZoomEye)
if keyPartA, keyPartB, ok := multipartKey(zoomEyeKeys); ok {
keys.ZoomEyeUsername = keyPartA
keys.ZoomEyePassword = keyPartB
}
}
if len(c.ZoomEyeApi) > 0 {
keys.ZoomEyeKey = c.ZoomEyeApi[rand.Intn(len(c.ZoomEyeApi))]
keys.ZoomEyeKey = sliceutil.PickRandom(c.ZoomEyeApi)
}
if len(c.Fofa) > 0 {
fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))]
parts := strings.Split(fofaKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.FofaUsername = parts[0]
keys.FofaSecret = parts[1]
fofaKeys := sliceutil.PickRandom(c.Fofa)
if keyPartA, keyPartB, ok := multipartKey(fofaKeys); ok {
keys.FofaUsername = keyPartA
keys.FofaSecret = keyPartB
}
}
if len(c.FullHunt) > 0 {
keys.FullHunt = c.FullHunt[rand.Intn(len(c.FullHunt))]
keys.FullHunt = sliceutil.PickRandom(c.FullHunt)
}
return keys
}

5
v2/pkg/runner/options.go
View File

@ -4,11 +4,13 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"os"
"os/user"
"path/filepath"
"reflect"
"strings"
"time"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/goflags"
@ -66,6 +68,9 @@ type Options struct {
// ParseOptions parses the command line flags provided by a user
func ParseOptions() *Options {
// Seed default random number generator
rand.Seed(time.Now().UnixNano())
// Migrate config to provider config
if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) {
gologger.Info().Msgf("Detected old %s config file, trying to migrate providers to %s\n", defaultConfigLocation, defaultProviderConfigLocation)

12
v2/pkg/runner/util.go
View File

@ -35,3 +35,15 @@ func sanitize(data string) (string, error) {
}
return data, nil
}
func multipartKey(key string) (keyPartA, keyPartB string, ok bool) {
parts := strings.Split(key, ":")
ok = len(parts) == MultipleKeyPartsLength
if ok {
keyPartA = parts[0]
keyPartB = parts[1]
}
return
}

108
v2/pkg/subscraping/sources/spyse/spyse.go
View File

@ -1,108 +0,0 @@
// Package spyse logic
package spyse
import (
"context"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
spyse "github.com/spyse-com/go-spyse/pkg"
)
const searchMethodResultsLimit = 10000
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.Spyse == "" {
return
}
client, err := spyse.NewClient(session.Keys.Spyse, nil)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
domainSvc := spyse.NewDomainService(client)
var searchDomain = "." + domain
var subdomainsSearchParams spyse.QueryBuilder
subdomainsSearchParams.AppendParam(spyse.QueryParam{
Name: domainSvc.Params().Name.Name,
Operator: domainSvc.Params().Name.Operator.EndsWith,
Value: searchDomain,
})
totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if totalResults == 0 {
return
}
// The default "Search" method returns only first 10 000 subdomains
// To obtain more than 10 000 subdomains the "Scroll" method should be using
// Note: The "Scroll" method is only available for "PRO" customers, so we need to check
// quota.IsScrollSearchEnabled param
if totalResults > searchMethodResultsLimit && client.Account().IsScrollSearchEnabled {
var scrollID string
var scrollResults *spyse.DomainScrollResponse
for {
scrollResults, err = domainSvc.ScrollSearch(ctx, subdomainsSearchParams.Query, scrollID)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if len(scrollResults.Items) > 0 {
scrollID = scrollResults.SearchID
for i := range scrollResults.Items {
results <- subscraping.Result{
Source: s.Name(),
Type: subscraping.Subdomain,
Value: scrollResults.Items[i].Name,
}
}
}
}
} else {
var limit = 100
var searchResults []spyse.Domain
for offset := 0; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit {
searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
for i := range searchResults {
results <- subscraping.Result{
Source: s.Name(),
Type: subscraping.Subdomain,
Value: searchResults[i].Name,
}
}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "spyse"
}

1
v2/pkg/subscraping/types.go
View File

@ -56,7 +56,6 @@ type Keys struct {
Robtex string `json:"robtex"`
Securitytrails string `json:"securitytrails"`
Shodan string `json:"shodan"`
Spyse string `json:"spyse"`
ThreatBook string `json:"threatbook"`
URLScan string `json:"urlscan"`
Virustotal string `json:"virustotal"`