daffainfo
/
subfinder
mirror of https://github.com/daffainfo/subfinder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into dependabot/docker/golang-1.17.0-alpine

dnsrepo-source
Sandeep Singh 2021-08-24 00:54:35 +05:30 committed by GitHub
parent 60e2010142 08671cf155
commit 410a19ac91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 491 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/dependabot.yml vendored
View File

@ -11,6 +11,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
@ -20,6 +21,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
@ -29,6 +31,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
include: "scope"

27
.github/workflows/build-test.yml vendored Normal file
View File

@ -0,0 +1,27 @@
name: 🔨 Build Test
on:
push:
pull_request:
workflow_dispatch:
jobs:
build:
name: Test Builds
runs-on: ubuntu-latest
steps:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: Check out code
uses: actions/checkout@v2
- name: Test
run: go test .
working-directory: v2/cmd/subfinder/
- name: Build
run: go build .
working-directory: v2/cmd/subfinder/

40
.github/workflows/build.yaml vendored
View File

@ -1,40 +0,0 @@
name: Build
on:
push:
branches:
- master
pull_request:
jobs:
golangci-lint:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v2.5.2
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.33
args: --timeout 5m
working-directory: v2/
build:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: Test
run: go test ./...
working-directory: v2/
- name: Build
run: go build .
working-directory: v2/cmd/subfinder/

38
.github/workflows/codeql-analysis.yml vendored Normal file
View File

@ -0,0 +1,38 @@
name: 🚨 CodeQL Analysis
on:
workflow_dispatch:
pull_request:
branches:
- dev
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'go' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@v1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1

17
.github/workflows/dockerhub-push-on-release.yml vendored
View File

@ -1,17 +0,0 @@
# dockerhub-push pushes docker build to dockerhub automatically
# on the creation of a new release
name: Publish to Dockerhub on creation of a new release
on:
release:
types: [published]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Publish to Dockerhub Registry
uses: elgohr/Publish-Docker-Github-Action@master
with:
name: projectdiscovery/subfinder
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

34
.github/workflows/dockerhub-push.yml vendored Normal file
View File

@ -0,0 +1,34 @@
name: 🌥 Docker Push
on:
release:
types: [published]
workflow_dispatch:
jobs:
docker:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
-
name: Build and push
uses: docker/build-push-action@v2
with:
context: .
platforms: linux/amd64,linux/arm64,linux/arm
push: true
tags: projectdiscovery/subfinder:latest

19
.github/workflows/lint-test.yml vendored Normal file
View File

@ -0,0 +1,19 @@
name: 🙏🏻 Lint Test
on:
push:
pull_request:
workflow_dispatch:
jobs:
lint:
name: Lint Test
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v2
with:
version: latest
args: --timeout 5m
working-directory: v2/

8
.github/workflows/release.yml → .github/workflows/release-binary.yml vendored
View File

@ -1,8 +1,9 @@
name: Release
name: 🎉 Release Binary
on:
create:
tags:
- v*
workflow_dispatch:
jobs:
release:
@ -17,7 +18,7 @@ jobs:
name: "Set up Go"
uses: actions/setup-go@v2
with:
go-version: 1.15
go-version: 1.16
-
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@ -26,5 +27,4 @@ jobs:
with:
args: "release --rm-dist"
version: latest
workdir: v2/
workdir: .

122
.golangci.yml
View File

@ -1,122 +0,0 @@
linters-settings:
dupl:
threshold: 100
exhaustive:
default-signifies-exhaustive: false
# funlen:
# lines: 100
# statements: 50
goconst:
min-len: 2
min-occurrences: 2
gocritic:
enabled-tags:
- diagnostic
- experimental
- opinionated
- performance
- style
disabled-checks:
- dupImport # https://github.com/go-critic/go-critic/issues/845
- ifElseChain
# gocyclo:
# min-complexity: 15
goimports:
local-prefixes: github.com/golangci/golangci-lint
golint:
min-confidence: 0
gomnd:
settings:
mnd:
# don't include the "operation" and "assign"
checks: argument,case,condition,return
govet:
check-shadowing: true
settings:
printf:
funcs:
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
# lll:
# line-length: 140
maligned:
suggest-new: true
misspell:
locale: US
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
linters:
# please, do not use `enable-all`: it's deprecated and will be removed soon.
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
enable:
- bodyclose
- deadcode
- dogsled
- dupl
# - errcheck
- exhaustive
- gochecknoinits
- goconst
- gocritic
- gofmt
- goimports
- golint
- gomnd
- goprintffuncname
- gosimple
- govet
- ineffassign
- interfacer
- maligned
- misspell
- nakedret
- noctx
# - nolintlint
- rowserrcheck
- scopelint
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
# don't enable:
# - depguard
# - asciicheck
# - funlen
# - gochecknoglobals
# - gocognit
# - gocyclo
# - godot
# - godox
# - goerr113
# - gosec
# - lll
# - nestif
# - prealloc
# - testpackage
# - wsl
issues:
exclude-use-default: false
exclude:
# should have a package comment, unless it's in another file for this package (golint)
- 'in another file for this package'
# golangci.com configuration
# https://github.com/golangci/golangci/wiki/Configuration
service:
golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
prepare:
- echo "here I can run custom commands, but no preparation needed for this repo"

1
Dockerfile
View File

@ -1,5 +1,6 @@
# Build
FROM golang:1.17.0-alpine AS build-env
RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
# Release

51
README.md
View File

@ -53,30 +53,32 @@ subfinder -h
```
This will display help for the tool. Here are all the switches it supports.
| Flag | Description | Example |
| ---------------- | ---------------------------------------------------------- | -------------------------------------- |
| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
| -config | Configuration file for API Keys, etc | subfinder -config config.yaml |
| -d | Domain to find subdomains for | subfinder -d uber.com |
| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt |
| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
| -nC | Don't Use colors in output | subfinder -nC |
| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
| -ls | List all available sources | subfinder -ls |
| -o | File to write output to (optional) | subfinder -o output.txt |
| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
| -oI | Write output in Host,IP format | subfinder -oI |
| -oJ | Write output in JSON lines Format | subfinder -oJ |
| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt |
| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive |
| -silent | Show only subdomains in output | subfinder -silent |
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
| -v | Show Verbose output | subfinder -v |
| -version | Show current program version | subfinder -version |
| Flag | Description | Example |
| ---------------- | ---------------------------------------------------------- | --------------------------------------------|
| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
| -config | Configuration file for API Keys, etc | subfinder -config config.yaml |
| -d | Domain to find subdomains for | subfinder -d uber.com |
| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt |
| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
| -nC | Don't Use colors in output | subfinder -nC |
| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
| -ls | List all available sources | subfinder -ls |
| -o | File to write output to (optional) | subfinder -o output.txt |
| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
| -oI | Write output in Host,IP format | subfinder -oI |
| -oJ | Write output in JSON lines Format | subfinder -oJ |
| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt |
| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive |
| -silent | Show only subdomains in output | subfinder -silent |
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 |
| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 |
| -v | Show Verbose output | subfinder -v |
| -version | Show current program version | subfinder -version |
# Installation
@ -93,6 +95,7 @@ Subfinder requires **go1.14+** to install successfully. Run the following comman
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
- [Binaryedge](https://binaryedge.io)
- [C99](https://api.c99.nl/)
- [Certspotter](https://sslmate.com/certspotter/api/)
- [Censys](https://censys.io)
- [Chaos](https://chaos.projectdiscovery.io)

46
v2/.goreleaser.yml
View File

@ -3,23 +3,31 @@ before:
- go mod tidy
builds:
- binary: subfinder
main: cmd/subfinder/main.go
goos:
- linux
- windows
- darwin
goarch:
- amd64
- 386
- arm
- arm64
- env:
- CGO_ENABLED=0
goos:
- windows
- linux
- darwin
goarch:
- amd64
- 386
- arm
- arm64
ignore:
- goos: darwin
goarch: '386'
- goos: windows
goarch: 'arm'
binary: '{{ .ProjectName }}'
main: cmd/subfinder/main.go
archives:
- id: tgz
format: tar.gz
replacements:
darwin: macOS
format_overrides:
- goos: windows
format: zip
- format: zip
replacements:
darwin: macOS
checksum:
algorithm: sha256

3
v2/go.mod
View File

@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2
go 1.16
require (
github.com/corpix/uarand v0.1.1
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
github.com/json-iterator/go v1.1.10
github.com/lib/pq v1.10.0
@ -11,7 +12,9 @@ require (
github.com/projectdiscovery/fdmax v0.0.3
github.com/projectdiscovery/gologger v1.1.4
github.com/rs/xid v1.3.0
github.com/spyse-com/go-spyse v1.2.1
github.com/stretchr/testify v1.7.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
go.uber.org/ratelimit v0.2.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

15
v2/go.sum
View File

@ -1,8 +1,15 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -51,12 +58,16 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -87,6 +98,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8=
github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@ -97,7 +110,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

4
v2/pkg/passive/passive.go
View File

@ -11,11 +11,11 @@ import (
)
// EnumerateSubdomains enumerates all the subdomains for a given domain
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
session, err := subscraping.NewSession(domain, keys, proxy, timeout)
session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout)
if err != nil {
results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
}

13
v2/pkg/passive/sources.go
View File

@ -7,14 +7,15 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
@ -45,6 +46,7 @@ var DefaultSources = []string{
"certspotter",
"censys",
"chaos",
"chinaz",
"crtsh",
"dnsdumpster",
"hackertarget",
@ -59,6 +61,7 @@ var DefaultSources = []string{
"threatcrowd",
"threatminer",
"virustotal",
"fofa",
}
// DefaultRecursiveSources contains list of default recursive sources
@ -84,7 +87,6 @@ var DefaultAllSources = []string{
"archiveis",
"binaryedge",
"bufferover",
"c99",
"censys",
"certspotter",
"chaos",
@ -112,6 +114,7 @@ var DefaultAllSources = []string{
"virustotal",
"waybackarchive",
"zoomeye",
"fofa",
}
// Agent is a struct for running passive subdomain enumeration
@ -146,14 +149,14 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &binaryedge.Source{}
case "bufferover":
a.sources[source] = &bufferover.Source{}
case "c99":
a.sources[source] = &c99.Source{}
case "censys":
a.sources[source] = &censys.Source{}
case "certspotter":
a.sources[source] = &certspotter.Source{}
case "chaos":
a.sources[source] = &chaos.Source{}
case "chinaz":
a.sources[source] = &chinaz.Source{}
case "commoncrawl":
a.sources[source] = &commoncrawl.Source{}
case "crtsh":
@ -202,6 +205,8 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &waybackarchive.Source{}
case "zoomeye":
a.sources[source] = &zoomeye.Source{}
case "fofa":
a.sources[source] = &fofa.Source{}
}
}
}

4
v2/pkg/runner/banners.go
View File

@ -11,11 +11,11 @@ const banner = `
_______ __/ /_ / __(_)___ ____/ /__ _____
/ ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
(__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.8
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9
`
// Version is the current version of subfinder
const Version = `2.4.8`
const Version = `v2.4.9`
// showBanner is used to show the banner to the user
func showBanner() {

17
v2/pkg/runner/config.go
View File

@ -30,10 +30,10 @@ type ConfigFile struct {
ExcludeSources []string `yaml:"exclude-sources,omitempty"`
// API keys for different sources
Binaryedge []string `yaml:"binaryedge"`
C99 []string `yaml:"c99"`
Censys []string `yaml:"censys"`
Certspotter []string `yaml:"certspotter"`
Chaos []string `yaml:"chaos"`
Chinaz []string `yaml:"chinaz"`
DNSDB []string `yaml:"dnsdb"`
GitHub []string `yaml:"github"`
IntelX []string `yaml:"intelx"`
@ -47,6 +47,7 @@ type ConfigFile struct {
URLScan []string `yaml:"urlscan"`
Virustotal []string `yaml:"virustotal"`
ZoomEye []string `yaml:"zoomeye"`
Fofa []string `yaml:"fofa"`
// Version indicates the version of subfinder installed.
Version string `yaml:"subfinder-version"`
}
@ -120,9 +121,6 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
if len(c.Binaryedge) > 0 {
keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
}
if len(c.C99) > 0 {
keys.C99 = c.C99[rand.Intn(len(c.C99))]
}
if len(c.Censys) > 0 {
censysKeys := c.Censys[rand.Intn(len(c.Censys))]
@ -139,6 +137,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
if len(c.Chaos) > 0 {
keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
}
if len(c.Chinaz) > 0 {
keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))]
}
if (len(c.DNSDB)) > 0 {
keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
}
@ -198,6 +199,14 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
keys.ZoomEyePassword = parts[1]
}
}
if len(c.Fofa) > 0 {
fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))]
parts := strings.Split(fofaKeys, ":")
if len(parts) == MultipleKeyPartsLength {
keys.FofaUsername = parts[0]
keys.FofaSecret = parts[1]
}
}
return keys
}

2
v2/pkg/runner/enumerate.go
View File

@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
// Run the passive subdomain enumeration
now := time.Now()
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
wg := &sync.WaitGroup{}
wg.Add(1)

2
v2/pkg/runner/options.go
View File

@ -40,6 +40,7 @@ type Options struct {
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
ConfigFile string // ConfigFile contains the location of the config file
Proxy string // HTTP proxy
RateLimit int // Maximum number of HTTP requests to send per second
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
}
@ -77,6 +78,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use")
flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second")
flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
flag.Parse()

15
v2/pkg/subscraping/agent.go
View File

@ -10,11 +10,13 @@ import (
"net/url"
"time"
"github.com/corpix/uarand"
"github.com/projectdiscovery/gologger"
"go.uber.org/ratelimit"
)
// NewSession creates a new session object for a domain
func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) {
func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) {
Transport := &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
@ -44,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session,
Keys: keys,
}
// Initiate rate limit instance
if rateLimit > 0 {
session.RateLimiter = ratelimit.New(rateLimit)
} else {
session.RateLimiter = ratelimit.NewUnlimited()
}
// Create a new extractor object for the current domain
extractor, err := NewSubdomainExtractor(domain)
session.Extractor = extractor
@ -78,7 +87,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
req.Header.Set("User-Agent", uarand.GetRandom())
req.Header.Set("Accept", "*/*")
req.Header.Set("Accept-Language", "en")
req.Header.Set("Connection", "close")
@ -95,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
req.Header.Set(key, value)
}
s.RateLimiter.Take()
return httpRequestWrapper(s.Client, req)
}

94
v2/pkg/subscraping/sources/archiveis/archiveis.go
View File

@ -5,74 +5,48 @@ import (
"context"
"fmt"
"io/ioutil"
"regexp"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type agent struct {
Results chan subscraping.Result
Session *subscraping.Session
}
var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">&rarr;</a>")
func (a *agent) enumerate(ctx context.Context, baseURL string) {
select {
case <-ctx.Done():
return
default:
}
resp, err := a.Session.SimpleGet(ctx, baseURL)
if err != nil {
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
a.Session.DiscardHTTPResponse(resp)
return
}
// Get the response body
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
src := string(body)
for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) {
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
}
match1 := reNext.FindStringSubmatch(src)
if len(match1) > 0 {
a.enumerate(ctx, match1[1])
}
}
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
a := agent{
Session: session,
Results: results,
}
go func() {
a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain))
close(a.Results)
}()
return a.Results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "archiveis"
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain))
if err != nil {
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
src := string(body)
for _, subdomain := range session.Extractor.FindAllString(src, -1) {
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
}
}()
return results
}

71
v2/pkg/subscraping/sources/c99/c99.go
View File

@ -1,71 +0,0 @@
// Package c99 logic
package c99
import (
"context"
"fmt"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct{}
type dnsdbLookupResponse struct {
Success bool `json:"success"`
Subdomains []struct {
Subdomain string `json:"subdomain"`
IP string `json:"ip"`
Cloudflare bool `json:"cloudflare"`
} `json:"subdomains"`
Error string `json:"error"`
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.C99 == "" {
return
}
searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain)
resp, err := session.SimpleGet(ctx, searchURL)
if err != nil {
session.DiscardHTTPResponse(resp)
return
}
defer resp.Body.Close()
var response dnsdbLookupResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if response.Error != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)}
return
}
for _, data := range response.Subdomains {
if !strings.HasPrefix(data.Subdomain, ".") {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "c99"
}

56
v2/pkg/subscraping/sources/chinaz/chinaz.go Normal file
View File

@ -0,0 +1,56 @@
package chinaz
// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi
import (
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
"io/ioutil"
)
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.Chinaz == "" {
return
}
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList")
if SubdomainList.ToBool() {
_data := []byte(SubdomainList.ToString())
for i := 0 ; i< SubdomainList.Size() ; i++{
subdomain := jsoniter.Get(_data,i,"DataUrl").ToString()
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
}
} else {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "chinaz"
}

2
v2/pkg/subscraping/sources/crtsh/crtsh.go
View File

@ -44,6 +44,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re
return 0
}
defer db.Close()
pattern := "%." + domain
query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci
WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))

1
v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go
View File

@ -30,6 +30,7 @@ func postForm(ctx context.Context, session *subscraping.Session, token, domain s
params := url.Values{
"csrfmiddlewaretoken": {token},
"targetip": {domain},
"user": {"free"},
}
resp, err := session.HTTPRequest(

74
v2/pkg/subscraping/sources/fofa/fofa.go Normal file
View File

@ -0,0 +1,74 @@
// Package fofa logic
package fofa
import (
"context"
"encoding/base64"
"fmt"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type fofaResponse struct {
Error bool `json:"error"`
ErrMsg string `json:"errmsg"`
Size int `json:"size"`
Results []string `json:"results"`
}
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" {
return
}
// fofa api doc https://fofa.so/static_pages/api_help
qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain)))
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.so/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
var response fofaResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
if response.Error {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)}
return
}
if response.Size > 0 {
for _, subdomain := range response.Results {
if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") {
subdomain = subdomain[strings.Index(subdomain, "//")+2:]
}
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "fofa"
}

10
v2/pkg/subscraping/sources/recon/recon.go
View File

@ -10,7 +10,11 @@ import (
)
type subdomain struct {
RawDomain string `json:"rawDomain"`
Domains []string `json:"domains"`
Ip string `json:"ip"`
RawDomains []string `json:"rawDomains"`
RawPort string `json:"rawPort"`
RawIp string `json:"rawIp"`
}
// Source is the passive scraping agent
@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
resp.Body.Close()
for _, subdomain := range subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain}
for _, dmn := range subdomain.RawDomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn}
}
}
}()

107
v2/pkg/subscraping/sources/spyse/spyse.go
View File

@ -3,31 +3,12 @@ package spyse
import (
"context"
"fmt"
"strconv"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
spyse "github.com/spyse-com/go-spyse/pkg"
)
type resultObject struct {
Name string `json:"name"`
}
type dataObject struct {
Items []resultObject `json:"items"`
TotalCount int `json:"total_count"`
}
type errorObject struct {
Code string `json:"code"`
Message string `json:"message"`
}
type spyseResult struct {
Data dataObject `json:"data"`
Error []errorObject `json:"error"`
}
const searchMethodResultsLimit = 10000
// Source is the passive scraping agent
type Source struct{}
@ -43,33 +24,81 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
maxCount := 100
client, err := spyse.NewClient(session.Keys.Spyse, nil)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
for offSet := 0; offSet <= maxCount; offSet += 100 {
resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse})
domainSvc := spyse.NewDomainService(client)
var searchDomain = "." + domain
var subdomainsSearchParams spyse.QueryBuilder
subdomainsSearchParams.AppendParam(spyse.QueryParam{
Name: domainSvc.Params().Name.Name,
Operator: domainSvc.Params().Name.Operator.EndsWith,
Value: searchDomain,
})
totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
if totalResults == 0 {
return
}
accountSvc := spyse.NewAccountService(client)
quota, err := accountSvc.Quota(context.Background())
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
// The default "Search" method returns only first 10 000 subdomains
// To obtain more than 10 000 subdomains the "Scroll" method should be using
// Note: The "Scroll" method is only available for "PRO" customers, so we need to check
// quota.IsScrollSearchEnabled param
if totalResults > searchMethodResultsLimit && quota.IsScrollSearchEnabled {
var scrollResponse *spyse.DomainScrollResponse
scrollResponse, err = domainSvc.ScrollSearch(
ctx, subdomainsSearchParams.Query, "")
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
var response spyseResult
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
for len(scrollResponse.Items) > 0 {
scrollResponse, err = domainSvc.ScrollSearch(
context.Background(), subdomainsSearchParams.Query, scrollResponse.SearchID)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
for i := range scrollResponse.Items {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: scrollResponse.Items[i].Name}
}
}
resp.Body.Close()
} else {
var limit = 100
var offset = 0
var searchResults []spyse.Domain
if response.Data.TotalCount == 0 {
return
}
for ; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit {
searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
maxCount = response.Data.TotalCount
for _, hostname := range response.Data.Items {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name}
for i := range searchResults {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: searchResults[i].Name}
}
}
}
}()

8
v2/pkg/subscraping/types.go
View File

@ -4,6 +4,8 @@ import (
"context"
"net/http"
"regexp"
"go.uber.org/ratelimit"
)
// BasicAuth request's Authorization header
@ -31,16 +33,18 @@ type Session struct {
Keys *Keys
// Client is the current http client
Client *http.Client
// Rate limit instance
RateLimiter ratelimit.Limiter
}
// Keys contains the current API Keys we have in store
type Keys struct {
Binaryedge string `json:"binaryedge"`
C99 string `json:"c99"`
CensysToken string `json:"censysUsername"`
CensysSecret string `json:"censysPassword"`
Certspotter string `json:"certspotter"`
Chaos string `json:"chaos"`
Chinaz string `json:"chinaz"`
DNSDB string `json:"dnsdb"`
GitHub []string `json:"github"`
IntelXHost string `json:"intelXHost"`
@ -57,6 +61,8 @@ type Keys struct {
Virustotal string `json:"virustotal"`
ZoomEyeUsername string `json:"zoomeye_username"`
ZoomEyePassword string `json:"zoomeye_password"`
FofaUsername string `json:"fofa_username"`
FofaSecret string `json:"fofa_secret"`
}
// Result is a result structure returned by a source