Merge branch 'dev' into dependabot/docker/golang-1.17.0-alpine
Sandeep Singh
GitHub
commit
410a19ac91
|
|
@ -11,6 +11,7 @@ updates:
|
||||||
directory: "/"
|
directory: "/"
|
||||||
schedule:
|
schedule:
|
||||||
interval: "weekly"
|
interval: "weekly"
|
||||||
|
target-branch: "dev"
|
||||||
commit-message:
|
commit-message:
|
||||||
prefix: "chore"
|
prefix: "chore"
|
||||||
include: "scope"
|
include: "scope"
|
||||||
|
|
@ -20,6 +21,7 @@ updates:
|
||||||
directory: "/"
|
directory: "/"
|
||||||
schedule:
|
schedule:
|
||||||
interval: "weekly"
|
interval: "weekly"
|
||||||
|
target-branch: "dev"
|
||||||
commit-message:
|
commit-message:
|
||||||
prefix: "chore"
|
prefix: "chore"
|
||||||
include: "scope"
|
include: "scope"
|
||||||
|
|
@ -29,6 +31,7 @@ updates:
|
||||||
directory: "/"
|
directory: "/"
|
||||||
schedule:
|
schedule:
|
||||||
interval: "weekly"
|
interval: "weekly"
|
||||||
|
target-branch: "dev"
|
||||||
commit-message:
|
commit-message:
|
||||||
prefix: "chore"
|
prefix: "chore"
|
||||||
include: "scope"
|
include: "scope"
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
name: 🔨 Build Test
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Test Builds
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v2
|
||||||
|
with:
|
||||||
|
go-version: 1.15
|
||||||
|
|
||||||
|
- name: Check out code
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
- name: Test
|
||||||
|
run: go test .
|
||||||
|
working-directory: v2/cmd/subfinder/
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: go build .
|
||||||
|
working-directory: v2/cmd/subfinder/
|
||||||
|
|
@ -1,40 +0,0 @@
|
||||||
name: Build
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
golangci-lint:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
- name: Run golangci-lint
|
|
||||||
uses: golangci/golangci-lint-action@v2.5.2
|
|
||||||
with:
|
|
||||||
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
|
|
||||||
version: v1.33
|
|
||||||
args: --timeout 5m
|
|
||||||
working-directory: v2/
|
|
||||||
|
|
||||||
build:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Check out code
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v2
|
|
||||||
with:
|
|
||||||
go-version: 1.15
|
|
||||||
|
|
||||||
- name: Test
|
|
||||||
run: go test ./...
|
|
||||||
working-directory: v2/
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: go build .
|
|
||||||
working-directory: v2/cmd/subfinder/
|
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
name: 🚨 CodeQL Analysis
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- dev
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
analyze:
|
||||||
|
name: Analyze
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
contents: read
|
||||||
|
security-events: write
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
language: [ 'go' ]
|
||||||
|
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
# Initializes the CodeQL tools for scanning.
|
||||||
|
- name: Initialize CodeQL
|
||||||
|
uses: github/codeql-action/init@v1
|
||||||
|
with:
|
||||||
|
languages: ${{ matrix.language }}
|
||||||
|
|
||||||
|
- name: Autobuild
|
||||||
|
uses: github/codeql-action/autobuild@v1
|
||||||
|
|
||||||
|
- name: Perform CodeQL Analysis
|
||||||
|
uses: github/codeql-action/analyze@v1
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
# dockerhub-push pushes docker build to dockerhub automatically
|
|
||||||
# on the creation of a new release
|
|
||||||
name: Publish to Dockerhub on creation of a new release
|
|
||||||
on:
|
|
||||||
release:
|
|
||||||
types: [published]
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@master
|
|
||||||
- name: Publish to Dockerhub Registry
|
|
||||||
uses: elgohr/Publish-Docker-Github-Action@master
|
|
||||||
with:
|
|
||||||
name: projectdiscovery/subfinder
|
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
name: 🌥 Docker Push
|
||||||
|
|
||||||
|
on:
|
||||||
|
release:
|
||||||
|
types: [published]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
docker:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
-
|
||||||
|
name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
-
|
||||||
|
name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v1
|
||||||
|
-
|
||||||
|
name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v1
|
||||||
|
-
|
||||||
|
name: Login to DockerHub
|
||||||
|
uses: docker/login-action@v1
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_TOKEN }}
|
||||||
|
-
|
||||||
|
name: Build and push
|
||||||
|
uses: docker/build-push-action@v2
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
platforms: linux/amd64,linux/arm64,linux/arm
|
||||||
|
push: true
|
||||||
|
tags: projectdiscovery/subfinder:latest
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
name: 🙏🏻 Lint Test
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
name: Lint Test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
- name: Run golangci-lint
|
||||||
|
uses: golangci/golangci-lint-action@v2
|
||||||
|
with:
|
||||||
|
version: latest
|
||||||
|
args: --timeout 5m
|
||||||
|
working-directory: v2/
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
name: Release
|
name: 🎉 Release Binary
|
||||||
on:
|
on:
|
||||||
create:
|
create:
|
||||||
tags:
|
tags:
|
||||||
- v*
|
- v*
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
release:
|
release:
|
||||||
|
|
@ -17,7 +18,7 @@ jobs:
|
||||||
name: "Set up Go"
|
name: "Set up Go"
|
||||||
uses: actions/setup-go@v2
|
uses: actions/setup-go@v2
|
||||||
with:
|
with:
|
||||||
go-version: 1.15
|
go-version: 1.16
|
||||||
-
|
-
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||||
|
|
@ -26,5 +27,4 @@ jobs:
|
||||||
with:
|
with:
|
||||||
args: "release --rm-dist"
|
args: "release --rm-dist"
|
||||||
version: latest
|
version: latest
|
||||||
workdir: v2/
|
workdir: .
|
||||||
|
|
||||||
122
.golangci.yml
122
.golangci.yml
|
|
@ -1,122 +0,0 @@
|
||||||
linters-settings:
|
|
||||||
dupl:
|
|
||||||
threshold: 100
|
|
||||||
exhaustive:
|
|
||||||
default-signifies-exhaustive: false
|
|
||||||
# funlen:
|
|
||||||
# lines: 100
|
|
||||||
# statements: 50
|
|
||||||
goconst:
|
|
||||||
min-len: 2
|
|
||||||
min-occurrences: 2
|
|
||||||
gocritic:
|
|
||||||
enabled-tags:
|
|
||||||
- diagnostic
|
|
||||||
- experimental
|
|
||||||
- opinionated
|
|
||||||
- performance
|
|
||||||
- style
|
|
||||||
disabled-checks:
|
|
||||||
- dupImport # https://github.com/go-critic/go-critic/issues/845
|
|
||||||
- ifElseChain
|
|
||||||
# gocyclo:
|
|
||||||
# min-complexity: 15
|
|
||||||
goimports:
|
|
||||||
local-prefixes: github.com/golangci/golangci-lint
|
|
||||||
golint:
|
|
||||||
min-confidence: 0
|
|
||||||
gomnd:
|
|
||||||
settings:
|
|
||||||
mnd:
|
|
||||||
# don't include the "operation" and "assign"
|
|
||||||
checks: argument,case,condition,return
|
|
||||||
govet:
|
|
||||||
check-shadowing: true
|
|
||||||
settings:
|
|
||||||
printf:
|
|
||||||
funcs:
|
|
||||||
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
|
|
||||||
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
|
|
||||||
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
|
|
||||||
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
|
|
||||||
# lll:
|
|
||||||
# line-length: 140
|
|
||||||
maligned:
|
|
||||||
suggest-new: true
|
|
||||||
misspell:
|
|
||||||
locale: US
|
|
||||||
nolintlint:
|
|
||||||
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
|
|
||||||
allow-unused: false # report any unused nolint directives
|
|
||||||
require-explanation: false # don't require an explanation for nolint directives
|
|
||||||
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
|
|
||||||
|
|
||||||
linters:
|
|
||||||
# please, do not use `enable-all`: it's deprecated and will be removed soon.
|
|
||||||
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
|
|
||||||
disable-all: true
|
|
||||||
enable:
|
|
||||||
- bodyclose
|
|
||||||
- deadcode
|
|
||||||
- dogsled
|
|
||||||
- dupl
|
|
||||||
# - errcheck
|
|
||||||
- exhaustive
|
|
||||||
- gochecknoinits
|
|
||||||
- goconst
|
|
||||||
- gocritic
|
|
||||||
- gofmt
|
|
||||||
- goimports
|
|
||||||
- golint
|
|
||||||
- gomnd
|
|
||||||
- goprintffuncname
|
|
||||||
- gosimple
|
|
||||||
- govet
|
|
||||||
- ineffassign
|
|
||||||
- interfacer
|
|
||||||
- maligned
|
|
||||||
- misspell
|
|
||||||
- nakedret
|
|
||||||
- noctx
|
|
||||||
# - nolintlint
|
|
||||||
- rowserrcheck
|
|
||||||
- scopelint
|
|
||||||
- staticcheck
|
|
||||||
- structcheck
|
|
||||||
- stylecheck
|
|
||||||
- typecheck
|
|
||||||
- unconvert
|
|
||||||
- unparam
|
|
||||||
- unused
|
|
||||||
- varcheck
|
|
||||||
- whitespace
|
|
||||||
|
|
||||||
# don't enable:
|
|
||||||
# - depguard
|
|
||||||
# - asciicheck
|
|
||||||
# - funlen
|
|
||||||
# - gochecknoglobals
|
|
||||||
# - gocognit
|
|
||||||
# - gocyclo
|
|
||||||
# - godot
|
|
||||||
# - godox
|
|
||||||
# - goerr113
|
|
||||||
# - gosec
|
|
||||||
# - lll
|
|
||||||
# - nestif
|
|
||||||
# - prealloc
|
|
||||||
# - testpackage
|
|
||||||
# - wsl
|
|
||||||
|
|
||||||
issues:
|
|
||||||
exclude-use-default: false
|
|
||||||
exclude:
|
|
||||||
# should have a package comment, unless it's in another file for this package (golint)
|
|
||||||
- 'in another file for this package'
|
|
||||||
|
|
||||||
# golangci.com configuration
|
|
||||||
# https://github.com/golangci/golangci/wiki/Configuration
|
|
||||||
service:
|
|
||||||
golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
|
|
||||||
prepare:
|
|
||||||
- echo "here I can run custom commands, but no preparation needed for this repo"
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
# Build
|
# Build
|
||||||
FROM golang:1.17.0-alpine AS build-env
|
FROM golang:1.17.0-alpine AS build-env
|
||||||
|
|
||||||
RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
|
RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
|
||||||
|
|
||||||
# Release
|
# Release
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ subfinder -h
|
||||||
This will display help for the tool. Here are all the switches it supports.
|
This will display help for the tool. Here are all the switches it supports.
|
||||||
|
|
||||||
| Flag | Description | Example |
|
| Flag | Description | Example |
|
||||||
| ---------------- | ---------------------------------------------------------- | -------------------------------------- |
|
| ---------------- | ---------------------------------------------------------- | --------------------------------------------|
|
||||||
| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
|
| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
|
||||||
| -config | Configuration file for API Keys, etc | subfinder -config config.yaml |
|
| -config | Configuration file for API Keys, etc | subfinder -config config.yaml |
|
||||||
| -d | Domain to find subdomains for | subfinder -d uber.com |
|
| -d | Domain to find subdomains for | subfinder -d uber.com |
|
||||||
|
|
@ -75,6 +75,8 @@ This will display help for the tool. Here are all the switches it supports.
|
||||||
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
|
| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
|
||||||
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
|
| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
|
||||||
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
|
| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
|
||||||
|
| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 |
|
||||||
|
| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 |
|
||||||
| -v | Show Verbose output | subfinder -v |
|
| -v | Show Verbose output | subfinder -v |
|
||||||
| -version | Show current program version | subfinder -version |
|
| -version | Show current program version | subfinder -version |
|
||||||
|
|
||||||
|
|
@ -93,6 +95,7 @@ Subfinder requires **go1.14+** to install successfully. Run the following comman
|
||||||
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
|
Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
|
||||||
|
|
||||||
- [Binaryedge](https://binaryedge.io)
|
- [Binaryedge](https://binaryedge.io)
|
||||||
|
- [C99](https://api.c99.nl/)
|
||||||
- [Certspotter](https://sslmate.com/certspotter/api/)
|
- [Certspotter](https://sslmate.com/certspotter/api/)
|
||||||
- [Censys](https://censys.io)
|
- [Censys](https://censys.io)
|
||||||
- [Chaos](https://chaos.projectdiscovery.io)
|
- [Chaos](https://chaos.projectdiscovery.io)
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,11 @@ before:
|
||||||
- go mod tidy
|
- go mod tidy
|
||||||
|
|
||||||
builds:
|
builds:
|
||||||
- binary: subfinder
|
- env:
|
||||||
main: cmd/subfinder/main.go
|
- CGO_ENABLED=0
|
||||||
goos:
|
goos:
|
||||||
- linux
|
|
||||||
- windows
|
- windows
|
||||||
|
- linux
|
||||||
- darwin
|
- darwin
|
||||||
goarch:
|
goarch:
|
||||||
- amd64
|
- amd64
|
||||||
|
|
@ -15,11 +15,19 @@ builds:
|
||||||
- arm
|
- arm
|
||||||
- arm64
|
- arm64
|
||||||
|
|
||||||
|
ignore:
|
||||||
|
- goos: darwin
|
||||||
|
goarch: '386'
|
||||||
|
- goos: windows
|
||||||
|
goarch: 'arm'
|
||||||
|
|
||||||
|
binary: '{{ .ProjectName }}'
|
||||||
|
main: cmd/subfinder/main.go
|
||||||
|
|
||||||
archives:
|
archives:
|
||||||
- id: tgz
|
- format: zip
|
||||||
format: tar.gz
|
|
||||||
replacements:
|
replacements:
|
||||||
darwin: macOS
|
darwin: macOS
|
||||||
format_overrides:
|
|
||||||
- goos: windows
|
checksum:
|
||||||
format: zip
|
algorithm: sha256
|
||||||
|
|
@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2
|
||||||
go 1.16
|
go 1.16
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/corpix/uarand v0.1.1
|
||||||
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
|
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
|
||||||
github.com/json-iterator/go v1.1.10
|
github.com/json-iterator/go v1.1.10
|
||||||
github.com/lib/pq v1.10.0
|
github.com/lib/pq v1.10.0
|
||||||
|
|
@ -11,7 +12,9 @@ require (
|
||||||
github.com/projectdiscovery/fdmax v0.0.3
|
github.com/projectdiscovery/fdmax v0.0.3
|
||||||
github.com/projectdiscovery/gologger v1.1.4
|
github.com/projectdiscovery/gologger v1.1.4
|
||||||
github.com/rs/xid v1.3.0
|
github.com/rs/xid v1.3.0
|
||||||
|
github.com/spyse-com/go-spyse v1.2.1
|
||||||
github.com/stretchr/testify v1.7.0
|
github.com/stretchr/testify v1.7.0
|
||||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
||||||
|
go.uber.org/ratelimit v0.2.0
|
||||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||||
)
|
)
|
||||||
|
|
|
||||||
15
v2/go.sum
15
v2/go.sum
|
|
@ -1,8 +1,15 @@
|
||||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
|
||||||
|
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
|
||||||
|
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
|
||||||
|
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
|
||||||
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
|
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
|
||||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||||
|
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
|
||||||
|
github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
|
||||||
|
github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
|
@ -51,12 +58,16 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
|
||||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||||
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
|
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
|
||||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||||
|
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||||
|
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
|
||||||
|
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
|
github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
|
||||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||||
|
|
@ -87,6 +98,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g
|
||||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
|
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
|
||||||
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||||
|
github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8=
|
||||||
|
github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||||
|
|
@ -97,7 +110,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
|
||||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
|
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
|
||||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
|
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
|
||||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||||
|
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
|
||||||
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
|
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
|
|
||||||
|
|
@ -11,11 +11,11 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// EnumerateSubdomains enumerates all the subdomains for a given domain
|
// EnumerateSubdomains enumerates all the subdomains for a given domain
|
||||||
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
|
func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
|
||||||
results := make(chan subscraping.Result)
|
results := make(chan subscraping.Result)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
session, err := subscraping.NewSession(domain, keys, proxy, timeout)
|
session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
|
results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,14 +7,15 @@ import (
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99"
|
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
|
||||||
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
|
||||||
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
|
||||||
|
|
@ -45,6 +46,7 @@ var DefaultSources = []string{
|
||||||
"certspotter",
|
"certspotter",
|
||||||
"censys",
|
"censys",
|
||||||
"chaos",
|
"chaos",
|
||||||
|
"chinaz",
|
||||||
"crtsh",
|
"crtsh",
|
||||||
"dnsdumpster",
|
"dnsdumpster",
|
||||||
"hackertarget",
|
"hackertarget",
|
||||||
|
|
@ -59,6 +61,7 @@ var DefaultSources = []string{
|
||||||
"threatcrowd",
|
"threatcrowd",
|
||||||
"threatminer",
|
"threatminer",
|
||||||
"virustotal",
|
"virustotal",
|
||||||
|
"fofa",
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultRecursiveSources contains list of default recursive sources
|
// DefaultRecursiveSources contains list of default recursive sources
|
||||||
|
|
@ -84,7 +87,6 @@ var DefaultAllSources = []string{
|
||||||
"archiveis",
|
"archiveis",
|
||||||
"binaryedge",
|
"binaryedge",
|
||||||
"bufferover",
|
"bufferover",
|
||||||
"c99",
|
|
||||||
"censys",
|
"censys",
|
||||||
"certspotter",
|
"certspotter",
|
||||||
"chaos",
|
"chaos",
|
||||||
|
|
@ -112,6 +114,7 @@ var DefaultAllSources = []string{
|
||||||
"virustotal",
|
"virustotal",
|
||||||
"waybackarchive",
|
"waybackarchive",
|
||||||
"zoomeye",
|
"zoomeye",
|
||||||
|
"fofa",
|
||||||
}
|
}
|
||||||
|
|
||||||
// Agent is a struct for running passive subdomain enumeration
|
// Agent is a struct for running passive subdomain enumeration
|
||||||
|
|
@ -146,14 +149,14 @@ func (a *Agent) addSources(sources []string) {
|
||||||
a.sources[source] = &binaryedge.Source{}
|
a.sources[source] = &binaryedge.Source{}
|
||||||
case "bufferover":
|
case "bufferover":
|
||||||
a.sources[source] = &bufferover.Source{}
|
a.sources[source] = &bufferover.Source{}
|
||||||
case "c99":
|
|
||||||
a.sources[source] = &c99.Source{}
|
|
||||||
case "censys":
|
case "censys":
|
||||||
a.sources[source] = &censys.Source{}
|
a.sources[source] = &censys.Source{}
|
||||||
case "certspotter":
|
case "certspotter":
|
||||||
a.sources[source] = &certspotter.Source{}
|
a.sources[source] = &certspotter.Source{}
|
||||||
case "chaos":
|
case "chaos":
|
||||||
a.sources[source] = &chaos.Source{}
|
a.sources[source] = &chaos.Source{}
|
||||||
|
case "chinaz":
|
||||||
|
a.sources[source] = &chinaz.Source{}
|
||||||
case "commoncrawl":
|
case "commoncrawl":
|
||||||
a.sources[source] = &commoncrawl.Source{}
|
a.sources[source] = &commoncrawl.Source{}
|
||||||
case "crtsh":
|
case "crtsh":
|
||||||
|
|
@ -202,6 +205,8 @@ func (a *Agent) addSources(sources []string) {
|
||||||
a.sources[source] = &waybackarchive.Source{}
|
a.sources[source] = &waybackarchive.Source{}
|
||||||
case "zoomeye":
|
case "zoomeye":
|
||||||
a.sources[source] = &zoomeye.Source{}
|
a.sources[source] = &zoomeye.Source{}
|
||||||
|
case "fofa":
|
||||||
|
a.sources[source] = &fofa.Source{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,11 +11,11 @@ const banner = `
|
||||||
_______ __/ /_ / __(_)___ ____/ /__ _____
|
_______ __/ /_ / __(_)___ ____/ /__ _____
|
||||||
/ ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
|
/ ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
|
||||||
(__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
|
(__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
|
||||||
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.8
|
/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9
|
||||||
`
|
`
|
||||||
|
|
||||||
// Version is the current version of subfinder
|
// Version is the current version of subfinder
|
||||||
const Version = `2.4.8`
|
const Version = `v2.4.9`
|
||||||
|
|
||||||
// showBanner is used to show the banner to the user
|
// showBanner is used to show the banner to the user
|
||||||
func showBanner() {
|
func showBanner() {
|
||||||
|
|
|
||||||
|
|
@ -30,10 +30,10 @@ type ConfigFile struct {
|
||||||
ExcludeSources []string `yaml:"exclude-sources,omitempty"`
|
ExcludeSources []string `yaml:"exclude-sources,omitempty"`
|
||||||
// API keys for different sources
|
// API keys for different sources
|
||||||
Binaryedge []string `yaml:"binaryedge"`
|
Binaryedge []string `yaml:"binaryedge"`
|
||||||
C99 []string `yaml:"c99"`
|
|
||||||
Censys []string `yaml:"censys"`
|
Censys []string `yaml:"censys"`
|
||||||
Certspotter []string `yaml:"certspotter"`
|
Certspotter []string `yaml:"certspotter"`
|
||||||
Chaos []string `yaml:"chaos"`
|
Chaos []string `yaml:"chaos"`
|
||||||
|
Chinaz []string `yaml:"chinaz"`
|
||||||
DNSDB []string `yaml:"dnsdb"`
|
DNSDB []string `yaml:"dnsdb"`
|
||||||
GitHub []string `yaml:"github"`
|
GitHub []string `yaml:"github"`
|
||||||
IntelX []string `yaml:"intelx"`
|
IntelX []string `yaml:"intelx"`
|
||||||
|
|
@ -47,6 +47,7 @@ type ConfigFile struct {
|
||||||
URLScan []string `yaml:"urlscan"`
|
URLScan []string `yaml:"urlscan"`
|
||||||
Virustotal []string `yaml:"virustotal"`
|
Virustotal []string `yaml:"virustotal"`
|
||||||
ZoomEye []string `yaml:"zoomeye"`
|
ZoomEye []string `yaml:"zoomeye"`
|
||||||
|
Fofa []string `yaml:"fofa"`
|
||||||
// Version indicates the version of subfinder installed.
|
// Version indicates the version of subfinder installed.
|
||||||
Version string `yaml:"subfinder-version"`
|
Version string `yaml:"subfinder-version"`
|
||||||
}
|
}
|
||||||
|
|
@ -120,9 +121,6 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
|
||||||
if len(c.Binaryedge) > 0 {
|
if len(c.Binaryedge) > 0 {
|
||||||
keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
|
keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
|
||||||
}
|
}
|
||||||
if len(c.C99) > 0 {
|
|
||||||
keys.C99 = c.C99[rand.Intn(len(c.C99))]
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(c.Censys) > 0 {
|
if len(c.Censys) > 0 {
|
||||||
censysKeys := c.Censys[rand.Intn(len(c.Censys))]
|
censysKeys := c.Censys[rand.Intn(len(c.Censys))]
|
||||||
|
|
@ -139,6 +137,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
|
||||||
if len(c.Chaos) > 0 {
|
if len(c.Chaos) > 0 {
|
||||||
keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
|
keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
|
||||||
}
|
}
|
||||||
|
if len(c.Chinaz) > 0 {
|
||||||
|
keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))]
|
||||||
|
}
|
||||||
if (len(c.DNSDB)) > 0 {
|
if (len(c.DNSDB)) > 0 {
|
||||||
keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
|
keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
|
||||||
}
|
}
|
||||||
|
|
@ -198,6 +199,14 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
|
||||||
keys.ZoomEyePassword = parts[1]
|
keys.ZoomEyePassword = parts[1]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(c.Fofa) > 0 {
|
||||||
|
fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))]
|
||||||
|
parts := strings.Split(fofaKeys, ":")
|
||||||
|
if len(parts) == MultipleKeyPartsLength {
|
||||||
|
keys.FofaUsername = parts[0]
|
||||||
|
keys.FofaSecret = parts[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
|
||||||
|
|
||||||
// Run the passive subdomain enumeration
|
// Run the passive subdomain enumeration
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
|
passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
|
||||||
|
|
||||||
wg := &sync.WaitGroup{}
|
wg := &sync.WaitGroup{}
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,7 @@ type Options struct {
|
||||||
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
|
ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
|
||||||
ConfigFile string // ConfigFile contains the location of the config file
|
ConfigFile string // ConfigFile contains the location of the config file
|
||||||
Proxy string // HTTP proxy
|
Proxy string // HTTP proxy
|
||||||
|
RateLimit int // Maximum number of HTTP requests to send per second
|
||||||
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
|
YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -77,6 +78,7 @@ func ParseOptions() *Options {
|
||||||
flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
|
flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
|
||||||
flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
|
flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
|
||||||
flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use")
|
flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use")
|
||||||
|
flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second")
|
||||||
flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
|
flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,11 +10,13 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/corpix/uarand"
|
||||||
"github.com/projectdiscovery/gologger"
|
"github.com/projectdiscovery/gologger"
|
||||||
|
"go.uber.org/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewSession creates a new session object for a domain
|
// NewSession creates a new session object for a domain
|
||||||
func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) {
|
func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) {
|
||||||
Transport := &http.Transport{
|
Transport := &http.Transport{
|
||||||
MaxIdleConns: 100,
|
MaxIdleConns: 100,
|
||||||
MaxIdleConnsPerHost: 100,
|
MaxIdleConnsPerHost: 100,
|
||||||
|
|
@ -44,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session,
|
||||||
Keys: keys,
|
Keys: keys,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Initiate rate limit instance
|
||||||
|
if rateLimit > 0 {
|
||||||
|
session.RateLimiter = ratelimit.New(rateLimit)
|
||||||
|
} else {
|
||||||
|
session.RateLimiter = ratelimit.NewUnlimited()
|
||||||
|
}
|
||||||
|
|
||||||
// Create a new extractor object for the current domain
|
// Create a new extractor object for the current domain
|
||||||
extractor, err := NewSubdomainExtractor(domain)
|
extractor, err := NewSubdomainExtractor(domain)
|
||||||
session.Extractor = extractor
|
session.Extractor = extractor
|
||||||
|
|
@ -78,7 +87,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
|
req.Header.Set("User-Agent", uarand.GetRandom())
|
||||||
req.Header.Set("Accept", "*/*")
|
req.Header.Set("Accept", "*/*")
|
||||||
req.Header.Set("Accept-Language", "en")
|
req.Header.Set("Accept-Language", "en")
|
||||||
req.Header.Set("Connection", "close")
|
req.Header.Set("Connection", "close")
|
||||||
|
|
@ -95,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
|
||||||
req.Header.Set(key, value)
|
req.Header.Set(key, value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.RateLimiter.Take()
|
||||||
|
|
||||||
return httpRequestWrapper(s.Client, req)
|
return httpRequestWrapper(s.Client, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,36 +5,36 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"regexp"
|
|
||||||
|
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||||
)
|
)
|
||||||
|
|
||||||
type agent struct {
|
// Source is the passive scraping agent
|
||||||
Results chan subscraping.Result
|
type Source struct{}
|
||||||
Session *subscraping.Session
|
|
||||||
|
// Name returns the name of the source
|
||||||
|
func (s *Source) Name() string {
|
||||||
|
return "archiveis"
|
||||||
}
|
}
|
||||||
|
|
||||||
var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">→</a>")
|
// Run function returns all subdomains found with the service
|
||||||
|
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
||||||
|
results := make(chan subscraping.Result)
|
||||||
|
|
||||||
func (a *agent) enumerate(ctx context.Context, baseURL string) {
|
go func() {
|
||||||
select {
|
defer close(results)
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain))
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := a.Session.SimpleGet(ctx, baseURL)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
|
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
|
||||||
a.Session.DiscardHTTPResponse(resp)
|
session.DiscardHTTPResponse(resp)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the response body
|
|
||||||
body, err := ioutil.ReadAll(resp.Body)
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
|
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
@ -42,37 +42,11 @@ func (a *agent) enumerate(ctx context.Context, baseURL string) {
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
|
|
||||||
src := string(body)
|
src := string(body)
|
||||||
for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) {
|
|
||||||
a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
|
for _, subdomain := range session.Extractor.FindAllString(src, -1) {
|
||||||
|
results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
|
||||||
}
|
}
|
||||||
|
|
||||||
match1 := reNext.FindStringSubmatch(src)
|
|
||||||
if len(match1) > 0 {
|
|
||||||
a.enumerate(ctx, match1[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Source is the passive scraping agent
|
|
||||||
type Source struct{}
|
|
||||||
|
|
||||||
// Run function returns all subdomains found with the service
|
|
||||||
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
|
||||||
results := make(chan subscraping.Result)
|
|
||||||
|
|
||||||
a := agent{
|
|
||||||
Session: session,
|
|
||||||
Results: results,
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain))
|
|
||||||
close(a.Results)
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return a.Results
|
return results
|
||||||
}
|
|
||||||
|
|
||||||
// Name returns the name of the source
|
|
||||||
func (s *Source) Name() string {
|
|
||||||
return "archiveis"
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
// Package c99 logic
|
|
||||||
package c99
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
jsoniter "github.com/json-iterator/go"
|
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Source is the passive scraping agent
|
|
||||||
type Source struct{}
|
|
||||||
|
|
||||||
type dnsdbLookupResponse struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
Subdomains []struct {
|
|
||||||
Subdomain string `json:"subdomain"`
|
|
||||||
IP string `json:"ip"`
|
|
||||||
Cloudflare bool `json:"cloudflare"`
|
|
||||||
} `json:"subdomains"`
|
|
||||||
Error string `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run function returns all subdomains found with the service
|
|
||||||
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
|
||||||
results := make(chan subscraping.Result)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(results)
|
|
||||||
|
|
||||||
if session.Keys.C99 == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain)
|
|
||||||
resp, err := session.SimpleGet(ctx, searchURL)
|
|
||||||
if err != nil {
|
|
||||||
session.DiscardHTTPResponse(resp)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
var response dnsdbLookupResponse
|
|
||||||
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
|
|
||||||
if err != nil {
|
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if response.Error != "" {
|
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, data := range response.Subdomains {
|
|
||||||
if !strings.HasPrefix(data.Subdomain, ".") {
|
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
// Name returns the name of the source
|
|
||||||
func (s *Source) Name() string {
|
|
||||||
return "c99"
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
package chinaz
|
||||||
|
// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
jsoniter "github.com/json-iterator/go"
|
||||||
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||||
|
"io/ioutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Source is the passive scraping agent
|
||||||
|
type Source struct{}
|
||||||
|
|
||||||
|
// Run function returns all subdomains found with the service
|
||||||
|
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
||||||
|
results := make(chan subscraping.Result)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(results)
|
||||||
|
|
||||||
|
if session.Keys.Chinaz == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain))
|
||||||
|
if err != nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
session.DiscardHTTPResponse(resp)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList")
|
||||||
|
|
||||||
|
if SubdomainList.ToBool() {
|
||||||
|
_data := []byte(SubdomainList.ToString())
|
||||||
|
for i := 0 ; i< SubdomainList.Size() ; i++{
|
||||||
|
subdomain := jsoniter.Get(_data,i,"DataUrl").ToString()
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of the source
|
||||||
|
func (s *Source) Name() string {
|
||||||
|
return "chinaz"
|
||||||
|
}
|
||||||
|
|
@ -44,6 +44,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
pattern := "%." + domain
|
pattern := "%." + domain
|
||||||
query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci
|
query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci
|
||||||
WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))
|
WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ func postForm(ctx context.Context, session *subscraping.Session, token, domain s
|
||||||
params := url.Values{
|
params := url.Values{
|
||||||
"csrfmiddlewaretoken": {token},
|
"csrfmiddlewaretoken": {token},
|
||||||
"targetip": {domain},
|
"targetip": {domain},
|
||||||
|
"user": {"free"},
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := session.HTTPRequest(
|
resp, err := session.HTTPRequest(
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
// Package fofa logic
|
||||||
|
package fofa
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
jsoniter "github.com/json-iterator/go"
|
||||||
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fofaResponse struct {
|
||||||
|
Error bool `json:"error"`
|
||||||
|
ErrMsg string `json:"errmsg"`
|
||||||
|
Size int `json:"size"`
|
||||||
|
Results []string `json:"results"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Source is the passive scraping agent
|
||||||
|
type Source struct{}
|
||||||
|
|
||||||
|
// Run function returns all subdomains found with the service
|
||||||
|
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
||||||
|
results := make(chan subscraping.Result)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(results)
|
||||||
|
|
||||||
|
if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// fofa api doc https://fofa.so/static_pages/api_help
|
||||||
|
qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain)))
|
||||||
|
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.so/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64))
|
||||||
|
if err != nil && resp == nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
session.DiscardHTTPResponse(resp)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var response fofaResponse
|
||||||
|
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
|
||||||
|
if err != nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
resp.Body.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
if response.Error {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if response.Size > 0 {
|
||||||
|
for _, subdomain := range response.Results {
|
||||||
|
if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") {
|
||||||
|
subdomain = subdomain[strings.Index(subdomain, "//")+2:]
|
||||||
|
}
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of the source
|
||||||
|
func (s *Source) Name() string {
|
||||||
|
return "fofa"
|
||||||
|
}
|
||||||
|
|
@ -10,7 +10,11 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type subdomain struct {
|
type subdomain struct {
|
||||||
RawDomain string `json:"rawDomain"`
|
Domains []string `json:"domains"`
|
||||||
|
Ip string `json:"ip"`
|
||||||
|
RawDomains []string `json:"rawDomains"`
|
||||||
|
RawPort string `json:"rawPort"`
|
||||||
|
RawIp string `json:"rawIp"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Source is the passive scraping agent
|
// Source is the passive scraping agent
|
||||||
|
|
@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
|
|
||||||
for _, subdomain := range subdomains {
|
for _, subdomain := range subdomains {
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain}
|
for _, dmn := range subdomain.RawDomains {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,31 +3,12 @@ package spyse
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
jsoniter "github.com/json-iterator/go"
|
|
||||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||||
|
spyse "github.com/spyse-com/go-spyse/pkg"
|
||||||
)
|
)
|
||||||
|
|
||||||
type resultObject struct {
|
const searchMethodResultsLimit = 10000
|
||||||
Name string `json:"name"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type dataObject struct {
|
|
||||||
Items []resultObject `json:"items"`
|
|
||||||
TotalCount int `json:"total_count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type errorObject struct {
|
|
||||||
Code string `json:"code"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type spyseResult struct {
|
|
||||||
Data dataObject `json:"data"`
|
|
||||||
Error []errorObject `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Source is the passive scraping agent
|
// Source is the passive scraping agent
|
||||||
type Source struct{}
|
type Source struct{}
|
||||||
|
|
@ -43,33 +24,81 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
maxCount := 100
|
client, err := spyse.NewClient(session.Keys.Spyse, nil)
|
||||||
|
|
||||||
for offSet := 0; offSet <= maxCount; offSet += 100 {
|
|
||||||
resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
session.DiscardHTTPResponse(resp)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
var response spyseResult
|
domainSvc := spyse.NewDomainService(client)
|
||||||
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
|
|
||||||
|
var searchDomain = "." + domain
|
||||||
|
var subdomainsSearchParams spyse.QueryBuilder
|
||||||
|
|
||||||
|
subdomainsSearchParams.AppendParam(spyse.QueryParam{
|
||||||
|
Name: domainSvc.Params().Name.Name,
|
||||||
|
Operator: domainSvc.Params().Name.Operator.EndsWith,
|
||||||
|
Value: searchDomain,
|
||||||
|
})
|
||||||
|
|
||||||
|
totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
resp.Body.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
|
|
||||||
if response.Data.TotalCount == 0 {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
maxCount = response.Data.TotalCount
|
if totalResults == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
for _, hostname := range response.Data.Items {
|
accountSvc := spyse.NewAccountService(client)
|
||||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name}
|
|
||||||
|
quota, err := accountSvc.Quota(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The default "Search" method returns only first 10 000 subdomains
|
||||||
|
// To obtain more than 10 000 subdomains the "Scroll" method should be using
|
||||||
|
// Note: The "Scroll" method is only available for "PRO" customers, so we need to check
|
||||||
|
// quota.IsScrollSearchEnabled param
|
||||||
|
if totalResults > searchMethodResultsLimit && quota.IsScrollSearchEnabled {
|
||||||
|
var scrollResponse *spyse.DomainScrollResponse
|
||||||
|
scrollResponse, err = domainSvc.ScrollSearch(
|
||||||
|
ctx, subdomainsSearchParams.Query, "")
|
||||||
|
if err != nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for len(scrollResponse.Items) > 0 {
|
||||||
|
scrollResponse, err = domainSvc.ScrollSearch(
|
||||||
|
context.Background(), subdomainsSearchParams.Query, scrollResponse.SearchID)
|
||||||
|
if err != nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range scrollResponse.Items {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: scrollResponse.Items[i].Name}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var limit = 100
|
||||||
|
var offset = 0
|
||||||
|
var searchResults []spyse.Domain
|
||||||
|
|
||||||
|
for ; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit {
|
||||||
|
searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset)
|
||||||
|
if err != nil {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range searchResults {
|
||||||
|
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: searchResults[i].Name}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
|
||||||
|
"go.uber.org/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BasicAuth request's Authorization header
|
// BasicAuth request's Authorization header
|
||||||
|
|
@ -31,16 +33,18 @@ type Session struct {
|
||||||
Keys *Keys
|
Keys *Keys
|
||||||
// Client is the current http client
|
// Client is the current http client
|
||||||
Client *http.Client
|
Client *http.Client
|
||||||
|
// Rate limit instance
|
||||||
|
RateLimiter ratelimit.Limiter
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keys contains the current API Keys we have in store
|
// Keys contains the current API Keys we have in store
|
||||||
type Keys struct {
|
type Keys struct {
|
||||||
Binaryedge string `json:"binaryedge"`
|
Binaryedge string `json:"binaryedge"`
|
||||||
C99 string `json:"c99"`
|
|
||||||
CensysToken string `json:"censysUsername"`
|
CensysToken string `json:"censysUsername"`
|
||||||
CensysSecret string `json:"censysPassword"`
|
CensysSecret string `json:"censysPassword"`
|
||||||
Certspotter string `json:"certspotter"`
|
Certspotter string `json:"certspotter"`
|
||||||
Chaos string `json:"chaos"`
|
Chaos string `json:"chaos"`
|
||||||
|
Chinaz string `json:"chinaz"`
|
||||||
DNSDB string `json:"dnsdb"`
|
DNSDB string `json:"dnsdb"`
|
||||||
GitHub []string `json:"github"`
|
GitHub []string `json:"github"`
|
||||||
IntelXHost string `json:"intelXHost"`
|
IntelXHost string `json:"intelXHost"`
|
||||||
|
|
@ -57,6 +61,8 @@ type Keys struct {
|
||||||
Virustotal string `json:"virustotal"`
|
Virustotal string `json:"virustotal"`
|
||||||
ZoomEyeUsername string `json:"zoomeye_username"`
|
ZoomEyeUsername string `json:"zoomeye_username"`
|
||||||
ZoomEyePassword string `json:"zoomeye_password"`
|
ZoomEyePassword string `json:"zoomeye_password"`
|
||||||
|
FofaUsername string `json:"fofa_username"`
|
||||||
|
FofaSecret string `json:"fofa_secret"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Result is a result structure returned by a source
|
// Result is a result structure returned by a source
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue