diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d5617f..69d9543 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,6 +11,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" @@ -20,6 +21,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" @@ -29,6 +31,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" - include: "scope" + include: "scope" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..ad0d347 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,27 @@ +name: 🔨 Build Test +on: + push: + pull_request: + workflow_dispatch: + + +jobs: + build: + name: Test Builds + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Test + run: go test . + working-directory: v2/cmd/subfinder/ + + - name: Build + run: go build . + working-directory: v2/cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index d0d0e47..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build - -on: - push: - branches: - - master - pull_request: - -jobs: - golangci-lint: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.33 - args: --timeout 5m - working-directory: v2/ - - build: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.15 - - - name: Test - run: go test ./... - working-directory: v2/ - - - name: Build - run: go build . - working-directory: v2/cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..545cdea --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: 🚨 CodeQL Analysis + +on: + workflow_dispatch: + pull_request: + branches: + - dev + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push-on-release.yml b/.github/workflows/dockerhub-push-on-release.yml deleted file mode 100644 index a5156cb..0000000 --- a/.github/workflows/dockerhub-push-on-release.yml +++ /dev/null @@ -1,17 +0,0 @@ -# dockerhub-push pushes docker build to dockerhub automatically -# on the creation of a new release -name: Publish to Dockerhub on creation of a new release -on: - release: - types: [published] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Publish to Dockerhub Registry - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: projectdiscovery/subfinder - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml new file mode 100644 index 0000000..ab9b9db --- /dev/null +++ b/.github/workflows/dockerhub-push.yml @@ -0,0 +1,34 @@ +name: 🌥 Docker Push + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm + push: true + tags: projectdiscovery/subfinder:latest \ No newline at end of file diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml new file mode 100644 index 0000000..ad6dd61 --- /dev/null +++ b/.github/workflows/lint-test.yml @@ -0,0 +1,19 @@ +name: 🙏🏻 Lint Test +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint: + name: Lint Test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: --timeout 5m + working-directory: v2/ \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release-binary.yml similarity index 85% rename from .github/workflows/release.yml rename to .github/workflows/release-binary.yml index 8229941..6fe8c82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-binary.yml @@ -1,8 +1,9 @@ -name: Release +name: 🎉 Release Binary on: create: tags: - v* + workflow_dispatch: jobs: release: @@ -17,7 +18,7 @@ jobs: name: "Set up Go" uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.16 - env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -26,5 +27,4 @@ jobs: with: args: "release --rm-dist" version: latest - workdir: v2/ - \ No newline at end of file + workdir: . \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index a16e0ce..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,122 +0,0 @@ -linters-settings: - dupl: - threshold: 100 - exhaustive: - default-signifies-exhaustive: false - # funlen: - # lines: 100 - # statements: 50 - goconst: - min-len: 2 - min-occurrences: 2 - gocritic: - enabled-tags: - - diagnostic - - experimental - - opinionated - - performance - - style - disabled-checks: - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - ifElseChain - # gocyclo: - # min-complexity: 15 - goimports: - local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 - gomnd: - settings: - mnd: - # don't include the "operation" and "assign" - checks: argument,case,condition,return - govet: - check-shadowing: true - settings: - printf: - funcs: - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - # lll: - # line-length: 140 - maligned: - suggest-new: true - misspell: - locale: US - nolintlint: - allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) - allow-unused: false # report any unused nolint directives - require-explanation: false # don't require an explanation for nolint directives - require-specific: false # don't require nolint directives to be specific about which linter is being skipped - -linters: - # please, do not use `enable-all`: it's deprecated and will be removed soon. - # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint - disable-all: true - enable: - - bodyclose - - deadcode - - dogsled - - dupl - # - errcheck - - exhaustive - - gochecknoinits - - goconst - - gocritic - - gofmt - - goimports - - golint - - gomnd - - goprintffuncname - - gosimple - - govet - - ineffassign - - interfacer - - maligned - - misspell - - nakedret - - noctx - # - nolintlint - - rowserrcheck - - scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - - # don't enable: - # - depguard - # - asciicheck - # - funlen - # - gochecknoglobals - # - gocognit - # - gocyclo - # - godot - # - godox - # - goerr113 - # - gosec - # - lll - # - nestif - # - prealloc - # - testpackage - # - wsl - -issues: - exclude-use-default: false - exclude: - # should have a package comment, unless it's in another file for this package (golint) - - 'in another file for this package' - -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/Dockerfile b/Dockerfile index 4f72364..fad32a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # Build FROM golang:1.17.0-alpine AS build-env + RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder # Release diff --git a/README.md b/README.md index 32603a4..7f55e62 100644 --- a/README.md +++ b/README.md @@ -53,30 +53,32 @@ subfinder -h ``` This will display help for the tool. Here are all the switches it supports. -| Flag | Description | Example | -| ---------------- | ---------------------------------------------------------- | -------------------------------------- | -| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | -| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | -| -d | Domain to find subdomains for | subfinder -d uber.com | -| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | -| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | -| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | -| -nC | Don't Use colors in output | subfinder -nC | -| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | -| -ls | List all available sources | subfinder -ls | -| -o | File to write output to (optional) | subfinder -o output.txt | -| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | -| -oI | Write output in Host,IP format | subfinder -oI | -| -oJ | Write output in JSON lines Format | subfinder -oJ | -| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | -| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | -| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | -| -silent | Show only subdomains in output | subfinder -silent | -| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | -| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | -| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | -| -v | Show Verbose output | subfinder -v | -| -version | Show current program version | subfinder -version | +| Flag | Description | Example | +| ---------------- | ---------------------------------------------------------- | --------------------------------------------| +| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | +| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | +| -d | Domain to find subdomains for | subfinder -d uber.com | +| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | +| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | +| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | +| -nC | Don't Use colors in output | subfinder -nC | +| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | +| -ls | List all available sources | subfinder -ls | +| -o | File to write output to (optional) | subfinder -o output.txt | +| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | +| -oI | Write output in Host,IP format | subfinder -oI | +| -oJ | Write output in JSON lines Format | subfinder -oJ | +| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | +| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | +| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | +| -silent | Show only subdomains in output | subfinder -silent | +| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | +| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | +| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | +| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 | +| -v | Show Verbose output | subfinder -v | +| -version | Show current program version | subfinder -version | # Installation @@ -93,6 +95,7 @@ Subfinder requires **go1.14+** to install successfully. Run the following comman Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: - [Binaryedge](https://binaryedge.io) +- [C99](https://api.c99.nl/) - [Certspotter](https://sslmate.com/certspotter/api/) - [Censys](https://censys.io) - [Chaos](https://chaos.projectdiscovery.io) diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 9b17be1..6006e6d 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -3,23 +3,31 @@ before: - go mod tidy builds: - - binary: subfinder - main: cmd/subfinder/main.go - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 - +- env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - 386 + - arm + - arm64 + + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' + + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go + archives: - - id: tgz - format: tar.gz - replacements: - darwin: macOS - format_overrides: - - goos: windows - format: zip \ No newline at end of file +- format: zip + replacements: + darwin: macOS + +checksum: + algorithm: sha256 \ No newline at end of file diff --git a/v2/go.mod b/v2/go.mod index 543cf9e..3ff93db 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2 go 1.16 require ( + github.com/corpix/uarand v0.1.1 github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd github.com/json-iterator/go v1.1.10 github.com/lib/pq v1.10.0 @@ -11,7 +12,9 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/gologger v1.1.4 github.com/rs/xid v1.3.0 + github.com/spyse-com/go-spyse v1.2.1 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + go.uber.org/ratelimit v0.2.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/v2/go.sum b/v2/go.sum index 0c27ad7..3bc2b69 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,8 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -51,12 +58,16 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -87,6 +98,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8= +github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -97,7 +110,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..0ed98d4 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index a3f65e3..72b9a3e 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -7,14 +7,15 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" @@ -45,6 +46,7 @@ var DefaultSources = []string{ "certspotter", "censys", "chaos", + "chinaz", "crtsh", "dnsdumpster", "hackertarget", @@ -59,6 +61,7 @@ var DefaultSources = []string{ "threatcrowd", "threatminer", "virustotal", + "fofa", } // DefaultRecursiveSources contains list of default recursive sources @@ -84,7 +87,6 @@ var DefaultAllSources = []string{ "archiveis", "binaryedge", "bufferover", - "c99", "censys", "certspotter", "chaos", @@ -112,6 +114,7 @@ var DefaultAllSources = []string{ "virustotal", "waybackarchive", "zoomeye", + "fofa", } // Agent is a struct for running passive subdomain enumeration @@ -146,14 +149,14 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &binaryedge.Source{} case "bufferover": a.sources[source] = &bufferover.Source{} - case "c99": - a.sources[source] = &c99.Source{} case "censys": a.sources[source] = &censys.Source{} case "certspotter": a.sources[source] = &certspotter.Source{} case "chaos": a.sources[source] = &chaos.Source{} + case "chinaz": + a.sources[source] = &chinaz.Source{} case "commoncrawl": a.sources[source] = &commoncrawl.Source{} case "crtsh": @@ -202,6 +205,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &waybackarchive.Source{} case "zoomeye": a.sources[source] = &zoomeye.Source{} + case "fofa": + a.sources[source] = &fofa.Source{} } } } diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index be0a5e6..292d227 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -11,11 +11,11 @@ const banner = ` _______ __/ /_ / __(_)___ ____/ /__ _____ / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / -/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.8 +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9 ` // Version is the current version of subfinder -const Version = `2.4.8` +const Version = `v2.4.9` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 547ecda..0d6d2f9 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -30,10 +30,10 @@ type ConfigFile struct { ExcludeSources []string `yaml:"exclude-sources,omitempty"` // API keys for different sources Binaryedge []string `yaml:"binaryedge"` - C99 []string `yaml:"c99"` Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` + Chinaz []string `yaml:"chinaz"` DNSDB []string `yaml:"dnsdb"` GitHub []string `yaml:"github"` IntelX []string `yaml:"intelx"` @@ -47,6 +47,7 @@ type ConfigFile struct { URLScan []string `yaml:"urlscan"` Virustotal []string `yaml:"virustotal"` ZoomEye []string `yaml:"zoomeye"` + Fofa []string `yaml:"fofa"` // Version indicates the version of subfinder installed. Version string `yaml:"subfinder-version"` } @@ -120,9 +121,6 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Binaryedge) > 0 { keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] } - if len(c.C99) > 0 { - keys.C99 = c.C99[rand.Intn(len(c.C99))] - } if len(c.Censys) > 0 { censysKeys := c.Censys[rand.Intn(len(c.Censys))] @@ -139,6 +137,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Chaos) > 0 { keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] } + if len(c.Chinaz) > 0 { + keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))] + } if (len(c.DNSDB)) > 0 { keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] } @@ -198,6 +199,14 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { keys.ZoomEyePassword = parts[1] } } + if len(c.Fofa) > 0 { + fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))] + parts := strings.Split(fofaKeys, ":") + if len(parts) == MultipleKeyPartsLength { + keys.FofaUsername = parts[0] + keys.FofaSecret = parts[1] + } + } return keys } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..5df3fb6 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..9413538 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..6797542 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -10,11 +10,13 @@ import ( "net/url" "time" + "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" + "go.uber.org/ratelimit" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -44,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, Keys: keys, } + // Initiate rate limit instance + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) + } else { + session.RateLimiter = ratelimit.NewUnlimited() + } + // Create a new extractor object for the current domain extractor, err := NewSubdomainExtractor(domain) session.Extractor = extractor @@ -78,7 +87,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + req.Header.Set("User-Agent", uarand.GetRandom()) req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") @@ -95,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s req.Header.Set(key, value) } + s.RateLimiter.Take() + return httpRequestWrapper(s.Client, req) } diff --git a/v2/pkg/subscraping/sources/archiveis/archiveis.go b/v2/pkg/subscraping/sources/archiveis/archiveis.go index c4e9d3a..9a2b535 100755 --- a/v2/pkg/subscraping/sources/archiveis/archiveis.go +++ b/v2/pkg/subscraping/sources/archiveis/archiveis.go @@ -5,74 +5,48 @@ import ( "context" "fmt" "io/ioutil" - "regexp" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -type agent struct { - Results chan subscraping.Result - Session *subscraping.Session -} - -var reNext = regexp.MustCompile("") - -func (a *agent) enumerate(ctx context.Context, baseURL string) { - select { - case <-ctx.Done(): - return - default: - } - - resp, err := a.Session.SimpleGet(ctx, baseURL) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - a.Session.DiscardHTTPResponse(resp) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - - resp.Body.Close() - - src := string(body) - for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} - } - - match1 := reNext.FindStringSubmatch(src) - if len(match1) > 0 { - a.enumerate(ctx, match1[1]) - } -} - // Source is the passive scraping agent type Source struct{} -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - a := agent{ - Session: session, - Results: results, - } - - go func() { - a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain)) - close(a.Results) - }() - - return a.Results -} - // Name returns the name of the source func (s *Source) Name() string { return "archiveis" } + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain)) + + if err != nil { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + src := string(body) + + for _, subdomain := range session.Extractor.FindAllString(src, -1) { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go deleted file mode 100644 index 211669f..0000000 --- a/v2/pkg/subscraping/sources/c99/c99.go +++ /dev/null @@ -1,71 +0,0 @@ -// Package c99 logic -package c99 - -import ( - "context" - "fmt" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -type dnsdbLookupResponse struct { - Success bool `json:"success"` - Subdomains []struct { - Subdomain string `json:"subdomain"` - IP string `json:"ip"` - Cloudflare bool `json:"cloudflare"` - } `json:"subdomains"` - Error string `json:"error"` -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - - if session.Keys.C99 == "" { - return - } - - searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain) - resp, err := session.SimpleGet(ctx, searchURL) - if err != nil { - session.DiscardHTTPResponse(resp) - return - } - - defer resp.Body.Close() - - var response dnsdbLookupResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - - if response.Error != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} - return - } - - for _, data := range response.Subdomains { - if !strings.HasPrefix(data.Subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} - } - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "c99" -} diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go new file mode 100644 index 0000000..b30e93e --- /dev/null +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -0,0 +1,56 @@ +package chinaz +// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi +import ( + "context" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "io/ioutil" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.Chinaz == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + + resp.Body.Close() + + SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList") + + if SubdomainList.ToBool() { + _data := []byte(SubdomainList.ToString()) + for i := 0 ; i< SubdomainList.Size() ; i++{ + subdomain := jsoniter.Get(_data,i,"DataUrl").ToString() + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "chinaz" +} diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 16400fa..4875b0c 100755 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -44,6 +44,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re return 0 } + defer db.Close() + pattern := "%." + domain query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 2a48fe3..08f0ebf 100755 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -30,6 +30,7 @@ func postForm(ctx context.Context, session *subscraping.Session, token, domain s params := url.Values{ "csrfmiddlewaretoken": {token}, "targetip": {domain}, + "user": {"free"}, } resp, err := session.HTTPRequest( diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go new file mode 100644 index 0000000..0250aee --- /dev/null +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -0,0 +1,74 @@ +// Package fofa logic +package fofa + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type fofaResponse struct { + Error bool `json:"error"` + ErrMsg string `json:"errmsg"` + Size int `json:"size"` + Results []string `json:"results"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" { + return + } + + // fofa api doc https://fofa.so/static_pages/api_help + qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.so/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response fofaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Error { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)} + return + } + + if response.Size > 0 { + for _, subdomain := range response.Results { + if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { + subdomain = subdomain[strings.Index(subdomain, "//")+2:] + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "fofa" +} diff --git a/v2/pkg/subscraping/sources/recon/recon.go b/v2/pkg/subscraping/sources/recon/recon.go index b1539b1..ff4b9d1 100644 --- a/v2/pkg/subscraping/sources/recon/recon.go +++ b/v2/pkg/subscraping/sources/recon/recon.go @@ -10,7 +10,11 @@ import ( ) type subdomain struct { - RawDomain string `json:"rawDomain"` + Domains []string `json:"domains"` + Ip string `json:"ip"` + RawDomains []string `json:"rawDomains"` + RawPort string `json:"rawPort"` + RawIp string `json:"rawIp"` } // Source is the passive scraping agent @@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain} + for _, dmn := range subdomain.RawDomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn} + } } }() diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 2ae271e..fbb7501 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -3,31 +3,12 @@ package spyse import ( "context" - "fmt" - "strconv" - jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + spyse "github.com/spyse-com/go-spyse/pkg" ) -type resultObject struct { - Name string `json:"name"` -} - -type dataObject struct { - Items []resultObject `json:"items"` - TotalCount int `json:"total_count"` -} - -type errorObject struct { - Code string `json:"code"` - Message string `json:"message"` -} - -type spyseResult struct { - Data dataObject `json:"data"` - Error []errorObject `json:"error"` -} +const searchMethodResultsLimit = 10000 // Source is the passive scraping agent type Source struct{} @@ -43,33 +24,81 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - maxCount := 100 + client, err := spyse.NewClient(session.Keys.Spyse, nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } - for offSet := 0; offSet <= maxCount; offSet += 100 { - resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse}) + domainSvc := spyse.NewDomainService(client) + + var searchDomain = "." + domain + var subdomainsSearchParams spyse.QueryBuilder + + subdomainsSearchParams.AppendParam(spyse.QueryParam{ + Name: domainSvc.Params().Name.Name, + Operator: domainSvc.Params().Name.Operator.EndsWith, + Value: searchDomain, + }) + + totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if totalResults == 0 { + return + } + + accountSvc := spyse.NewAccountService(client) + + quota, err := accountSvc.Quota(context.Background()) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + // The default "Search" method returns only first 10 000 subdomains + // To obtain more than 10 000 subdomains the "Scroll" method should be using + // Note: The "Scroll" method is only available for "PRO" customers, so we need to check + // quota.IsScrollSearchEnabled param + if totalResults > searchMethodResultsLimit && quota.IsScrollSearchEnabled { + var scrollResponse *spyse.DomainScrollResponse + scrollResponse, err = domainSvc.ScrollSearch( + ctx, subdomainsSearchParams.Query, "") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) return } - var response spyseResult - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return + for len(scrollResponse.Items) > 0 { + scrollResponse, err = domainSvc.ScrollSearch( + context.Background(), subdomainsSearchParams.Query, scrollResponse.SearchID) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + for i := range scrollResponse.Items { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: scrollResponse.Items[i].Name} + } } - resp.Body.Close() + } else { + var limit = 100 + var offset = 0 + var searchResults []spyse.Domain - if response.Data.TotalCount == 0 { - return - } + for ; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { + searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } - maxCount = response.Data.TotalCount - - for _, hostname := range response.Data.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name} + for i := range searchResults { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: searchResults[i].Name} + } } } }() diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..5d95252 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -4,6 +4,8 @@ import ( "context" "net/http" "regexp" + + "go.uber.org/ratelimit" ) // BasicAuth request's Authorization header @@ -31,16 +33,18 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Rate limit instance + RateLimiter ratelimit.Limiter } // Keys contains the current API Keys we have in store type Keys struct { Binaryedge string `json:"binaryedge"` - C99 string `json:"c99"` CensysToken string `json:"censysUsername"` CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` Chaos string `json:"chaos"` + Chinaz string `json:"chinaz"` DNSDB string `json:"dnsdb"` GitHub []string `json:"github"` IntelXHost string `json:"intelXHost"` @@ -57,6 +61,8 @@ type Keys struct { Virustotal string `json:"virustotal"` ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` + FofaUsername string `json:"fofa_username"` + FofaSecret string `json:"fofa_secret"` } // Result is a result structure returned by a source