Merge branch 'dev' into feature-dns-to-dnsx
commit
807b49b847
|
@ -0,0 +1,34 @@
|
|||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
|
||||
# Maintain dependencies for GitHub Actions
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
commit-message:
|
||||
prefix: "chore"
|
||||
include: "scope"
|
||||
|
||||
# Maintain dependencies for go modules
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
commit-message:
|
||||
prefix: "chore"
|
||||
include: "scope"
|
||||
|
||||
# Maintain dependencies for docker
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
commit-message:
|
||||
prefix: "chore"
|
||||
include: "scope"
|
|
@ -108,9 +108,15 @@ linters:
|
|||
# - testpackage
|
||||
# - wsl
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
# should have a package comment, unless it's in another file for this package (golint)
|
||||
- 'in another file for this package'
|
||||
|
||||
# golangci.com configuration
|
||||
# https://github.com/golangci/golangci/wiki/Configuration
|
||||
service:
|
||||
golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
|
||||
golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
|
||||
prepare:
|
||||
- echo "here I can run custom commands, but no preparation needed for this repo"
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services:
|
||||
|
||||
- Project Sonar / Bufferover: https://opendata.rapid7.com/about
|
||||
- Project Sonar / Bufferover: https://opendata.rapid7.com/about / [https://tls.bufferover.run](https://tls.bufferover.run/dns?q)
|
||||
- CommonCrawl: https://commoncrawl.org/terms-of-use/full
|
||||
- certspotter: https://sslmate.com/terms
|
||||
- dnsdumpster: https://hackertarget.com/terms
|
||||
|
|
|
@ -2,8 +2,8 @@
|
|||
FROM golang:1.13.4-alpine3.10 AS build-env
|
||||
MAINTAINER Ice3man (nizamul@projectdiscovery.io)
|
||||
RUN apk add --no-cache --upgrade git openssh-client ca-certificates
|
||||
RUN go get -u github.com/golang/dep/cmd/dep
|
||||
RUN go get -u github.com/projectdiscovery/subfinder/v2/cmd/subfinder
|
||||
RUN go get -v github.com/golang/dep/cmd/dep
|
||||
RUN GO111MODULE=on go get -v github.com/projectdiscovery/v2/cmd/subfinder
|
||||
|
||||
FROM alpine:latest
|
||||
COPY --from=build-env /go/bin/subfinder /usr/local/bin/
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) Exposed Atoms Pvt Ltd
|
||||
Copyright (c) 2021 ProjectDiscovery, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
@ -101,7 +101,7 @@ The installation is easy. You can download the pre-built binaries for different
|
|||
subfinder requires **go1.14+** to install successfully. Run the following command to get the repo -
|
||||
|
||||
```sh
|
||||
GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
|
||||
GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
|
||||
```
|
||||
|
||||
### From Github
|
||||
|
@ -288,6 +288,10 @@ For example, this runs the tool against uber.com and output the results to your
|
|||
docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d uber.com > uber.com.txt
|
||||
```
|
||||
|
||||
### Resources
|
||||
|
||||
- [Recon with Me !!!](https://dhiyaneshgeek.github.io/bug/bounty/2020/02/06/recon-with-me/)
|
||||
|
||||
# License
|
||||
|
||||
subfinder is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details.
|
||||
|
|
|
@ -15,11 +15,11 @@ func main() {
|
|||
|
||||
newRunner, err := runner.NewRunner(options)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not create runner: %s\n", err)
|
||||
gologger.Fatal().Msgf("Could not create runner: %s\n", err)
|
||||
}
|
||||
|
||||
err = newRunner.RunEnumeration(context.Background())
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not run enumeration: %s\n", err)
|
||||
gologger.Fatal().Msgf("Could not run enumeration: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
|
15
v2/go.mod
15
v2/go.mod
|
@ -13,8 +13,13 @@ require (
|
|||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/chaos-client v0.1.6
|
||||
github.com/projectdiscovery/dnsx v1.0.1
|
||||
github.com/miekg/dns v1.1.38
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/chaos-client v0.1.7
|
||||
github.com/projectdiscovery/fdmax v0.0.3
|
||||
github.com/projectdiscovery/gologger v1.0.1
|
||||
github.com/projectdiscovery/gologger v1.1.3
|
||||
github.com/rs/xid v1.2.1
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
||||
|
@ -24,4 +29,12 @@ require (
|
|||
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
|
||||
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
|
||||
golang.org/x/sys v0.0.0-20210217105451-b926d437f341 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
)
|
||||
|
|
25
v2/go.sum
25
v2/go.sum
|
@ -27,6 +27,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
|
|||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
|
||||
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
|
||||
github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
|
@ -55,6 +57,13 @@ github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLY
|
|||
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
|
||||
github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
|
||||
github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
|
||||
github.com/projectdiscovery/chaos-client v0.1.7 h1:8M1LazT+lD3qJP1ERWXiVzQn9lPMO3p6v3qsiGNKUlQ=
|
||||
github.com/projectdiscovery/chaos-client v0.1.7/go.mod h1:F5omaoJh/vMvWnZhKD4zFFA5ti+RPwUletwepKSyfxk=
|
||||
github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg=
|
||||
github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI=
|
||||
github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
|
||||
github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU=
|
||||
github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
@ -64,6 +73,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
|||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
|
@ -75,6 +86,8 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0a
|
|||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 h1:sYNJzB4J8toYPQTM6pAkcmBRgw9SnQKP9oXCHfgy604=
|
||||
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
|
||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
|
@ -84,6 +97,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY=
|
||||
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
|
||||
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
|
||||
|
@ -99,6 +114,11 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU
|
|||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
|
||||
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU=
|
||||
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
@ -118,3 +138,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
@ -47,7 +47,7 @@ func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeo
|
|||
wg.Wait()
|
||||
|
||||
for source, data := range timeTaken {
|
||||
gologger.Verbosef(data, source)
|
||||
gologger.Verbose().Msgf(data, source)
|
||||
}
|
||||
|
||||
close(results)
|
||||
|
|
|
@ -7,10 +7,8 @@ import (
|
|||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/cebaidu"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotterold"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
|
||||
|
@ -45,9 +43,7 @@ var DefaultSources = []string{
|
|||
"alienvault",
|
||||
"anubis",
|
||||
"bufferover",
|
||||
"cebaidu",
|
||||
"certspotter",
|
||||
"certspotterold",
|
||||
"censys",
|
||||
"chaos",
|
||||
"crtsh",
|
||||
|
@ -72,9 +68,7 @@ var DefaultRecursiveSources = []string{
|
|||
"alienvault",
|
||||
"binaryedge",
|
||||
"bufferover",
|
||||
"cebaidu",
|
||||
"certspotter",
|
||||
"certspotterold",
|
||||
"crtsh",
|
||||
"dnsdumpster",
|
||||
"hackertarget",
|
||||
|
@ -94,10 +88,8 @@ var DefaultAllSources = []string{
|
|||
"archiveis",
|
||||
"binaryedge",
|
||||
"bufferover",
|
||||
"cebaidu",
|
||||
"censys",
|
||||
"certspotter",
|
||||
"certspotterold",
|
||||
"chaos",
|
||||
"commoncrawl",
|
||||
"crtsh",
|
||||
|
@ -159,14 +151,10 @@ func (a *Agent) addSources(sources []string) {
|
|||
a.sources[source] = &binaryedge.Source{}
|
||||
case "bufferover":
|
||||
a.sources[source] = &bufferover.Source{}
|
||||
case "cebaidu":
|
||||
a.sources[source] = &cebaidu.Source{}
|
||||
case "censys":
|
||||
a.sources[source] = &censys.Source{}
|
||||
case "certspotter":
|
||||
a.sources[source] = &certspotter.Source{}
|
||||
case "certspotterold":
|
||||
a.sources[source] = &certspotterold.Source{}
|
||||
case "chaos":
|
||||
a.sources[source] = &chaos.Source{}
|
||||
case "commoncrawl":
|
||||
|
|
|
@ -104,14 +104,18 @@ func (r *ResolutionPool) resolveWorker() {
|
|||
continue
|
||||
}
|
||||
|
||||
var skip bool
|
||||
for _, host := range hosts {
|
||||
// Ignore the host if it exists in wildcard ips map
|
||||
if _, ok := r.wildcardIPs[host]; ok { //nolint:staticcheck //search alternatives for "comma ok"
|
||||
continue
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
|
||||
if !skip {
|
||||
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
|
||||
}
|
||||
}
|
||||
r.wg.Done()
|
||||
}
|
||||
|
|
|
@ -10,27 +10,27 @@ const banner = `
|
|||
_ __ _ _
|
||||
____ _| |__ / _(_)_ _ __| |___ _ _
|
||||
(_-< || | '_ \ _| | ' \/ _ / -_) '_|
|
||||
/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.5
|
||||
/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.6
|
||||
`
|
||||
|
||||
// Version is the current version of subfinder
|
||||
const Version = `2.4.5`
|
||||
const Version = `2.4.6`
|
||||
|
||||
// showBanner is used to show the banner to the user
|
||||
func showBanner() {
|
||||
gologger.Printf("%s\n", banner)
|
||||
gologger.Printf("\t\tprojectdiscovery.io\n\n")
|
||||
gologger.Print().Msgf("%s\n", banner)
|
||||
gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
|
||||
|
||||
gologger.Labelf("Use with caution. You are responsible for your actions\n")
|
||||
gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
|
||||
gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
|
||||
gologger.Print().Msgf("Use with caution. You are responsible for your actions\n")
|
||||
gologger.Print().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
|
||||
gologger.Print().Msgf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
|
||||
}
|
||||
|
||||
// normalRunTasks runs the normal startup tasks
|
||||
func (options *Options) normalRunTasks() {
|
||||
configFile, err := UnmarshalRead(options.ConfigFile)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
|
||||
gologger.Fatal().Msgf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
|
||||
}
|
||||
|
||||
// If we have a different version of subfinder installed
|
||||
|
@ -43,7 +43,7 @@ func (options *Options) normalRunTasks() {
|
|||
|
||||
err = configFile.MarshalWrite(options.ConfigFile)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
|
||||
gologger.Fatal().Msgf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
|
||||
}
|
||||
}
|
||||
options.YAMLConfig = configFile
|
||||
|
@ -67,9 +67,9 @@ func (options *Options) firstRunTasks() {
|
|||
|
||||
err := config.MarshalWrite(options.ConfigFile)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
|
||||
gologger.Fatal().Msgf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
|
||||
}
|
||||
options.YAMLConfig = config
|
||||
|
||||
gologger.Infof("Configuration file saved to %s\n", options.ConfigFile)
|
||||
gologger.Info().Msgf("Configuration file saved to %s\n", options.ConfigFile)
|
||||
}
|
||||
|
|
|
@ -42,7 +42,7 @@ func (r *Runner) UploadToChaos(ctx context.Context, reader io.Reader) error {
|
|||
defer func() {
|
||||
_, err := io.Copy(ioutil.Discard, resp.Body)
|
||||
if err != nil {
|
||||
gologger.Warningf("Could not discard response body: %s\n", err)
|
||||
gologger.Warning().Msgf("Could not discard response body: %s\n", err)
|
||||
return
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
|
|
@ -18,7 +18,7 @@ const maxNumCount = 2
|
|||
|
||||
// EnumerateSingleDomain performs subdomain enumeration against a single domain
|
||||
func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output string, appendToFile bool) error {
|
||||
gologger.Infof("Enumerating subdomains for %s\n", domain)
|
||||
gologger.Info().Msgf("Enumerating subdomains for %s\n", domain)
|
||||
|
||||
// Get the API keys for sources from the configuration
|
||||
// and also create the active resolving engine for the domain.
|
||||
|
@ -32,7 +32,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
err := resolutionPool.InitWildcards(domain)
|
||||
if err != nil {
|
||||
// Log the error but don't quit.
|
||||
gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err)
|
||||
gologger.Warning().Msgf("Could not get wildcards for domain %s: %s\n", domain, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -51,7 +51,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
for result := range passiveResults {
|
||||
switch result.Type {
|
||||
case subscraping.Error:
|
||||
gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error)
|
||||
gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error)
|
||||
case subscraping.Subdomain:
|
||||
// Validate the subdomain found and remove wildcards from
|
||||
if !strings.HasSuffix(result.Value, "."+domain) {
|
||||
|
@ -65,7 +65,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
|
||||
// Log the verbose message about the found subdomain per source
|
||||
if _, ok := sourceMap[subdomain][result.Source]; !ok {
|
||||
gologger.Verbosef("%s\n", result.Source, subdomain)
|
||||
gologger.Verbose().Msgf("%s\n", result.Source, subdomain)
|
||||
}
|
||||
|
||||
sourceMap[subdomain][result.Source] = struct{}{}
|
||||
|
@ -103,7 +103,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
for result := range resolutionPool.Results {
|
||||
switch result.Type {
|
||||
case resolve.Error:
|
||||
gologger.Warningf("Could not resolve host: %s\n", result.Error)
|
||||
gologger.Warning().Msgf("Could not resolve host: %s\n", result.Error)
|
||||
case resolve.Subdomain:
|
||||
// Add the found subdomain to a map.
|
||||
if _, ok := foundResults[result.Host]; !ok {
|
||||
|
@ -133,16 +133,16 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
}
|
||||
}
|
||||
if err != nil {
|
||||
gologger.Errorf("Could not verbose results for %s: %s\n", domain, err)
|
||||
gologger.Error().Msgf("Could not verbose results for %s: %s\n", domain, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Show found subdomain count in any case.
|
||||
duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String()
|
||||
if r.options.RemoveWildcard {
|
||||
gologger.Infof("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration)
|
||||
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration)
|
||||
} else {
|
||||
gologger.Infof("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
|
||||
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
|
||||
}
|
||||
|
||||
// In case the user has specified to upload to chaos, write everything to a temporary buffer and upload
|
||||
|
@ -151,14 +151,14 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
err := outputter.WriteForChaos(uniqueMap, buf)
|
||||
// If an error occurs, do not interrupt, continue to check if user specified an output file
|
||||
if err != nil {
|
||||
gologger.Errorf("Could not prepare results for chaos %s\n", err)
|
||||
gologger.Error().Msgf("Could not prepare results for chaos %s\n", err)
|
||||
} else {
|
||||
// no error in writing host output, upload to chaos
|
||||
err = r.UploadToChaos(ctx, buf)
|
||||
if err != nil {
|
||||
gologger.Errorf("Could not upload results to chaos %s\n", err)
|
||||
gologger.Error().Msgf("Could not upload results to chaos %s\n", err)
|
||||
} else {
|
||||
gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
|
||||
gologger.Info().Msgf("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
|
||||
}
|
||||
// clear buffer
|
||||
buf.Reset()
|
||||
|
@ -168,7 +168,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
if output != "" {
|
||||
file, err := outputter.createFile(output, appendToFile)
|
||||
if err != nil {
|
||||
gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err)
|
||||
gologger.Error().Msgf("Could not create file %s for %s: %s\n", output, domain, err)
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -188,7 +188,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
|
|||
}
|
||||
}
|
||||
if err != nil {
|
||||
gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err)
|
||||
gologger.Error().Msgf("Could not write results to file %s for %s: %s\n", output, domain, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -49,7 +49,7 @@ func ParseOptions() *Options {
|
|||
config, err := GetConfigDirectory()
|
||||
if err != nil {
|
||||
// This should never be reached
|
||||
gologger.Fatalf("Could not get user home: %s\n", err)
|
||||
gologger.Fatal().Msgf("Could not get user home: %s\n", err)
|
||||
}
|
||||
|
||||
flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
|
||||
|
@ -89,7 +89,7 @@ func ParseOptions() *Options {
|
|||
showBanner()
|
||||
|
||||
if options.Version {
|
||||
gologger.Infof("Current Version: %s\n", Version)
|
||||
gologger.Info().Msgf("Current Version: %s\n", Version)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
|
@ -111,7 +111,7 @@ func ParseOptions() *Options {
|
|||
// invalid options have been used, exit.
|
||||
err = options.validateOptions()
|
||||
if err != nil {
|
||||
gologger.Fatalf("Program exiting: %s\n", err)
|
||||
gologger.Fatal().Msgf("Program exiting: %s\n", err)
|
||||
}
|
||||
|
||||
return options
|
||||
|
@ -130,9 +130,9 @@ func hasStdin() bool {
|
|||
}
|
||||
|
||||
func listSources(options *Options) {
|
||||
gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources))
|
||||
gologger.Infof("Sources marked with an * needs key or token in order to work.\n")
|
||||
gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile)
|
||||
gologger.Info().Msgf("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources))
|
||||
gologger.Info().Msgf("Sources marked with an * needs key or token in order to work.\n")
|
||||
gologger.Info().Msgf("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile)
|
||||
|
||||
keys := options.YAMLConfig.GetKeys()
|
||||
needsKey := make(map[string]interface{})
|
||||
|
@ -146,6 +146,6 @@ func listSources(options *Options) {
|
|||
if _, ok := needsKey[source]; ok {
|
||||
message = "%s *\n"
|
||||
}
|
||||
gologger.Silentf(message, source)
|
||||
gologger.Silent().Msgf(message, source)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,6 +4,8 @@ import (
|
|||
"errors"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/formatter"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
)
|
||||
|
||||
// validateOptions validates the configuration options passed
|
||||
|
@ -39,12 +41,12 @@ func (options *Options) validateOptions() error {
|
|||
func (options *Options) configureOutput() {
|
||||
// If the user desires verbose output, show verbose output
|
||||
if options.Verbose {
|
||||
gologger.MaxLevel = gologger.Verbose
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
|
||||
}
|
||||
if options.NoColor {
|
||||
gologger.UseColors = false
|
||||
gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
|
||||
}
|
||||
if options.Silent {
|
||||
gologger.MaxLevel = gologger.Silent
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -90,7 +90,7 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) {
|
|||
if response != nil {
|
||||
_, err := io.Copy(ioutil.Discard, response.Body)
|
||||
if err != nil {
|
||||
gologger.Warningf("Could not discard response body: %s\n", err)
|
||||
gologger.Warning().Msgf("Could not discard response body: %s\n", err)
|
||||
return
|
||||
}
|
||||
response.Body.Close()
|
||||
|
|
|
@ -1,63 +0,0 @@
|
|||
package cebaidu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||
)
|
||||
|
||||
// Source is the passive scraping agent
|
||||
type Source struct{}
|
||||
|
||||
type domain struct {
|
||||
Domain string `json:"domain"`
|
||||
}
|
||||
|
||||
type cebaiduResponse struct {
|
||||
Code int64 `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data []domain `json:"data"`
|
||||
}
|
||||
|
||||
// Run function returns all subdomains found with the service
|
||||
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
||||
results := make(chan subscraping.Result)
|
||||
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://ce.baidu.com/index/getRelatedSites?site_address=%s", domain))
|
||||
if err != nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
session.DiscardHTTPResponse(resp)
|
||||
return
|
||||
}
|
||||
|
||||
var response cebaiduResponse
|
||||
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
|
||||
if err != nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
resp.Body.Close()
|
||||
return
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if response.Code > 0 {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)}
|
||||
return
|
||||
}
|
||||
|
||||
for _, result := range response.Data {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain}
|
||||
}
|
||||
}()
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// Name returns the name of the source
|
||||
func (s *Source) Name() string {
|
||||
return "cebaidu"
|
||||
}
|
|
@ -1,75 +0,0 @@
|
|||
package certspotterold
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||
)
|
||||
|
||||
type errorResponse struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"Message"`
|
||||
}
|
||||
|
||||
type subdomain struct {
|
||||
DNSNames []string `json:"dns_names"`
|
||||
}
|
||||
|
||||
// Source is the passive scraping agent
|
||||
type Source struct{}
|
||||
|
||||
// Run function returns all subdomains found with the service
|
||||
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
||||
results := make(chan subscraping.Result)
|
||||
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain))
|
||||
if err != nil && resp == nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
session.DiscardHTTPResponse(resp)
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResponse errorResponse
|
||||
err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse)
|
||||
if err != nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
resp.Body.Close()
|
||||
return
|
||||
}
|
||||
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s: %s", errResponse.Code, errResponse.Message)}
|
||||
resp.Body.Close()
|
||||
return
|
||||
}
|
||||
|
||||
var subdomains []subdomain
|
||||
err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
|
||||
if err != nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
resp.Body.Close()
|
||||
return
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
|
||||
for _, subdomain := range subdomains {
|
||||
for _, dnsname := range subdomain.DNSNames {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dnsname}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
// Name returns the name of the source
|
||||
func (s *Source) Name() string {
|
||||
return "certspotterold"
|
||||
}
|
|
@ -69,7 +69,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
|
|||
|
||||
if token.RetryAfter > 0 {
|
||||
if len(tokens.pool) == 1 {
|
||||
gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter)
|
||||
gologger.Verbose().Msgf("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter)
|
||||
time.Sleep(time.Duration(token.RetryAfter) * time.Second)
|
||||
} else {
|
||||
token = tokens.Get()
|
||||
|
|
|
@ -3,6 +3,7 @@ package passivetotal
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||
|
@ -54,6 +55,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
|
|||
resp.Body.Close()
|
||||
|
||||
for _, subdomain := range data.Subdomains {
|
||||
// skip entries like xxx.xxx.xxx.xxx\032domain.tld
|
||||
if passiveTotalFilterRegex.MatchString(subdomain) {
|
||||
continue
|
||||
}
|
||||
finalSubdomain := subdomain + "." + domain
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain}
|
||||
}
|
||||
|
@ -66,3 +71,5 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
|
|||
func (s *Source) Name() string {
|
||||
return "passivetotal"
|
||||
}
|
||||
|
||||
var passiveTotalFilterRegex *regexp.Regexp = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`)
|
||||
|
|
|
@ -12,14 +12,10 @@ import (
|
|||
type Source struct{}
|
||||
|
||||
type dnsdbLookupResponse struct {
|
||||
Domain string `json:"domain"`
|
||||
Data []struct {
|
||||
Subdomain string `json:"subdomain"`
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
} `json:"data"`
|
||||
Result int `json:"result"`
|
||||
Error string `json:"error"`
|
||||
Domain string `json:"domain"`
|
||||
Subdomains []string `json:"subdomains"`
|
||||
Result int `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// Run function returns all subdomains found with the service
|
||||
|
@ -54,14 +50,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
|
|||
return
|
||||
}
|
||||
|
||||
for _, data := range response.Data {
|
||||
if data.Subdomain != "" {
|
||||
if data.Type == "CNAME" {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Value}
|
||||
} else if data.Type == "A" {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data.Subdomain, domain)}
|
||||
}
|
||||
}
|
||||
for _, data := range response.Subdomains {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain)}
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
Loading…
Reference in New Issue