Merge branch 'dev' into feature-dns-to-dnsx

master
Mzack9999 2021-02-18 20:17:37 +01:00 committed by GitHub
commit 807b49b847
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 150 additions and 215 deletions

34
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,34 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
# Maintain dependencies for GitHub Actions
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
commit-message:
prefix: "chore"
include: "scope"
# Maintain dependencies for go modules
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"
commit-message:
prefix: "chore"
include: "scope"
# Maintain dependencies for docker
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "weekly"
commit-message:
prefix: "chore"
include: "scope"

View File

@ -108,9 +108,15 @@ linters:
# - testpackage
# - wsl
issues:
exclude-use-default: false
exclude:
# should have a package comment, unless it's in another file for this package (golint)
- 'in another file for this package'
# golangci.com configuration
# https://github.com/golangci/golangci/wiki/Configuration
service:
golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly
prepare:
- echo "here I can run custom commands, but no preparation needed for this repo"

View File

@ -2,7 +2,7 @@
Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services:
- Project Sonar / Bufferover: https://opendata.rapid7.com/about
- Project Sonar / Bufferover: https://opendata.rapid7.com/about / [https://tls.bufferover.run](https://tls.bufferover.run/dns?q)
- CommonCrawl: https://commoncrawl.org/terms-of-use/full
- certspotter: https://sslmate.com/terms
- dnsdumpster: https://hackertarget.com/terms

View File

@ -2,8 +2,8 @@
FROM golang:1.13.4-alpine3.10 AS build-env
MAINTAINER Ice3man (nizamul@projectdiscovery.io)
RUN apk add --no-cache --upgrade git openssh-client ca-certificates
RUN go get -u github.com/golang/dep/cmd/dep
RUN go get -u github.com/projectdiscovery/subfinder/v2/cmd/subfinder
RUN go get -v github.com/golang/dep/cmd/dep
RUN GO111MODULE=on go get -v github.com/projectdiscovery/v2/cmd/subfinder
FROM alpine:latest
COPY --from=build-env /go/bin/subfinder /usr/local/bin/

View File

@ -1,6 +1,6 @@
MIT License
Copyright (c) Exposed Atoms Pvt Ltd
Copyright (c) 2021 ProjectDiscovery, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -101,7 +101,7 @@ The installation is easy. You can download the pre-built binaries for different
subfinder requires **go1.14+** to install successfully. Run the following command to get the repo -
```sh
GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
```
### From Github
@ -288,6 +288,10 @@ For example, this runs the tool against uber.com and output the results to your
docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d uber.com > uber.com.txt
```
### Resources
- [Recon with Me !!!](https://dhiyaneshgeek.github.io/bug/bounty/2020/02/06/recon-with-me/)
# License
subfinder is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details.

View File

@ -15,11 +15,11 @@ func main() {
newRunner, err := runner.NewRunner(options)
if err != nil {
gologger.Fatalf("Could not create runner: %s\n", err)
gologger.Fatal().Msgf("Could not create runner: %s\n", err)
}
err = newRunner.RunEnumeration(context.Background())
if err != nil {
gologger.Fatalf("Could not run enumeration: %s\n", err)
gologger.Fatal().Msgf("Could not run enumeration: %s\n", err)
}
}

View File

@ -13,8 +13,13 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/chaos-client v0.1.6
github.com/projectdiscovery/dnsx v1.0.1
github.com/miekg/dns v1.1.38
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/chaos-client v0.1.7
github.com/projectdiscovery/fdmax v0.0.3
github.com/projectdiscovery/gologger v1.0.1
github.com/projectdiscovery/gologger v1.1.3
github.com/rs/xid v1.2.1
github.com/stretchr/testify v1.6.1
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
@ -24,4 +29,12 @@ require (
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
github.com/stretchr/testify v1.7.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
golang.org/x/sys v0.0.0-20210217105451-b926d437f341 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

View File

@ -27,6 +27,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@ -55,6 +57,13 @@ github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLY
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI=
github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/chaos-client v0.1.7 h1:8M1LazT+lD3qJP1ERWXiVzQn9lPMO3p6v3qsiGNKUlQ=
github.com/projectdiscovery/chaos-client v0.1.7/go.mod h1:F5omaoJh/vMvWnZhKD4zFFA5ti+RPwUletwepKSyfxk=
github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg=
github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI=
github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU=
github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -64,6 +73,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@ -75,6 +86,8 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0a
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 h1:sYNJzB4J8toYPQTM6pAkcmBRgw9SnQKP9oXCHfgy604=
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -84,6 +97,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
@ -99,6 +114,11 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU=
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -118,3 +138,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -47,7 +47,7 @@ func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeo
wg.Wait()
for source, data := range timeTaken {
gologger.Verbosef(data, source)
gologger.Verbose().Msgf(data, source)
}
close(results)

View File

@ -7,10 +7,8 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/cebaidu"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotterold"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
@ -45,9 +43,7 @@ var DefaultSources = []string{
"alienvault",
"anubis",
"bufferover",
"cebaidu",
"certspotter",
"certspotterold",
"censys",
"chaos",
"crtsh",
@ -72,9 +68,7 @@ var DefaultRecursiveSources = []string{
"alienvault",
"binaryedge",
"bufferover",
"cebaidu",
"certspotter",
"certspotterold",
"crtsh",
"dnsdumpster",
"hackertarget",
@ -94,10 +88,8 @@ var DefaultAllSources = []string{
"archiveis",
"binaryedge",
"bufferover",
"cebaidu",
"censys",
"certspotter",
"certspotterold",
"chaos",
"commoncrawl",
"crtsh",
@ -159,14 +151,10 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &binaryedge.Source{}
case "bufferover":
a.sources[source] = &bufferover.Source{}
case "cebaidu":
a.sources[source] = &cebaidu.Source{}
case "censys":
a.sources[source] = &censys.Source{}
case "certspotter":
a.sources[source] = &certspotter.Source{}
case "certspotterold":
a.sources[source] = &certspotterold.Source{}
case "chaos":
a.sources[source] = &chaos.Source{}
case "commoncrawl":

View File

@ -104,14 +104,18 @@ func (r *ResolutionPool) resolveWorker() {
continue
}
var skip bool
for _, host := range hosts {
// Ignore the host if it exists in wildcard ips map
if _, ok := r.wildcardIPs[host]; ok { //nolint:staticcheck //search alternatives for "comma ok"
continue
skip = true
break
}
}
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
if !skip {
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
}
}
r.wg.Done()
}

View File

@ -10,27 +10,27 @@ const banner = `
_ __ _ _
____ _| |__ / _(_)_ _ __| |___ _ _
(_-< || | '_ \ _| | ' \/ _ / -_) '_|
/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.5
/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.6
`
// Version is the current version of subfinder
const Version = `2.4.5`
const Version = `2.4.6`
// showBanner is used to show the banner to the user
func showBanner() {
gologger.Printf("%s\n", banner)
gologger.Printf("\t\tprojectdiscovery.io\n\n")
gologger.Print().Msgf("%s\n", banner)
gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
gologger.Labelf("Use with caution. You are responsible for your actions\n")
gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
gologger.Print().Msgf("Use with caution. You are responsible for your actions\n")
gologger.Print().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
gologger.Print().Msgf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
}
// normalRunTasks runs the normal startup tasks
func (options *Options) normalRunTasks() {
configFile, err := UnmarshalRead(options.ConfigFile)
if err != nil {
gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
gologger.Fatal().Msgf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
}
// If we have a different version of subfinder installed
@ -43,7 +43,7 @@ func (options *Options) normalRunTasks() {
err = configFile.MarshalWrite(options.ConfigFile)
if err != nil {
gologger.Fatalf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
gologger.Fatal().Msgf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
}
}
options.YAMLConfig = configFile
@ -67,9 +67,9 @@ func (options *Options) firstRunTasks() {
err := config.MarshalWrite(options.ConfigFile)
if err != nil {
gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
gologger.Fatal().Msgf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
}
options.YAMLConfig = config
gologger.Infof("Configuration file saved to %s\n", options.ConfigFile)
gologger.Info().Msgf("Configuration file saved to %s\n", options.ConfigFile)
}

View File

@ -42,7 +42,7 @@ func (r *Runner) UploadToChaos(ctx context.Context, reader io.Reader) error {
defer func() {
_, err := io.Copy(ioutil.Discard, resp.Body)
if err != nil {
gologger.Warningf("Could not discard response body: %s\n", err)
gologger.Warning().Msgf("Could not discard response body: %s\n", err)
return
}
resp.Body.Close()

View File

@ -18,7 +18,7 @@ const maxNumCount = 2
// EnumerateSingleDomain performs subdomain enumeration against a single domain
func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output string, appendToFile bool) error {
gologger.Infof("Enumerating subdomains for %s\n", domain)
gologger.Info().Msgf("Enumerating subdomains for %s\n", domain)
// Get the API keys for sources from the configuration
// and also create the active resolving engine for the domain.
@ -32,7 +32,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
err := resolutionPool.InitWildcards(domain)
if err != nil {
// Log the error but don't quit.
gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err)
gologger.Warning().Msgf("Could not get wildcards for domain %s: %s\n", domain, err)
}
}
@ -51,7 +51,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
for result := range passiveResults {
switch result.Type {
case subscraping.Error:
gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error)
gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error)
case subscraping.Subdomain:
// Validate the subdomain found and remove wildcards from
if !strings.HasSuffix(result.Value, "."+domain) {
@ -65,7 +65,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
// Log the verbose message about the found subdomain per source
if _, ok := sourceMap[subdomain][result.Source]; !ok {
gologger.Verbosef("%s\n", result.Source, subdomain)
gologger.Verbose().Msgf("%s\n", result.Source, subdomain)
}
sourceMap[subdomain][result.Source] = struct{}{}
@ -103,7 +103,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
for result := range resolutionPool.Results {
switch result.Type {
case resolve.Error:
gologger.Warningf("Could not resolve host: %s\n", result.Error)
gologger.Warning().Msgf("Could not resolve host: %s\n", result.Error)
case resolve.Subdomain:
// Add the found subdomain to a map.
if _, ok := foundResults[result.Host]; !ok {
@ -133,16 +133,16 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
}
}
if err != nil {
gologger.Errorf("Could not verbose results for %s: %s\n", domain, err)
gologger.Error().Msgf("Could not verbose results for %s: %s\n", domain, err)
return err
}
// Show found subdomain count in any case.
duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String()
if r.options.RemoveWildcard {
gologger.Infof("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration)
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration)
} else {
gologger.Infof("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
}
// In case the user has specified to upload to chaos, write everything to a temporary buffer and upload
@ -151,14 +151,14 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
err := outputter.WriteForChaos(uniqueMap, buf)
// If an error occurs, do not interrupt, continue to check if user specified an output file
if err != nil {
gologger.Errorf("Could not prepare results for chaos %s\n", err)
gologger.Error().Msgf("Could not prepare results for chaos %s\n", err)
} else {
// no error in writing host output, upload to chaos
err = r.UploadToChaos(ctx, buf)
if err != nil {
gologger.Errorf("Could not upload results to chaos %s\n", err)
gologger.Error().Msgf("Could not upload results to chaos %s\n", err)
} else {
gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
gologger.Info().Msgf("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
}
// clear buffer
buf.Reset()
@ -168,7 +168,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
if output != "" {
file, err := outputter.createFile(output, appendToFile)
if err != nil {
gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err)
gologger.Error().Msgf("Could not create file %s for %s: %s\n", output, domain, err)
return err
}
@ -188,7 +188,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin
}
}
if err != nil {
gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err)
gologger.Error().Msgf("Could not write results to file %s for %s: %s\n", output, domain, err)
return err
}
}

View File

@ -49,7 +49,7 @@ func ParseOptions() *Options {
config, err := GetConfigDirectory()
if err != nil {
// This should never be reached
gologger.Fatalf("Could not get user home: %s\n", err)
gologger.Fatal().Msgf("Could not get user home: %s\n", err)
}
flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
@ -89,7 +89,7 @@ func ParseOptions() *Options {
showBanner()
if options.Version {
gologger.Infof("Current Version: %s\n", Version)
gologger.Info().Msgf("Current Version: %s\n", Version)
os.Exit(0)
}
@ -111,7 +111,7 @@ func ParseOptions() *Options {
// invalid options have been used, exit.
err = options.validateOptions()
if err != nil {
gologger.Fatalf("Program exiting: %s\n", err)
gologger.Fatal().Msgf("Program exiting: %s\n", err)
}
return options
@ -130,9 +130,9 @@ func hasStdin() bool {
}
func listSources(options *Options) {
gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources))
gologger.Infof("Sources marked with an * needs key or token in order to work.\n")
gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile)
gologger.Info().Msgf("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources))
gologger.Info().Msgf("Sources marked with an * needs key or token in order to work.\n")
gologger.Info().Msgf("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile)
keys := options.YAMLConfig.GetKeys()
needsKey := make(map[string]interface{})
@ -146,6 +146,6 @@ func listSources(options *Options) {
if _, ok := needsKey[source]; ok {
message = "%s *\n"
}
gologger.Silentf(message, source)
gologger.Silent().Msgf(message, source)
}
}

View File

@ -4,6 +4,8 @@ import (
"errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
)
// validateOptions validates the configuration options passed
@ -39,12 +41,12 @@ func (options *Options) validateOptions() error {
func (options *Options) configureOutput() {
// If the user desires verbose output, show verbose output
if options.Verbose {
gologger.MaxLevel = gologger.Verbose
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
}
if options.NoColor {
gologger.UseColors = false
gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
}
if options.Silent {
gologger.MaxLevel = gologger.Silent
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
}
}

View File

@ -90,7 +90,7 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) {
if response != nil {
_, err := io.Copy(ioutil.Discard, response.Body)
if err != nil {
gologger.Warningf("Could not discard response body: %s\n", err)
gologger.Warning().Msgf("Could not discard response body: %s\n", err)
return
}
response.Body.Close()

View File

@ -1,63 +0,0 @@
package cebaidu
import (
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct{}
type domain struct {
Domain string `json:"domain"`
}
type cebaiduResponse struct {
Code int64 `json:"code"`
Message string `json:"message"`
Data []domain `json:"data"`
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://ce.baidu.com/index/getRelatedSites?site_address=%s", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
var response cebaiduResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
if response.Code > 0 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)}
return
}
for _, result := range response.Data {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "cebaidu"
}

View File

@ -1,75 +0,0 @@
package certspotterold
import (
"context"
"fmt"
"net/http"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type errorResponse struct {
Code string `json:"code"`
Message string `json:"Message"`
}
type subdomain struct {
DNSNames []string `json:"dns_names"`
}
// Source is the passive scraping agent
type Source struct{}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
go func() {
defer close(results)
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}
if resp.StatusCode != http.StatusOK {
var errResponse errorResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s: %s", errResponse.Code, errResponse.Message)}
resp.Body.Close()
return
}
var subdomains []subdomain
err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()
for _, subdomain := range subdomains {
for _, dnsname := range subdomain.DNSNames {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dnsname}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "certspotterold"
}

View File

@ -69,7 +69,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
if token.RetryAfter > 0 {
if len(tokens.pool) == 1 {
gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter)
gologger.Verbose().Msgf("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter)
time.Sleep(time.Duration(token.RetryAfter) * time.Second)
} else {
token = tokens.Get()

View File

@ -3,6 +3,7 @@ package passivetotal
import (
"bytes"
"context"
"regexp"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
@ -54,6 +55,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
resp.Body.Close()
for _, subdomain := range data.Subdomains {
// skip entries like xxx.xxx.xxx.xxx\032domain.tld
if passiveTotalFilterRegex.MatchString(subdomain) {
continue
}
finalSubdomain := subdomain + "." + domain
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain}
}
@ -66,3 +71,5 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
func (s *Source) Name() string {
return "passivetotal"
}
var passiveTotalFilterRegex *regexp.Regexp = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`)

View File

@ -12,14 +12,10 @@ import (
type Source struct{}
type dnsdbLookupResponse struct {
Domain string `json:"domain"`
Data []struct {
Subdomain string `json:"subdomain"`
Type string `json:"type"`
Value string `json:"value"`
} `json:"data"`
Result int `json:"result"`
Error string `json:"error"`
Domain string `json:"domain"`
Subdomains []string `json:"subdomains"`
Result int `json:"result"`
Error string `json:"error"`
}
// Run function returns all subdomains found with the service
@ -54,14 +50,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
for _, data := range response.Data {
if data.Subdomain != "" {
if data.Type == "CNAME" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Value}
} else if data.Type == "A" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data.Subdomain, domain)}
}
}
for _, data := range response.Subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain)}
}
}()