diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index bcb87f94..af5edfc3 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -3,67 +3,22 @@ name: 🎉 Release Binary on: push: tags: - - v* + - 'v*' workflow_dispatch: jobs: - build-mac: - runs-on: macos-latest - steps: - - name: Code checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 - with: - version: latest - args: release -f .goreleaser/mac.yml --rm-dist - workdir: . - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - build-linux: + release: runs-on: ubuntu-latest steps: - - name: Code checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 - with: - version: latest - args: release -f .goreleaser/linux.yml --rm-dist - workdir: . - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - build-windows: - runs-on: windows-latest - steps: - - name: Code checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 + uses: projectdiscovery/actions/setup/go@v1 + + - uses: projectdiscovery/actions/goreleaser@v1 with: - version: latest - args: release -f .goreleaser/windows.yml --rm-dist - workdir: . + release: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 2a7d0fe2..1fee1281 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -5,67 +5,19 @@ on: paths: - '**.go' - '**.mod' - - '**.yml' workflow_dispatch: jobs: - release-test-mac: - runs-on: macos-latest + release-test: + runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - - name: release test - uses: goreleaser/goreleaser-action@v4 - with: - args: "release --clean --snapshot -f .goreleaser/mac.yml" - version: latest - workdir: . - - release-test-linux: - runs-on: ubuntu-latest - steps: - - name: "Check out code" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - + uses: projectdiscovery/actions/setup/go@v1 - - name: release test - uses: goreleaser/goreleaser-action@v4 - with: - args: "release --clean --snapshot -f .goreleaser/linux.yml" - version: latest - workdir: . - - release-test-windows: - runs-on: windows-latest - steps: - - name: "Check out code" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - - name: release test - uses: goreleaser/goreleaser-action@v4 - with: - args: "release --clean --snapshot -f .goreleaser/windows.yml" - version: latest - workdir: . \ No newline at end of file + - name: Release snapshot + uses: projectdiscovery/actions/goreleaser@v1 \ No newline at end of file diff --git a/.goreleaser/linux.yml b/.goreleaser/linux.yml deleted file mode 100644 index dcf121a3..00000000 --- a/.goreleaser/linux.yml +++ /dev/null @@ -1,25 +0,0 @@ -env: - - GO111MODULE=on -before: - hooks: - - go mod tidy -project_name: uncover -builds: - - id: uncover-linux - binary: '{{ .ProjectName }}' - env: - - CGO_ENABLED=0 - main: ./cmd/uncover/uncover.go - goos: - - linux - goarch: - - amd64 - - arm64 - - arm - -archives: - - format: zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' - -checksum: - name_template: "{{ .ProjectName }}-linux-checksums.txt" diff --git a/.goreleaser/mac.yml b/.goreleaser/mac.yml deleted file mode 100644 index 21bf7d90..00000000 --- a/.goreleaser/mac.yml +++ /dev/null @@ -1,24 +0,0 @@ -env: - - GO111MODULE=on -before: - hooks: - - go mod tidy -project_name: uncover -builds: - - id: uncover-darwin - binary: '{{ .ProjectName }}' - env: - - CGO_ENABLED=0 - main: ./cmd/uncover/uncover.go - goos: - - darwin - goarch: - - amd64 - - arm64 -archives: - - format: zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' - - -checksum: - name_template: "{{ .ProjectName }}-mac-checksums.txt" \ No newline at end of file diff --git a/.goreleaser/windows.yml b/.goreleaser/windows.yml deleted file mode 100644 index d2e79f7f..00000000 --- a/.goreleaser/windows.yml +++ /dev/null @@ -1,23 +0,0 @@ -env: - - GO111MODULE=on -before: - hooks: - - go mod tidy -project_name: uncover -builds: - - id: uncover-windows - env: - - CGO_ENABLED=0 - binary: '{{ .ProjectName }}' - main: ./cmd/uncover/uncover.go - goos: - - windows - goarch: - - amd64 - - 386 -archives: - - format: zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' - -checksum: - name_template: "{{ .ProjectName }}-windows-checksums.txt" \ No newline at end of file diff --git a/runner/banners.go b/runner/banners.go index 9fd9cb32..411f0907 100644 --- a/runner/banners.go +++ b/runner/banners.go @@ -13,7 +13,7 @@ const banner = ` ` // Version is the current version of uncover -const version = `v1.1.3` +const version = `v1.1.4` // showBanner is used to show the banner to the user func showBanner() { diff --git a/sources/agent/anubis-spider/anubis.go b/sources/agent/anubis-spider/anubis.go index 4574fc8d..774d08e7 100644 --- a/sources/agent/anubis-spider/anubis.go +++ b/sources/agent/anubis-spider/anubis.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "io" "net/http" "strings" @@ -69,7 +70,8 @@ func (agent *Agent) query(URL string, session *sources.Session, anubis *anubisRe sub = sources.MatchSubdomains(anubis.Domain, body.String(), true) for _, anu := range sub { result := sources.Result{Source: agent.Name()} - result.Host = anu + _, result.Host, result.Port = util.GetProtocolHostAndPort(anu) + result.IP = result.Host raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/sources/agent/binaryedge/binary.go b/sources/agent/binaryedge/binary.go index b712d39e..5bf0c4ec 100644 --- a/sources/agent/binaryedge/binary.go +++ b/sources/agent/binaryedge/binary.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/pkg/errors" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "io" "net/http" "net/url" @@ -89,9 +90,8 @@ func (agent *Agent) query(session *sources.Session, binaryRequest *BinaryRequest } for _, binaryResult := range binaryResponse.Data { result := sources.Result{Source: agent.Name()} - result.Host = binaryResult - result.Port = 80 - result.IP = binaryResult + _, result.Host, result.Port = util.GetProtocolHostAndPort(binaryResult) + result.IP = result.Host raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/sources/agent/bing-spider/query.go b/sources/agent/bing-spider/query.go index 748a5344..df915d86 100644 --- a/sources/agent/bing-spider/query.go +++ b/sources/agent/bing-spider/query.go @@ -102,6 +102,7 @@ func (q *query) updates(subdomains []string) { protocol, host, port := util.GetProtocolHostAndPort(subdomain) result := sources.Result{Source: q.agent.Name()} result.Host = host + result.IP = host result.Port = port portStr := fmt.Sprintf("%d", port) result.Url = protocol + "://" + host + ":" + portStr diff --git a/sources/agent/censys/censys.go b/sources/agent/censys/censys.go index 87600571..42d9ea23 100644 --- a/sources/agent/censys/censys.go +++ b/sources/agent/censys/censys.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + util "github.com/wjlin0/uncover/utils" "net/http" "net/url" @@ -97,7 +98,7 @@ func (agent *Agent) query(URL string, session *sources.Session, censysRequest *C result.IP = ip.(string) } if name, ok := censysResult["name"]; ok { - result.Host = name.(string) + _, result.Host, _ = util.GetProtocolHostAndPort(name.(string)) } if services, ok := censysResult["services"]; ok { for _, serviceData := range services.([]interface{}) { diff --git a/sources/agent/chinaz-spider/chinaz.go b/sources/agent/chinaz-spider/chinaz.go index 1d14e81d..63e5ee30 100644 --- a/sources/agent/chinaz-spider/chinaz.go +++ b/sources/agent/chinaz-spider/chinaz.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "io" "net/http" "strings" @@ -70,7 +71,8 @@ func (agent *Agent) query(URL string, session *sources.Session, chinaz *chinazRe sub = sources.MatchSubdomains(chinaz.Domain, body.String(), true) for _, ch := range sub { result := sources.Result{Source: agent.Name()} - result.Host = ch + _, result.Host, result.Port = util.GetProtocolHostAndPort(ch) + result.IP = result.Host raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/sources/agent/fullhunt/fullhunt.go b/sources/agent/fullhunt/fullhunt.go index a05ae846..5242b12c 100644 --- a/sources/agent/fullhunt/fullhunt.go +++ b/sources/agent/fullhunt/fullhunt.go @@ -103,6 +103,7 @@ func (agent *Agent) query(session *sources.Session, URL string, fullhunt *fullhu protocol, host, port := util.GetProtocolHostAndPort(host) result.Url = fmt.Sprintf("%s://%s:%d", protocol, host, port) result.Host = host + result.IP = host result.Port = port raw, _ := json.Marshal(result) result.Raw = raw diff --git a/sources/agent/github/github.go b/sources/agent/github/github.go index 93d2b3d4..fe9be693 100644 --- a/sources/agent/github/github.go +++ b/sources/agent/github/github.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + util "github.com/wjlin0/uncover/utils" "net/http" "github.com/wjlin0/uncover/sources" @@ -88,7 +89,8 @@ func (agent *Agent) query(URL string, session *sources.Session, githubRequest *g subdomains := sources.MatchSubdomains(githubRequest.Query, body.String(), true) for _, sub := range subdomains { result := sources.Result{Source: agent.Name()} - result.Host = sub + _, result.Host, result.Port = util.GetProtocolHostAndPort(sub) + result.IP = result.Host raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/sources/agent/hunter/hunter.go b/sources/agent/hunter/hunter.go index 652af6cb..6b9cab12 100644 --- a/sources/agent/hunter/hunter.go +++ b/sources/agent/hunter/hunter.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "net/http" ) @@ -75,7 +76,11 @@ func (agent *Agent) query(URL string, session *sources.Session, hunterRequest *R for _, hunterResult := range hunterResponse.Data.Arr { result := sources.Result{Source: agent.Name()} result.IP = hunterResult.IP - result.Port = hunterResult.Port + _, host, port := util.GetProtocolHostAndPort(hunterResult.Domain) + result.Host = host + if result.Port = hunterResult.Port; result.Port == 0 { + result.Port = port + } result.Host = hunterResult.Domain raw, _ := json.Marshal(result) result.Raw = raw diff --git a/sources/agent/qianxun-spider/qianxun.go b/sources/agent/qianxun-spider/qianxun.go index 5d0a7686..7e2549af 100644 --- a/sources/agent/qianxun-spider/qianxun.go +++ b/sources/agent/qianxun-spider/qianxun.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/pkg/errors" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "io" "net/http" "strings" @@ -92,7 +93,8 @@ func (agent *Agent) query(session *sources.Session, domain string, URL string, D continue } result := sources.Result{Source: Source} - result.Host = qianxun + _, result.Host, result.Port = util.GetProtocolHostAndPort(qianxun) + result.IP = result.Host raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/sources/agent/quake/quake.go b/sources/agent/quake/quake.go index c39d1a8d..bbe6e8c5 100644 --- a/sources/agent/quake/quake.go +++ b/sources/agent/quake/quake.go @@ -7,6 +7,7 @@ import ( "fmt" errorutil "github.com/projectdiscovery/utils/errors" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "io" "net/http" ) @@ -93,12 +94,14 @@ func (agent *Agent) query(URL string, session *sources.Session, quakeRequest *Re for _, quakeResult := range quakeResponse.Data { result := sources.Result{Source: agent.Name()} result.IP = quakeResult.IP - if result.Port = quakeResult.Port; result.Port == 0 { - result.Port = 80 - } + result.Port = quakeResult.Port host := quakeResult.Hostname if host == "" { - host = fmt.Sprintf("%s:%d", result.IP, result.Port) + host = fmt.Sprintf("%s", result.IP) + } + _, host, port := util.GetProtocolHostAndPort(host) + if result.Port == 0 { + result.Port = port } result.Host = host raw, _ := json.Marshal(result) diff --git a/sources/agent/rapiddns-spider/rapiddns.go b/sources/agent/rapiddns-spider/rapiddns.go index 4090ae37..cf59532a 100644 --- a/sources/agent/rapiddns-spider/rapiddns.go +++ b/sources/agent/rapiddns-spider/rapiddns.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/wjlin0/uncover/sources" + util "github.com/wjlin0/uncover/utils" "io" "net/http" "strings" @@ -70,7 +71,7 @@ func (agent *Agent) query(URL string, session *sources.Session, rapid *rapidDNS, sub = sources.MatchSubdomains(rapid.Domain, body.String(), true) for _, ra := range sub { result := sources.Result{Source: agent.Name()} - result.Host = ra + _, result.Host, result.Port = util.GetProtocolHostAndPort(ra) raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/sources/agent/shodan/shodan.go b/sources/agent/shodan/shodan.go index 0359d4b3..3b5ba487 100644 --- a/sources/agent/shodan/shodan.go +++ b/sources/agent/shodan/shodan.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + util "github.com/wjlin0/uncover/utils" "net/http" "net/url" @@ -101,7 +102,8 @@ func (agent *Agent) query(URL string, session *sources.Session, shodanRequest *S if hostnames, ok := shodanResult["hostnames"]; ok { if _, ok := hostnames.([]interface{}); ok { for _, hostname := range hostnames.([]interface{}) { - result.Host = fmt.Sprint(hostname) + _, host, _ := util.GetProtocolHostAndPort(fmt.Sprint(hostname)) + result.Host = host } } raw, _ := json.Marshal(shodanResult) diff --git a/sources/agent/zoomeye/zoomeye.go b/sources/agent/zoomeye/zoomeye.go index f3790e8c..2f5edd11 100644 --- a/sources/agent/zoomeye/zoomeye.go +++ b/sources/agent/zoomeye/zoomeye.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + util "github.com/wjlin0/uncover/utils" "net/http" "net/url" "strconv" @@ -102,7 +103,8 @@ func (agent *Agent) query(URL string, session *sources.Session, zoomeyeRequest * if result.Port == 0 { continue } - result.Host = port["hostname"].(string) + _, result.Host, _ = util.GetProtocolHostAndPort(port["hostname"].(string)) + result.IP = result.Host raw, _ := json.Marshal(zoomeyeResult) result.Raw = raw results <- result diff --git a/uncover.go b/uncover.go index 3b860f40..d73e42e5 100644 --- a/uncover.go +++ b/uncover.go @@ -179,7 +179,7 @@ func (s *Service) Execute(ctx context.Context) (<-chan sources.Result, error) { } keys := s.Provider.GetKeys() if keys.Empty() && !(stringsutil.EqualFoldAny(agent.Name(), AnonymousAgents()...)) { - gologger.Error().Msgf(agent.Name(), "agent given but keys not found") + gologger.Error().Msgf("%s agent given but keys not found", agent.Name()) continue agentLabel } ch, err := agent.Query(s.Session, &sources.Query{