Merge pull request projectdiscovery#176 from projectdiscovery/dev

uncover v1.0.3

.github/workflows/build-test.yml

@@ -2,6 +2,9 @@ name: 🔨 Build Test
 
 on:
   pull_request:
+    paths:
+      - '**.go'
+      - '**.mod'
   workflow_dispatch:
 
 jobs:

.github/workflows/codeql-analysis.yml

@@ -1,10 +1,11 @@
 name: 🚨 CodeQL Analysis
 
 on:
-  workflow_dispatch:
   pull_request:
-    branches:
-      - dev
+    paths:
+      - '**.go'
+      - '**.mod'
+  workflow_dispatch:
 
 jobs:
   analyze:

.github/workflows/dockerhub-push.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   docker:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Git Checkout
         uses: actions/checkout@v3
@@ -32,7 +32,7 @@ jobs:
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm

.github/workflows/lint-test.yml

@@ -2,6 +2,9 @@ name: 🙏🏻 Lint Test
 
 on:
   pull_request:
+    paths:
+      - '**.go'
+      - '**.mod'
   workflow_dispatch:
 
 jobs:
@@ -18,7 +21,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.1
+        uses: golangci/golangci-lint-action@v3.4.0
         with:
           version: latest
           args: --timeout 5m

.github/workflows/release-binary.yml

@@ -8,7 +8,7 @@ on:
 
 jobs: 
   release: 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps: 
       - name: "Check out code"
         uses: actions/checkout@v3
@@ -28,4 +28,6 @@ jobs:
           workdir: .
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-          SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
+          SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
+          DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}"
+          DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}"

.goreleaser.yml

@@ -11,7 +11,7 @@ builds:
     - darwin
   goarch:
     - amd64
-    - 386
+    - '386'
     - arm
     - arm64
 
@@ -39,4 +39,8 @@ announce:
     enabled: true
     channel: '#release'
     username: GoReleaser
-    message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' 
+    message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}'
+
+  discord:
+    enabled: true
+    message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}'

Dockerfile

@@ -1,7 +1,15 @@
-FROM golang:1.18.2-alpine3.14 AS build-env
-RUN go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest
+# Base
+FROM golang:1.20.2-alpine AS builder
+RUN apk add --no-cache git build-base
+WORKDIR /app
+COPY . /app
+RUN go mod download
+RUN go build -o ./cmd/uncover ./cmd/uncover
 
-FROM alpine:3.17.0
-RUN apk add --no-cache bind-tools ca-certificates
-COPY --from=build-env /go/bin/uncover /usr/local/bin/uncover
-ENTRYPOINT ["uncover"]
+# Release
+FROM alpine:3.17.2
+RUN apk -U upgrade --no-cache \
+    && apk add --no-cache bind-tools ca-certificates
+COPY --from=builder /app/cmd/uncover/uncover /usr/local/bin/
+
+ENTRYPOINT ["uncover"]

README.md

@@ -45,6 +45,8 @@
   - **[Zoomeye](https://www.zoomeye.org)**
   - **[Netlas](https://netlas.io/)**
   - **[CriminalIP](https://www.criminalip.io)**
+  - **[PublicWWW](https://publicwww.com)**
+  - **[HunterHow](https://hunter.how)**
 - Multiple API key input support
 - Automatic API key randomization
 - **stdin** / **stdout** support for input
@@ -72,7 +74,7 @@ Usage:
 Flags:
 INPUT:
    -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
-   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan)
+   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan)
 
 SEARCH-ENGINE:
    -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')
@@ -84,13 +86,16 @@ SEARCH-ENGINE:
    -ze, -zoomeye string[]     search query for zoomeye (example: -zoomeye 'query.txt')
    -ne, -netlas string[]      search query for netlas (example: -netlas 'query.txt')
    -cl, -criminalip string[]  search query for criminalip (example: -criminalip 'query.txt')
+   -pw, -publicwww string[]   search query for publicwww (example: -publicwww 'query.txt')
+   -hh, -hunterhow string[]   search query for hunterhow (example: -hunterhow 'query.txt')
 
 CONFIG:
-   -pc, -provider string  provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
-   -config string         flag configuration file (default "$HOME/.config/uncover/config.yaml")
-   -timeout int           timeout in seconds (default 30)
-   -delay int             delay between requests in seconds (0 to disable) (default 1)
-   -retry int             number of times to retry a failed request (default 2)
+   -pc, -provider string         provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
+   -config string                flag configuration file (default "$HOME/.config/uncover/config.yaml")
+   -timeout int                  timeout in seconds (default 30)
+   -rl, -rate-limit int          maximum number of http requests to send per second
+   -rlm, -rate-limit-minute int  maximum number of requests to send per minute
+   -retry int                    number of times to retry a failed request (default 2)
 
 OUTPUT:
    -o, -output string  output file to write found results
@@ -138,6 +143,9 @@ netlas:
 criminalip:
   - CRIMINALIP_API_KEY_1
   - CRIMINALIP_API_KEY_2
+publicwww:
+  - PUBLICWWW_API_KEY_1
+  - PUBLICWWW_API_KEY_2
 ```
 
 When multiple keys/credentials are specified for same provider in the config file, random key will be used for each execution.
@@ -155,9 +163,10 @@ export HUNTER_API_KEY=xxx
 export ZOOMEYE_API_KEY=xxx
 export NETLAS_API_KEY=xxx
 export CRIMINALIP_API_KEY=xxx
+export PUBLICWWW_API_KEY=xxx
 ```
 
-Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.org/login), [Netlas](https://app.netlas.io/registration/) and [CriminalIP](https://www.criminalip.io/register).
+Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.org/login), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register) and [Publicwww](https://publicwww.com/profile/signup.html).
 
 ## Running Uncover
 

go.mod

@@ -1,60 +1,85 @@
 module github.com/projectdiscovery/uncover
 
-go 1.18
+go 1.19
 
 require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/fdmax v0.0.4
-	github.com/projectdiscovery/goflags v0.1.6
-	github.com/projectdiscovery/gologger v1.1.5
-	github.com/projectdiscovery/mapcidr v1.0.3
-	github.com/projectdiscovery/ratelimit v0.0.4
-	github.com/projectdiscovery/retryablehttp-go v1.0.7
-	github.com/projectdiscovery/stringsutil v0.0.2
-	github.com/stretchr/testify v1.8.1
+	github.com/projectdiscovery/goflags v0.1.8
+	github.com/projectdiscovery/gologger v1.1.8
+	github.com/projectdiscovery/mapcidr v1.1.0
+	github.com/projectdiscovery/ratelimit v0.0.6
+	github.com/projectdiscovery/retryablehttp-go v1.0.13
+	github.com/stretchr/testify v1.8.2
 )
 
 require (
-	github.com/miekg/dns v1.1.50 // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 // indirect
-	golang.org/x/mod v0.6.0 // indirect
-	golang.org/x/tools v0.2.0 // indirect
+	github.com/miekg/dns v1.1.52 // indirect
+	golang.org/x/exp v0.0.0-20230314191032-db074128a8ec // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
 )
 
 require (
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.21 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.23 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect
-	github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54
-	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect
+	github.com/projectdiscovery/utils v0.0.16
+	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
+	aead.dev/minisign v0.2.0 // indirect
+	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/alecthomas/chroma v0.10.0 // indirect
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/charmbracelet/glamour v0.6.0 // indirect
+	github.com/cheggaaa/pb/v3 v3.1.2 // indirect
+	github.com/dlclark/regexp2 v1.8.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
+	github.com/fatih/color v1.14.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-github/v30 v30.1.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mholt/archiver v3.1.1+incompatible // indirect
+	github.com/minio/selfupdate v0.6.0 // indirect
+	github.com/muesli/reflow v0.3.0 // indirect
+	github.com/muesli/termenv v0.15.1 // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	github.com/yuin/goldmark v1.5.4 // indirect
+	github.com/yuin/goldmark-emoji v1.0.1 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 )