feat: added tracking for status code, waf-detection & grouped errors

cmd/nuclei/main.go

@@ -446,6 +446,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
 		flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"),
+		flagSet.BoolVarP(&options.HTTPStats, "http-stats", "hps", false, "enable http status capturing (experimental)"),
 	)
 
 	flagSet.CreateGroup("cloud", "Cloud",

internal/runner/runner.go

@@ -19,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
+	outputstats "github.com/projectdiscovery/nuclei/v3/pkg/output/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
@@ -92,6 +93,8 @@ type Runner struct {
 	pdcpUploadErrMsg   string
 	inputProvider      provider.InputProvider
 	fuzzFrequencyCache *frequency.Tracker
+	httpStats          *outputstats.Tracker
+
 	//general purpose temporary directory
 	tmpDir          string
 	parser          parser.Parser
@@ -256,6 +259,10 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	// setup a proxy writer to automatically upload results to PDCP
 	runner.output = runner.setupPDCPUpload(outputWriter)
+	if options.HTTPStats {
+		runner.httpStats = outputstats.NewTracker()
+		runner.output = output.NewMultiWriter(runner.output, output.NewTrackerWriter(runner.httpStats))
+	}
 
 	if options.JSONL && options.EnableProgressBar {
 		options.StatsJSON = true
@@ -362,6 +369,9 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
+	if r.httpStats != nil {
+		r.httpStats.DisplayTopStats(r.options.NoColor)
+	}
 	// dump hosterrors cache
 	if r.hostErrors != nil {
 		r.hostErrors.Close()

pkg/output/multi_writer.go

@@ -8,6 +8,8 @@ type MultiWriter struct {
 	writers []Writer
 }
 
+var _ Writer = &MultiWriter{}
+
 // NewMultiWriter creates a new MultiWriter instance
 func NewMultiWriter(writers ...Writer) *MultiWriter {
 	return &MultiWriter{writers: writers}
@@ -57,3 +59,9 @@ func (mw *MultiWriter) WriteStoreDebugData(host, templateID, eventType string, d
 		writer.WriteStoreDebugData(host, templateID, eventType, data)
 	}
 }
+
+func (mw *MultiWriter) RequestStatsLog(statusCode, response string) {
+	for _, writer := range mw.writers {
+		writer.RequestStatsLog(statusCode, response)
+	}
+}

pkg/output/output.go

@@ -50,6 +50,8 @@ type Writer interface {
 	WriteFailure(*InternalWrappedEvent) error
 	// Request logs a request in the trace log
 	Request(templateID, url, requestType string, err error)
+	// RequestStatsLog logs a request stats log
+	RequestStatsLog(statusCode, response string)
 	//  WriteStoreDebugData writes the request/response debug data to file
 	WriteStoreDebugData(host, templateID, eventType string, data string)
 }
@@ -75,6 +77,8 @@ type StandardWriter struct {
 	KeysToRedact          []string
 }
 
+var _ Writer = &StandardWriter{}
+
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
 
 // InternalEvent is an internal output generation structure for nuclei.
@@ -351,15 +355,33 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 	if w.traceFile == nil && w.errorFile == nil {
 		return
 	}
+
+	request := getJSONLogRequestFromError(templatePath, input, requestType, requestErr)
+	if w.timestamp {
+		ts := time.Now()
+		request.Timestamp = &ts
+	}
+	data, err := jsoniter.Marshal(request)
+	if err != nil {
+		return
+	}
+
+	if w.traceFile != nil {
+		_, _ = w.traceFile.Write(data)
+	}
+
+	if requestErr != nil && w.errorFile != nil {
+		_, _ = w.errorFile.Write(data)
+	}
+}
+
+func getJSONLogRequestFromError(templatePath, input, requestType string, requestErr error) *JSONLogRequest {
 	request := &JSONLogRequest{
 		Template: templatePath,
 		Input:    input,
 		Type:     requestType,
 	}
-	if w.timestamp {
-		ts := time.Now()
-		request.Timestamp = &ts
-	}
+
 	parsed, _ := urlutil.ParseAbsoluteURL(input, false)
 	if parsed != nil {
 		request.Address = parsed.Hostname()
@@ -397,18 +419,7 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 	if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil {
 		request.Address = val.String()
 	}
-	data, err := jsoniter.Marshal(request)
-	if err != nil {
-		return
-	}
-
-	if w.traceFile != nil {
-		_, _ = w.traceFile.Write(data)
-	}
-
-	if requestErr != nil && w.errorFile != nil {
-		_, _ = w.errorFile.Write(data)
-	}
+	return request
 }
 
 // Colorizer returns the colorizer instance for writer
@@ -540,12 +551,14 @@ func tryParseCause(err error) error {
 	if strings.HasPrefix(msg, "ReadStatusLine:") {
 		// last index is actual error (from rawhttp)
 		parts := strings.Split(msg, ":")
-		return errkit.New("%s", strings.TrimSpace(parts[len(parts)-1]))
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
 	}
 	if strings.Contains(msg, "read ") {
 		// same here
 		parts := strings.Split(msg, ":")
-		return errkit.New("%s", strings.TrimSpace(parts[len(parts)-1]))
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
 	}
 	return err
 }
+
+func (w *StandardWriter) RequestStatsLog(statusCode, response string) {}

pkg/output/output_stats.go

@@ -0,0 +1,51 @@
+package output
+
+import (
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output/stats"
+)
+
+// StatsOutputWriter implements writer interface for stats observation
+type StatsOutputWriter struct {
+	colorizer aurora.Aurora
+	Tracker   *stats.Tracker
+}
+
+var _ Writer = &StatsOutputWriter{}
+
+// NewStatsOutputWriter returns a new StatsOutputWriter instance.
+func NewTrackerWriter(t *stats.Tracker) *StatsOutputWriter {
+	return &StatsOutputWriter{
+		colorizer: aurora.NewAurora(true),
+		Tracker:   t,
+	}
+}
+
+func (tw *StatsOutputWriter) Close() {}
+
+func (tw *StatsOutputWriter) Colorizer() aurora.Aurora {
+	return tw.colorizer
+}
+
+func (tw *StatsOutputWriter) Write(event *ResultEvent) error {
+	return nil
+}
+
+func (tw *StatsOutputWriter) WriteFailure(event *InternalWrappedEvent) error {
+	return nil
+}
+
+func (tw *StatsOutputWriter) Request(templateID, url, requestType string, err error) {
+	if err == nil {
+		return
+	}
+	jsonReq := getJSONLogRequestFromError(templateID, url, requestType, err)
+	tw.Tracker.TrackErrorKind(jsonReq.Error)
+}
+
+func (tw *StatsOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {}
+
+func (tw *StatsOutputWriter) RequestStatsLog(statusCode, response string) {
+	tw.Tracker.TrackStatusCode(statusCode)
+	tw.Tracker.TrackWAFDetected(response)
+}

pkg/output/stats/stats.go

@@ -0,0 +1,181 @@
+// Package stats provides a stats tracker for tracking Status Codes,
+// Errors & WAF detection events.
+//
+// It is wrapped and called by output.Writer interface.
+package stats
+
+import (
+	_ "embed"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync/atomic"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output/stats/waf"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+)
+
+// Tracker is a stats tracker instance for nuclei scans
+type Tracker struct {
+	// counters for various stats
+	statusCodes *mapsutil.SyncLockMap[string, *atomic.Int32]
+	errorCodes  *mapsutil.SyncLockMap[string, *atomic.Int32]
+	wafDetected *mapsutil.SyncLockMap[string, *atomic.Int32]
+
+	// internal stuff
+	wafDetector *waf.WafDetector
+}
+
+// NewTracker creates a new Tracker instance.
+func NewTracker() *Tracker {
+	return &Tracker{
+		statusCodes: mapsutil.NewSyncLockMap[string, *atomic.Int32](),
+		errorCodes:  mapsutil.NewSyncLockMap[string, *atomic.Int32](),
+		wafDetected: mapsutil.NewSyncLockMap[string, *atomic.Int32](),
+		wafDetector: waf.NewWafDetector(),
+	}
+}
+
+// TrackStatusCode tracks the status code of a request
+func (t *Tracker) TrackStatusCode(statusCode string) {
+	t.incrementCounter(t.statusCodes, statusCode)
+}
+
+// TrackErrorKind tracks the error kind of a request
+func (t *Tracker) TrackErrorKind(errKind string) {
+	t.incrementCounter(t.errorCodes, errKind)
+}
+
+// TrackWAFDetected tracks the waf detected of a request
+//
+// First it detects if a waf is running and if so, it increments
+// the counter for the waf.
+func (t *Tracker) TrackWAFDetected(httpResponse string) {
+	waf, ok := t.wafDetector.DetectWAF(httpResponse)
+	if !ok {
+		return
+	}
+
+	t.incrementCounter(t.wafDetected, waf)
+}
+
+func (t *Tracker) incrementCounter(m *mapsutil.SyncLockMap[string, *atomic.Int32], key string) {
+	if counter, ok := m.Get(key); ok {
+		counter.Add(1)
+	} else {
+		newCounter := new(atomic.Int32)
+		newCounter.Store(1)
+		_ = m.Set(key, newCounter)
+	}
+}
+
+type StatsOutput struct {
+	StatusCodeStats map[string]int `json:"status_code_stats"`
+	ErrorStats      map[string]int `json:"error_stats"`
+	WAFStats        map[string]int `json:"waf_stats"`
+}
+
+func (t *Tracker) GetStats() *StatsOutput {
+	stats := &StatsOutput{
+		StatusCodeStats: make(map[string]int),
+		ErrorStats:      make(map[string]int),
+		WAFStats:        make(map[string]int),
+	}
+	_ = t.errorCodes.Iterate(func(k string, v *atomic.Int32) error {
+		stats.ErrorStats[k] = int(v.Load())
+		return nil
+	})
+	_ = t.statusCodes.Iterate(func(k string, v *atomic.Int32) error {
+		stats.StatusCodeStats[k] = int(v.Load())
+		return nil
+	})
+	_ = t.wafDetected.Iterate(func(k string, v *atomic.Int32) error {
+		waf, ok := t.wafDetector.GetWAF(k)
+		if !ok {
+			return nil
+		}
+		stats.WAFStats[waf.Name] = int(v.Load())
+		return nil
+	})
+	return stats
+}
+
+// DisplayTopStats prints the most relevant statistics for CLI
+func (t *Tracker) DisplayTopStats(noColor bool) {
+	stats := t.GetStats()
+
+	fmt.Printf("\n%s\n", aurora.Bold(aurora.Blue("Top Status Codes:")))
+	topStatusCodes := getTopN(stats.StatusCodeStats, 6)
+	for _, item := range topStatusCodes {
+		if noColor {
+			fmt.Printf("  %s: %d\n", item.Key, item.Value)
+		} else {
+			color := getStatusCodeColor(item.Key)
+			fmt.Printf("  %s: %d\n", aurora.Colorize(item.Key, color), item.Value)
+		}
+	}
+
+	if len(stats.ErrorStats) > 0 {
+		fmt.Printf("\n%s\n", aurora.Bold(aurora.Red("Top Errors:")))
+		topErrors := getTopN(stats.ErrorStats, 5)
+		for _, item := range topErrors {
+			if noColor {
+				fmt.Printf("  %s: %d\n", item.Key, item.Value)
+			} else {
+				fmt.Printf("  %s: %d\n", aurora.Red(item.Key), item.Value)
+			}
+		}
+	}
+
+	if len(stats.WAFStats) > 0 {
+		fmt.Printf("\n%s\n", aurora.Bold(aurora.Yellow("WAF Detections:")))
+		for name, count := range stats.WAFStats {
+			if noColor {
+				fmt.Printf("  %s: %d\n", name, count)
+			} else {
+				fmt.Printf("  %s: %d\n", aurora.Yellow(name), count)
+			}
+		}
+	}
+}
+
+// Helper struct for sorting
+type kv struct {
+	Key   string
+	Value int
+}
+
+// getTopN returns top N items from a map, sorted by value
+func getTopN(m map[string]int, n int) []kv {
+	var items []kv
+	for k, v := range m {
+		items = append(items, kv{k, v})
+	}
+
+	sort.Slice(items, func(i, j int) bool {
+		return items[i].Value > items[j].Value
+	})
+
+	if len(items) > n {
+		items = items[:n]
+	}
+	return items
+}
+
+// getStatusCodeColor returns appropriate color for status code
+func getStatusCodeColor(statusCode string) aurora.Color {
+	code, _ := strconv.Atoi(statusCode)
+	switch {
+	case code >= 200 && code < 300:
+		return aurora.GreenFg
+	case code >= 300 && code < 400:
+		return aurora.BlueFg
+	case code >= 400 && code < 500:
+		return aurora.YellowFg
+	case code >= 500:
+		return aurora.RedFg
+	default:
+		return aurora.WhiteFg
+	}
+}