Append partial hash of URL path + fragment to base file names

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - The Nmap/Masscan XML report parser did not ignore closed/filtered ports. It now only works on ports with state `open`.
 
+### Added
+- Support for processing of multiple URLs on the same host by appending hash of URL path and fragment to file names
+
 ## [1.5.0]
 
 ### Added

agents/util.go

@@ -1,8 +1,10 @@
 package agents
 
 import (
+	"crypto/sha1"
 	"crypto/tls"
 	"fmt"
+	"io"
 	"math/rand"
 	"net/url"
 	"strconv"
@@ -105,8 +107,14 @@ func BaseFilenameFromURL(s string) string {
 	if err != nil {
 		return ""
 	}
+
+	h := sha1.New()
+	io.WriteString(h, u.Path)
+	io.WriteString(h, u.Fragment)
+
+	pathHash := fmt.Sprintf("%x", h.Sum(nil))[0:16]
 	host := strings.Replace(u.Host, ":", "__", 1)
-	filename := fmt.Sprintf("%s__%s", u.Scheme, strings.Replace(host, ".", "_", -1))
+	filename := fmt.Sprintf("%s__%s__%s", u.Scheme, strings.Replace(host, ".", "_", -1), pathHash)
 	return strings.ToLower(filename)
 }
 

core/session.go

@@ -1,7 +1,9 @@
 package core
 
 import (
+	"crypto/sha1"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -244,8 +246,14 @@ func (s *Session) BaseFilenameFromURL(stru string) string {
 	if err != nil {
 		return ""
 	}
+
+	h := sha1.New()
+	io.WriteString(h, u.Path)
+	io.WriteString(h, u.Fragment)
+
+	pathHash := fmt.Sprintf("%x", h.Sum(nil))[0:16]
 	host := strings.Replace(u.Host, ":", "__", 1)
-	filename := fmt.Sprintf("%s__%s", u.Scheme, strings.Replace(host, ".", "_", -1))
+	filename := fmt.Sprintf("%s__%s__%s", u.Scheme, strings.Replace(host, ".", "_", -1), pathHash)
 	return strings.ToLower(filename)
 }