diff --git a/README.md b/README.md index 71f3be4..94add9e 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ More detailed documentation on usage can be found in [doc](doc/README.md). ## What is scanned for? -An list of privacy and security problems which are detected by OnionScan can be +A list of privacy and security problems which are detected by OnionScan can be found [here](doc/what-is-scanned-for.md). You can also directly configure the types of scanning that onionscan does using diff --git a/deanonymization/common_correlations.go b/deanonymization/common_correlations.go index ad474ef..865f4e0 100644 --- a/deanonymization/common_correlations.go +++ b/deanonymization/common_correlations.go @@ -4,6 +4,7 @@ import ( "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "strconv" + "strings" ) // CommonCorrelations extracts relationships from other non-web protocols @@ -31,7 +32,22 @@ func CommonCorrelations(osreport *report.OnionScanReport, anonreport *report.Ano } // Adding all Crawl Ids to Common Correlations (this is a bit of a hack to make the webui nicer) - for _, crawlID := range osreport.Crawls { + for uri, crawlID := range osreport.Crawls { + + if strings.HasSuffix(uri, "/") { + cr,err := osc.Database.GetCrawlRecord(crawlID) + if err == nil { + page := cr.Page + for key,val := range page.Headers { + + osc.Database.InsertRelationship(osreport.HiddenService, "crawl", "http-header", key+":"+strings.Join(val,";")) + } + osc.Database.InsertRelationship(osreport.HiddenService, "crawl", "page-info", page.Title) + } else { + osc.LogError(err) + } + } + osc.Database.InsertRelationship(osreport.HiddenService, "crawl", "database-id", strconv.Itoa(crawlID)) } diff --git a/doc/correlation-lab.md b/doc/correlation-lab.md index 97cf329..323b34b 100644 --- a/doc/correlation-lab.md +++ b/doc/correlation-lab.md @@ -4,4 +4,41 @@ # Discovery Identity Correlations -# Tagging Correltions +The OnionScan Correlation Lab is a rather unique environment. The Lab provides +you with a way of uncovering relationships between different onion sites. + +The best way to often start is to enter the name of an onion service you are +interested in, in the search bar: + + + +If you have scanned the site with OnionScan then the search should result in a +page displaying all kinds of correlations that OnionScan has detected: + + + +You can look around this page and find identifiers and other information that +may indicate potential deanonymization vectors. + +OnionScan also attempts to highlight the most important information at the top +of the page - for example, in the screen above OnionScan has added the page title +along with two tags indicating that OnionScan found a mod_status leak on the +service in question. + + + +# Tagging Correlations + +To help with investigations, The Correlation Lab supports the tagging of search +results - you can tag any given search results, including the results for other +tags, in the left-hand column. + + + +You can then search for all tagged pages using the search feature - or by clicking +on the tag: + + + + + diff --git a/doc/images/correlation-custom-tag.png b/doc/images/correlation-custom-tag.png new file mode 100644 index 0000000..bb4d886 Binary files /dev/null and b/doc/images/correlation-custom-tag.png differ diff --git a/doc/images/correlation-search.png b/doc/images/correlation-search.png new file mode 100644 index 0000000..723fe02 Binary files /dev/null and b/doc/images/correlation-search.png differ diff --git a/doc/images/correlation-summary.png b/doc/images/correlation-summary.png new file mode 100644 index 0000000..2b7941c Binary files /dev/null and b/doc/images/correlation-summary.png differ diff --git a/doc/images/correlation-tagging.png b/doc/images/correlation-tagging.png new file mode 100644 index 0000000..90e17bb Binary files /dev/null and b/doc/images/correlation-tagging.png differ diff --git a/doc/images/correlation-title.png b/doc/images/correlation-title.png new file mode 100644 index 0000000..5874c91 Binary files /dev/null and b/doc/images/correlation-title.png differ diff --git a/doc/what-is-scanned-for.md b/doc/what-is-scanned-for.md index 4c0fa1a..a3d607c 100644 --- a/doc/what-is-scanned-for.md +++ b/doc/what-is-scanned-for.md @@ -1,13 +1,13 @@ # What is scanned for? -Listed below are a few of the more serious privacy problems that may be found -during a scan, ordered per scan type. +Below is an incomplete list of the kinds of scans and correlations that OnionScan +supports. ## Web sites When OnionScan detects a web server, it is scanned for the issues described in this section. -### Apache mod_status Protection +### Apache mod_status Leak This [should not be news](http://arstechnica.com/security/2016/02/default-settings-in-apache-may-decloak-tor-hidden-services/), you should not have it enabled. If you do have it enabled, attacks can: @@ -47,4 +47,42 @@ Sometimes, even without mod_status we can determine if two sites are hosted on * Technology Stack (e.g. php, jquery version etc.) * Website folder layout e.g. do you use `/style` or `/css` or do you use wordpress. * Fingerprints of images -* GPG Versions being used. + +### Analytics IDs + +Some onion services use 3rd party analytics providers to track usage of their +site. These providers often require a unique code to be embedded within the +site - this code can be used to determine if two sites share a common operator + or to find clearnet sites using the same code. + +### PGP Identities + +OnionScan extracts PGP identities from webpages in order to grab identifiers +like email address / identities & GPG versions. + +## SSH + +OnionScan collected information about SSH endpoints including software versions +and the SSH public key fingerprint. These can be correlated against other onion +services or clearnet servers in order to try and identifier the actual sever +location. + +## FTP & SMTP + +OnionScan collected information from other non-web servers, most notably software +banners. These banners are often misconfigured to reveal information about the +target server - including OS version, and sometimes hostnames and IP addresses. + +The software version itself can also be a correlation vector. + +## Cryptocurrency Clients + +OnionScan scans for common cryptocurrency clients including Bitcoin and Litecoin. + +From these it extract other connected onion services as well as the user agent. + +## Protocol Detection + +OnionScan also detects for the presence of many other protocols including IRC, +XMPP, VNC & Ricochet. + diff --git a/templates/index.html b/templates/index.html index cc79ff6..e5af22a 100644 --- a/templates/index.html +++ b/templates/index.html @@ -263,7 +263,7 @@

Tag Search Term

-
Summary for {{.SearchTerm}}   +
Summary for {{.SearchTerm}} {{if ne "" .Summary.Title}}({{.Summary.Title}}){{end}}   {{range .Tags}} {{.}}  diff --git a/webui/webui.go b/webui/webui.go index ba3ba76..384e836 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -29,6 +29,7 @@ type SummaryField struct { type Summary struct { Fields []SummaryField Total int + Title string } type Content struct { @@ -238,6 +239,10 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) { results = append(results, results_identifier...) for _, rel := range results { + if rel.Type == "page-info" { + content.Summary.Title = rel.Identifier + } + if rel.From == "onionscan://user-data" { if rel.Type == "tag" { content.UserTags = append(content.UserTags, rel.Identifier) @@ -332,7 +337,7 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) { } } else if rel.Type == "database-id" { uriCount++ - } + } } // AutoTag our content @@ -354,6 +359,8 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) { for _, v := range tables { content.Summary.Total += len(v.Rows) } + + for k, v := range tables { log.Printf("Adding Table %s %v", k, v) @@ -384,6 +391,12 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) { alt = "Tag Relationships" case "onion": alt = "Co-Hosted Onion Sites" + case "search-results": + alt = "Search Results" + case "http-header": + alt = "HTTP Headers" + case "page-info": + alt = "Webpage Information" } total := (float32(len(v.Rows)) / float32(content.Summary.Total)) * float32(100)