Fixing up Web GUI to add Page Title and Headers

Also updating the Correlation Document.

README.md

@@ -74,7 +74,7 @@ More detailed documentation on usage can be found in [doc](doc/README.md).
 
 ## What is scanned for?
 
-An list of privacy and security problems which are detected by OnionScan can be
+A list of privacy and security problems which are detected by OnionScan can be
 found [here](doc/what-is-scanned-for.md).
 
 You can also directly configure the types of scanning that onionscan does using

deanonymization/common_correlations.go

@@ -4,6 +4,7 @@ import (
  "github.com/s-rah/onionscan/config"
  "github.com/s-rah/onionscan/report"
  "strconv"
+ "strings"
 )
 
 // CommonCorrelations extracts relationships from other non-web protocols
@@ -31,7 +32,22 @@ func CommonCorrelations(osreport *report.OnionScanReport, anonreport *report.Ano
  }
 
  // Adding all Crawl Ids to Common Correlations (this is a bit of a hack to make the webui nicer)
- for _, crawlID := range osreport.Crawls {
+ for uri, crawlID := range osreport.Crawls {
+
+ if strings.HasSuffix(uri, "/") {
+ cr,err := osc.Database.GetCrawlRecord(crawlID)
+ if err == nil {
+ page := cr.Page
+ for key,val := range page.Headers {
+
+ osc.Database.InsertRelationship(osreport.HiddenService, "crawl", "http-header", key+":"+strings.Join(val,";"))
+ }
+ osc.Database.InsertRelationship(osreport.HiddenService, "crawl", "page-info", page.Title)
+ } else {
+ osc.LogError(err)
+ }
+ }
+
  osc.Database.InsertRelationship(osreport.HiddenService, "crawl", "database-id", strconv.Itoa(crawlID))
  }
 

doc/correlation-lab.md

@@ -4,4 +4,41 @@
 
 # Discovery Identity Correlations
 
-# Tagging Correltions
+The OnionScan Correlation Lab is a rather unique environment. The Lab provides
+you with a way of uncovering relationships between different onion sites.
+
+The best way to often start is to enter the name of an onion service you are
+interested in, in the search bar:
+
+<img src="./images/correlation-search.png" title="Searching for an OnionSite"/>
+
+If you have scanned the site with OnionScan then the search should result in a
+page displaying all kinds of correlations that OnionScan has detected:
+
+<img src="./images/correlation-summary.png" title="Correlation Lab Summary"/>
+
+You can look around this page and find identifiers and other information that
+may indicate potential deanonymization vectors.
+
+OnionScan also attempts to highlight the most important information at the top
+of the page - for example, in the screen above OnionScan has added the page title
+along with two tags indicating that OnionScan found a mod_status leak on the 
+service in question.
+
+<img src="./images/correlation-title.png" title="Correlation Lab Summary"/>
+
+# Tagging Correlations
+
+To help with investigations, The Correlation Lab supports the tagging of search
+results - you can tag any given search results, including the results for other
+tags, in the left-hand column. 
+
+<img src="./images/correlation-tagging.png" title="Correlation Lab Options Menu, showing the Tagging feature."/>
+
+You can then search for all tagged pages using the search feature - or by clicking
+on the tag:
+
+<img src="./images/correlation-custom-tag.png" title="Correlation Lab Tagging Summary, showing two sites with the same tag"/>
+
+
+

doc/what-is-scanned-for.md

@@ -1,13 +1,13 @@
 # What is scanned for?
 
-Listed below are a few of the more serious privacy problems that may be found
-during a scan, ordered per scan type.
+Below is an incomplete list of the kinds of scans and correlations that OnionScan
+supports.
 
 ## Web sites
 
 When OnionScan detects a web server, it is scanned for the issues described in this section.
 
-### Apache mod_status Protection
+### Apache mod_status Leak
 
 This [should not be news](http://arstechnica.com/security/2016/02/default-settings-in-apache-may-decloak-tor-hidden-services/), you should not have it enabled. If you do have it enabled, attacks can:
 
@@ -47,4 +47,42 @@ Sometimes, even without mod_status we can determine if two sites are hosted on
 * Technology Stack (e.g. php, jquery version etc.)
 * Website folder layout e.g. do you use `/style` or `/css` or do you use wordpress.
 * Fingerprints of images
-* GPG Versions being used.
+
+### Analytics IDs
+
+Some onion services use 3rd party analytics providers to track usage of their
+site. These providers often require a unique code to be embedded within the
+site - this code can be used to determine if two sites share a common operator
+ or to find clearnet sites using the same code.
+
+### PGP Identities
+
+OnionScan extracts PGP identities from webpages in order to grab identifiers
+like email address / identities & GPG versions.
+
+## SSH
+
+OnionScan collected information about SSH endpoints including software versions
+and the SSH public key fingerprint. These can be correlated against other onion
+services or clearnet servers in order to try and identifier the actual sever
+location.
+
+## FTP & SMTP
+
+OnionScan collected information from other non-web servers, most notably software
+banners. These banners are often misconfigured to reveal information about the 
+target server - including OS version, and sometimes hostnames and IP addresses.
+
+The software version itself can also be a correlation vector.
+
+## Cryptocurrency Clients
+
+OnionScan scans for common cryptocurrency clients including Bitcoin and Litecoin.
+
+From these it extract other connected onion services as well as the user agent.
+
+## Protocol Detection
+
+OnionScan also detects for the presence of many other protocols including IRC,
+XMPP, VNC & Ricochet.
+

templates/index.html

@@ -263,7 +263,7 @@ <h3>Tag Search Term</h3>
 
 
  <div class="panel panel-default">
- <div class="panel-heading">Summary for {{.SearchTerm}}&nbsp;&nbsp;
+ <div class="panel-heading">Summary for {{.SearchTerm}} {{if ne "" .Summary.Title}}({{.Summary.Title}}){{end}}&nbsp;&nbsp;
 
  {{range .Tags}}
  <span class="label label-{{if eq . "mod_status"}}danger{{else}}primary{{end}}"><a href="/?search={{.}}">{{.}}</a></span>&nbsp;

webui/webui.go

@@ -29,6 +29,7 @@ type SummaryField struct {
 type Summary struct {
  Fields []SummaryField
  Total int
+ Title string
 }
 
 type Content struct {
@@ -238,6 +239,10 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) {
  results = append(results, results_identifier...)
 
  for _, rel := range results {
+ if rel.Type == "page-info" {
+ content.Summary.Title = rel.Identifier
+ }
+
  if rel.From == "onionscan://user-data" {
  if rel.Type == "tag" {
  content.UserTags = append(content.UserTags, rel.Identifier)
@@ -332,7 +337,7 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) {
  }
  } else if rel.Type == "database-id" {
  uriCount++
- }
+ } 
  }
 
  // AutoTag our content
@@ -354,6 +359,8 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) {
  for _, v := range tables {
  content.Summary.Total += len(v.Rows)
  }
+
+
 
  for k, v := range tables {
  log.Printf("Adding Table %s %v", k, v)
@@ -384,6 +391,12 @@ func (wui *WebUI) Index(w http.ResponseWriter, r *http.Request) {
  alt = "Tag Relationships"
  case "onion":
  alt = "Co-Hosted Onion Sites"
+ case "search-results":
+ alt = "Search Results"
+ case "http-header":
+ alt = "HTTP Headers"
+ case "page-info":
+ alt = "Webpage Information"
  }
 
  total := (float32(len(v.Rows)) / float32(content.Summary.Total)) * float32(100)