Added the ability to completely override the object responsible for r…

…etrieving HTML content for goose to parse.

pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.gravity</groupId>
   <artifactId>goose</artifactId>
-  <version>2.1.19</version>
+  <version>2.1.20</version>
   <packaging>jar</packaging>
   <name>goose</name>
   <url>http://maven.apache.org</url>

src/main/scala/com/gravity/goose/Configuration.scala

@@ -18,6 +18,7 @@
 
 package com.gravity.goose
 
+import network.{DefaultHtmlFetcher, HtmlFetcher}
 import org.jsoup.nodes.Element
 import java.util.Date
 import reflect.BeanProperty
@@ -114,5 +115,13 @@ class Configuration {
     this.additionalDataExtractor = extractor
   }
 
+  var htmlFetcher: HtmlFetcher = DefaultHtmlFetcher
+
+  def setHtmlFetcher(fetcher: HtmlFetcher) {
+    require(fetcher != null, "fetcher MUST NOT be null!")
+    this.htmlFetcher = fetcher
+  }
+
+  def getHtmlFetcher: HtmlFetcher = htmlFetcher
 
 }

src/main/scala/com/gravity/goose/Crawler.scala

@@ -21,7 +21,6 @@ package com.gravity.goose
 import cleaners.{StandardDocumentCleaner, DocumentCleaner}
 import extractors.ContentExtractor
 import images.{UpgradedImageIExtractor, ImageExtractor}
-import network.HtmlFetcher
 import org.apache.http.client.HttpClient
 import org.jsoup.nodes.{Document, Element}
 import org.jsoup.Jsoup
@@ -48,7 +47,7 @@ class Crawler(config: Configuration) {
       rawHtml <- getHTML(crawlCandidate, parseCandidate)
       doc <- getDocument(parseCandidate.url.toString, rawHtml)
     } {
-      trace("Crawling url: %s".format(parseCandidate.url))
+      trace("Crawling url: " + parseCandidate.url)
 
       val extractor = getExtractor
       val docCleaner = getDocCleaner
@@ -96,10 +95,9 @@ class Crawler(config: Configuration) {
 
           article.cleanedArticleText = outputFormatter.getFormattedText(article.topNode)
         }
-        case _ => trace("NO ARTICLE FOUND");
+        case _ => trace("NO ARTICLE FOUND")
       }
       releaseResources(article)
-      //      self.reply(article)
       article
     }
 
@@ -110,7 +108,7 @@ class Crawler(config: Configuration) {
     if (crawlCandidate.rawHTML != null) {
       Some(crawlCandidate.rawHTML)
     } else {
-      HtmlFetcher.getHtml(config, parsingCandidate.url.toString) match {
+      config.getHtmlFetcher.getHtml(config, parsingCandidate.url.toString) match {
         case Some(html) => {
           Some(html)
         }
@@ -121,7 +119,7 @@ class Crawler(config: Configuration) {
 
 
   def getImageExtractor(article: Article): ImageExtractor = {
-    val httpClient: HttpClient = HtmlFetcher.getHttpClient
+    val httpClient: HttpClient = config.getHtmlFetcher.getHttpClient
     new UpgradedImageIExtractor(httpClient, article, config)
   }
 
@@ -139,7 +137,7 @@ class Crawler(config: Configuration) {
       Some(Jsoup.parse(rawlHtml))
     } catch {
       case e: Exception => {
-        trace("Unable to parse %s properly into JSoup Doc".format(url))
+        trace("Unable to parse " + url + " properly into JSoup Doc")
         None
       }
     }
@@ -153,7 +151,7 @@ class Crawler(config: Configuration) {
   * cleans up any temp files we have laying around like temp images
   * removes any image in the temp dir that starts with the linkhash of the url we just parsed
   */
-  def releaseResources(article: Article) = {
+  def releaseResources(article: Article) {
     trace(logPrefix + "STARTING TO RELEASE ALL RESOURCES")
 
     val dir: File = new File(config.localStoragePath)

src/main/scala/com/gravity/goose/Goose.scala

@@ -18,7 +18,7 @@
 
 package com.gravity.goose
 
-import network.HtmlFetcher
+import network.DefaultHtmlFetcher
 import java.io.File
 
 /**
@@ -45,7 +45,7 @@ class Goose(config: Configuration = new Configuration) {
   }
 
   def shutdownNetwork() {
-    HtmlFetcher.getHttpClient.getConnectionManager.shutdown()
+    DefaultHtmlFetcher.getHttpClient.getConnectionManager.shutdown()
   }
 
   def sendToActor(crawlCandidate: CrawlCandidate) = {

src/main/scala/com/gravity/goose/images/ImageSaver.scala

@@ -37,7 +37,7 @@ import java.io._
 import java.util.Random
 import com.gravity.goose.utils.Logging
 import com.gravity.goose.Configuration
-import com.gravity.goose.network.HtmlFetcher
+import com.gravity.goose.network.DefaultHtmlFetcher
 
 /**
 * This class will be responsible for storing images to disk
@@ -87,7 +87,7 @@ object ImageSaver extends Logging {
   def fetchEntity(httpClient: HttpClient, imageSrc: String): Option[HttpEntity] = {
 
     val localContext: HttpContext = new BasicHttpContext
-    localContext.setAttribute(ClientContext.COOKIE_STORE, HtmlFetcher.emptyCookieStore)
+    localContext.setAttribute(ClientContext.COOKIE_STORE, DefaultHtmlFetcher.emptyCookieStore)
     val httpget = new HttpGet(imageSrc)
     val response = httpClient.execute(httpget, localContext)
     val respStatus: String = response.getStatusLine.toString

src/main/scala/com/gravity/goose/images/ImageUtils.scala

@@ -38,7 +38,7 @@ import com.gravity.goose.Configuration
 import com.gravity.goose.text.HashUtils
 import org.apache.http.util.EntityUtils
 import org.apache.commons.io.IOUtils
-import com.gravity.goose.network.{ImageFetchException, HtmlFetcher}
+import com.gravity.goose.network.{ImageFetchException, DefaultHtmlFetcher}
 
 object ImageUtils extends Logging {
   /**
@@ -169,7 +169,7 @@ object ImageUtils extends Logging {
       }
 
       trace("Not found locally...starting to download image: " + imageSrc)
-      fetchEntity(httpClient, imageSrc) match {
+      fetchEntity(httpClient, imageSrc, config) match {
         case Some(entity) => {
           trace("Got entity for %s".format(imageSrc))
           writeEntityContentsToDisk(entity, linkhash, imageSrc, config) match {
@@ -259,13 +259,13 @@ object ImageUtils extends Logging {
     imgSrc.replace(" ", "%20")
   }
 
-  def fetchEntity(httpClient: HttpClient, imageSrc: String): Option[HttpEntity] = {
+  def fetchEntity(httpClient: HttpClient, imageSrc: String, config: Configuration): Option[HttpEntity] = {
 
     val localContext: HttpContext = new BasicHttpContext
-    localContext.setAttribute(ClientContext.COOKIE_STORE, HtmlFetcher.emptyCookieStore)
+    localContext.setAttribute(ClientContext.COOKIE_STORE, DefaultHtmlFetcher.emptyCookieStore)
     val httpget = new HttpGet(imageSrc)
     val response = try {
-      HtmlFetcher.getHttpClient.execute(httpget, localContext)
+      config.getHtmlFetcher.getHttpClient.execute(httpget, localContext)
     }
     catch {
       case ex: Exception => throw new ImageFetchException(imageSrc, ex)

src/main/scala/com/gravity/goose/images/StandardImageExtractor.scala

@@ -29,7 +29,7 @@ import collection.mutable.HashMap
 import scala.collection.JavaConversions._
 import com.gravity.goose.text.string
 import java.net.{MalformedURLException, URL}
-import com.gravity.goose.network.HtmlFetcher
+import com.gravity.goose.network.DefaultHtmlFetcher
 import java.io.{IOException, File}
 import java.util.regex.{Pattern, Matcher}
 import org.apache.http.client.methods.HttpGet
@@ -473,7 +473,7 @@ class StandardImageExtractor(httpClient: HttpClient, article: Article, config: C
       var link: String = this.buildImagePath(src)
       link = link.replace(" ", "%20")
       val localContext: HttpContext = new BasicHttpContext
-      localContext.setAttribute(ClientContext.COOKIE_STORE, HtmlFetcher.emptyCookieStore)
+      localContext.setAttribute(ClientContext.COOKIE_STORE, DefaultHtmlFetcher.emptyCookieStore)
       httpget = new HttpGet(link)
       var response: HttpResponse = null
       response = httpClient.execute(httpget, localContext)