Merge pull request #92 from devopsgroup-io/develop

Develop
devopsgroup-io · Nov 27, 2017 · e51fa22 · e51fa22
2 parents c0eecf1 + 3d98fec
commit e51fa22
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -264,6 +264,14 @@ if( jQuery('video').length >0 ){
 }
 ```
 
+* SimpleCrawler TypeError: The header content contains invalid characters
+    * Try setting the acceptCookies option to false
+
+```yml
+sitecrawler_options:
+  acceptCookies: false
+```
+
 ## Code of Conduct
 
 Take a moment to read or [Code of Conduct](CODE_OF_CONDUCT.md)

diff --git a/lib/crawler.js b/lib/crawler.js
@@ -63,8 +63,16 @@ var Crawler = function(options) {
 
     // check siteshooter config file for cache boolen option
     if (this.options.sitecrawler.cache){
-        // cache results
-        this.crawler.cache = new SimpleCrawler.cache(process.cwd());
+
+        var self = this;
+
+         utils.checkDirectory(path.join(config.process.get('working_directory'), '.siteshooter'), function(){
+
+            utils.checkDirectory(path.join(config.process.get('working_directory'), '.siteshooter', 'cache'), function(){
+                // cache results
+                self.crawler.cache = new SimpleCrawler.cache(path.join(config.process.get('working_directory'), '.siteshooter', 'cache'));
+            });
+        });
     }
 
     // Treat self-signed SSL certificates as valid