ready

src/crawler.ts

@@ -39,7 +39,7 @@ class Crawler extends EventEmitter {
             retries: 3,
             retryInterval: 2000,
             timeout: 15000,
-            isJson: true,
+            isJson: false,
         };
         this.options = { ...defaultOptions, ...options };
 
@@ -148,7 +148,7 @@ class Crawler extends EventEmitter {
             try {
                 options.preRequest!(options, async (err?: Error | null) => {
                     if (err) {
-                        log.error(err);
+                        log.debug(err);
                         return this._handler(err, options);
                     }
                     return await request();
@@ -222,7 +222,7 @@ class Crawler extends EventEmitter {
                 try {
                     response.$ = load(response.body);
                 } catch (err) {
-                    log.error(err);
+                    log.warn("HTML detected failed. Set jQuery to false to mute this warning.");
                 }
             }
         }

src/options.ts

@@ -108,7 +108,7 @@ export const alignOptions = (options: any): any => {
             gotOptions.headers.referer = options.referer;
         }
         else {
-            const domain = options.url.match(/^(\w+):\/\/([^\/]+)/);
+            const domain = gotOptions.url.match(/^(\w+):\/\/([^\/]+)/);
             if (domain) gotOptions.headers.referer = domain[0];
         }
     }

src/types/crawler.ts

@@ -42,7 +42,7 @@ type requestOptions = {
     forceUTF8?: boolean;
     /**
      * crawlerOption
-     * @default false
+     * @default true
      * @description If true, the crawler will use the cheerio library to parse the HTML content.
      * @see cheerio.load()
      * @example If inject successfully, the response object will have "$" property, which is a function to use jQuery.
@@ -123,7 +123,7 @@ type requestOptions = {
 
     /**
      * @description If true, the crawler will parse the response body as JSON.
-     * @default true
+     * @default false
      */
     isJson?: boolean;
 

tests/callback.test.js

@@ -17,9 +17,6 @@ describe('Callback test', function() {
 			retryInterval:0,
 			retries:0,
 			timeout:100,
-			logger: {
-				log:() => {}
-			},
 		});
 	});
 

tests/encoding.test.js

@@ -6,7 +6,7 @@ import Crawler from '../dist/index.js';
 import { expect } from 'chai';
 import nock from 'nock';
 import fs from 'fs';
-var __dirname = "./";
+var __dirname = "./tests";
 
 describe('Encoding', function () {
     before(function () {

tests/requests.test.js

@@ -20,7 +20,7 @@ describe('Request tests', function () {
     beforeEach(function () {
         crawler = new Crawler({
             retries: 0,
-            json: true,
+            isJson: true,
             jQuery: false,
         });
 
@@ -95,7 +95,7 @@ describe('Request tests', function () {
         crawler = new Crawler({
             headers: { 'User-Agent': 'test/1.2' },
             jQuery: false,
-            json: true,
+            isJson: true,
             callback: function (error, res, done) {
                 expect(error).to.be.null;
                 expect(res.body['user-agent']).to.equal('foo/bar');
@@ -113,7 +113,7 @@ describe('Request tests', function () {
                 'user-agent': 'test/1.2'
             },
             jQuery: false,
-            json: true,
+            isJson: true,
             callback: function (error, res, done) {
                 expect(error).to.be.null;
                 expect(res.body['user-agent']).to.equal('foo/bar');