step

bda-research · Apr 29, 2024 · 18ebe93 · 18ebe93
1 parent 40f5589
commit 18ebe93
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 14 deletions.
diff --git a/src/crawler.ts b/src/crawler.ts
@@ -34,7 +34,7 @@ class Crawler extends EventEmitter {
             skipDuplicates: false,
             rotateUA: false,
             homogeneous: false,
-
+            method: 'GET',
             forceUTF8: true,
             incomingEncoding: null,
             jQuery: true,
@@ -156,20 +156,20 @@ class Crawler extends EventEmitter {
 
         try {
             const response = await got(alignOptions(reqOptions));
-            this._handler(null, reqOptions, response);
+            return this._handler(null, reqOptions, response);
         } catch (error) {
             console.log("error:", error);
-            this._handler(error, reqOptions);
+            return this._handler(error, reqOptions);
         }
     };
 
-    private _handler = (error: any | null, options: requestOptions, response?: any): void => {
+    private _handler = (error: any | null, options: requestOptions, response?: any): any => {
         if (error) {
             console.log(
                 `Error: ${error} when fetching ${options.uri} ${options.retries ? `(${options.retries} retries left)` : ""
                 }`
             );
-            if (options.retries){
+            if (options.retries) {
                 setTimeout(() => {
                     options.retries!--;
                     this._execute(options as crawlerOptions);
@@ -196,6 +196,7 @@ class Crawler extends EventEmitter {
                 console.debug("Charset: " + charset);
                 if (charset && charset !== "utf-8" && charset != "ascii") {
                     response.body = iconv.decode(response.body, charset);
+                    response.body = response.body.toString();
                 }
             }
         } catch (error) {
@@ -206,14 +207,17 @@ class Crawler extends EventEmitter {
         }
 
         response.options = options;
-
-        // @todo: jQuery injection
-        if (options.method === "HEAD" || !options.jQuery) {
-            if (options.callback && typeof options.callback === "function") {
-                return options.callback(null, response, options.release);
-            }
-            return response;
+        if (options.callback && typeof options.callback === "function") {
+            return options.callback(null, response, options.release);
         }
+        return response;
+        // @todo: jQuery injection
+        // if (options.method === "HEAD" || !options.jQuery) {
+        //     if (options.callback && typeof options.callback === "function") {
+        //         return options.callback(null, response, options.release);
+        //     }
+        //     return response;
+        // }
 
         // const injectableTypes = ["html", "xhtml", "text/xml", "application/xml", "+xml"];
         // if (!options.html && !typeis(contentType(response), injectableTypes)) {

diff --git a/src/options.ts b/src/options.ts
@@ -27,7 +27,7 @@ export const alignOptions = (options: any): any => {
 
     const gotOptions = {
         ...options,
-        url: options.uri,
+        url: options.url ?? options.uri,
         searchParams: options.qs,
         rejectUnauthorized: options.strictSSL,
         decompress: options.gzip,

diff --git a/src/types/crawler.d.ts b/src/types/crawler.d.ts
@@ -41,7 +41,12 @@ type globalOnlyOptions = {
 type requestOptions = {
     forceUTF8?: boolean;
     jQuery?: boolean;
+    /**
+     * @deprecated
+     * @description Use "encoding" instead.
+     */
     incomingEncoding?: string | null;
+    encoding?: string | null;
     retries?: number;
     retryTimeout?: number;
     timeout?: number;

diff --git a/test.js b/test.js
@@ -1,5 +1,9 @@
 import { crawler } from './src/index.ts';
-const result = await crawler.send('http://www.google.com');
+const result = await crawler.send({
+    url: "http://www.google.com",
+    method: "GET",
+    incomingEncoding: "utf8",
+});
 console.log(result)
 // import got from "got";
 // import fs from "fs";