fix video download

Talhamehar007 · Oct 25, 2020 · a12dff5 · a12dff5
1 parent 30300bc
commit a12dff5
Show file tree

Hide file tree

Showing 8 changed files with 63 additions and 34 deletions.
diff --git a/bin/cli.js b/bin/cli.js
@@ -36,29 +36,33 @@ const startScraper = async argv => {
             argv.asyncBulk = argv.async;
         }
 
-        const scraper = await TikTokScraper[argv.type](argv.input, argv);
+        try {
+            const scraper = await TikTokScraper[argv.type](argv.input, argv);
 
-        if (scraper.zip) {
-            console.log(argv.zip ? `ZIP path: ${scraper.zip}` : `Folder Path: ${scraper.zip}`);
-        }
-        if (scraper.json) {
-            console.log(`JSON path: ${scraper.json}`);
-        }
-        if (scraper.csv) {
-            console.log(`CSV path: ${scraper.csv}`);
-        }
-        if (scraper.message) {
-            console.log(scraper.message);
-        }
-        if (scraper.webhook) {
-            console.log('HTTP REQUEST: ');
-            console.table(scraper.webhook);
-        }
-        if (scraper.table) {
-            console.table(scraper.table);
-        }
-        if (argv.cli && argv.type === 'getUserProfileInfo') {
-            console.log(scraper);
+            if (scraper.zip) {
+                console.log(argv.zip ? `ZIP path: ${scraper.zip}` : `Folder Path: ${scraper.zip}`);
+            }
+            if (scraper.json) {
+                console.log(`JSON path: ${scraper.json}`);
+            }
+            if (scraper.csv) {
+                console.log(`CSV path: ${scraper.csv}`);
+            }
+            if (scraper.message) {
+                console.log(scraper.message);
+            }
+            if (scraper.webhook) {
+                console.log('HTTP REQUEST: ');
+                console.table(scraper.webhook);
+            }
+            if (scraper.table) {
+                console.table(scraper.table);
+            }
+            if (argv.cli && argv.type === 'getUserProfileInfo') {
+                console.log(scraper);
+            }
+        } catch (error) {
+            console.error(error.message);
         }
     } catch (error) {
         console.log(error);

diff --git a/src/constant/index.ts b/src/constant/index.ts
@@ -18,5 +18,5 @@ export = {
         music: 11,
         trend: 12,
     },
-    userAgentList: ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'],
+    userAgentList: ['Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.80 Safari/537.36'],
 };
diff --git a/src/core/Downloader.ts b/src/core/Downloader.ts
@@ -30,7 +30,9 @@ export class Downloader {
 
     public bulk: boolean;
 
-    constructor({ progress, proxy, noWaterMark, userAgent, filepath, bulk }: DownloaderConstructor) {
+    public tt_webid_v2: string;
+
+    constructor({ progress, proxy, noWaterMark, userAgent, filepath, bulk, tt_webid_v2 }: DownloaderConstructor) {
         this.progress = true || progress;
         this.progressBar = [];
         this.noWaterMark = noWaterMark;
@@ -39,6 +41,7 @@ export class Downloader {
         this.mbars = new MultipleBar();
         this.proxy = proxy;
         this.bulk = bulk;
+        this.tt_webid_v2 = tt_webid_v2;
     }
 
     /**
@@ -187,9 +190,11 @@ export class Downloader {
         }
         const options = ({
             uri: url,
+            method: 'GET',
             headers: {
-                'user-agent': 'okhttp',
-                referer: 'https://www.tiktok.com/',
+                'user-agent': this.userAgent,
+                Referer: 'https://www.tiktok.com/',
+                Cookie: `tt_webid_v2=${this.tt_webid_v2}`,
             },
             encoding: null,
             ...(proxy.proxy && proxy.socks ? { agent: proxy.proxy } : {}),

diff --git a/src/core/TikTok.ts b/src/core/TikTok.ts
@@ -12,7 +12,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import { forEachLimit } from 'async';
 
 import CONST from '../constant';
-import { sign } from '../helpers';
+import { makeid, sign } from '../helpers';
 
 import {
     PostCollector,
@@ -103,6 +103,8 @@ export class TikTokScraper extends EventEmitter {
 
     private method: string;
 
+    private tt_webid_v2: string;
+
     private httpRequests: {
         good: number;
         bad: number;
@@ -174,13 +176,15 @@ export class TikTokScraper extends EventEmitter {
         this.noDuplicates = [];
         this.timeout = timeout;
         this.bulk = bulk;
+        this.tt_webid_v2 = `68${makeid(16)}`;
         this.Downloader = new Downloader({
             progress,
             proxy,
             noWaterMark,
             userAgent,
             filepath: process.env.SCRAPING_FROM_DOCKER ? '/usr/app/files' : filepath || '',
             bulk,
+            tt_webid_v2: this.tt_webid_v2,
         });
         this.webHookUrl = webHookUrl;
         this.method = method;
@@ -860,7 +864,6 @@ export class TikTokScraper extends EventEmitter {
         if (musicIdRegex) {
             this.input = musicIdRegex[1] as string;
         }
-        console.log(this.input);
         return {
             id: this.input,
             secUid: '',
@@ -1064,19 +1067,21 @@ export class TikTokScraper extends EventEmitter {
         if (!this.input) {
             throw `Url is missing`;
         }
-        const query = {
+        const options = {
             uri: this.input,
             headers: {
                 'user-agent': this.userAgent,
-                referer: 'https://www.tiktok.com/',
+                Referer: 'https://www.tiktok.com/',
+                Cookie: `tt_webid_v2=${this.tt_webid_v2}`,
             },
             method: 'GET',
             json: true,
         };
+
         try {
             let short = false;
             let regex: RegExpExecArray | null;
-            const response = await this.request<string>(query);
+            const response = await this.request<string>(options);
             if (!response) {
                 throw new Error(`Can't extract video meta data`);
             }

diff --git a/src/entry.ts b/src/entry.ts
@@ -210,6 +210,14 @@ export const video = async (input: string, options = {} as Options): Promise<any
     const path = options?.filepath ? `${options?.filepath}/${result.id}` : result.id;
     let outputMessage = {};
 
+    if (options?.download) {
+        try {
+            await scraper.Downloader.downloadSingleVideo(result);
+        } catch {
+            throw new Error('Unable to download the video');
+        }
+    }
+
     if (options?.filetype) {
         await scraper.saveMetadata({ json: `${path}.json`, csv: `${path}.csv` });
 
@@ -220,9 +228,6 @@ export const video = async (input: string, options = {} as Options): Promise<any
         };
     }
 
-    if (options?.download) {
-        await scraper.Downloader.downloadSingleVideo(result);
-    }
     return {
         ...(options?.download ? { message: `Video location: ${contructor.filepath}/${result.id}.mp4` } : {}),
         ...outputMessage,

diff --git a/src/helpers/Random.ts b/src/helpers/Random.ts
@@ -0,0 +1,8 @@
+export const makeid = (len: number) => {
+    let text = '';
+    const char_list = '0123456789';
+    for (let i = 0; i < len; i += 1) {
+        text += char_list.charAt(Math.floor(Math.random() * char_list.length));
+    }
+    return text;
+};
diff --git a/src/helpers/index.ts b/src/helpers/index.ts
@@ -1,2 +1,3 @@
 export * from './Bar';
 export * from './Signature';
+export * from './Random';
diff --git a/src/types/Downloader.ts b/src/types/Downloader.ts
@@ -7,6 +7,7 @@ export interface DownloaderConstructor {
     userAgent: string;
     filepath: string;
     bulk: boolean;
+    tt_webid_v2: string;
 }
 
 export interface DownloadParams {