CLI set path to the file with the session list and other changes

Talhamehar007 · Dec 24, 2020 · 38194ad · 38194ad
1 parent 1459b7f
commit 38194ad
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -126,8 +126,10 @@ Commands:
 
 Options:
   --version            Show version number                             [boolean]
-  --session            Set session cookie value. Session is required to scrape
-                       user/trending/hashtag/music feed
+  --session            Set session cookie value. Sometimes session can be
+                       helpful when scraping data from any method  [default: ""]
+  --session-file       Set path to the file with list of active sessions. One
+                       session per line!                           [default: ""]
   --timeout            Set timeout between requests. Timeout is in Milliseconds:
                        1000 mls = 1 s                               [default: 0]
   --number, -n         Number of posts to scrape. If you will set 0 then all
@@ -219,10 +221,10 @@ docker run -v /User/blah/downloads:/usr/app/files tiktok-scraper user tiktok -d
 ### Methods
 
 ```javascript
-.user(id, options) //Scrape posts from a specific user (Promise) <--- REQUIRES SESSION
-.hashtag(id, options) //Scrape posts from hashtag section (Promise) <--- REQUIRES SESSION
-.trend('', options) // Scrape posts from a trends section (Promise) <--- REQUIRES SESSION
-.music(id, options) // Scrape posts by music id (Promise) <--- REQUIRES SESSION
+.user(id, options) //Scrape posts from a specific user (Promise)
+.hashtag(id, options) //Scrape posts from hashtag section (Promise)
+.trend('', options) // Scrape posts from a trends section (Promise)
+.music(id, options) // Scrape posts by music id (Promise)
 
 .userEvent(id, options) //Scrape posts from a specific user (Event)
 .hashtagEvent(id, options) //Scrape posts from hashtag section (Event)
@@ -435,7 +437,9 @@ hashtag.on('error', error => {
 hashtag.scrape();
 ```
 ### Get Set Session
-In order to scrape user/hashtag/music/trending feed you need to set authenticated session cookie value!
+**NOT REQUIRED**
+
+**Very common problem is when tiktok is blacklisting your IP/PROXY and in such case you can try to set session and there will be higher chances for success**
 
 Get the session:
 - Open https://www.tiktok.com/ in any browser
@@ -446,7 +450,17 @@ Get the session:
 - **sid_tt=521kkadkasdaskdj4j213j12j312;** - this will be your authenticated session cookie value that should be used to scrape user/hashtag/music/trending feed
 
 Set the session:
-- In the **CLI** you can set session by using the option --session. For example **--session sid_tt=521kkadkasdaskdj4j213j12j312;**
+- **CLI**:
+    -  Set single session by using option **--session**. For example **--session sid_tt=521kkadkasdaskdj4j213j12j312;**
+    -  Set path to the file with the list of sessions by using option **--session-file**. For example **--session-file /var/bob/sessionList.txt**
+        - Example content /var/bob/sessionList.txt:
+        ```
+        sid_tt=521kkadkasdaskdj4j213j12j312;
+        sid_tt=521kkadkasdaskdj4j213j12j312;
+        sid_tt=521kkadkasdaskdj4j213j12j312;
+        sid_tt=521kkadkasdaskdj4j213j12j312;
+        ```
+         
 - In the **MODULE** you can set session by setting the option value sessionList . For example **sessionList:["sid_tt=521kkadkasdaskdj4j213j12j312;", "sid_tt=12312312312312;"]**
 
 ### Download Video

diff --git a/bin/cli.js b/bin/cli.js
@@ -66,7 +66,7 @@ const startScraper = async argv => {
                 console.log(scraper);
             }
         } catch (error) {
-            console.error(error.message);
+            console.error(error.message ? error.message : error);
         }
     } catch (error) {
         console.log(error);
@@ -114,7 +114,12 @@ yargs
             describe: 'help',
         },
         session: {
-            describe: 'Set session cookie value. Session is required to scrape user/trending/hashtag/music feed',
+            default: '',
+            describe: 'Set session cookie value. Sometimes session can be helpful when scraping data from any method',
+        },
+        'session-file': {
+            default: '',
+            describe: 'Set path to the file with list of active sessions. One session per line!',
         },
         timeout: {
             default: 0,
@@ -266,14 +271,6 @@ yargs
             argv._[0] = 'getUserProfileInfo';
         }
 
-        if (CONST.requiredSession.indexOf(argv._[0]) > -1) {
-            if (!argv.session) {
-                throw new Error(
-                    'In order to scrape user/trending/music/hashtag feed you need to set authenticated session cookie value --session .Please read the github readme',
-                );
-            }
-        }
-
         return true;
     })
     .demandCommand()

diff --git a/src/entry.ts b/src/entry.ts
@@ -66,6 +66,23 @@ const proxyFromFile = async (file: string) => {
     }
 };
 
+/**
+ * Load session list from a file
+ * @param file
+ */
+const sessionFromFile = async (file: string) => {
+    try {
+        const data = (await fromCallback(cb => readFile(file, { encoding: 'utf-8' }, cb))) as string;
+        const proxyList = data.split('\n');
+        if (!proxyList.length || proxyList[0] === '') {
+            throw new Error('Session file is empty');
+        }
+        return proxyList;
+    } catch (error) {
+        throw error.message;
+    }
+};
+
 const promiseScraper = async (input: string, type: ScrapeType, options = {} as Options): Promise<Result> => {
     if (options && typeof options !== 'object') {
         throw new TypeError('Object is expected');
@@ -75,6 +92,10 @@ const promiseScraper = async (input: string, type: ScrapeType, options = {} as O
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
 
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
+
     const constructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type, input } };
 
     const scraper = new TikTokScraper(constructor);
@@ -109,7 +130,9 @@ export const getHashtagInfo = async (input: string, options = {} as Options): Pr
     if (options?.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
-
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const contructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type: 'signle_hashtag' as ScrapeType, input } };
     const scraper = new TikTokScraper(contructor);
 
@@ -124,7 +147,9 @@ export const getMusicInfo = async (input: string, options = {} as Options): Prom
     if (options?.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
-
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const contructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type: 'single_music' as ScrapeType, input } };
     const scraper = new TikTokScraper(contructor);
 
@@ -140,6 +165,9 @@ export const getUserProfileInfo = async (input: string, options = {} as Options)
     if (options?.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const contructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type: 'sinsgle_user' as ScrapeType, input } };
     const scraper = new TikTokScraper(contructor);
 
@@ -154,7 +182,9 @@ export const signUrl = async (input: string, options = {} as Options): Promise<s
     if (options.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
-
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const contructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type: 'signature' as ScrapeType, input } };
     const scraper = new TikTokScraper(contructor);
 
@@ -170,6 +200,9 @@ export const getVideoMeta = async (input: string, options = {} as Options): Prom
     if (options?.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const contructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type: 'video_meta' as ScrapeType, input } };
     const scraper = new TikTokScraper(contructor);
 
@@ -188,6 +221,9 @@ export const video = async (input: string, options = {} as Options): Promise<any
     if (options?.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const contructor: TikTokConstructor = { ...getInitOptions(), ...options, ...{ type: 'video' as ScrapeType, input } };
     const scraper = new TikTokScraper(contructor);
     const result: PostCollector = await scraper.getVideoMeta();
@@ -372,6 +408,9 @@ export const fromfile = async (input: string, options = {} as Options) => {
                     by_user_id: true,
                 };
             }
+            if (item.indexOf('@') > -1) {
+                item = item.replace(/@/g, '');
+            }
             return {
                 type: 'user',
                 input: item,
@@ -384,6 +423,9 @@ export const fromfile = async (input: string, options = {} as Options) => {
     if (options?.proxyFile) {
         options.proxy = await proxyFromFile(options?.proxyFile);
     }
+    if (options?.sessionFile) {
+        options.sessionList = await sessionFromFile(options?.sessionFile);
+    }
     const result = await batchProcessor(batch, options);
 
     return { table: result };

diff --git a/src/types/TikTok.ts b/src/types/TikTok.ts
@@ -24,6 +24,7 @@ export interface Options {
     proxy?: string[] | string;
     sessionList?: string[];
     proxyFile?: string;
+    sessionFile?: string;
     event?: boolean;
     by_user_id?: boolean;
     download?: boolean;