Merge pull request 78#23 from 78/js

Js

spider/ecosystem.config.js

@@ -4,7 +4,7 @@ module.exports = {
     script : "./spider.js",
     env: {
         PORT: 6881,
-        TABLE_CAPTION: 200,
+        TABLE_CAPTION: 1000,
         MAX_CONCURRENT: 500
     },
     cron_restart: "0 */3 * * *"
@@ -15,6 +15,10 @@ module.exports = {
       env: {
           PORT: 3000
       }
+  },
+  {
+      name: 'task',
+      script: './task.js'
   }
   ]
 }

spider/package.json

@@ -6,6 +6,7 @@
   "dependencies": {
     "bittorrent-protocol": "^3.0.1",
     "buffer-crc32": "^0.2.13",
+    "cron": "^1.7.1",
     "dhtspider": "^0.1.0",
     "iconv": "^2.3.4",
     "iconv-lite": "^0.4.24",

spider/reduce.js

@@ -6,7 +6,7 @@ MongoClient.connect('mongodb://localhost:27017/admin', {useNewUrlParser: true},
         process.exit(1)
     }
     const torrentdb = mconn.db('torrent')
-    const stream = torrentdb.collection('log').find({date: process.argv[2]}).stream()
+    const stream = torrentdb.collection('log').find({date: process.env.DATE}).stream()
     const d = []
     let ii = 0
     stream.on('error', (err) => {
@@ -20,7 +20,7 @@ MongoClient.connect('mongodb://localhost:27017/admin', {useNewUrlParser: true},
             }
         })
         ii += 1
-        if(d.length >= 10000) {
+        if(d.length >= 100000) {
             stream.pause()
             console.log(ii)
             torrentdb.collection('hash').bulkWrite(d, (err, r) => {
@@ -39,7 +39,7 @@ MongoClient.connect('mongodb://localhost:27017/admin', {useNewUrlParser: true},
                 if(err) {
                     console.error(err)
                 }else{
-                    console.log('done')
+                    console.log('done', process.env.DATE)
                     mconn.close()
                 }
             })

spider/spider.js

@@ -21,7 +21,7 @@ MongoClient.connect('mongodb://localhost:27017/admin', {useNewUrlParser: true},
     torrentdb.collection('hash').createIndex({hash: 1}, {unique: 1})
     torrentdb.collection('hash').find().sort({_id: -1}).limit(1).next((err, r) => {
         nextTorrentId = r._id + 1
-        console.log('Next torrent _id is', nextTorrentId)
+        console.log('Next torrent _id is', nextTorrentId, 'tableCaption', process.env.TABLE_CAPTION)
     })
 })
 

spider/task.js

@@ -0,0 +1,41 @@
+const CronJob = require('cron').CronJob
+const moment = require('moment')
+const assert=require('assert')
+const { exec } = require('child_process')
+
+function doTask(date) {
+    const env = {DATE: date}
+    console.log('DATE=', date)
+
+    const p1 = exec('node reduce.js', {env: env}, (err) => {
+        assert.ifError(err)
+        const p2 = exec('indexer -c sphinx.conf hash_delta --rotate', {env: env}, (err) => {
+            assert.ifError(err)
+            const p3 = exec('indexer -c sphinx.conf --merge hash hash_delta --rotate', (err) => {
+                assert.ifError(err)
+                console.log(new Date(), date, 'task done')
+            })
+            p3.stdout.pipe(process.stdout)
+        })
+        p2.stdout.pipe(process.stdout)
+    })
+    p1.stdout.pipe(process.stdout)
+}
+
+/* 每天凌晨1点执行索引任务 */
+if(!module.parent) {
+    if(process.env.DATE) {
+        doTask(process.env.DATE)
+    }else{
+        const cj = new CronJob('0 0 1 * * *', ()=> {
+            console.log(new Date(), 'Starting indexing task in 10 seconds...')
+            setTimeout(() => {
+                const date = moment().add(-1, 'days').format('YYYY-MM-DD')
+                doTask(date)
+            }, 10*1000)
+        }, null, true)
+
+        console.log(new Date(), 'CronJob started, next task time is', cj.nextDates().toString())
+    }
+}
+