asyncronous task pooling with maxConcurrency=50 in run-tests.js

mallchin · Sep 24, 2017 · d4d8fd9 · d4d8fd9
1 parent 4d5a469
commit d4d8fd9
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 4 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -27,7 +27,6 @@ script:
 - npm run build
 - tox -e qa,doc
 - npm run test-base
-- python3 test/test_async.py cryptopia
 - node run-tests --js --python --python3
 - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then env COMMIT_MESSAGE=${NPM_VERSION:1} GITHUB_TOKEN=${GITHUB_TOKEN} ./push.sh; fi
 - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then env COMMIT_MESSAGE=${NPM_VERSION:1} GITHUB_TOKEN=${GITHUB_TOKEN} ./push-wiki.sh; fi

diff --git a/run-tests.js b/run-tests.js
@@ -157,13 +157,67 @@ const testExchange = async (exchange) => {
 
 /*  ------------------------------------------------------------------------ */
 
+function TaskPool ({ maxTime, maxConcurrency }) {
+
+    const pending = []
+        , queue   = []
+
+    let numActive = 0
+
+    return {
+
+        all: () => Promise.all (pending),
+
+        run (task) {
+
+            if (numActive >= maxConcurrency) { // queue task
+
+                return new Promise (resolve => queue.push (() => this.run (task).then (resolve)))
+
+            } else { // execute task
+
+                let p = timeout (maxTime, task ()).then (x => {
+                    numActive--
+                    console.log (numActive)
+                    return (queue.length && (numActive < maxConcurrency))
+                                ? queue.shift () ().then (() => x)
+                                : x
+                })
+                numActive++
+                console.log (numActive)             
+                pending.push (p)
+                return p
+            }
+        }
+    }
+}
+
+/*  ------------------------------------------------------------------------ */
+
+async function testAllExchanges () {
+
+    // NOTE: naive impl crashes with out-of-memory-error eventually (in Travis), so need some pooling...
+    //
+    // return Promise.all (exchanges.map (testExchange))
+
+    const taskPool = TaskPool ({ maxTime: 120*1000, maxConcurrency: 50 })
+
+    for (const ex of exchanges) {
+        taskPool.run (() => testExchange (ex))
+    }
+
+    return taskPool.all ()
+}
+
+/*  ------------------------------------------------------------------------ */
+
 (async function () {
 
     log.bright.magenta.noPretty ('Testing'.white, { exchanges, symbol, keys })
 
-    const tested   = await Promise.all (exchanges.map (testExchange))
-        , warnings = tested.filter  (t => !t.failed && t.hasWarnings)
-        , failed   = tested.filter  (t =>  t.failed)
+    const tested    = await testAllExchanges ()
+        , warnings  = tested.filter (t => !t.failed && t.hasWarnings)
+        , failed    = tested.filter (t =>  t.failed)
         , succeeded = tested.filter (t => !t.failed && !t.hasWarnings)
 
     log.newline ()