allow custom allocation of split memory chunks using an existing buffer

src/preamble.js

@@ -1270,9 +1270,15 @@ if (totalMemory !== TOTAL_MEMORY) {
 
 var buffers = [], HEAP8s = [], HEAP16s = [], HEAP32s = [], HEAPU8s = [], HEAPU16s = [], HEAPU32s = [], HEAPF32s = [], HEAPF64s = [];
 
-function allocateSplitChunk(i) {
-  assert(!buffers[i] && !HEAP8s[i]);
-  var curr = new ArrayBuffer(SPLIT_MEMORY);
+// Allocates a split chunk, a range of memory of size SPLIT_MEMORY. Generally data is not provided, and a new
+// buffer is allocated, this is what happens when malloc works. However, you can provide your own buffer,
+// which then lets you access it at address [ i*SPLIT_MEMORY, (i+1)*SPLIT_MEMORY ).
+// The function returns true if it succeeds. It can also throw an exception if no data is provided and
+// the browser fails to allocate the buffer.
+function allocateSplitChunk(i, data) {
+  if (buffers[i]) return false; // already taken
+  var curr = data ? data : new ArrayBuffer(SPLIT_MEMORY);
+  assert(curr instanceof ArrayBuffer);
   buffers[i] = curr;
   HEAP8s[i] = new Int8Array(curr);
   HEAP16s[i] = new Int16Array(curr);
@@ -1282,6 +1288,7 @@ function allocateSplitChunk(i) {
   HEAPU32s[i] = new Uint32Array(curr);
   HEAPF32s[i] = new Float32Array(curr);
   HEAPF64s[i] = new Float64Array(curr);
+  return true;
 }
 function freeSplitChunk(i) {
   assert(buffers[i] && HEAP8s[i]);

system/lib/split_malloc.cpp

@@ -29,6 +29,12 @@ static size_t total_memory = 0;
 static size_t split_memory = 0;
 static size_t num_spaces = 0;
 
+enum AllocateResult {
+  OK = 0,
+  NO_MEMORY = 1,
+  ALREADY_USED = 2
+};
+
 struct Space {
   mspace space;
   bool allocated; // whether storage is allocated for this chunk, both an ArrayBuffer in JS and an mspace here
@@ -42,29 +48,26 @@ struct Space {
     index = i;
   }
 
-  bool allocate() {
+  AllocateResult allocate() {
     assert(!allocated);
     assert(count == 0);
     allocated = true;
     int start;
     if (index > 0) {
       if (int(split_memory*(index+1)) < 0) {
         // 32-bit pointer overflow. we could support more than this 2G, up to 4GB, if we made all pointer shifts >>>. likely slower though
-        return false;
+        return NO_MEMORY;
       }
-      int success = EM_ASM_INT({
-        if (!ALLOW_MEMORY_GROWTH) {
-          allocateSplitChunk($0);
-          return 1;
-        }
+      AllocateResult result = (AllocateResult)EM_ASM_INT({
+        // can fail due to the browser not have enough memory for the chunk, or if the slice
+        // is already used, which can happen if other code allocated it
         try {
-          allocateSplitChunk($0);
-          return 1;
+          return allocateSplitChunk($0) ? $1 : $3;
         } catch(e) {
-          return 0; // failed to allocate
+          return $2; // failed to allocate
         }
-      }, index);
-      if (!success) return false;
+      }, index, OK, NO_MEMORY, ALREADY_USED);
+      if (result != OK) return result;
       start = split_memory*index;
     } else {
       // small area in existing chunk 0
@@ -80,7 +83,7 @@ struct Space {
       }
     }
     assert(space);
-    return true;
+    return OK;
   }
 
   void free() {
@@ -137,22 +140,28 @@ static void* get_memory(size_t size, bool malloc=true, size_t alignment=-1, bool
   static int next = 0;
   int start = next;
   while (1) { // simple round-robin, while keeping to use the same one as long as it keeps succeeding
+    AllocateResult result = OK;
     if (!spaces[next].allocated) {
-      if (!spaces[next].allocate()) return 0; // might fail to allocate in memory growth mode
+      result = spaces[next].allocate();
+      if (result == NO_MEMORY) return 0; // mallocation failure
     }
-    void *ret;
-    if (malloc) {
-      ret = mspace_malloc(spaces[next].space, size);
+    if (result == OK) {
+      void *ret;
+      if (malloc) {
+        ret = mspace_malloc(spaces[next].space, size);
+      } else {
+        ret = mspace_memalign(spaces[next].space, alignment, size);
+      }
+      if (ret) {
+        spaces[next].count++;
+        return ret;
+      }
+      if (must_succeed) {
+        EM_ASM({ Module.printErr("failed to allocate in a new space after memory growth, perhaps increase SPLIT_MEMORY?"); });
+        abort();
+      }
     } else {
-      ret = mspace_memalign(spaces[next].space, alignment, size);
-    }
-    if (ret) {
-      spaces[next].count++;
-      return ret;
-    }
-    if (must_succeed) {
-      EM_ASM({ Module.printErr("failed to allocate in a new space after memory growth, perhaps increase SPLIT_MEMORY?"); });
-      abort();
+      assert(result == ALREADY_USED); // continue on to the next space
     }
     next++;
     if (next == num_spaces) next = 0;

tests/test_other.py

@@ -5483,6 +5483,47 @@ def test_split_memory_release(self):
     Module.print('success.');
   });
 }
+''')
+    for opts in [0, 1, 2]:
+      print opts
+      check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts)])
+      self.assertContained('success.', run_js('a.out.js'))
+
+  def test_split_memory_use_existing(self):
+    open('src.c', 'w').write(r'''
+#include <emscripten.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+int main() {
+  EM_ASM({
+    function getIndex(x) {
+      return x >> SPLIT_MEMORY_BITS;
+    }
+    var t;
+    do {
+      t = Module._malloc(1024*1024);
+    } while (getIndex(t) === 0);
+    assert(getIndex(t) === 1, 'allocated into first chunk');
+    assert(!buffers[2]);
+    var existing = new Uint8Array(1024); // ok to be smaller
+    allocateSplitChunk(2, existing.buffer);
+    assert(buffers[2]);
+    existing[0] = 12;
+    existing[50] = 98;
+    var p = SPLIT_MEMORY*2;
+    assert(HEAPU8[p+0] === 12 && HEAPU8[p+50] === 98); // mapped into the normal memory space!
+    HEAPU8[p+33] = 201;
+    assert(existing[33] === 201); // works both ways
+    do {
+      t = Module._malloc(1024*1024);
+    } while (getIndex(t) === 1);
+    assert(getIndex(t) === 3, 'should skip chunk 2, since it is used by us, but seeing ' + getIndex(t));
+    assert(HEAPU8[p+0] === 12 && HEAPU8[p+50] === 98);
+    assert(existing[33] === 201);
+    Module.print('success.');
+  });
+}
 ''')
     for opts in [0, 1, 2]:
       print opts