Skip to content

Commit

Permalink
Self-extracting wasm wrapper
Browse files Browse the repository at this point in the history
  • Loading branch information
@texodus
texodus committed Nov 12, 2023
1 parent 9ad1b1e commit ba2ebf0
Show file tree
Hide file tree
Showing 21 changed files with 644 additions and 411 deletions.
25 changes: 14 additions & 11 deletions cpp/perspective/build.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,36 +13,39 @@
const { execSync } = require("child_process");
const os = require("os");
const path = require("path");
const fflate = require("fflate");
const fs = require("fs");

const stdio = "inherit";
const env = process.PSP_DEBUG ? "debug" : "release";
const env = process.env.PSP_DEBUG ? "debug" : "release";
const cwd = path.join(process.cwd(), "dist", env);

delete process.env.NODE;

function bootstrap(file) {
execSync(`cargo run -p perspective-bootstrap -- ${file}`, {
cwd: path.join(process.cwd(), "..", "..", "rust", "perspective-viewer"),
stdio,
});
}

try {
execSync(`mkdirp ${cwd}`, { stdio });
process.env.CLICOLOR_FORCE = 1;
execSync(`emcmake cmake ${__dirname} -Wno-dev -DCMAKE_BUILD_TYPE=${env}`, {
cwd,
stdio,
});

execSync(`emmake make -j${process.env.PSP_NUM_CPUS || os.cpus().length}`, {
cwd,
stdio,
});

execSync(`cpy web/**/* ../web`, { cwd, stdio });
execSync(`cpy node/**/* ../node`, { cwd, stdio });

const wasm = fs.readFileSync("dist/web/perspective.cpp.wasm");
const compressed = fflate.compressSync(wasm);
fs.writeFileSync("dist/web/perspective.cpp.wasm", compressed);

const wasm2 = fs.readFileSync("dist/node/perspective.cpp.wasm");
const compressed2 = fflate.compressSync(wasm2);
fs.writeFileSync("dist/node/perspective.cpp.wasm", compressed2);
if (!process.env.PSP_DEBUG) {
bootstrap(`../../cpp/perspective/dist/web/perspective.cpp.wasm`);
bootstrap(`../../cpp/perspective/dist/node/perspective.cpp.wasm`);
}
} catch (e) {
console.error(e);
process.exit(1);
Expand Down
3 changes: 0 additions & 3 deletions cpp/perspective/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,5 @@
"scripts": {
"build": "node ../../tools/perspective-scripts/run_emsdk.mjs node ./build.js",
"clean": "rimraf dist"
},
"devDependencies": {
"fflate": "^0.7.4"
}
}
86 changes: 43 additions & 43 deletions cpp/perspective/src/include/perspective/scalar.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,49 +200,49 @@ struct PERSPECTIVE_EXPORT t_tscalar {
bool m_inplace;
};

inline t_tscalar operator"" _ts(long double v) {
t_tscalar rv;
double tmp = v;
rv.set(tmp);
return rv;
}

inline t_tscalar operator"" _ts(unsigned long long int v) {
t_tscalar rv;
std::int64_t tmp = v;
rv.set(tmp);
return rv;
}

inline t_tscalar operator"" _ts(const char* v, std::size_t len) {
t_tscalar rv;
rv.set(v);
return rv;
}

inline t_tscalar operator"" _ns(long double v) {
t_tscalar rv;
rv.m_data.m_uint64 = 0;
rv.m_type = DTYPE_FLOAT64;
rv.m_status = STATUS_INVALID;
return rv;
}

inline t_tscalar operator"" _ns(unsigned long long int v) {
t_tscalar rv;
rv.m_data.m_uint64 = 0;
rv.m_type = DTYPE_INT64;
rv.m_status = STATUS_INVALID;
return rv;
}

inline t_tscalar operator"" _ns(const char* v, std::size_t len) {
t_tscalar rv;
rv.m_data.m_uint64 = 0;
rv.m_type = DTYPE_STR;
rv.m_status = STATUS_INVALID;
return rv;
}
// inline t_tscalar operator"" _ts(long double v) {
// t_tscalar rv;
// double tmp = v;
// rv.set(tmp);
// return rv;
// }

// inline t_tscalar operator"" _ts(unsigned long long int v) {
// t_tscalar rv;
// std::int64_t tmp = v;
// rv.set(tmp);
// return rv;
// }

// inline t_tscalar operator"" _ts(const char* v, std::size_t len) {
// t_tscalar rv;
// rv.set(v);
// return rv;
// }

// inline t_tscalar operator"" _ns(long double v) {
// t_tscalar rv;
// rv.m_data.m_uint64 = 0;
// rv.m_type = DTYPE_FLOAT64;
// rv.m_status = STATUS_INVALID;
// return rv;
// }

// inline t_tscalar operator"" _ns(unsigned long long int v) {
// t_tscalar rv;
// rv.m_data.m_uint64 = 0;
// rv.m_type = DTYPE_INT64;
// rv.m_status = STATUS_INVALID;
// return rv;
// }

// inline t_tscalar operator"" _ns(const char* v, std::size_t len) {
// t_tscalar rv;
// rv.m_data.m_uint64 = 0;
// rv.m_type = DTYPE_STR;
// rv.m_status = STATUS_INVALID;
// return rv;
// }

PERSPECTIVE_EXPORT t_tscalar mknone();
PERSPECTIVE_EXPORT t_tscalar mknull(t_dtype dtype);
Expand Down
1 change: 0 additions & 1 deletion packages/perspective/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
"clean": "rimraf dist"
},
"dependencies": {
"fflate": "^0.7.4",
"stoppable": "1.1.0",
"ws": "^6.1.2"
},
Expand Down
51 changes: 4 additions & 47 deletions packages/perspective/src/js/perspective.browser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,13 @@ import * as defaults from "./config/constants.js";
import { get_config, get_type_config as _get_type_config } from "./config";
import { Client } from "./api/client.js";
import { WebSocketClient } from "./websocket/client";

import { override_config } from "./config/index.js";
import { Decompress } from "fflate";

import wasm_worker from "../../src/js/perspective.worker.js";
import wasm from "../../dist/pkg/web/perspective.cpp.wasm";

let IS_INLINE = false;

function is_gzip(buffer) {
return new Uint32Array(buffer.slice(0, 4))[0] == 559903;
}

/**
* Singleton WASM file download cache.
*/
Expand All @@ -41,59 +35,22 @@ const _override = /* @__PURE__ */ (function () {

async wasm() {
let _wasm = await wasm;

let parts = [];
let length = 0;
const decompressor = new Decompress((chunk) => {
if (chunk) {
length += chunk.byteLength;
parts.push(chunk);
}
});

if (_wasm instanceof ArrayBuffer && !_wasm.buffer) {
_wasm = new Uint8Array(_wasm);
}

if (_wasm.buffer && _wasm.buffer instanceof ArrayBuffer) {
IS_INLINE = true;
if (is_gzip(_wasm.buffer)) {
decompressor.push(_wasm, true);
} else {
length = _wasm.byteLength;
parts = [_wasm];
}
length = _wasm.byteLength;
this._wasm = _wasm;
} else if (_wasm instanceof ArrayBuffer) {
length = _wasm.byteLength;
parts = [new Uint8Array(_wasm)];
this._wasm = new Uint8Array(_wasm);
} else {
const resp = await fetch(_wasm);
const reader = resp.body.getReader();
let state = 0;
while (true) {
const { value, done } = await reader.read();
if (done) break;
if (
(state === 0 && is_gzip(value.buffer)) ||
state === 1
) {
state = 1;
decompressor.push(value, done);
} else {
state = 2;
length += value.byteLength;
parts.push(value);
}
}
this._wasm = await resp.arrayBuffer();
}

let offset = 0;
const buffer = new Uint8Array(length);
for (const part of parts) {
buffer.set(part, offset);
offset += part.byteLength;
}
this._wasm = buffer.buffer;
return this._wasm;
}
})();
Expand Down
32 changes: 18 additions & 14 deletions packages/perspective/src/js/perspective.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2195,20 +2195,24 @@ export default function (Module) {
* @param {ArrayBuffer} buffer an ArrayBuffer or Buffer containing the
* Perspective WASM code
*/
init(msg) {
if (typeof WebAssembly === "undefined") {
throw new Error("WebAssembly not supported");
} else {
__MODULE__({
wasmBinary: msg.buffer,
wasmJSMethod: "native-wasm",
locateFile: (x) => x,
}).then((mod) => {
__MODULE__ = mod;
__MODULE__.init();
super.init(msg);
});
}
async init(msg) {
let wasmBinary = msg.buffer;
try {
const mod = await WebAssembly.instantiate(wasmBinary);
const exports = mod.instance.exports;
const size = exports.size();
const offset = exports.offset();
const array = new Uint8Array(exports.memory.buffer);
wasmBinary = array.slice(offset, offset + size);
} catch {}
__MODULE__ = await __MODULE__({
wasmBinary,
wasmJSMethod: "native-wasm",
locateFile: (x) => x,
});

__MODULE__.init();
super.init(msg);
}
}

Expand Down
55 changes: 18 additions & 37 deletions packages/perspective/src/js/perspective.node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,52 +24,33 @@ const http = require("http");
const WebSocket = require("ws");
const process = require("process");
const path = require("path");
const { Decompress } = require("fflate");

const load_perspective = require("../../dist/pkg/node/perspective.cpp.js");

const LOCAL_PATH = path.join(process.cwd(), "node_modules");
const buffer = require("../../dist/pkg/node/perspective.cpp.wasm").default;

function deflate(buffer) {
let parts = [];
let length = 0;
const decompressor = new Decompress((chunk) => {
if (chunk) {
length += chunk.byteLength;
parts.push(chunk);
}
});

decompressor.push(buffer, true);
let offset = 0;
const buffer2 = new Uint8Array(length);
for (const part of parts) {
buffer2.set(part, offset);
offset += part.byteLength;
}

return buffer2.buffer;
}
const SYNC_SERVER = new (class extends Server {
init(msg) {
let done;
this._loaded_promise = new Promise((x) => {
done = x;
});
buffer
.then((buffer) => {
return load_perspective({
wasmBinary: deflate(buffer),
wasmJSMethod: "native-wasm",
});
})
.then((core) => {
core.init();
this.perspective = perspective(core);
super.init(msg);
done();
this._loaded_promise = (async () => {
let wasmBinary = await buffer;
try {
const mod = await WebAssembly.instantiate(wasmBinary);
const exports = mod.instance.exports;
const size = exports.size();
const offset = exports.offset();
const array = new Uint8Array(exports.memory.buffer);
wasmBinary = array.slice(offset, offset + size);
} catch {}
const core = await load_perspective({
wasmBinary,
wasmJSMethod: "native-wasm",
});

core.init();
this.perspective = perspective(core);
super.init(msg);
})();
}

post(msg) {
Expand Down
Loading

0 comments on commit ba2ebf0

Please sign in to comment.