max/nodes
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve hash functions

Browse Source
This commit is contained in:
max_richter
2024-04-16 13:30:14 +02:00
parent dec205b234
commit 3d3ea5b5f8
33 changed files with 1416 additions and 963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
packages/node-registry/package.json
View File

@ -1,39 +1,40 @@
{
"name": "node-registry",
"version": "0.0.1",
"private": true,
"scripts": {
"dev": "vite dev",
"build": "vite build",
"preview": "vite preview",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"test": "vitest",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.0.0",
"@sveltejs/kit": "^2.0.0",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/eslint": "^8.56.0",
"@typescript-eslint/eslint-plugin": "^7.0.0",
"@typescript-eslint/parser": "^7.0.0",
"eslint": "^8.56.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.35.1",
"prettier": "^3.1.1",
"prettier-plugin-svelte": "^3.1.2",
"svelte": "^4.2.7",
"svelte-check": "^3.6.0",
"tslib": "^2.4.1",
"typescript": "^5.0.0",
"vite": "^5.0.3",
"vite-plugin-wasm": "^3.3.0",
"vitest": "^1.2.0"
},
"type": "module",
"dependencies": {
"math": "link:../../nodes/math/pkg"
}
"name": "node-registry",
"version": "0.0.1",
"private": true,
"scripts": {
"dev": "vite dev",
"build": "vite build",
"preview": "vite preview",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"test": "vitest",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/vite-plugin-svelte": "^3.1.0",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint": "^9.0.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.37.0",
"prettier": "^3.2.5",
"prettier-plugin-svelte": "^3.2.3",
"svelte": "^4.2.14",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.9",
"vite-plugin-wasm": "^3.3.0",
"vitest": "^1.5.0"
},
"type": "module",
"dependencies": {
"@nodes/utils": "link:../utils",
"utils": "link:../utils"
}
}

28
packages/node-registry/src/lib/registry.ts
View File

@ -1,19 +1,8 @@
export async function getNodeWrapper(id: `${string}/${string}/${string}`) {
import { createWasmWrapper } from "@nodes/utils"
const wrapperReponse = await fetch(`/n/${id}/wrapper`);
if (!wrapperReponse.ok) {
throw new Error(`Failed to load node ${id}`);
}
let wrapperCode = await wrapperReponse.text();
wrapperCode = wrapperCode.replace("wasm = val;", `if(wasm) return;
wasm = val;`);
const wasmWrapper = await import(/*@vite-ignore*/`data:text/javascript;base64,${btoa(wrapperCode)}#${id}${Math.random().toString().slice(2)}`);
return wasmWrapper;
}
export async function getNodeWasm(id: `${string}/${string}/${string}`): Promise<WebAssembly.Instance> {
export async function getNodeWasm(id: `${string}/${string}/${string}`) {
const wasmResponse = await fetch(`/n/${id}/wasm`);
@ -21,13 +10,12 @@ export async function getNodeWasm(id: `${string}/${string}/${string}`): Promise<
throw new Error(`Failed to load node ${id}`);
}
const wasmWrapper = await getNodeWrapper(id);
const wrapper = createWasmWrapper();
const module = new WebAssembly.Module(await wasmResponse.arrayBuffer());
const instance = new WebAssembly.Instance(module, { ["./index_bg.js"]: wasmWrapper });
wasmWrapper.__wbg_set_wasm(instance.exports);
const instance = new WebAssembly.Instance(module, { ["./index_bg.js"]: wrapper });
wrapper.setInstance(instance)
return wasmWrapper;
return wrapper;
}
@ -35,8 +23,8 @@ export async function getNode(id: `${string}/${string}/${string}`) {
const wrapper = await getNodeWasm(id);
const outputs = wrapper.get_outputs();
const rawInputs = wrapper.get_input_types();
const outputs = wrapper?.get_outputs?.() || [];
const rawInputs = wrapper.get_inputs();
try {
const inputTypes = JSON.parse(rawInputs);
return { id, outputs, inputs: inputTypes }

19
packages/node-registry/src/routes/n/[user]/[collection]/[node]/wrapper/+server.ts
View File

@ -1,19 +0,0 @@
import type { RequestHandler } from "./$types";
import fs from "fs/promises";
import path from "path";
export const GET: RequestHandler = async function GET({ params }) {
const filePath = path.resolve(`../../nodes/${params.user}/${params.collection}/${params.node}/pkg/index_bg.js`);
try {
await fs.access(filePath);
} catch (e) {
console.log("Not Found", filePath);
return new Response("Not found", { status: 404 });
}
const file = await fs.readFile(filePath);
return new Response(file, { status: 200, headers: { "Content-Type": "text/javascript" } });
}

32
packages/ui/package.json
View File

@ -27,22 +27,22 @@
"svelte": "^4.0.0"
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.0.0",
"@sveltejs/kit": "^2.0.0",
"@sveltejs/package": "^2.0.0",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/eslint": "^8.56.0",
"@typescript-eslint/eslint-plugin": "^7.0.0",
"@typescript-eslint/parser": "^7.0.0",
"eslint": "^8.56.0",
"eslint-plugin-svelte": "^2.35.1",
"publint": "^0.1.9",
"svelte": "^4.2.7",
"svelte-check": "^3.6.0",
"tslib": "^2.4.1",
"typescript": "^5.0.0",
"vite": "^5.0.11",
"vitest": "^1.2.0"
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/package": "^2.3.1",
"@sveltejs/vite-plugin-svelte": "^3.1.0",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint": "^9.0.0",
"eslint-plugin-svelte": "^2.37.0",
"publint": "^0.2.7",
"svelte": "^4.2.14",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.9",
"vitest": "^1.5.0"
},
"svelte": "./dist/index.js",
"types": "./dist/index.d.ts",

2
packages/utils/package.json
View File

@ -2,7 +2,7 @@
"name": "@nodes/utils",
"version": "1.0.0",
"description": "",
"main": "index.js",
"main": "src/index.ts",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},

1
packages/utils/src/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./wasm-wrapper";

260
packages/utils/src/wasm-wrapper.ts Normal file
View File

@ -0,0 +1,260 @@
const cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
const cachedTextEncoder = new TextEncoder('utf-8');
const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
? function (arg, view) {
return cachedTextEncoder.encodeInto(arg, view);
}
: function (arg, view) {
const buf = cachedTextEncoder.encode(arg);
view.set(buf);
return {
read: arg.length,
written: buf.length
};
});
export function createWasmWrapper() {
let wasm: any;
let cachedUint8Memory0: Uint8Array | null = null;
let cachedInt32Memory0: Int32Array | null = null;
let cachedUint32Memory0: Uint32Array | null = null;
const heap = new Array(128).fill(undefined);
heap.push(undefined, null, true, false);
let heap_next = heap.length;
function getUint8Memory0() {
if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8Memory0;
}
function getInt32Memory0() {
if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) {
cachedInt32Memory0 = new Int32Array(wasm.memory.buffer);
}
return cachedInt32Memory0;
}
function getUint32Memory0() {
if (cachedUint32Memory0 === null || cachedUint32Memory0.byteLength === 0) {
cachedUint32Memory0 = new Uint32Array(wasm.memory.buffer);
}
return cachedUint32Memory0;
}
function getStringFromWasm0(ptr: number, len: number) {
return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
}
function getObject(idx: number) { return heap[idx]; }
function addHeapObject(obj: any) {
if (heap_next === heap.length) heap.push(heap.length + 1);
const idx = heap_next;
heap_next = heap[idx];
heap[idx] = obj;
return idx;
}
let WASM_VECTOR_LEN = 0;
function passArray32ToWasm0(arg: ArrayLike<number>, malloc: (arg0: number, arg1: number) => number) {
const ptr = malloc(arg.length * 4, 4) >>> 0;
getUint32Memory0().set(arg, ptr / 4);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
function getArrayI32FromWasm0(ptr: number, len: number) {
ptr = ptr >>> 0;
return getInt32Memory0().subarray(ptr / 4, ptr / 4 + len);
}
function dropObject(idx: number) {
if (idx < 132) return;
heap[idx] = heap_next;
heap_next = idx;
}
function takeObject(idx: number) {
const ret = getObject(idx);
dropObject(idx);
return ret;
}
function getArrayJsValueFromWasm0(ptr: number, len: number) {
ptr = ptr >>> 0;
const mem = getUint32Memory0();
const slice = mem.subarray(ptr / 4, ptr / 4 + len);
const result = [];
for (let i = 0; i < slice.length; i++) {
result.push(takeObject(slice[i]));
}
return result;
}
function __wbindgen_string_new(arg0: number, arg1: number) {
const ret = getStringFromWasm0(arg0, arg1);
return addHeapObject(ret);
};
function get_outputs() {
if (wasm === undefined || !wasm.get_outputs) {
return [];
}
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.get_outputs(retptr);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
var v1 = getArrayJsValueFromWasm0(r0, r1).slice();
wasm.__wbindgen_free(r0, r1 * 4, 4);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
// Additional methods and their internal helpers can also be refactored in a similar manner.
function get_inputs() {
let deferred1_0: number;
let deferred1_1: number;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.get_input_types(retptr);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred1_0 = r0;
deferred1_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
}
}
function execute(args) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passArray32ToWasm0(args, wasm.__wbindgen_malloc);
const len0 = WASM_VECTOR_LEN;
wasm.execute(retptr, ptr0, len0);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
var v2 = getArrayI32FromWasm0(r0, r1).slice();
wasm.__wbindgen_free(r0, r1 * 4, 4);
return v2;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
function passStringToWasm0(arg: string, malloc: (arg0: any, arg1: number) => number, realloc: ((arg0: number, arg1: any, arg2: number, arg3: number) => number) | undefined) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8Memory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
function __wbg_new_abda76e883ba8a5f() {
const ret = new Error();
return addHeapObject(ret);
};
function __wbg_stack_658279fe44541cf6(arg0, arg1) {
const ret = getObject(arg1).stack;
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len1 = WASM_VECTOR_LEN;
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
};
function __wbg_error_f851667af71bcfc6(arg0, arg1) {
let deferred0_0;
let deferred0_1;
try {
deferred0_0 = arg0;
deferred0_1 = arg1;
console.error(getStringFromWasm0(arg0, arg1));
} finally {
wasm.__wbindgen_free(deferred0_0, deferred0_1, 1);
}
};
function __wbindgen_object_drop_ref(arg0) {
takeObject(arg0);
};
function __wbg_log_5bb5f88f245d7762(arg0) {
console.log(getObject(arg0));
};
function __wbindgen_throw(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
};
return {
setInstance(instance: WebAssembly.Instance) {
wasm = instance.exports;
},
// Expose other methods that interact with the wasm instance
execute,
get_outputs,
get_inputs,
__wbindgen_string_new,
__wbindgen_object_drop_ref,
__wbg_new_abda76e883ba8a5f,
__wbg_error_f851667af71bcfc6,
__wbg_stack_658279fe44541cf6,
__wbg_log_5bb5f88f245d7762,
__wbindgen_throw,
};
}