max/nodes
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve hash functions

Browse Source
This commit is contained in:
max_richter
2024-04-16 13:30:14 +02:00
parent dec205b234
commit 3d3ea5b5f8
33 changed files with 1416 additions and 963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
app/package.json
View File

@ -14,34 +14,37 @@
"story:preview": "histoire preview"
},
"dependencies": {
"@nodes/graph-interface": "link:../packages/graph-interface",
"@nodes/ui": "link:../packages/ui",
"@sveltejs/kit": "^2.5.0",
"@nodes/utils": "link:../packages/utils",
"@sveltejs/kit": "^2.5.6",
"@tauri-apps/api": "2.0.0-beta.2",
"@tauri-apps/plugin-shell": "^2.0.0-beta.0",
"@threlte/core": "^7.1.0",
"@threlte/extras": "^8.7.5",
"@threlte/flex": "^1.0.1",
"@types/three": "^0.159.0",
"three": "^0.159.0"
"@tauri-apps/plugin-shell": "2.0.0-beta.2",
"@threlte/core": "^7.3.0",
"@threlte/extras": "^8.11.2",
"@threlte/flex": "^1.0.2",
"@types/three": "^0.163.0",
"comlink": "^4.4.1",
"jsondiffpatch": "^0.6.0",
"three": "^0.163.0"
},
"devDependencies": {
"@histoire/plugin-svelte": "^0.17.9",
"@histoire/plugin-svelte": "^0.17.17",
"@nodes/types": "link:../packages/types",
"@sveltejs/adapter-static": "^3.0.1",
"@sveltejs/vite-plugin-svelte": "^3.0.1",
"@sveltejs/vite-plugin-svelte": "^3.1.0",
"@tauri-apps/cli": "2.0.0-beta.3",
"@tsconfig/svelte": "^5.0.2",
"@tsconfig/svelte": "^5.0.4",
"@zerodevx/svelte-json-view": "^1.0.9",
"histoire": "^0.17.9",
"internal-ip": "^7.0.0",
"svelte": "^4.2.8",
"svelte-check": "^3.4.6",
"tslib": "^2.6.0",
"typescript": "^5.0.2",
"vite": "^5.1.4",
"vite-plugin-glsl": "^1.2.1",
"histoire": "^0.17.17",
"internal-ip": "^8.0.0",
"svelte": "^4.2.14",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.9",
"vite-plugin-comlink": "^4.0.3",
"vite-plugin-glsl": "^1.3.0",
"vite-plugin-wasm": "^3.3.0",
"vitest": "^1.2.0"
"vitest": "^1.5.0"
}
}

53
app/src/lib/helpers/fastHash.test.ts
View File

@ -1,11 +1,54 @@
import { test, expect } from 'vitest';
import fastHash from './fastHash';
import { fastHashArray, fastHashString } from './fastHash';
test('Hashes dont clash', () => {
const hashA = fastHash('abcdef');
const hashB = fastHash('abcde');
const hashC = fastHash('abcde');
test('fastHashString doesnt produce clashes', () => {
const hashA = fastHashString('abcdef');
const hashB = fastHashString('abcdeg');
const hashC = fastHashString('abcdeg');
expect(hashA).not.toEqual(hashB);
expect(hashB).toEqual(hashC);
});
test("fastHashArray doesnt product collisions", () => {
const a = new Int32Array(1000);
const hash_a = fastHashArray(a);
a[0] = 1;
const hash_b = fastHashArray(a);
expect(hash_a).not.toEqual(hash_b);
});
test('fastHashArray is fast(ish) < 20ms', () => {
const a = new Int32Array(10_000);
const t0 = performance.now();
fastHashArray(a);
const t1 = performance.now();
a[0] = 1;
fastHashArray(a);
const t2 = performance.now();
expect(t1 - t0).toBeLessThan(20);
expect(t2 - t1).toBeLessThan(20);
});
// test if the fastHashArray function is deterministic
test('fastHashArray is deterministic', () => {
const a = new Int32Array(1000);
a[42] = 69;
const b = new Int32Array(1000);
b[42] = 69;
const hashA = fastHashArray(a);
const hashB = fastHashArray(b);
expect(hashA).toEqual(hashB);
});

112
app/src/lib/helpers/fastHash.ts
View File

@ -1,7 +1,95 @@
// https://github.com/6502/sha256/blob/main/sha256.js
function sha256(data?: string | Uint8Array) {
let h0 = 0x6a09e667, h1 = 0xbb67ae85, h2 = 0x3c6ef372, h3 = 0xa54ff53a,
h4 = 0x510e527f, h5 = 0x9b05688c, h6 = 0x1f83d9ab, h7 = 0x5be0cd19,
tsz = 0, bp = 0;
const k = [0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2],
rrot = (x, n) => (x >>> n) | (x << (32 - n)),
w = new Uint32Array(64),
buf = new Uint8Array(64),
process = () => {
for (let j = 0, r = 0; j < 16; j++, r += 4) {
w[j] = (buf[r] << 24) | (buf[r + 1] << 16) | (buf[r + 2] << 8) | buf[r + 3];
}
for (let j = 16; j < 64; j++) {
let s0 = rrot(w[j - 15], 7) ^ rrot(w[j - 15], 18) ^ (w[j - 15] >>> 3);
let s1 = rrot(w[j - 2], 17) ^ rrot(w[j - 2], 19) ^ (w[j - 2] >>> 10);
w[j] = (w[j - 16] + s0 + w[j - 7] + s1) | 0;
}
let a = h0, b = h1, c = h2, d = h3, e = h4, f = h5, g = h6, h = h7;
for (let j = 0; j < 64; j++) {
let S1 = rrot(e, 6) ^ rrot(e, 11) ^ rrot(e, 25),
ch = (e & f) ^ ((~e) & g),
t1 = (h + S1 + ch + k[j] + w[j]) | 0,
S0 = rrot(a, 2) ^ rrot(a, 13) ^ rrot(a, 22),
maj = (a & b) ^ (a & c) ^ (b & c),
t2 = (S0 + maj) | 0;
h = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
}
h0 = (h0 + a) | 0; h1 = (h1 + b) | 0; h2 = (h2 + c) | 0; h3 = (h3 + d) | 0;
h4 = (h4 + e) | 0; h5 = (h5 + f) | 0; h6 = (h6 + g) | 0; h7 = (h7 + h) | 0;
bp = 0;
},
add = data => {
if (typeof data === "string") {
data = typeof TextEncoder === "undefined" ? Buffer.from(data) : (new TextEncoder).encode(data);
}
for (let i = 0; i < data.length; i++) {
buf[bp++] = data[i];
if (bp === 64) process();
}
tsz += data.length;
},
digest = () => {
buf[bp++] = 0x80; if (bp == 64) process();
if (bp + 8 > 64) {
while (bp < 64) buf[bp++] = 0x00;
process();
}
while (bp < 58) buf[bp++] = 0x00;
// Max number of bytes is 35,184,372,088,831
let L = tsz * 8;
buf[bp++] = (L / 1099511627776.) & 255;
buf[bp++] = (L / 4294967296.) & 255;
buf[bp++] = L >>> 24;
buf[bp++] = (L >>> 16) & 255;
buf[bp++] = (L >>> 8) & 255;
buf[bp++] = L & 255;
process();
let reply = new Uint8Array(32);
reply[0] = h0 >>> 24; reply[1] = (h0 >>> 16) & 255; reply[2] = (h0 >>> 8) & 255; reply[3] = h0 & 255;
reply[4] = h1 >>> 24; reply[5] = (h1 >>> 16) & 255; reply[6] = (h1 >>> 8) & 255; reply[7] = h1 & 255;
reply[8] = h2 >>> 24; reply[9] = (h2 >>> 16) & 255; reply[10] = (h2 >>> 8) & 255; reply[11] = h2 & 255;
reply[12] = h3 >>> 24; reply[13] = (h3 >>> 16) & 255; reply[14] = (h3 >>> 8) & 255; reply[15] = h3 & 255;
reply[16] = h4 >>> 24; reply[17] = (h4 >>> 16) & 255; reply[18] = (h4 >>> 8) & 255; reply[19] = h4 & 255;
reply[20] = h5 >>> 24; reply[21] = (h5 >>> 16) & 255; reply[22] = (h5 >>> 8) & 255; reply[23] = h5 & 255;
reply[24] = h6 >>> 24; reply[25] = (h6 >>> 16) & 255; reply[26] = (h6 >>> 8) & 255; reply[27] = h6 & 255;
reply[28] = h7 >>> 24; reply[29] = (h7 >>> 16) & 255; reply[30] = (h7 >>> 8) & 255; reply[31] = h7 & 255;
let res = "";
reply.forEach(x => res += ("0" + x.toString(16)).slice(-2));
return res;
};
if (data) add(data);
return { add, digest };
}
export function fastHashArray(arr: Int32Array): string {
return sha256(new Uint8Array(arr.buffer)).digest();
}
// Shamelessly copied from
// https://stackoverflow.com/a/8831937
export default function (input: string) {
export function fastHashString(input: string) {
if (input.length === 0) return 0;
let hash = 0;
@ -12,3 +100,23 @@ export default function (input: string) {
return hash;
}
export function fastHash(input: (string | Int32Array | number)[]) {
const s = sha256();
for (let i = 0; i < input.length; i++) {
const v = input[i]
if (typeof v === "string") {
s.add(v);
} else if (v instanceof Int32Array) {
s.add(new Uint8Array(v.buffer));
} else {
s.add(v.toString());
}
}
return s.digest()
}

27
app/src/lib/node-registry.ts
View File

@ -1,4 +1,5 @@
import type { NodeRegistry, NodeType } from "@nodes/types";
import { createWasmWrapper } from "@nodes/utils";
import { createLogger } from "./helpers";
@ -49,31 +50,20 @@ export class RemoteNodeRegistry implements NodeRegistry {
private async loadNode(id: string) {
const nodeUrl = `${this.url}/n/${id}`;
const response = await fetch(nodeUrl);
const wasmResponse = await fetch(`${nodeUrl}/wasm`);
const wrapperReponse = await fetch(`${nodeUrl}/wrapper`);
if (!wrapperReponse.ok) {
const [response, wasmResponse] = await Promise.all([fetch(nodeUrl), fetch(`${nodeUrl}/wasm`)]);
if (!wasmResponse.ok || !response.ok) {
this.status = "error";
throw new Error(`Failed to load node ${id}`);
}
let wrapperCode = await wrapperReponse.text();
wrapperCode = wrapperCode.replace("wasm = val;", `if(wasm) return;
wasm = val;`);
const wasmWrapper = await import(/*@vite-ignore*/`data:text/javascript;base64,${btoa(wrapperCode)}#${id}`);
// Setup Wasm wrapper
const wrapper = createWasmWrapper();
const module = new WebAssembly.Module(await wasmResponse.arrayBuffer());
const instance = new WebAssembly.Instance(module, { ["./index_bg.js"]: wasmWrapper });
wasmWrapper.__wbg_set_wasm(instance.exports);
const instance = new WebAssembly.Instance(module, { ["./index_bg.js"]: wrapper });
wrapper.setInstance(instance);
if (!response.ok) {
this.status = "error";
throw new Error(`Failed to load node ${id}`);
} else {
log.log("loaded node", id);
}
const node = await response.json();
node.execute = wasmWrapper.execute;
node.execute = wrapper.execute;
return node;
}
@ -85,6 +75,7 @@ wasm = val;`);
nodeIds.push("max/plantarium/output");
nodeIds.push("max/plantarium/array");
nodeIds.push("max/plantarium/sum");
nodeIds.push("max/plantarium/stem");
const nodes = await Promise.all(nodeIds.map(id => this.loadNode(id)));

21
app/src/lib/runtime-executor.ts
View File

@ -1,16 +1,9 @@
import type { Graph, NodeRegistry, NodeType, RuntimeExecutor } from "@nodes/types";
import { encodeFloat } from "./helpers/encode";
import { concat_encoded, encode } from "./helpers/flat_tree";
import fastHash from "./helpers/fastHash";
import { fastHash, fastHashString } from "./helpers/fastHash";
async function hashIntArray(arr: Int32Array): Promise<string> {
const hashBuffer = await crypto.subtle.digest('SHA-256', arr.buffer);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map(byte => byte.toString(16).padStart(2, '0')).join('');
return hashHex;
}
export class MemoryRuntimeExecutor implements RuntimeExecutor {
@ -162,18 +155,14 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
const a0 = performance.now();
const node_inputs = Object.entries(inputs);
const cacheKey = `${node.id}/${fastHash(node_inputs.map(([_, value]: [string, any]) => {
if (value instanceof Int32Array) {
return hashIntArray(value);
}
console.log(value);
return `${value}`
}).join("/"))}`;
const cacheKey = "123" || `${node.id}/${fastHash(node_inputs.map(([_, value]: [string, any]) => {
return value
}))}`;
const a1 = performance.now();
console.log(`${a1 - a0}ms hashed inputs: ${node.id} -> ${cacheKey}`);
if (this.cache[cacheKey] && this.cache[cacheKey].eol > Date.now()) {
if (false && this.cache[cacheKey] && this.cache[cacheKey].eol > Date.now()) {
results[node.id] = this.cache[cacheKey].value;
console.log(`Using cached value`);
continue;

3
app/tsconfig.json
View File

@ -11,7 +11,8 @@
"strict": true,
"moduleResolution": "bundler",
"types": [
"vite-plugin-glsl/ext"
"vite-plugin-glsl/ext",
"vite-plugin-comlink/client"
]
}
// Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias

13
app/vite.config.ts
View File

@ -2,9 +2,20 @@ import { sveltekit } from '@sveltejs/kit/vite'
import { defineConfig } from 'vite'
import glsl from "vite-plugin-glsl";
import wasm from "vite-plugin-wasm";
import comlink from 'vite-plugin-comlink';
export default defineConfig({
plugins: [sveltekit(), glsl(), wasm()],
plugins: [
comlink(),
sveltekit(),
glsl(),
wasm()
],
worker: {
plugins: () => ([
comlink()
])
},
ssr: {
noExternal: ['three'],
}