max/nodarium
1
0
Fork 0
Code Issues 12 Pull Requests 2 Actions Packages Projects Releases 3 Wiki Activity

feat: make all nodes work with new runtime

Browse Source
This commit is contained in:
Max Richter
2026-01-23 01:14:09 +01:00
parent 47882a832d
commit a497a46674
23 changed files with 494 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/Dockerfile
View File

@@ -39,5 +39,6 @@ server {
EOF
COPY --from=builder /app/app/build /app
COPY --from=builder /app/packages/ui/build /app/ui
EXPOSE 80

19
app/src/lib/graph-interface/graph-manager.svelte.ts
View File

@@ -606,11 +606,14 @@ export class GraphManager extends EventEmitter<{
return;
}
const fromType = from.state.type || this.registry.getNode(from.type);
const toType = to.state.type || this.registry.getNode(to.type);
// check if socket types match
const fromSocketType = from.state?.type?.outputs?.[fromSocket];
const toSocketType = [to.state?.type?.inputs?.[toSocket]?.type];
if (to.state?.type?.inputs?.[toSocket]?.accepts) {
toSocketType.push(...(to?.state?.type?.inputs?.[toSocket]?.accepts || []));
const fromSocketType = fromType?.outputs?.[fromSocket];
const toSocketType = [toType?.inputs?.[toSocket]?.type];
if (toType?.inputs?.[toSocket]?.accepts) {
toSocketType.push(...(toType?.inputs?.[toSocket]?.accepts || []));
}
if (!areSocketsCompatible(fromSocketType, toSocketType)) {
@@ -733,9 +736,9 @@ export class GraphManager extends EventEmitter<{
}
getPossibleSockets({ node, index }: Socket): [NodeInstance, string | number][] {
const nodeType = node?.state?.type;
console.log({ node: $state.snapshot(node), index, nodeType });
const nodeType = this.registry.getNode(node.type);
if (!nodeType) return [];
console.log({ index });
const sockets: [NodeInstance, string | number][] = [];
@@ -750,7 +753,7 @@ export class GraphManager extends EventEmitter<{
const ownType = nodeType?.inputs?.[index].type;
for (const node of nodes) {
const nodeType = node?.state?.type;
const nodeType = this.registry.getNode(node.type);
const inputs = nodeType?.outputs;
if (!inputs) continue;
for (let index = 0; index < inputs.length; index++) {
@@ -778,7 +781,7 @@ export class GraphManager extends EventEmitter<{
const ownType = nodeType.outputs?.[index];
for (const node of nodes) {
const inputs = node?.state?.type?.inputs;
const inputs = this.registry.getNode(node.type)?.inputs;
if (!inputs) continue;
for (const key in inputs) {
const otherType = [inputs[key].type];

2
app/src/lib/graph-interface/node/NodeHeader.svelte
View File

@@ -43,7 +43,7 @@
<div class="wrapper" data-node-id={node.id} data-node-type={node.type}>
<div class="content">
{node.type.split("/").pop()}
{node.type.split("/").pop()} ({node.id})
</div>
<div
class="click-target"

121
app/src/lib/runtime/runtime-executor.ts
View File

@@ -57,9 +57,18 @@ function getValue(input: NodeInput, value?: unknown) {
throw new Error(`Unknown input type ${input.type}`);
}
type Pointer = {
function compareInt32(a: Int32Array, b: Int32Array) {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) return false;
}
return true;
}
export type Pointer = {
start: number;
end: number;
_title?: string;
};
export class MemoryRuntimeExecutor implements RuntimeExecutor {
@@ -69,15 +78,24 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
> = new Map();
private offset = 0;
private isRunning = false;
private memory = new WebAssembly.Memory({
initial: 1024,
maximum: 8192
});
private memoryView = new Int32Array();
results: Record<number, Pointer> = {};
inputPtrs: Record<number, Pointer[]> = {};
seed = 123123;
perf?: PerformanceStore;
public getMemory() {
return new Int32Array(this.memory.buffer);
}
constructor(
private registry: NodeRegistry,
public cache?: SyncCache<Int32Array>
@@ -129,7 +147,8 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
const outputNode = graphNodes.find((node) => node.type.endsWith('/output'));
if (!outputNode) {
throw new Error('No output node found');
// throw new Error('No output node found');
console.log('No output node found');
}
const nodeMap = new Map(
@@ -151,7 +170,7 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
const nodes = [];
// loop through all the nodes and assign each nodes its depth
const stack = [outputNode];
const stack = [outputNode || graphNodes[0]];
while (stack.length) {
const node = stack.pop();
if (!node) continue;
@@ -166,14 +185,15 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
return [outputNode, nodes] as const;
}
private writeToMemory(v: number | number[] | Int32Array) {
private writeToMemory(v: number | number[] | Int32Array, title?: string) {
let length = 1;
const view = new Int32Array(this.memory.buffer);
if (typeof v === 'number') {
view[this.offset] = v;
this.memoryView[this.offset] = v;
console.log('MEM: writing number', v, ' to', this.offset);
length = 1;
} else {
view.set(v, this.offset);
this.memoryView.set(v, this.offset);
length = v.length;
}
@@ -184,15 +204,25 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
return {
start,
end
end,
_title: title
};
}
private printMemory() {
this.memoryView = new Int32Array(this.memory.buffer);
console.log('MEMORY', this.memoryView.slice(0, 10));
}
async execute(graph: Graph, _settings: Record<string, unknown>) {
this.offset = 0;
this.inputPtrs = {};
this.results = {};
if (this.isRunning) return undefined as unknown as Int32Array;
this.isRunning = true;
// Then we add some metadata to the graph
const [outputNode, nodes] = await this.addMetaData(graph);
const [_outputNode, nodes] = await this.addMetaData(graph);
/*
* Here we sort the nodes into buckets, which we then execute one by one
@@ -210,14 +240,14 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
(a, b) => (b.state?.depth || 0) - (a.state?.depth || 0)
);
// here we store the intermediate results of the nodes
const results: Record<number, Pointer> = {};
for (const node of sortedNodes) {
const node_type = this.nodes.get(node.type)!;
console.log('EXECUTING NODE', node_type.definition.id);
console.log(node_type.definition.inputs);
console.log('---------------');
console.log('STARTING NODE EXECUTION', node_type.definition.id);
this.printMemory();
// console.log(node_type.definition.inputs);
const inputs = Object.entries(node_type.definition.inputs || {}).map(
([key, input]) => {
// We should probably initially write this to memory
@@ -225,6 +255,8 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
return this.writeToMemory(this.seed);
}
const title = `${node.id}.${key}`;
// We should probably initially write this to memory
// If the input is linked to a setting, we use that value
// if (input.setting) {
@@ -234,56 +266,85 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
// check if the input is connected to another node
const inputNode = node.state.inputNodes[key];
if (inputNode) {
if (results[inputNode.id] === undefined) {
if (this.results[inputNode.id] === undefined) {
throw new Error(
`Node ${node.type} is missing input from node ${inputNode.type}`
);
}
return results[inputNode.id];
return this.results[inputNode.id];
}
// If the value is stored in the node itself, we use that value
if (node.props?.[key] !== undefined) {
return this.writeToMemory(getValue(input, node.props[key]));
const value = getValue(input, node.props[key]);
console.log(`Writing prop for ${node.id} -> ${key} to memory`, node.props[key], value);
return this.writeToMemory(value, title);
}
return this.writeToMemory(getValue(input));
return this.writeToMemory(getValue(input), title);
}
);
this.printMemory();
if (!node_type || !node.state || !node_type.execute) {
log.warn(`Node ${node.id} has no definition`);
continue;
}
this.inputPtrs[node.id] = inputs;
const args = inputs.map(s => [s.start, s.end]).flat();
console.log('ARGS', args);
console.log('ARGS', inputs);
this.printMemory();
try {
const bytesWritten = node_type.execute(this.offset, args);
results[node.id] = {
start: this.offset,
end: this.offset + bytesWritten
};
this.offset += bytesWritten;
console.log('EXECUTING NODE, writing output of node to ->', this.offset);
const bytesWritten = node_type.execute(this.offset * 4, args.map(a => a * 4));
const view = new Int32Array(this.memory.buffer);
const input = view.slice(args[0], args[1]);
const output = view.slice(this.offset, this.offset + bytesWritten / 4);
console.log('RESULT', { args, input, output });
// Optimization
// If the input arg is the same length as the output arg
if (
args.length === 2 && args[1] - args[0] == bytesWritten / 4 && compareInt32(input, output)
) {
console.log('INPUT === OUTPUT');
this.results[node.id] = {
start: args[0],
end: args[1],
_title: `${node.id} ->`
};
} else {
this.results[node.id] = {
start: this.offset,
end: this.offset + bytesWritten / 4,
_title: `${node.id} ->`
};
this.offset += bytesWritten / 4;
}
console.log('FINISHED EXECUTION', {
bytesWritten,
offset: this.offset
});
} catch (e) {
console.error(e);
console.error(`Failed to execute node ${node.type}/${node.id}`, e);
this.isRunning = false;
}
}
const mem = new Int32Array(this.memory.buffer);
console.log('OUT', mem.slice(0, 10));
// const mem = new Int32Array(this.memory.buffer);
// console.log('OUT', mem.slice(0, 10));
// return the result of the parent of the output node
const res = results[outputNode.id];
// const res = this.results[outputNode.id];
this.perf?.endPoint('runtime');
return res as unknown as Int32Array;
this.isRunning = false;
return undefined as unknown as Int32Array;
}
getPerformanceData() {

163
app/src/routes/dev/+page.svelte
View File

@@ -1,16 +1,14 @@
<script lang="ts">
import * as templates from "$lib/graph-templates";
import Panel from "$lib/sidebar/Panel.svelte";
import Sidebar from "$lib/sidebar/Sidebar.svelte";
import GraphInterface from "$lib/graph-interface";
import { RemoteNodeRegistry } from "@nodarium/registry";
import { type Graph } from "@nodarium/types";
import { type Graph, type NodeInstance } from "@nodarium/types";
import Grid from "$lib/grid";
import { MemoryRuntimeExecutor } from "$lib/runtime";
import { MemoryRuntimeExecutor, type Pointer } from "$lib/runtime";
import devPlant from "./dev-graph.json";
import { decodeNestedArray } from "@nodarium/utils";
let result = $state<Int32Array>();
import { decodeFloat } from "@nodarium/utils";
import { localState } from "$lib/helpers/localState.svelte";
const nodeRegistry = new RemoteNodeRegistry("");
nodeRegistry.overwriteNode("max/plantarium/output", {
@@ -24,21 +22,63 @@
type: "*",
},
},
execute(input: Int32Array) {
result = input;
return input;
execute(outputPos: number, args: number[]) {
return 0;
},
});
const runtimeExecutor = new MemoryRuntimeExecutor(nodeRegistry);
let inputPtrs: Record<number, Pointer[]>;
let activeNode = $state<NodeInstance>();
let isCalculating = $state<boolean>(false);
let windowHeight = $state(500);
let start = $state(0);
const rowHeight = 40;
const numRows = $derived(Math.floor(windowHeight / rowHeight));
let memory = $state<Int32Array>();
const visibleRows = $derived(memory?.slice(start, start + numRows));
const ptrs = $derived.by(() => {
if (!inputPtrs) return [];
const seen = new Set();
const ptrs = [...Object.values(inputPtrs)]
.flat()
.sort((a, b) => (a.start > b.start ? 1 : -1))
.filter((ptr) => {
const id = `${ptr.start}-${ptr.end}`;
if (seen.has(id)) return false;
seen.add(id);
return true;
});
if (!ptrs) return [];
let out = [];
for (let i = 0; i < numRows; i++) {
let rowIndex = start + i;
const activePtr = ptrs.find(
(ptr) => ptr.start < rowIndex && ptr.end >= rowIndex,
);
if (activePtr) {
out.push({
start: rowIndex,
end: rowIndex + 1,
_title: activePtr._title,
});
}
}
return out;
});
let graph = $state(
localStorage.getItem("nodes.dev.graph")
? JSON.parse(localStorage.getItem("nodes.dev.graph")!)
: devPlant,
);
function handleSave(graph: Graph) {
localStorage.setItem("nodes.dev.graph", JSON.stringify(graph));
function handleSave(g: Graph) {
localStorage.setItem("nodes.dev.graph", JSON.stringify(g));
}
let graphSettings = $state<Record<string, any>>({});
@@ -46,27 +86,116 @@
randomSeed: { type: "boolean", value: false },
});
async function handleResult(res: unknown) {
const result = await runtimeExecutor.execute(graph, graphSettings);
console.log({ res, result });
let calcTimeout: ReturnType<typeof setTimeout>;
async function handleResult(res?: Graph) {
console.clear();
isCalculating = true;
if (res) handleSave(res);
try {
await runtimeExecutor.execute(res || graph, graphSettings);
} catch (e) {
console.log(e);
}
memory = runtimeExecutor.getMemory();
inputPtrs = runtimeExecutor.inputPtrs;
clearTimeout(calcTimeout);
calcTimeout = setTimeout(() => {
isCalculating = false;
}, 500);
}
const rowIsFloat = localState<boolean[]>("node.dev.isFloat", []);
function decodeValue(value: number, isFloat?: boolean) {
return isFloat ? decodeFloat(value) : value;
}
</script>
<svelte:window
bind:innerHeight={windowHeight}
onkeydown={(ev) => ev.key === "r" && handleResult()}
/>
<Grid.Row>
<Grid.Cell>
{#if result}
<pre><code>{JSON.stringify(decodeNestedArray(result))}</code></pre>
{#if visibleRows?.length}
<table
class="min-w-full select-none overflow-hidden text-left text-sm flex-1"
>
<thead class="">
<tr>
<th class="px-4 py-2 border-b border-[var(--outline)]">i</th>
<th
class="px-4 py-2 border-b border-[var(--outline)] w-[50px]"
style:width="50px">Ptrs</th
>
<th class="px-4 py-2 border-b border-[var(--outline)]">Value</th>
<th class="px-4 py-2 border-b border-[var(--outline)]">Float</th>
</tr>
</thead>
<tbody>
{#each visibleRows as r, i}
{@const index = i + start}
{@const ptr = ptrs[i]}
<tr class="h-[40px] odd:bg-[var(--layer-1)]">
<td class="px-4 border-b border-[var(--outline)] w-8">{index}</td>
<td
class="w-[50px] border-b border-[var(--outline)]
{ptr?._title?.includes('->')
? 'bg-red-500'
: 'bg-blue-500'}"
>
<span>{ptr?._title}</span>
</td>
<td
class="px-4 border-b border-[var(--outline)] cursor-pointer text-blue-600 hover:text-blue-800"
onclick={() =>
(rowIsFloat.value[index] = !rowIsFloat.value[index])}
>
{decodeValue(r, rowIsFloat.value[index])}
</td>
<td class="px-4 border-b border-[var(--outline)] italic w-5">
<input
type="checkbox"
checked={rowIsFloat.value[index]}
onclick={() =>
(rowIsFloat.value[index] = !rowIsFloat.value[index])}
/>
</td>
</tr>
{/each}
</tbody>
</table>
<input
class="absolute bottom-4 left-4 bg-white"
bind:value={start}
min="0"
type="number"
step="1"
/>
{/if}
</Grid.Cell>
<Grid.Cell>
{#if isCalculating}
<span
class="opacity-50 top-4 left-4 i-[tabler--loader-2] w-10 h-10 absolute animate-spin z-100"
></span>
{/if}
<button
onclick={() => handleResult()}
class="flex items-center cursor-pointer absolute bottom-4 left-4 z-100"
>
Execute Graph (R)
</button>
<GraphInterface
{graph}
bind:activeNode
registry={nodeRegistry}
bind:settings={graphSettings}
bind:settingTypes={graphSettingTypes}
onsave={(g) => handleSave(g)}
onresult={(result) => handleResult(result)}
onresult={(res) => handleResult(res)}
/>
</Grid.Cell>
</Grid.Row>