From 841b447ac34afe095f697ccb9dd8f280433a61cc Mon Sep 17 00:00:00 2001 From: Max Richter Date: Thu, 22 Jan 2026 15:54:08 +0100 Subject: [PATCH] feat: add "*"/any type input for dev page --- .../graph-interface/graph-manager.svelte.ts | 74 ++++++---- .../lib/graph-interface/graph-state.svelte.ts | 15 +- .../lib/graph-interface/graph/Graph.svelte | 2 +- .../lib/graph-interface/graph/Wrapper.svelte | 2 +- .../lib/graph-interface/history-manager.ts | 32 ++--- app/src/lib/node-registry.ts | 2 +- app/src/lib/runtime/runtime-executor.ts | 100 ++++++------- app/src/routes/+page.svelte | 5 - app/src/routes/dev/+layout.svelte | 2 +- app/src/routes/dev/+page.svelte | 136 +++++++----------- app/src/routes/dev/dev-graph.json | 53 +++++++ packages/registry/src/node-registry-client.ts | 10 +- packages/types/src/inputs.ts | 60 ++++---- 13 files changed, 273 insertions(+), 220 deletions(-) create mode 100644 app/src/routes/dev/dev-graph.json diff --git a/app/src/lib/graph-interface/graph-manager.svelte.ts b/app/src/lib/graph-interface/graph-manager.svelte.ts index d0134d7..65d7012 100644 --- a/app/src/lib/graph-interface/graph-manager.svelte.ts +++ b/app/src/lib/graph-interface/graph-manager.svelte.ts @@ -25,14 +25,14 @@ const clone = 'structuredClone' in self ? self.structuredClone : (args: any) => JSON.parse(JSON.stringify(args)); -function areSocketsCompatible( +export function areSocketsCompatible( output: string | undefined, inputs: string | (string | undefined)[] | undefined ) { if (Array.isArray(inputs) && output) { - return inputs.includes(output); + return inputs.includes('*') || inputs.includes(output); } - return inputs === output; + return inputs === output || inputs === '*'; } function areEdgesEqual(firstEdge: Edge, secondEdge: Edge) { @@ -268,14 +268,7 @@ export class GraphManager extends EventEmitter<{ private _init(graph: Graph) { const nodes = new Map( graph.nodes.map((node) => { - const nodeType = this.registry.getNode(node.type); - const n = node as NodeInstance; - if (nodeType) { - n.state = { - type: nodeType - }; - } - return [node.id, n]; + return [node.id, node as NodeInstance]; }) ); @@ -300,6 +293,30 @@ export class GraphManager extends EventEmitter<{ this.execute(); } + private async loadAllCollections() { + // Fetch all nodes from all collections of the loaded nodes + const nodeIds = Array.from(new Set([...this.graph.nodes.map((n) => n.type)])); + const allCollections = new Set<`${string}/${string}`>(); + for (const id of nodeIds) { + const [user, collection] = id.split('/'); + allCollections.add(`${user}/${collection}`); + } + + const allCollectionIds = await Promise + .all([...allCollections] + .map(async (collection) => + remoteRegistry + .fetchCollection(collection) + .then((collection: { nodes: { id: NodeId }[] }) => { + return collection.nodes.map(n => n.id.replace(/\.wasm$/, '') as NodeId); + }) + )); + + const missingNodeIds = [...new Set(allCollectionIds.flat())]; + + this.registry.load(missingNodeIds); + } + async load(graph: Graph) { const a = performance.now(); @@ -308,25 +325,16 @@ export class GraphManager extends EventEmitter<{ this.status = 'loading'; this.id = graph.id; - logger.info('loading graph', { nodes: graph.nodes, edges: graph.edges, id: graph.id }); - const nodeIds = Array.from(new Set([...graph.nodes.map((n) => n.type)])); - await this.registry.load(nodeIds); - // Fetch all nodes from all collections of the loaded nodes - const allCollections = new Set<`${string}/${string}`>(); - for (const id of nodeIds) { - const [user, collection] = id.split('/'); - allCollections.add(`${user}/${collection}`); - } - for (const collection of allCollections) { - remoteRegistry - .fetchCollection(collection) - .then((collection: { nodes: { id: NodeId }[] }) => { - const ids = collection.nodes.map((n) => n.id); - return this.registry.load(ids); - }); - } + logger.info('loading graph', { + nodes: graph.nodes, + edges: graph.edges, + id: graph.id, + ids: nodeIds + }); + + await this.registry.load(nodeIds); logger.info('loaded node types', this.registry.getAllNodes()); @@ -384,7 +392,9 @@ export class GraphManager extends EventEmitter<{ this.loaded = true; logger.log(`Graph loaded in ${performance.now() - a}ms`); + setTimeout(() => this.execute(), 100); + this.loadAllCollections(); // lazily load all nodes from all collections } getAllNodes() { @@ -491,10 +501,10 @@ export class GraphManager extends EventEmitter<{ const inputs = Object.entries(to.state?.type?.inputs ?? {}); const outputs = from.state?.type?.outputs ?? []; for (let i = 0; i < inputs.length; i++) { - const [inputName, input] = inputs[0]; + const [inputName, input] = inputs[i]; for (let o = 0; o < outputs.length; o++) { - const output = outputs[0]; - if (input.type === output) { + const output = outputs[o]; + if (input.type === output || input.type === '*') { return this.createEdge(from, o, to, inputName); } } @@ -724,6 +734,7 @@ export class GraphManager extends EventEmitter<{ getPossibleSockets({ node, index }: Socket): [NodeInstance, string | number][] { const nodeType = node?.state?.type; + console.log({ node: $state.snapshot(node), index, nodeType }); if (!nodeType) return []; const sockets: [NodeInstance, string | number][] = []; @@ -783,6 +794,7 @@ export class GraphManager extends EventEmitter<{ } } + console.log(`Found ${sockets.length} possible sockets`, sockets); return sockets; } diff --git a/app/src/lib/graph-interface/graph-state.svelte.ts b/app/src/lib/graph-interface/graph-state.svelte.ts index c75d4f7..a67d7f8 100644 --- a/app/src/lib/graph-interface/graph-state.svelte.ts +++ b/app/src/lib/graph-interface/graph-state.svelte.ts @@ -58,7 +58,9 @@ export class GraphState { wrapper = $state(null!); rect: DOMRect = $derived( - (this.wrapper && this.width && this.height) ? this.wrapper.getBoundingClientRect() : new DOMRect(0, 0, 0, 0) + (this.wrapper && this.width && this.height) + ? this.wrapper.getBoundingClientRect() + : new DOMRect(0, 0, 0, 0) ); camera = $state(null!); @@ -168,11 +170,14 @@ export class GraphState { (node?.state?.y ?? node.position[1]) + 2.5 + 10 * index ]; } else { - const _index = Object.keys(node.state?.type?.inputs || {}).indexOf(index); - return [ + const inputs = node.state.type?.inputs || this.graph.registry.getNode(node.type)?.inputs + || {}; + const _index = Object.keys(inputs).indexOf(index); + const pos = [ node?.state?.x ?? node.position[0], (node?.state?.y ?? node.position[1]) + 10 + 10 * _index - ]; + ] as [number, number]; + return pos; } } @@ -248,7 +253,7 @@ export class GraphState { let { node, index, position } = socket; - // remove existing edge + // if the socket is an input socket -> remove existing edges if (typeof index === 'string') { const edges = this.graph.getEdgesToNode(node); for (const edge of edges) { diff --git a/app/src/lib/graph-interface/graph/Graph.svelte b/app/src/lib/graph-interface/graph/Graph.svelte index 052af80..f615676 100644 --- a/app/src/lib/graph-interface/graph/Graph.svelte +++ b/app/src/lib/graph-interface/graph/Graph.svelte @@ -57,7 +57,7 @@ ]; const input = Object.entries(newNode?.state?.type?.inputs || {}).find( - (inp) => inp[1].type === socketType, + (inp) => inp[1].type === socketType || inp[1].type === "*", ); if (input) { diff --git a/app/src/lib/graph-interface/graph/Wrapper.svelte b/app/src/lib/graph-interface/graph/Wrapper.svelte index 3e08e8e..d4f3497 100644 --- a/app/src/lib/graph-interface/graph/Wrapper.svelte +++ b/app/src/lib/graph-interface/graph/Wrapper.svelte @@ -29,11 +29,11 @@ let { graph, registry, - settings = $bindable(), activeNode = $bindable(), showGrid = $bindable(true), snapToGrid = $bindable(true), showHelp = $bindable(false), + settings = $bindable(), settingTypes = $bindable(), onsave, onresult, diff --git a/app/src/lib/graph-interface/history-manager.ts b/app/src/lib/graph-interface/history-manager.ts index e86441e..5f42d92 100644 --- a/app/src/lib/graph-interface/history-manager.ts +++ b/app/src/lib/graph-interface/history-manager.ts @@ -1,21 +1,21 @@ -import { create, type Delta } from "jsondiffpatch"; -import type { Graph } from "@nodarium/types"; -import { clone } from "./helpers/index.js"; -import { createLogger } from "@nodarium/utils"; +import type { Graph } from '@nodarium/types'; +import { createLogger } from '@nodarium/utils'; +import { create, type Delta } from 'jsondiffpatch'; +import { clone } from './helpers/index.js'; const diff = create({ objectHash: function (obj, index) { if (obj === null) return obj; - if ("id" in obj) return obj.id as string; - if ("_id" in obj) return obj._id as string; + if ('id' in obj) return obj.id as string; + if ('_id' in obj) return obj._id as string; if (Array.isArray(obj)) { - return obj.join("-"); + return obj.join('-'); } - return "$$index:" + index; - }, + return '$$index:' + index; + } }); -const log = createLogger("history"); +const log = createLogger('history'); log.mute(); export class HistoryManager { @@ -26,7 +26,7 @@ export class HistoryManager { private opts = { debounce: 400, - maxHistory: 100, + maxHistory: 100 }; constructor({ maxHistory = 100, debounce = 100 } = {}) { @@ -40,12 +40,12 @@ export class HistoryManager { if (!this.state) { this.state = clone(state); this.initialState = this.state; - log.log("initial state saved"); + log.log('initial state saved'); } else { const newState = state; const delta = diff.diff(this.state, newState); if (delta) { - log.log("saving state"); + log.log('saving state'); // Add the delta to history if (this.index < this.history.length - 1) { // Clear the history after the current index if new changes are made @@ -61,7 +61,7 @@ export class HistoryManager { } this.state = newState; } else { - log.log("no changes"); + log.log('no changes'); } } } @@ -75,7 +75,7 @@ export class HistoryManager { undo() { if (this.index === -1 && this.initialState) { - log.log("reached start, loading initial state"); + log.log('reached start, loading initial state'); return clone(this.initialState); } else { const delta = this.history[this.index]; @@ -95,7 +95,7 @@ export class HistoryManager { this.state = nextState; return clone(nextState); } else { - log.log("reached end"); + log.log('reached end'); } } } diff --git a/app/src/lib/node-registry.ts b/app/src/lib/node-registry.ts index b6d03e6..99197bf 100644 --- a/app/src/lib/node-registry.ts +++ b/app/src/lib/node-registry.ts @@ -20,7 +20,7 @@ export async function getNodeWasm(id: `${string}/${string}/${string}`) { const wasmBytes = await getWasm(id); if (!wasmBytes) return null; - const wrapper = createWasmWrapper(wasmBytes); + const wrapper = createWasmWrapper(wasmBytes.buffer); return wrapper; } diff --git a/app/src/lib/runtime/runtime-executor.ts b/app/src/lib/runtime/runtime-executor.ts index 31d8baa..28faccc 100644 --- a/app/src/lib/runtime/runtime-executor.ts +++ b/app/src/lib/runtime/runtime-executor.ts @@ -4,47 +4,47 @@ import type { NodeInput, NodeRegistry, RuntimeExecutor, - SyncCache, -} from "@nodarium/types"; + SyncCache +} from '@nodarium/types'; import { concatEncodedArrays, createLogger, encodeFloat, fastHashArrayBuffer, - type PerformanceStore, -} from "@nodarium/utils"; -import type { RuntimeNode } from "./types"; + type PerformanceStore +} from '@nodarium/utils'; +import type { RuntimeNode } from './types'; -const log = createLogger("runtime-executor"); +const log = createLogger('runtime-executor'); log.mute(); function getValue(input: NodeInput, value?: unknown) { - if (value === undefined && "value" in input) { + if (value === undefined && 'value' in input) { value = input.value; } - if (input.type === "float") { + if (input.type === 'float') { return encodeFloat(value as number); } if (Array.isArray(value)) { - if (input.type === "vec3") { + if (input.type === 'vec3') { return [ 0, value.length + 1, ...value.map((v) => encodeFloat(v)), 1, - 1, + 1 ] as number[]; } return [0, value.length + 1, ...value, 1, 1] as number[]; } - if (typeof value === "boolean") { + if (typeof value === 'boolean') { return value ? 1 : 0; } - if (typeof value === "number") { + if (typeof value === 'number') { return value; } @@ -52,6 +52,7 @@ function getValue(input: NodeInput, value?: unknown) { return value; } + console.error({ input, value }); throw new Error(`Unknown input type ${input.type}`); } @@ -62,16 +63,18 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { perf?: PerformanceStore; + private results: Record = {}; + constructor( private registry: NodeRegistry, - public cache?: SyncCache, + public cache?: SyncCache ) { this.cache = undefined; } private async getNodeDefinitions(graph: Graph) { - if (this.registry.status !== "ready") { - throw new Error("Node registry is not ready"); + if (this.registry.status !== 'ready') { + throw new Error('Node registry is not ready'); } await this.registry.load(graph.nodes.map((node) => node.type)); @@ -98,21 +101,18 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { depth: 0, children: [], parents: [], - inputNodes: {}, - } - return n - }) + inputNodes: {} + }; + return n; + }); - - const outputNode = graphNodes.find((node) => - node.type.endsWith("/output"), - ); + const outputNode = graphNodes.find((node) => node.type.endsWith('/output')); if (!outputNode) { - throw new Error("No output node found"); + throw new Error('No output node found'); } const nodeMap = new Map( - graphNodes.map((node) => [node.id, node]), + graphNodes.map((node) => [node.id, node]) ); // loop through all edges and assign the parent and child nodes to each node @@ -146,7 +146,7 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { } async execute(graph: Graph, settings: Record) { - this.perf?.addPoint("runtime"); + this.perf?.addPoint('runtime'); let a = performance.now(); @@ -154,7 +154,7 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { const [outputNode, nodes] = await this.addMetaData(graph); let b = performance.now(); - this.perf?.addPoint("collect-metadata", b - a); + this.perf?.addPoint('collect-metadata', b - a); /* * Here we sort the nodes into buckets, which we then execute one by one @@ -169,13 +169,13 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { // we execute the nodes from the bottom up const sortedNodes = nodes.sort( - (a, b) => (b.state?.depth || 0) - (a.state?.depth || 0), + (a, b) => (b.state?.depth || 0) - (a.state?.depth || 0) ); // here we store the intermediate results of the nodes - const results: Record = {}; + this.results = {}; - if (settings["randomSeed"]) { + if (settings['randomSeed']) { this.seed = Math.floor(Math.random() * 100000000); } @@ -192,7 +192,7 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { // Collect the inputs for the node const inputs = Object.entries(node_type.inputs || {}).map( ([key, input]) => { - if (input.type === "seed") { + if (input.type === 'seed') { return this.seed; } @@ -204,12 +204,12 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { // check if the input is connected to another node const inputNode = node.state.inputNodes[key]; if (inputNode) { - if (results[inputNode.id] === undefined) { + if (this.results[inputNode.id] === undefined) { throw new Error( - `Node ${node.type} is missing input from node ${inputNode.type}`, + `Node ${node.type} is missing input from node ${inputNode.type}` ); } - return results[inputNode.id]; + return this.results[inputNode.id]; } // If the value is stored in the node itself, we use that value @@ -218,45 +218,45 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { } return getValue(input); - }, + } ); b = performance.now(); - this.perf?.addPoint("collected-inputs", b - a); + this.perf?.addPoint('collected-inputs', b - a); try { a = performance.now(); const encoded_inputs = concatEncodedArrays(inputs); b = performance.now(); - this.perf?.addPoint("encoded-inputs", b - a); + this.perf?.addPoint('encoded-inputs', b - a); a = performance.now(); let inputHash = `node-${node.id}-${fastHashArrayBuffer(encoded_inputs)}`; b = performance.now(); - this.perf?.addPoint("hash-inputs", b - a); + this.perf?.addPoint('hash-inputs', b - a); let cachedValue = this.cache?.get(inputHash); if (cachedValue !== undefined) { log.log(`Using cached value for ${node_type.id || node.id}`); - this.perf?.addPoint("cache-hit", 1); - results[node.id] = cachedValue as Int32Array; + this.perf?.addPoint('cache-hit', 1); + this.results[node.id] = cachedValue as Int32Array; continue; } - this.perf?.addPoint("cache-hit", 0); + this.perf?.addPoint('cache-hit', 0); log.group(`executing ${node_type.id}-${node.id}`); log.log(`Inputs:`, inputs); a = performance.now(); - results[node.id] = node_type.execute(encoded_inputs); - log.log("Executed", node.type, node.id) + this.results[node.id] = node_type.execute(encoded_inputs); + log.log('Executed', node.type, node.id); b = performance.now(); if (this.cache && node.id !== outputNode.id) { - this.cache.set(inputHash, results[node.id]); + this.cache.set(inputHash, this.results[node.id]); } - this.perf?.addPoint("node/" + node_type.id, b - a); - log.log("Result:", results[node.id]); + this.perf?.addPoint('node/' + node_type.id, b - a); + log.log('Result:', this.results[node.id]); log.groupEnd(); } catch (e) { log.groupEnd(); @@ -265,17 +265,21 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor { } // return the result of the parent of the output node - const res = results[outputNode.id]; + const res = this.results[outputNode.id]; if (this.cache) { this.cache.size = sortedNodes.length * 2; } - this.perf?.endPoint("runtime"); + this.perf?.endPoint('runtime'); return res as unknown as Int32Array; } + getIntermediateResults() { + return this.results; + } + getPerformanceData() { return this.perf?.get(); } diff --git a/app/src/routes/+page.svelte b/app/src/routes/+page.svelte index 6d89f4a..1699fdb 100644 --- a/app/src/routes/+page.svelte +++ b/app/src/routes/+page.svelte @@ -90,11 +90,6 @@ let graphSettingTypes = $state({ randomSeed: { type: "boolean", value: false }, }); - $effect(() => { - if (graphSettings && graphSettingTypes) { - manager?.setSettings($state.snapshot(graphSettings)); - } - }); async function update( g: Graph, diff --git a/app/src/routes/dev/+layout.svelte b/app/src/routes/dev/+layout.svelte index ddf9ccd..69deb21 100644 --- a/app/src/routes/dev/+layout.svelte +++ b/app/src/routes/dev/+layout.svelte @@ -3,6 +3,6 @@ const { children } = $props<{ children?: Snippet }>(); -
+
{@render children()}
diff --git a/app/src/routes/dev/+page.svelte b/app/src/routes/dev/+page.svelte index 6166fa2..24e01b5 100644 --- a/app/src/routes/dev/+page.svelte +++ b/app/src/routes/dev/+page.svelte @@ -1,88 +1,73 @@ -
- {#if nodeInstance} - - {/if} -
- - 
-      
-        {JSON.stringify(nodeInstance?.props)}
-      
-
+ {#if result} + 
{JSON.stringify(decodeNestedArray(result))}
+ {/if} -
- {#if nodeWasm} - - {/if} -
+ handleSave(g)} + onresult={(result) => handleResult(result)} + /> @@ -92,22 +77,7 @@ classes="text-green-400" title="Node Store" icon="i-[tabler--database]" - > -
- {#await nodeRegistry.fetchCollection("max/plantarium")} -

Loading Nodes...

- {:then result} - {#each result.nodes as n} - - {/each} - {/await} -
- + >