feat: move registry and runtime into separate packages

README.md

@@ -12,6 +12,8 @@ Nodarium
 
 </div>
 
+Currently this visual programming language is used to develop https://nodes.max-richter.dev, a procedural modelling tool for 3d-plants.
+
 # Table of contents
 
 - [Architecture](./docs/ARCHITECTURE.md)

app/package.json

@@ -10,6 +10,8 @@
     "preview": "vite preview"
   },
   "dependencies": {
+    "@nodes/registry": "link:../packages/registry",
+    "@nodes/runtime": "link:../packages/runtime",
     "@nodes/ui": "link:../packages/ui",
     "@nodes/utils": "link:../packages/utils",
     "@sveltejs/kit": "^2.5.7",

app/src/lib/helpers/index.ts

@@ -73,35 +73,6 @@ export const debounce = (fn: Function, ms = 300) => {
 
 export const clone: <T>(v: T) => T = "structedClone" in globalThis ? globalThis.structuredClone : (obj) => JSON.parse(JSON.stringify(obj));
 
-export const createLogger = (() => {
-  let maxLength = 5;
-  return (scope: string) => {
-    maxLength = Math.max(maxLength, scope.length);
-    let muted = false;
-
-    let isGrouped = false;
-
-    function s(color: string, ...args: any) {
-      return isGrouped ? [...args] : [`[%c${scope.padEnd(maxLength, " ")}]:`, `color: ${color}`, ...args];
-    }
-
-    return {
-      log: (...args: any[]) => !muted && console.log(...s("#888", ...args)),
-      info: (...args: any[]) => !muted && console.info(...s("#888", ...args)),
-      warn: (...args: any[]) => !muted && console.warn(...s("#888", ...args)),
-      error: (...args: any[]) => console.error(...s("#f88", ...args)),
-      group: (...args: any[]) => { if (!muted) { console.groupCollapsed(...s("#888", ...args)); isGrouped = true; } },
-      groupEnd: () => { if (!muted) { console.groupEnd(); isGrouped = false } },
-      mute() {
-        muted = true;
-      },
-      unmute() {
-        muted = false;
-      }
-
-    }
-  }
-})();
 
 
 export function withSubComponents<A, B extends Record<string, any>>(

app/src/lib/performance/PerformanceViewer.svelte

@@ -3,7 +3,7 @@
   import { humanizeNumber } from "$lib/helpers";
   import { Checkbox } from "@nodes/ui";
   import localStore from "$lib/helpers/localStore";
-  import { type PerformanceData } from "./store";
+  import { type PerformanceData } from "@nodes/utils";
   import BarSplit from "./BarSplit.svelte";
 
   export let data: PerformanceData;

app/src/lib/performance/SmallPerformanceViewer.svelte

@@ -2,15 +2,13 @@
   import { humanizeDuration, humanizeNumber } from "$lib/helpers";
   import localStore from "$lib/helpers/localStore";
   import SmallGraph from "./SmallGraph.svelte";
-  import type { PerformanceData, PerformanceStore } from "./store";
+  import type { PerformanceData, PerformanceStore } from "@nodes/utils";
 
   export let store: PerformanceStore;
 
   const open = localStore("node.performance.small.open", {
     runtime: false,
     fps: false,
-    vertices: false,
-    faces: false,
   });
 
   $: vertices = $store?.at(-1)?.["total-vertices"][0] || 0;
@@ -54,29 +52,15 @@
       </tr>
     {/if}
 
-    <tr on:click={() => ($open.vertices = !$open.vertices)}>
+    <tr>
-      <td>{$open.vertices ? "-" : "+"} vertices </td>
+      <td>vertices </td>
       <td>{humanizeNumber(vertices || 0)}</td>
     </tr>
-    {#if $open.vertices}
-      <tr>
-        <td colspan="2">
-          <SmallGraph points={getPoints($store, "total-vertices")} />
-        </td>
-      </tr>
-    {/if}
 
-    <tr on:click={() => ($open.faces = !$open.faces)}>
+    <tr>
-      <td>{$open.faces ? "-" : "+"} faces </td>
+      <td>faces </td>
       <td>{humanizeNumber(faces || 0)}</td>
     </tr>
-    {#if $open.faces}
-      <tr>
-        <td colspan="2">
-          <SmallGraph points={getPoints($store, "total-faces")} />
-        </td>
-      </tr>
-    {/if}
   </table>
 </div>
 

app/src/lib/performance/index.ts

@@ -1,2 +1 @@
-export * from "./store";
 export { default as PerformanceViewer } from "./PerformanceViewer.svelte";

app/src/lib/result-viewer/Viewer.svelte

@@ -5,7 +5,7 @@
 
   import { updateGeometries } from "./updateGeometries";
   import { decodeFloat, splitNestedArray } from "@nodes/utils";
-  import type { PerformanceStore } from "$lib/performance";
+  import type { PerformanceStore } from "@nodes/utils";
   import { AppSettings } from "$lib/settings/app-settings";
   import SmallPerformanceViewer from "$lib/performance/SmallPerformanceViewer.svelte";
 

app/src/lib/settings/panels/BenchmarkPanel.svelte

@@ -136,16 +136,6 @@
     border: solid thin var(--outline);
     border-bottom: none;
   }
-  textarea {
-    width: 100%;
-    height: 1em;
-    font-size: 1em;
-    padding: 0.5em;
-    border: solid thin var(--outline);
-    background: var(--layer-2);
-    box-sizing: border-box;
-    height: 2.5em;
-  }
   i {
     opacity: 0.5;
     font-size: 0.8em;

app/src/routes/+page.svelte

@@ -1,8 +1,6 @@
 <script lang="ts">
   import Grid from "$lib/grid";
   import GraphInterface from "$lib/graph-interface";
-  import { WorkerRuntimeExecutor } from "$lib/worker-runtime-executor";
-  import { RemoteNodeRegistry } from "$lib/node-registry-client";
   import * as templates from "$lib/graph-templates";
   import type { Graph, Node } from "@nodes/types";
   import Viewer from "$lib/result-viewer/Viewer.svelte";
@@ -18,17 +16,17 @@
   import Panel from "$lib/settings/Panel.svelte";
   import GraphSettings from "$lib/settings/panels/GraphSettings.svelte";
   import NestedSettings from "$lib/settings/panels/NestedSettings.svelte";
-  import { createPerformanceStore } from "$lib/performance";
+  import type { Group } from "three";
-  import type { Group, Scene } from "three";
   import ExportSettings from "$lib/settings/panels/ExportSettings.svelte";
   import {
     MemoryRuntimeCache,
+    WorkerRuntimeExecutor,
     MemoryRuntimeExecutor,
-  } from "$lib/runtime-executor";
+  } from "@nodes/runtime";
-  import { IndexDBCache } from "$lib/node-registry-cache";
+  import { IndexDBCache, RemoteNodeRegistry } from "@nodes/registry";
-  import { decodeNestedArray, fastHashString } from "@nodes/utils";
+  import { decodeNestedArray, createPerformanceStore } from "@nodes/utils";
   import BenchmarkPanel from "$lib/settings/panels/BenchmarkPanel.svelte";
-  import { debounceAsyncFunction, withArgsChangeOnly } from "$lib/helpers";
+  import { debounceAsyncFunction } from "$lib/helpers";
 
   let performanceStore = createPerformanceStore("page");
 

app/src/routes/runtime/server.ts

@@ -1,34 +0,0 @@
-import type { RequestHandler } from "./$types";
-import { MemoryRuntimeExecutor } from "$lib/runtime-executor";
-import { RemoteNodeRegistry } from "$lib/node-registry-client";
-import { createPerformanceStore } from "$lib/performance";
-
-const registry = new RemoteNodeRegistry("");
-const runtime = new MemoryRuntimeExecutor(registry);
-const performanceStore = createPerformanceStore();
-runtime.perf = performanceStore;
-
-export const prerender = false;
-
-export const POST: RequestHandler = async ({ request, fetch }) => {
-
-  const { graph, settings } = await request.json();
-
-  if (!graph || !settings) {
-    return new Response("Invalid request", { status: 400 });
-  }
-
-  registry.fetch = fetch;
-
-  await registry.load(graph.nodes.map(node => node.type))
-
-  const res = await runtime.execute(graph, settings);
-
-  let headers: Record<string, string> = { "Content-Type": "application/octet-stream" };
-  if (runtime.perf) {
-    headers["performance"] = JSON.stringify(runtime.perf.get());
-  }
-
-  return new Response(res, { headers });
-
-}

packages/registry/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@nodes/registry",
+  "version": "0.0.0",
+  "description": "",
+  "main": "src/index.ts",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@nodes/types": "link:../types",
+    "@nodes/utils": "link:../utils",
+    "idb": "^8.0.0"
+  }
+}

packages/registry/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./node-registry-cache";
+export * from "./node-registry-client";

packages/registry/src/node-registry-cache.ts

@@ -1,7 +1,7 @@
-import type { RuntimeCache } from '@nodes/types';
+import type { AsyncCache } from '@nodes/types';
 import { openDB, type IDBPDatabase } from 'idb';
 
-export class IndexDBCache implements RuntimeCache<ArrayBuffer> {
+export class IndexDBCache implements AsyncCache<ArrayBuffer> {
 
   size: number = 100;
 

packages/registry/src/node-registry-client.ts

@@ -1,6 +1,5 @@
-import { type NodeRegistry, type NodeDefinition, NodeDefinitionSchema, type RuntimeCache } from "@nodes/types";
+import { type NodeRegistry, type NodeDefinition, NodeDefinitionSchema, type AsyncCache } from "@nodes/types";
-import { createWasmWrapper } from "@nodes/utils";
+import { createWasmWrapper, createLogger } from "@nodes/utils";
-import { createLogger } from "./helpers";
 
 const log = createLogger("node-registry");
 log.mute();
@@ -10,7 +9,7 @@ export class RemoteNodeRegistry implements NodeRegistry {
   status: "loading" | "ready" | "error" = "loading";
   private nodes: Map<string, NodeDefinition> = new Map();
 
-  cache?: RuntimeCache<ArrayBuffer>;
+  cache?: AsyncCache<ArrayBuffer>;
 
   fetch: typeof fetch = globalThis.fetch.bind(globalThis);
 

packages/runtime/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@nodes/runtime",
+  "version": "0.0.0",
+  "description": "",
+  "main": "src/index.ts",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@nodes/registry": "link:../registry",
+    "@nodes/types": "link:../types",
+    "@nodes/utils": "link:../utils"
+  },
+  "devDependencies": {
+    "comlink": "^4.4.1",
+    "vite-plugin-comlink": "^4.0.3"
+  }
+}

packages/runtime/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./runtime-executor"
+export * from "./runtime-executor-cache"
+export * from "./worker-runtime-executor"

packages/runtime/src/runtime-executor-cache.ts

@@ -0,0 +1,19 @@
+import { type SyncCache } from "@nodes/types";
+
+export class MemoryRuntimeCache implements SyncCache {
+
+  private cache: [string, unknown][] = [];
+  size = 50;
+
+  get<T>(key: string): T | undefined {
+    return this.cache.find(([k]) => k === key)?.[1] as T;
+  }
+  set<T>(key: string, value: T): void {
+    this.cache.push([key, value]);
+    this.cache = this.cache.slice(-this.size);
+  }
+  clear(): void {
+    this.cache = [];
+  }
+
+}

packages/runtime/src/runtime-executor.ts

@@ -1,8 +1,6 @@
 import type { Graph, NodeRegistry, NodeDefinition, RuntimeExecutor, NodeInput } from "@nodes/types";
-import { concatEncodedArrays, encodeFloat, fastHashArrayBuffer } from "@nodes/utils"
+import { concatEncodedArrays, encodeFloat, fastHashArrayBuffer, createLogger, type PerformanceStore } from "@nodes/utils"
-import { createLogger } from "./helpers";
+import type { SyncCache } from "@nodes/types";
-import type { RuntimeCache } from "@nodes/types";
-import type { PerformanceStore } from "./performance";
 
 const log = createLogger("runtime-executor");
 log.mute()
@@ -45,7 +43,7 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
 
   perf?: PerformanceStore;
 
-  constructor(private registry: NodeRegistry, private cache?: RuntimeCache<Int32Array>) { }
+  constructor(private registry: NodeRegistry, private cache?: SyncCache<Int32Array>) { }
 
   private async getNodeDefinitions(graph: Graph) {
 
@@ -263,21 +261,3 @@ export class MemoryRuntimeExecutor implements RuntimeExecutor {
   }
 
 }
-
-export class MemoryRuntimeCache implements RuntimeCache {
-
-  private cache: [string, unknown][] = [];
-  size = 50;
-
-  get<T>(key: string): T | undefined {
-    return this.cache.find(([k]) => k === key)?.[1] as T;
-  }
-  set<T>(key: string, value: T): void {
-    this.cache.push([key, value]);
-    this.cache = this.cache.slice(-this.size);
-  }
-  clear(): void {
-    this.cache = [];
-  }
-
-}

packages/runtime/src/worker-runtime-executor-backend.ts

@@ -1,8 +1,8 @@
-import { MemoryRuntimeExecutor, MemoryRuntimeCache } from "./runtime-executor";
+import { MemoryRuntimeExecutor } from "./runtime-executor";
-import { RemoteNodeRegistry } from "./node-registry-client";
+import { RemoteNodeRegistry, IndexDBCache } from "@nodes/registry";
 import type { Graph } from "@nodes/types";
-import { createPerformanceStore } from "./performance/store";
+import { createPerformanceStore } from "@nodes/utils";
-import { IndexDBCache } from "./node-registry-cache";
+import { MemoryRuntimeCache } from "./runtime-executor-cache";
 
 const cache = new MemoryRuntimeCache();
 const indexDbCache = new IndexDBCache("node-registry");

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nodes/types",
-  "version": "1.0.0",
+  "version": "0.0.0",
   "description": "",
   "main": "src/index.ts",
   "exports": {

packages/types/src/components.ts

@@ -1,8 +1,6 @@
 import { Graph, NodeDefinition, NodeId } from "./types";
 
 export interface NodeRegistry {
-
-
   /**
   * The status of the node registry
   * @remarks The status should be "loading" when the registry is loading, "ready" when the registry is ready, and "error" if an error occurred while loading the registry
@@ -36,7 +34,7 @@ export interface NodeRegistry {
   register: (wasmBuffer: ArrayBuffer) => Promise<NodeDefinition>;
 
 
-  cache?: RuntimeCache<ArrayBuffer>;
+  cache?: AsyncCache<ArrayBuffer>;
 }
 
 export interface RuntimeExecutor {
@@ -48,8 +46,7 @@ export interface RuntimeExecutor {
   execute: (graph: Graph, settings: Record<string, unknown>) => Promise<Int32Array>;
 }
 
-export interface RuntimeCache<T = unknown> {
+export interface SyncCache<T = unknown> {
-
   /**
   * The maximum number of items that can be stored in the cache
   * @remarks When the cache size exceeds this value, the oldest items should be removed
@@ -61,16 +58,41 @@ export interface RuntimeCache<T = unknown> {
    * @param key - The key to get the value for
    * @returns The value for the given key, or undefined if no such value exists
    */
-  get: (key: string) => T | Promise<T> | undefined;
+  get: (key: string) => T | undefined;
   /**
    * Set the value for the given key
    * @param key - The key to set the value for
    * @param value - The value to set
    */
-  set: (key: string, value: T) => void | Promise<void>;
+  set: (key: string, value: T) => void;
   /**
    * Clear the cache
    */
   clear: () => void;
 
 }
+
+export interface AsyncCache<T = unknown> {
+  /**
+  * The maximum number of items that can be stored in the cache
+  * @remarks When the cache size exceeds this value, the oldest items should be removed
+  */
+  size: number;
+
+  /**
+   * Get the value for the given key
+   * @param key - The key to get the value for
+   * @returns The value for the given key, or undefined if no such value exists
+   */
+  get: (key: string) => Promise<T | undefined>;
+  /**
+   * Set the value for the given key
+   * @param key - The key to set the value for
+   * @param value - The value to set
+   */
+  set: (key: string, value: T) => Promise<void>;
+  /**
+   * Clear the cache
+   */
+  clear: () => void;
+}

packages/types/src/index.ts

@@ -1,5 +1,5 @@
 export type { NodeInput } from "./inputs";
-export type { NodeRegistry, RuntimeExecutor, RuntimeCache } from "./components";
+export type { NodeRegistry, RuntimeExecutor, SyncCache, AsyncCache } from "./components";
 export type { Node, NodeDefinition, Socket, NodeId, Edge, Graph } from "./types";
 export { NodeSchema, GraphSchema } from "./types";
 export { NodeDefinitionSchema } from "./types";

packages/utils/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nodes/utils",
-  "version": "1.0.0",
+  "version": "0.0.1",
   "description": "",
   "main": "src/index.ts",
   "scripts": {

packages/utils/src/index.ts

@@ -2,3 +2,5 @@ export * from "./wasm-wrapper";
 export * from "./flatTree"
 export * from "./encoding"
 export * from "./fastHash"
+export * from "./logger"
+export * from "./performance"

packages/utils/src/logger.ts

@@ -0,0 +1,29 @@
+export const createLogger = (() => {
+  let maxLength = 5;
+  return (scope: string) => {
+    maxLength = Math.max(maxLength, scope.length);
+    let muted = false;
+
+    let isGrouped = false;
+
+    function s(color: string, ...args: any) {
+      return isGrouped ? [...args] : [`[%c${scope.padEnd(maxLength, " ")}]:`, `color: ${color}`, ...args];
+    }
+
+    return {
+      log: (...args: any[]) => !muted && console.log(...s("#888", ...args)),
+      info: (...args: any[]) => !muted && console.info(...s("#888", ...args)),
+      warn: (...args: any[]) => !muted && console.warn(...s("#888", ...args)),
+      error: (...args: any[]) => console.error(...s("#f88", ...args)),
+      group: (...args: any[]) => { if (!muted) { console.groupCollapsed(...s("#888", ...args)); isGrouped = true; } },
+      groupEnd: () => { if (!muted) { console.groupEnd(); isGrouped = false } },
+      mute() {
+        muted = true;
+      },
+      unmute() {
+        muted = false;
+      }
+
+    }
+  }
+})();

packages/utils/src/performance.ts

@@ -1,17 +1,16 @@
-import { readable, type Readable } from "svelte/store";
-
 export type PerformanceData = Record<string, number[]>[];
 
-export interface PerformanceStore extends Readable<PerformanceData> {
+export interface PerformanceStore {
   startRun(): void;
   stopRun(): void;
   addPoint(name: string, value?: number): void;
   endPoint(name?: string): void;
   mergeData(data: PerformanceData[number]): void;
   get: () => PerformanceData;
+  subscribe: (cb: (v: PerformanceData) => void) => () => void;
 }
 
-export function createPerformanceStore(id?: string): PerformanceStore {
+export function createPerformanceStore(): PerformanceStore {
 
   let data: PerformanceData = [];
 
@@ -19,11 +18,18 @@ export function createPerformanceStore(id?: string): PerformanceStore {
   let temp: Record<string, number> | undefined;
   let lastPoint: string | undefined;
 
-  let set: (v: PerformanceData) => void;
+  const listeners: ((v: PerformanceData) => void)[] = [];
+  function subscribe(cb: (v: PerformanceData) => void) {
+    listeners.push(cb);
+    return () => {
+      const i = listeners.indexOf(cb);
+      if (i > -1) listeners.splice(i, 1);
+    }
+  }
 
-  const { subscribe } = readable<PerformanceData>([], (_set) => {
+  function set(v: PerformanceData) {
-    set = _set;
+    listeners.forEach((l) => l(v));
-  });
+  }
 
   function startRun() {
     if (currentRun) return;

pnpm-lock.yaml

@@ -10,6 +10,12 @@ importers:
 
   app:
     dependencies:
+      '@nodes/registry':
+        specifier: link:../packages/registry
+        version: link:../packages/registry
+      '@nodes/runtime':
+        specifier: link:../packages/runtime
+        version: link:../packages/runtime
       '@nodes/ui':
         specifier: link:../packages/ui
         version: link:../packages/ui
@@ -126,6 +132,37 @@ importers:
 
   nodes/max/plantarium/vec3: {}
 
+  packages/registry:
+    dependencies:
+      '@nodes/types':
+        specifier: link:../types
+        version: link:../types
+      '@nodes/utils':
+        specifier: link:../utils
+        version: link:../utils
+      idb:
+        specifier: ^8.0.0
+        version: 8.0.0
+
+  packages/runtime:
+    dependencies:
+      '@nodes/registry':
+        specifier: link:../registry
+        version: link:../registry
+      '@nodes/types':
+        specifier: link:../types
+        version: link:../types
+      '@nodes/utils':
+        specifier: link:../utils
+        version: link:../utils
+    devDependencies:
+      comlink:
+        specifier: ^4.4.1
+        version: 4.4.1
+      vite-plugin-comlink:
+        specifier: ^4.0.3
+        version: 4.0.3(comlink@4.4.1)(vite@5.2.10(@types/node@20.12.7)(sass@1.75.0))
+
   packages/types:
     dependencies:
       zod: