refactor(backend): split log files into separate file

lib/cache.ts

@@ -6,12 +6,18 @@ interface SetCacheOptions {
   expires?: number; // Override expiration for individual cache entries
 }
 
+export const caches = new Map<
+  string,
+  { info: () => { count: number; sizeInKB: number } }
+>();
+
 export function createCache<T>(
+  cacheName: string,
   createOpts: CreateCacheOptions = {},
 ) {
   const cache = new Map<string, { value: T; expiresAt?: number }>();
 
-  return {
+  const api = {
     get(key: string): T | undefined {
       const entry = cache.get(key);
       if (!entry) return undefined;
@@ -86,4 +92,9 @@ export function createCache<T>(
       return cache.size;
     },
   };
+
+  caches.set(cacheName, {
+    info: api.info.bind(api),
+  });
+  return api;
 }

lib/crud.ts

@@ -71,7 +71,7 @@ export function createCrud<T extends GenericResource>(
     parse: (doc: string, id: string) => T;
   },
 ) {
-  const cache = createCache<T>({ expires: 60 * 1000 });
+  const cache = createCache<T>(`crud/${prefix}`, { expires: 60 * 1000 });
 
   function pathFromId(id: string) {
     return `${prefix}${id.replaceAll(":", "")}.md`;

lib/documents.ts

@@ -10,7 +10,7 @@ import remarkFrontmatter, {
 } from "https://esm.sh/remark-frontmatter@4.0.1";
 import { SILVERBULLET_SERVER } from "@lib/env.ts";
 import { fixRenderedMarkdown } from "@lib/helpers.ts";
-import { createLogger } from "@lib/log.ts";
+import { createLogger } from "@lib/log/index.ts";
 import { db } from "@lib/db/sqlite.ts";
 import { documentTable } from "@lib/db/schema.ts";
 import { eq } from "drizzle-orm/sql";

lib/env.ts

@@ -1,8 +1,6 @@
 import path from "node:path";
 
 export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER");
-export const REDIS_HOST = Deno.env.get("REDIS_HOST");
-export const REDIS_PASS = Deno.env.get("REDIS_PASS");
 export const TMDB_API_KEY = Deno.env.get("TMDB_API_KEY");
 export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY");
 export const YOUTUBE_API_KEY = Deno.env.get("YOUTUBE_API_KEY");

lib/helpers.ts

@@ -9,7 +9,7 @@ export function json(content: unknown) {
 export const isValidUrl = (urlString: string) => {
   try {
     return Boolean(new URL(urlString));
-  } catch (e) {
+  } catch (_e) {
     return false;
   }
 };
@@ -87,6 +87,8 @@ export const createStreamResponse = () => {
   };
 };
 
+export type StreamResponse = ReturnType<typeof createStreamResponse>;
+
 export function debounce<T extends (...args: Parameters<T>) => void>(
   this: ThisParameterType<T>,
   fn: T,

lib/image.ts

@@ -1,5 +1,5 @@
 import { rgbToHex } from "@lib/string.ts";
-import { createLogger } from "@lib/log.ts";
+import { createLogger } from "@lib/log/index.ts";
 import { generateThumbhash } from "@lib/thumbhash.ts";
 import { parseMediaType } from "https://deno.land/std@0.224.0/media_types/parse_media_type.ts";
 import path from "node:path";

lib/index.ts

@@ -1 +0,0 @@
-export * from "./documents.ts";

lib/log.ts

@@ -1,61 +0,0 @@
-import { EventEmitter } from "https://deno.land/x/evtemitter@v3.0.0/mod.ts";
-import { LOG_LEVEL as _LOG_LEVEL } from "@lib/env.ts";
-
-enum LOG_LEVEL {
-  DEBUG = 0,
-  INFO = 1,
-  WARN = 2,
-  ERROR = 3,
-}
-
-const logMap = {
-  "debug": LOG_LEVEL.DEBUG,
-  "info": LOG_LEVEL.INFO,
-  "warn": LOG_LEVEL.WARN,
-  "error": LOG_LEVEL.ERROR,
-} as const;
-
-const logFuncs = {
-  [LOG_LEVEL.DEBUG]: console.debug,
-  [LOG_LEVEL.INFO]: console.info,
-  [LOG_LEVEL.WARN]: console.warn,
-  [LOG_LEVEL.ERROR]: console.error,
-} as const;
-
-let longestScope = 0;
-let logLevel = (_LOG_LEVEL && _LOG_LEVEL in logMap)
-  ? logMap[_LOG_LEVEL]
-  : LOG_LEVEL.DEBUG;
-
-const ee = new EventEmitter<{
-  log: { level: LOG_LEVEL; scope: string; args: unknown[] };
-}>();
-
-export function setLogLevel(level: LOG_LEVEL) {
-  logLevel = level;
-}
-
-type LoggerOptions = {
-  enabled?: boolean;
-};
-
-const createLogFunction =
-  (scope: string, level: LOG_LEVEL) => (...data: unknown[]) => {
-    ee.emit("log", { level, scope, args: data });
-    if (level < logLevel) return;
-    logFuncs[level](`[${scope.padEnd(longestScope, " ")}]`, ...data);
-  };
-
-export function createLogger(scope: string, _options?: LoggerOptions) {
-  longestScope = Math.max(scope.length, longestScope);
-
-  return {
-    debug: createLogFunction(scope, LOG_LEVEL.DEBUG),
-    info: createLogFunction(scope, LOG_LEVEL.INFO),
-    error: createLogFunction(scope, LOG_LEVEL.ERROR),
-    warn: createLogFunction(scope, LOG_LEVEL.WARN),
-    addEventListener:
-      ((type, cb) =>
-        ee.addEventListener(type, cb)) as typeof ee.addEventListener,
-  };
-}

lib/log/constants.ts

@@ -0,0 +1,14 @@
+import * as env from "@lib/env.ts";
+import { ensureDir } from "https://deno.land/std@0.224.0/fs/mod.ts";
+import { join } from "node:path";
+import { getLogLevel, LOG_LEVEL } from "@lib/log/types.ts";
+
+export const LOG_DIR = join(env.DATA_DIR, "logs");
+
+// Ensure the log directory exists
+await ensureDir(LOG_DIR);
+
+export let logLevel = getLogLevel(env.LOG_LEVEL);
+export function setLogLevel(level: LOG_LEVEL) {
+  logLevel = level;
+}

lib/log/fs.ts

@@ -1,7 +1,6 @@
-import { createLogger } from "@lib/log.ts";
 import { join } from "node:path";
-import { DATA_DIR } from "@lib/env.ts";
+import { LOG_DIR } from "@lib/log/constants.ts";
-import { ensureDir } from "fs";
+import { Log } from "@lib/log/types.ts";
 
 function getLogFileName() {
   const d = new Date();
@@ -11,30 +10,17 @@ function getLogFileName() {
   return `${year}-${month}-${day}.log`;
 }
 
-const LOG_DIR = join(DATA_DIR, "logs");
+export function writeLogEntry(entry: Log) {
-
+  const logEntry = JSON.stringify(entry);
-// Ensure the log directory exists
-await ensureDir(LOG_DIR);
-
-const log = createLogger("");
-log.addEventListener("log", async (data) => {
-  const logEntry = JSON.stringify(data.detail);
   const logFilePath = join(LOG_DIR, getLogFileName());
 
   // Append the log entry to the file (creating it if it doesn't exist)
-  await Deno.writeTextFile(
+  Deno.writeTextFile(
     logFilePath,
     new Date().toISOString() + " | " + logEntry + "\n",
     { append: true },
   );
-});
+}
-
-export type Log = {
-  scope: string;
-  level: number;
-  date: Date;
-  args: unknown[];
-};
 
 export async function getLogs() {
   const logFilePath = join(LOG_DIR, getLogFileName());
@@ -59,7 +45,7 @@ export async function getLogs() {
 
     // Return the logs sorted by date
     return logs.sort((a, b) => a.date.getTime() - b.date.getTime());
-  } catch (error) {
+  } catch (_error) {
     // If file does not exist, return an empty array
     return [];
   }

lib/log/index.ts

@@ -0,0 +1,52 @@
+import { StreamResponse } from "@lib/helpers.ts";
+import { writeLogEntry } from "@lib/log/fs.ts";
+import { LOG_LEVEL, Logger } from "@lib/log/types.ts";
+import { logLevel } from "@lib/log/constants.ts";
+
+let longestScope = 0;
+
+type LoggerOptions = {
+  enabled?: boolean;
+};
+
+const createLogFunction = (scope: string, level: LOG_LEVEL) => {
+  return (...data: unknown[]) => {
+    writeLogEntry({ level, scope, args: data, date: new Date() });
+    if (level < logLevel) return;
+    const logFunc = {
+      [LOG_LEVEL.DEBUG]: console.debug,
+      [LOG_LEVEL.INFO]: console.info,
+      [LOG_LEVEL.WARN]: console.warn,
+      [LOG_LEVEL.ERROR]: console.error,
+    }[level];
+
+    logFunc(`[${scope.padEnd(longestScope, " ")}]`, ...data);
+  };
+};
+
+export function createLogger(scope: string, _options?: LoggerOptions): Logger {
+  longestScope = Math.max(scope.length, longestScope);
+
+  return {
+    debug: createLogFunction(scope, LOG_LEVEL.DEBUG),
+    info: createLogFunction(scope, LOG_LEVEL.INFO),
+    error: createLogFunction(scope, LOG_LEVEL.ERROR),
+    warn: createLogFunction(scope, LOG_LEVEL.WARN),
+  };
+}
+
+export function loggerFromStream(stream: StreamResponse) {
+  return {
+    debug: (...data: unknown[]) =>
+      stream.enqueue(`${data.length > 1 ? data.join(" ") : data[0]}`),
+    info: (...data: unknown[]) =>
+      stream.enqueue(`${data.length > 1 ? data.join(" ") : data[0]}`),
+    error: (...data: unknown[]) =>
+      stream.enqueue(`[ERROR]: ${data.length > 1 ? data.join(" ") : data[0]}`),
+    warn: (...data: unknown[]) =>
+      stream.enqueue(`[WARN]: ${data.length > 1 ? data.join(" ") : data[0]}`),
+  };
+}
+
+export type { Log } from "@lib/log/types.ts";
+export { getLogs } from "@lib/log/fs.ts";

lib/log/types.ts

@@ -0,0 +1,32 @@
+export enum LOG_LEVEL {
+  DEBUG = 0,
+  INFO = 1,
+  WARN = 2,
+  ERROR = 3,
+}
+export function getLogLevel(level: string): LOG_LEVEL {
+  switch (level) {
+    case "debug":
+      return LOG_LEVEL.DEBUG;
+    case "warn":
+      return LOG_LEVEL.WARN;
+    case "error":
+      return LOG_LEVEL.ERROR;
+    default:
+      return LOG_LEVEL.INFO;
+  }
+}
+
+export type Log = {
+  scope: string;
+  level: number;
+  date: Date;
+  args: unknown[];
+};
+
+export interface Logger {
+  debug: (...data: unknown[]) => void;
+  info: (...data: unknown[]) => void;
+  error: (...data: unknown[]) => void;
+  warn: (...data: unknown[]) => void;
+}

lib/openai.ts

@@ -12,7 +12,7 @@ interface MovieRecommendation {
   title: string;
 }
 
-const cache = createCache<MovieRecommendation[]>();
+const cache = createCache<MovieRecommendation[]>("movie-recommendations");
 
 function extractListFromResponse(response?: string): string[] {
   if (!response) return [];

lib/recommendation.ts

@@ -15,7 +15,7 @@ type RecommendationResource = {
   year?: number;
 };
 
-const cache = createCache<RecommendationResource>();
+const cache = createCache<RecommendationResource>("recommendations");
 
 export async function createRecommendationResource(
   res: GenericResource,
@@ -88,7 +88,7 @@ export async function getSimilarMovies(id: string) {
 export async function getAllRecommendations(): Promise<
   RecommendationResource[]
 > {
-  const keys = cache.keys("recommendations:movie:*");
+  const keys = cache.keys();
   const res = await Promise.all(keys.map((k) => cache.get(k)));
   return res.map((r) => JSON.parse(r));
 }

lib/tmdb.ts

@@ -10,7 +10,7 @@ import { createCache } from "@lib/cache.ts";
 const moviedb = new MovieDb(Deno.env.get("TMDB_API_KEY") || "");
 
 const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
-const cache = createCache({ expires: CACHE_INTERVAL });
+const cache = createCache("the-movie-db", { expires: CACHE_INTERVAL });
 
 export const searchMovie = async (query: string, year?: number) => {
   const id = `query:moviesearch:${query}${year ? `-${year}` : ""}`;

lib/webScraper.ts

@@ -0,0 +1,2 @@
+export function webScrape(url: URL) {
+}

routes/_app.tsx

@@ -5,9 +5,11 @@ export default function App({ Component }: PageProps) {
   const globalCss = Deno
     .readTextFileSync("./static/global.css")
     .replaceAll("\n", "");
+
   return (
     <>
       <Head>
+        <link rel="stylesheet" href="/prism-material-dark.css" />
         <link rel="stylesheet" href="/styles.css" />
         <link
           rel="icon"

routes/admin/log/index.tsx

@@ -1,13 +1,13 @@
 import { MainLayout } from "@components/layouts/main.tsx";
 import { Handlers, PageProps } from "$fresh/server.ts";
 import { AccessDeniedError } from "@lib/errors.ts";
-import { getLogs, Log } from "@lib/logs.ts";
+import { getLogs, Log } from "@lib/log/index.ts";
 import { formatDate } from "@lib/string.ts";
 import { renderMarkdown } from "@lib/documents.ts";
 
 const renderLog = (t: unknown) =>
   renderMarkdown(`\`\`\`js
-${typeof t === "string" ? t : JSON.stringify(t).trim()}
+${typeof t === "string" ? t : JSON.stringify(t, null, 2).trim()}
 \`\`\``);
 
 export const handler: Handlers = {
@@ -16,11 +16,12 @@ export const handler: Handlers = {
     if (!("session" in ctx.state)) {
       throw new AccessDeniedError();
     }
+    console.log({ logs });
     return ctx.render({
       logs: logs.map((l) => {
         return {
           ...l,
-          html: l.args.map(renderLog).join("\n"),
+          html: l.args.map(renderLog).join("<br/>"),
         };
       }),
     });
@@ -34,7 +35,7 @@ function LogLine(
 ) {
   return (
     <div
-      class="mt-4 flex flex-col gap-2 bg-gray-900 px-4 py-2 rounded-2xl"
+      class="mt-4 flex flex-col gap-2 bg-gray-900 px-4 py-2 rounded-2xl max-w-3xl"
       style={{ background: "var(--light)" }}
     >
       <div class="flex gap-2">
@@ -51,13 +52,9 @@ function LogLine(
         </span>
       </div>
       <div
-        class="flex gap-1 text-white"
+        class="text-white"
         dangerouslySetInnerHTML={{ __html: log.html }}
-      >
+      />
-        <pre>
-          {log.html}
-        </pre>
-      </div>
     </div>
   );
 }
@@ -67,7 +64,7 @@ export default function Greet(
 ) {
   return (
     <MainLayout url={url}>
-      <h1 class="text-white text-4xl ">Performance</h1>
+      <h1 class="text-white text-4xl">Logs</h1>
 
       {logs.map((r) => {
         return (

routes/api/articles/create/index.ts

@@ -9,7 +9,7 @@ import tds from "https://cdn.skypack.dev/turndown@7.2.0";
 import { Article, createArticle } from "@lib/resource/articles.ts";
 import { getYoutubeVideoDetails } from "@lib/youtube.ts";
 import { extractYoutubeId, isYoutubeLink } from "@lib/string.ts";
-import { createLogger } from "@lib/log.ts";
+import { createLogger } from "@lib/log/index.ts";
 
 const parser = new DOMParser();
 

routes/api/images/index.ts

@@ -1,7 +1,7 @@
 import { FreshContext, Handlers } from "$fresh/server.ts";
 import { getImageContent } from "@lib/image.ts";
 import { SILVERBULLET_SERVER } from "@lib/env.ts";
-import { createLogger } from "@lib/log.ts";
+import { createLogger } from "@lib/log/index.ts";
 import { isLocalImage } from "@lib/string.ts";
 
 const log = createLogger("api/image");

routes/api/recipes/create/index.ts

@@ -5,7 +5,7 @@ import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
 import { createStreamResponse, isValidUrl } from "@lib/helpers.ts";
 import * as openai from "@lib/openai.ts";
 import tds from "https://cdn.skypack.dev/turndown@7.2.0";
-import { createLogger } from "@lib/log.ts";
+import { createLogger } from "@lib/log/index.ts";
 import { createRecipe, Recipe } from "@lib/resource/recipes.ts";
 import recipeSchema from "@lib/recipeSchema.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
@@ -136,8 +136,8 @@ async function processCreateRecipeFromUrl(
   const jsonLds = Array.from(
     document?.querySelectorAll(
       "script[type='application/ld+json']",
-    ) as HTMLScriptElement[],
+    ),
-  );
+  ) as unknown as HTMLScriptElement[];
 
   let recipe: z.infer<typeof recipeSchema> | undefined = undefined;
   if (jsonLds.length > 0) {

routes/api/tmdb/[id].ts

@@ -9,7 +9,9 @@ type CachedMovieCredits = {
 };
 
 const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
-const cache = createCache<CachedMovieCredits>({ expires: CACHE_INTERVAL });
+const cache = createCache<CachedMovieCredits>("movie-credits", {
+  expires: CACHE_INTERVAL,
+});
 
 const GET = async (
   _req: Request,

routes/api/tmdb/credits/[id].ts

@@ -1,7 +1,7 @@
 import { FreshContext } from "$fresh/server.ts";
 import { getMovieCredits } from "@lib/tmdb.ts";
 import { json } from "@lib/helpers.ts";
-import { createLogger } from "@lib/log.ts";
+import { createLogger } from "@lib/log/index.ts";
 import { createCache } from "@lib/cache.ts";
 
 type CachedMovieCredits = {
@@ -10,7 +10,9 @@ type CachedMovieCredits = {
 };
 
 const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
-const cache = createCache<CachedMovieCredits>({ expires: CACHE_INTERVAL });
+const cache = createCache<CachedMovieCredits>("movie-credits", {
+  expires: CACHE_INTERVAL,
+});
 
 const log = createLogger("api/tmdb");
 

static/global.css

@@ -112,3 +112,7 @@ input[type=number] {
 .items-52>* {
   height: 52px;
 }
+
+.highlight>pre {
+  text-wrap: wrap;
+}