max/memorium
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

refactor(backend): split log files into separate file

Browse Source
This commit is contained in:
max_richter
2025-01-19 16:43:00 +01:00
parent e9cc56d7ee
commit f106460502
24 changed files with 155 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/cache.ts
View File

@@ -6,12 +6,18 @@ interface SetCacheOptions {
expires?: number; // Override expiration for individual cache entries
}
export const caches = new Map<
string,
{ info: () => { count: number; sizeInKB: number } }
>();
export function createCache<T>(
cacheName: string,
createOpts: CreateCacheOptions = {},
) {
const cache = new Map<string, { value: T; expiresAt?: number }>();
return {
const api = {
get(key: string): T | undefined {
const entry = cache.get(key);
if (!entry) return undefined;
@@ -86,4 +92,9 @@ export function createCache<T>(
return cache.size;
},
};
caches.set(cacheName, {
info: api.info.bind(api),
});
return api;
}

2
lib/crud.ts
View File

@@ -71,7 +71,7 @@ export function createCrud<T extends GenericResource>(
parse: (doc: string, id: string) => T;
},
) {
const cache = createCache<T>({ expires: 60 * 1000 });
const cache = createCache<T>(`crud/${prefix}`, { expires: 60 * 1000 });
function pathFromId(id: string) {
return `${prefix}${id.replaceAll(":", "")}.md`;

2
lib/documents.ts
View File

@@ -10,7 +10,7 @@ import remarkFrontmatter, {
} from "https://esm.sh/remark-frontmatter@4.0.1";
import { SILVERBULLET_SERVER } from "@lib/env.ts";
import { fixRenderedMarkdown } from "@lib/helpers.ts";
import { createLogger } from "@lib/log.ts";
import { createLogger } from "@lib/log/index.ts";
import { db } from "@lib/db/sqlite.ts";
import { documentTable } from "@lib/db/schema.ts";
import { eq } from "drizzle-orm/sql";

2
lib/env.ts
View File

@@ -1,8 +1,6 @@
import path from "node:path";
export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER");
export const REDIS_HOST = Deno.env.get("REDIS_HOST");
export const REDIS_PASS = Deno.env.get("REDIS_PASS");
export const TMDB_API_KEY = Deno.env.get("TMDB_API_KEY");
export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY");
export const YOUTUBE_API_KEY = Deno.env.get("YOUTUBE_API_KEY");

4
lib/helpers.ts
View File

@@ -9,7 +9,7 @@ export function json(content: unknown) {
export const isValidUrl = (urlString: string) => {
try {
return Boolean(new URL(urlString));
} catch (e) {
} catch (_e) {
return false;
}
};
@@ -87,6 +87,8 @@ export const createStreamResponse = () => {
};
};
export type StreamResponse = ReturnType<typeof createStreamResponse>;
export function debounce<T extends (...args: Parameters<T>) => void>(
this: ThisParameterType<T>,
fn: T,

2
lib/image.ts
View File

@@ -1,5 +1,5 @@
import { rgbToHex } from "@lib/string.ts";
import { createLogger } from "@lib/log.ts";
import { createLogger } from "@lib/log/index.ts";
import { generateThumbhash } from "@lib/thumbhash.ts";
import { parseMediaType } from "https://deno.land/std@0.224.0/media_types/parse_media_type.ts";
import path from "node:path";

1
lib/index.ts
View File

@@ -1 +0,0 @@
export * from "./documents.ts";

61
lib/log.ts
View File

@@ -1,61 +0,0 @@
import { EventEmitter } from "https://deno.land/x/evtemitter@v3.0.0/mod.ts";
import { LOG_LEVEL as _LOG_LEVEL } from "@lib/env.ts";
enum LOG_LEVEL {
DEBUG = 0,
INFO = 1,
WARN = 2,
ERROR = 3,
}
const logMap = {
"debug": LOG_LEVEL.DEBUG,
"info": LOG_LEVEL.INFO,
"warn": LOG_LEVEL.WARN,
"error": LOG_LEVEL.ERROR,
} as const;
const logFuncs = {
[LOG_LEVEL.DEBUG]: console.debug,
[LOG_LEVEL.INFO]: console.info,
[LOG_LEVEL.WARN]: console.warn,
[LOG_LEVEL.ERROR]: console.error,
} as const;
let longestScope = 0;
let logLevel = (_LOG_LEVEL && _LOG_LEVEL in logMap)
? logMap[_LOG_LEVEL]
: LOG_LEVEL.DEBUG;
const ee = new EventEmitter<{
log: { level: LOG_LEVEL; scope: string; args: unknown[] };
}>();
export function setLogLevel(level: LOG_LEVEL) {
logLevel = level;
}
type LoggerOptions = {
enabled?: boolean;
};
const createLogFunction =
(scope: string, level: LOG_LEVEL) => (...data: unknown[]) => {
ee.emit("log", { level, scope, args: data });
if (level < logLevel) return;
logFuncs[level](`[${scope.padEnd(longestScope, " ")}]`, ...data);
};
export function createLogger(scope: string, _options?: LoggerOptions) {
longestScope = Math.max(scope.length, longestScope);
return {
debug: createLogFunction(scope, LOG_LEVEL.DEBUG),
info: createLogFunction(scope, LOG_LEVEL.INFO),
error: createLogFunction(scope, LOG_LEVEL.ERROR),
warn: createLogFunction(scope, LOG_LEVEL.WARN),
addEventListener:
((type, cb) =>
ee.addEventListener(type, cb)) as typeof ee.addEventListener,
};
}

14
lib/log/constants.ts Normal file
View File

@@ -0,0 +1,14 @@
import * as env from "@lib/env.ts";
import { ensureDir } from "https://deno.land/std@0.224.0/fs/mod.ts";
import { join } from "node:path";
import { getLogLevel, LOG_LEVEL } from "@lib/log/types.ts";
export const LOG_DIR = join(env.DATA_DIR, "logs");
// Ensure the log directory exists
await ensureDir(LOG_DIR);
export let logLevel = getLogLevel(env.LOG_LEVEL);
export function setLogLevel(level: LOG_LEVEL) {
logLevel = level;
}

28
lib/logs.ts → lib/log/fs.ts
View File

@@ -1,7 +1,6 @@
import { createLogger } from "@lib/log.ts";
import { join } from "node:path";
import { DATA_DIR } from "@lib/env.ts";
import { ensureDir } from "fs";
import { LOG_DIR } from "@lib/log/constants.ts";
import { Log } from "@lib/log/types.ts";
function getLogFileName() {
const d = new Date();
@@ -11,30 +10,17 @@ function getLogFileName() {
return `${year}-${month}-${day}.log`;
}
const LOG_DIR = join(DATA_DIR, "logs");
// Ensure the log directory exists
await ensureDir(LOG_DIR);
const log = createLogger("");
log.addEventListener("log", async (data) => {
const logEntry = JSON.stringify(data.detail);
export function writeLogEntry(entry: Log) {
const logEntry = JSON.stringify(entry);
const logFilePath = join(LOG_DIR, getLogFileName());
// Append the log entry to the file (creating it if it doesn't exist)
await Deno.writeTextFile(
Deno.writeTextFile(
logFilePath,
new Date().toISOString() + " | " + logEntry + "\n",
{ append: true },
);
});
export type Log = {
scope: string;
level: number;
date: Date;
args: unknown[];
};
}
export async function getLogs() {
const logFilePath = join(LOG_DIR, getLogFileName());
@@ -59,7 +45,7 @@ export async function getLogs() {
// Return the logs sorted by date
return logs.sort((a, b) => a.date.getTime() - b.date.getTime());
} catch (error) {
} catch (_error) {
// If file does not exist, return an empty array
return [];
}

52
lib/log/index.ts Normal file
View File

@@ -0,0 +1,52 @@
import { StreamResponse } from "@lib/helpers.ts";
import { writeLogEntry } from "@lib/log/fs.ts";
import { LOG_LEVEL, Logger } from "@lib/log/types.ts";
import { logLevel } from "@lib/log/constants.ts";
let longestScope = 0;
type LoggerOptions = {
enabled?: boolean;
};
const createLogFunction = (scope: string, level: LOG_LEVEL) => {
return (...data: unknown[]) => {
writeLogEntry({ level, scope, args: data, date: new Date() });
if (level < logLevel) return;
const logFunc = {
[LOG_LEVEL.DEBUG]: console.debug,
[LOG_LEVEL.INFO]: console.info,
[LOG_LEVEL.WARN]: console.warn,
[LOG_LEVEL.ERROR]: console.error,
}[level];
logFunc(`[${scope.padEnd(longestScope, " ")}]`, ...data);
};
};
export function createLogger(scope: string, _options?: LoggerOptions): Logger {
longestScope = Math.max(scope.length, longestScope);
return {
debug: createLogFunction(scope, LOG_LEVEL.DEBUG),
info: createLogFunction(scope, LOG_LEVEL.INFO),
error: createLogFunction(scope, LOG_LEVEL.ERROR),
warn: createLogFunction(scope, LOG_LEVEL.WARN),
};
}
export function loggerFromStream(stream: StreamResponse) {
return {
debug: (...data: unknown[]) =>
stream.enqueue(`${data.length > 1 ? data.join(" ") : data[0]}`),
info: (...data: unknown[]) =>
stream.enqueue(`${data.length > 1 ? data.join(" ") : data[0]}`),
error: (...data: unknown[]) =>
stream.enqueue(`[ERROR]: ${data.length > 1 ? data.join(" ") : data[0]}`),
warn: (...data: unknown[]) =>
stream.enqueue(`[WARN]: ${data.length > 1 ? data.join(" ") : data[0]}`),
};
}
export type { Log } from "@lib/log/types.ts";
export { getLogs } from "@lib/log/fs.ts";

32
lib/log/types.ts Normal file
View File

@@ -0,0 +1,32 @@
export enum LOG_LEVEL {
DEBUG = 0,
INFO = 1,
WARN = 2,
ERROR = 3,
}
export function getLogLevel(level: string): LOG_LEVEL {
switch (level) {
case "debug":
return LOG_LEVEL.DEBUG;
case "warn":
return LOG_LEVEL.WARN;
case "error":
return LOG_LEVEL.ERROR;
default:
return LOG_LEVEL.INFO;
}
}
export type Log = {
scope: string;
level: number;
date: Date;
args: unknown[];
};
export interface Logger {
debug: (...data: unknown[]) => void;
info: (...data: unknown[]) => void;
error: (...data: unknown[]) => void;
warn: (...data: unknown[]) => void;
}

2
lib/openai.ts
View File

@@ -12,7 +12,7 @@ interface MovieRecommendation {
title: string;
}
const cache = createCache<MovieRecommendation[]>();
const cache = createCache<MovieRecommendation[]>("movie-recommendations");
function extractListFromResponse(response?: string): string[] {
if (!response) return [];

4
lib/recommendation.ts
View File

@@ -15,7 +15,7 @@ type RecommendationResource = {
year?: number;
};
const cache = createCache<RecommendationResource>();
const cache = createCache<RecommendationResource>("recommendations");
export async function createRecommendationResource(
res: GenericResource,
@@ -88,7 +88,7 @@ export async function getSimilarMovies(id: string) {
export async function getAllRecommendations(): Promise<
RecommendationResource[]
> {
const keys = cache.keys("recommendations:movie:*");
const keys = cache.keys();
const res = await Promise.all(keys.map((k) => cache.get(k)));
return res.map((r) => JSON.parse(r));
}

2
lib/tmdb.ts
View File

@@ -10,7 +10,7 @@ import { createCache } from "@lib/cache.ts";
const moviedb = new MovieDb(Deno.env.get("TMDB_API_KEY") || "");
const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
const cache = createCache({ expires: CACHE_INTERVAL });
const cache = createCache("the-movie-db", { expires: CACHE_INTERVAL });
export const searchMovie = async (query: string, year?: number) => {
const id = `query:moviesearch:${query}${year ? `-${year}` : ""}`;

2
lib/webScraper.ts Normal file
View File

@@ -0,0 +1,2 @@
export function webScrape(url: URL) {
}