max/memorium
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat: move perf,logs and user into sqlite

Browse Source
This commit is contained in:
max_richter
2025-01-05 18:03:59 +01:00
parent 1937ef55bb
commit bf7d88a588
25 changed files with 524 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
lib/cache/logs.ts vendored
View File

@ -1,37 +0,0 @@
import { createLogger } from "@lib/log.ts";
import * as cache from "@lib/cache/cache.ts";
import { getTimeCacheKey, parseTimeCacheKey } from "@lib/string.ts";
const log = createLogger("");
log.addEventListener("log", (data) => {
cache.set(`log:${getTimeCacheKey()}`, JSON.stringify(data.detail), {
noLog: true,
});
});
export type Log = {
scope: string;
level: number;
date: Date;
args: unknown[];
};
export async function getLogs() {
const d = new Date();
const year = d.getFullYear();
const month = d.getMonth().toString();
const day = d.getDate().toString();
const keys = await cache.keys(
`log:${year}:${month}:${day}:*`,
);
const logs = await Promise.all(
keys.map(async (key) => {
const date = parseTimeCacheKey(key);
return { ...JSON.parse(await cache.get<string>(key)), date } as Log;
}),
);
return logs.sort((a, b) => a.date.getTime() > b.date.getTime() ? -1 : 1);
}

18
lib/db.ts
View File

@ -1,18 +0,0 @@
import z from "https://deno.land/x/zod@v3.21.4/index.ts";
import { createSchema } from "@lib/db/createSchema.ts";
const UserSchema = z.object({
id: z.string().optional().default(() => crypto.randomUUID()),
createdAt: z.date().default(() => new Date()),
email: z.string().email(),
name: z.string(),
});
export const userDB = createSchema("user", UserSchema);
const SessionSchema = z.object({
id: z.string().default(() => crypto.randomUUID()),
createdAt: z.date().default(() => new Date()),
expiresAt: z.date().default(() => new Date()),
userId: z.string(),
});
export const sessionDB = createSchema("session", SessionSchema);

32
lib/db/createSchema.ts
View File

@ -1,32 +0,0 @@
import { z } from "zod";
import * as cache from "@lib/cache/cache.ts";
export function createSchema<T extends z.ZodSchema>(name: string, schema: T) {
type Data = z.infer<T>;
return {
async create(input: Omit<Data, "id">): Promise<Data> {
const data = schema.safeParse(input);
if (data.success) {
const d = data.data;
const id = d["id"];
if (!id) return d;
await cache.set(`${name}:${id}`, JSON.stringify(d));
return d;
}
return null;
},
async findAll(): Promise<Data[]> {
const keys = await cache.keys(`${name}:*`);
return Promise.all(keys.map((k) => {
return cache.get<string>(k);
})).then((values) => values.map((v) => JSON.parse(v || "null")));
},
async find(id: string) {
const k = await cache.get<string>(`${name}:${id}`);
return JSON.parse(k || "null") as Data | null;
},
delete(id: string) {
return cache.del(`${name}:${id}`);
},
};
}

11
lib/env.ts
View File

@ -1,4 +1,4 @@
import "@std/dotenv/load";
import path from "node:path";
export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER");
export const REDIS_HOST = Deno.env.get("REDIS_HOST");
@ -8,7 +8,10 @@ export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY");
export const YOUTUBE_API_KEY = Deno.env.get("YOUTUBE_API_KEY");
export const GITEA_SERVER = Deno.env.get("GITEA_SERVER");
export const GITEA_CLIENT_ID = Deno.env.get("GITEA_CLIENT_ID");
export const GITEA_CLIENT_ID = Deno.env.get("GITEA_CLIENT_ID")!;
if (!GITEA_CLIENT_ID) {
throw new Error("GITEA_CLIENT_ID is required");
}
export const GITEA_CLIENT_SECRET = Deno.env.get("GITEA_CLIENT_SECRET");
export const GITEA_REDIRECT_URL = Deno.env.get("GITEA_REDIRECT_URL");
@ -23,5 +26,9 @@ export const TYPESENSE_URL = Deno.env.get("TYPESENSE_URL") ||
"http://localhost:8108";
export const TYPESENSE_API_KEY = Deno.env.get("TYPESENSE_API_KEY");
export const DATA_DIR = Deno.env.has("DATA_DIR")
? path.resolve(Deno.env.get("DATA_DIR")!)
: path.resolve(Deno.cwd(), "data");
export const LOG_LEVEL: string = Deno.env.get("LOG_LEVEL") ||
"warn";

16
lib/errors.ts
View File

@ -1,30 +1,30 @@
import { MiddlewareHandlerContext } from "$fresh/server.ts";
import { FreshContext } from "$fresh/server.ts";
class DomainError extends Error {
status = 500;
render?: (ctx: MiddlewareHandlerContext) => void;
render?: (ctx: FreshContext) => void;
constructor(public statusText = "Internal Server Error") {
super();
}
}
class NotFoundError extends DomainError {
status = 404;
constructor(public statusText = "Not Found") {
override status = 404;
constructor(public override statusText = "Not Found") {
super();
}
}
class BadRequestError extends DomainError {
status = 400;
constructor(public statusText = "Bad Request") {
override status = 400;
constructor(public override statusText = "Bad Request") {
super();
}
}
class AccessDeniedError extends DomainError {
status = 403;
constructor(public statusText = "Access Denied") {
override status = 403;
constructor(public override statusText = "Access Denied") {
super();
}
}

2
lib/hooks/useEventListener.ts
View File

@ -3,7 +3,7 @@ import { useEffect, useRef } from "preact/hooks";
export function useEventListener<T extends Event>(
eventName: string,
handler: (event: T) => void,
element: Window | HTMLElement = globalThis,
element: typeof globalThis | HTMLElement = globalThis,
) {
// Create a ref that stores handler
const savedHandler = useRef<(event: Event) => void>();

6
lib/log.ts
View File

@ -23,9 +23,9 @@ const logFuncs = {
} as const;
let longestScope = 0;
let logLevel = (_LOG_LEVEL && _LOG_LEVEL in logMap && logMap[_LOG_LEVEL]) ??
LOG_LEVEL.WARN;
let logLevel = _LOG_LEVEL && _LOG_LEVEL in logMap && _LOG_LEVEL in logMap
? logMap[_LOG_LEVEL]
: LOG_LEVEL.WARN;
const ee = new EventEmitter<{
log: { level: LOG_LEVEL; scope: string; args: unknown[] };
}>();

66
lib/logs.ts Normal file
View File

@ -0,0 +1,66 @@
import { createLogger } from "@lib/log.ts";
import { join } from "node:path";
import { DATA_DIR } from "@lib/env.ts";
import { ensureDir } from "fs";
function getLogFileName() {
const d = new Date();
const year = d.getFullYear();
const month = (d.getMonth() + 1).toString().padStart(2, "0"); // Ensure two digits
const day = d.getDate().toString().padStart(2, "0"); // Ensure two digits
return `${year}-${month}-${day}.log`;
}
const LOG_DIR = join(DATA_DIR, "logs");
// Ensure the log directory exists
await ensureDir(LOG_DIR);
const log = createLogger("");
log.addEventListener("log", async (data) => {
const logEntry = JSON.stringify(data.detail);
const logFilePath = join(LOG_DIR, getLogFileName());
// Append the log entry to the file (creating it if it doesn't exist)
await Deno.writeTextFile(
logFilePath,
new Date().toISOString() + " | " + logEntry + "\n",
{ append: true },
);
});
export type Log = {
scope: string;
level: number;
date: Date;
args: unknown[];
};
export async function getLogs() {
const logFilePath = join(LOG_DIR, getLogFileName());
try {
// Read the log file content
const logFileContent = await Deno.readTextFile(logFilePath);
// Split by lines and parse logs
const logs: Log[] = logFileContent
.split("\n")
.filter((line) => line.trim() !== "")
.map((line) => {
const [date, ...rest] = line.split(" | ");
const parsed = JSON.parse(rest.join(" | ")) as Log;
return {
...parsed,
date: new Date(date),
} as Log;
});
console.log(logs);
// Return the logs sorted by date
return logs.sort((a, b) => a.date.getTime() - b.date.getTime());
} catch (error) {
// If file does not exist, return an empty array
return [];
}
}

44
lib/cache/performance.ts → lib/performance.ts
View File

@ -1,5 +1,6 @@
import * as cache from "@lib/cache/cache.ts";
import { getTimeCacheKey } from "@lib/string.ts";
import { db } from "@lib/sqlite/sqlite.ts";
import { performanceTable } from "@lib/sqlite/schema.ts";
import { between } from "drizzle-orm/sql";
export type PerformancePoint = {
path: string;
@ -16,40 +17,29 @@ export type PerformanceRes = {
}[];
};
export const savePerformance = (url: string, milliseconds: number) => {
const cacheKey = getTimeCacheKey();
export const savePerformance = async (url: string, seconds: number) => {
const u = new URL(url);
if (u.pathname.includes("_frsh/")) return;
u.searchParams.delete("__frsh_c");
cache.set(
`performance:${cacheKey}`,
JSON.stringify({
path: decodeURIComponent(u.pathname),
search: u.search,
time: Math.floor(milliseconds * 1000),
}),
);
console.log("Saving performance", u.pathname, u.search, seconds);
const res = await db.insert(performanceTable).values({
path: decodeURIComponent(u.pathname),
search: u.search,
time: Math.floor(seconds * 1000),
});
console.log({ res });
};
export async function getPerformances(): Promise<PerformanceRes> {
const d = new Date();
const year = d.getFullYear();
const month = d.getMonth().toString();
const day = d.getDay().toString();
const now = new Date();
const startOfDay = new Date(now.setHours(0, 0, 0, 0));
const endOfDay = new Date(now.setHours(23, 59, 59, 999));
const keys = await cache.keys(
`performance:${year}:${month}:${day}:*`,
);
console.log(`performance:${year}:${month}:${day}:*`);
const performances = await Promise.all(
keys.map(async (key) =>
JSON.parse(await cache.get<string>(key)) as PerformancePoint
),
const performances = await db.select().from(performanceTable).where(
between(performanceTable.createdAt, startOfDay, endOfDay),
);
console.log({ performances });
let maximum = 0;

45
lib/sqlite/schema.ts Normal file
View File

@ -0,0 +1,45 @@
import { int, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
import { sql } from "drizzle-orm/sql";
export const userTable = sqliteTable("user", {
id: text()
.primaryKey(),
createdAt: integer("created_at", { mode: "timestamp" })
.default(sql`(current_timestamp)`)
.notNull(),
email: text()
.notNull(),
name: text()
.notNull(),
});
export const sessionTable = sqliteTable("session", {
id: text("id")
.primaryKey(),
createdAt: integer("created_at", { mode: "timestamp_ms" }).default(
sql`(current_timestamp)`,
),
expiresAt: integer("expires_at", { mode: "timestamp" })
.notNull(),
userId: text("user_id")
.notNull(),
});
export const performanceTable = sqliteTable("performance", {
path: text().notNull(),
search: text(),
time: int().notNull(),
createdAt: integer("created_at", {
mode: "timestamp_ms",
}).default(sql`(STRFTIME('%s', 'now') * 1000)`),
});
export const imageTable = sqliteTable("image", {
createdAt: integer("created_at", { mode: "timestamp" }).default(
sql`(current_timestamp)`,
),
path: text().notNull(),
average: text().notNull(),
blurhash: text().notNull(),
mime: text().notNull(),
});

14
lib/sqlite/sqlite.ts Normal file
View File

@ -0,0 +1,14 @@
import { drizzle } from "drizzle-orm/libsql/node";
import { DATA_DIR } from "@lib/env.ts";
import path from "node:path";
// const DB_FILE = "file://" + path.resolve(DATA_DIR, "db.sqlite");
const DB_FILE = "file:data-dev/db.sqlite";
// You can specify any property from the libsql connection options
export const db = drizzle({
logger: true,
connection: {
url: DB_FILE,
},
});