From 53c4d5b129b06deac607348345758919eb36211e Mon Sep 17 00:00:00 2001 From: Max Richter Date: Mon, 6 Jan 2025 16:14:29 +0100 Subject: [PATCH] feat: completely remove redis --- docker-compose.yml | 14 -- drizzle.config.ts | 2 +- drizzle/0009_free_robin_chapel.sql | 12 + drizzle/meta/0009_snapshot.json | 309 ++++++++++++++++++++++++ drizzle/meta/_journal.json | 7 + lib/cache.ts | 90 +++++++ lib/cache/cache.ts | 127 ---------- lib/crud.ts | 37 ++- lib/{sqlite => db}/schema.ts | 30 ++- lib/{sqlite => db}/sqlite.ts | 1 - lib/documents.ts | 4 +- lib/image.ts | 4 +- lib/openai.ts | 86 ++++--- lib/performance.ts | 4 +- lib/recommendation.ts | 28 +-- lib/tmdb.ts | 111 ++++----- routes/api/auth/callback.ts | 4 +- routes/api/movies/enhance/[name].ts | 24 +- routes/api/recommendation/all.ts | 7 +- routes/api/recommendation/movie/[id].ts | 1 + routes/api/series/enhance/[name].ts | 19 +- routes/api/tmdb/[id].ts | 5 +- routes/api/tmdb/credits/[id].ts | 9 +- routes/api/tmdb/query.ts | 5 +- 24 files changed, 629 insertions(+), 311 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 drizzle/0009_free_robin_chapel.sql create mode 100644 drizzle/meta/0009_snapshot.json create mode 100644 lib/cache.ts delete mode 100644 lib/cache/cache.ts rename lib/{sqlite => db}/schema.ts (69%) rename lib/{sqlite => db}/sqlite.ts (95%) diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 8fc20b9..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.8' - -services: - redis: - image: redis:latest - ports: - - "6379:6379" - volumes: - - ./data/redis-data:/data - -volumes: - redis-data: - typesense-data: - diff --git a/drizzle.config.ts b/drizzle.config.ts index 3dc9f00..5484958 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -9,7 +9,7 @@ const DB_FILE = "file:" + path.resolve(DATA_DIR, "db.sqlite"); export default defineConfig({ out: "./drizzle", - schema: "./lib/sqlite/schema.ts", + schema: "./lib/db/schema.ts", dialect: "turso", dbCredentials: { url: DB_FILE, diff --git a/drizzle/0009_free_robin_chapel.sql b/drizzle/0009_free_robin_chapel.sql new file mode 100644 index 0000000..c9f6443 --- /dev/null +++ b/drizzle/0009_free_robin_chapel.sql @@ -0,0 +1,12 @@ +CREATE TABLE `cache` ( + `scope` text NOT NULL, + `key` text PRIMARY KEY NOT NULL, + `json` text, + `binary` blob, + `created_at` integer DEFAULT (current_timestamp), + `expires_at` integer +); +--> statement-breakpoint +CREATE INDEX `key_idx` ON `cache` (`key`);--> statement-breakpoint +CREATE INDEX `scope_idx` ON `cache` (`scope`);--> statement-breakpoint +CREATE INDEX `name_idx` ON `document` (`name`); \ No newline at end of file diff --git a/drizzle/meta/0009_snapshot.json b/drizzle/meta/0009_snapshot.json new file mode 100644 index 0000000..e35e811 --- /dev/null +++ b/drizzle/meta/0009_snapshot.json @@ -0,0 +1,309 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5694a345-e55c-4aa3-9f29-1045b28f5203", + "prevId": "11e0dfc0-1020-46a9-9b26-061a2238a2d5", + "tables": { + "cache": { + "name": "cache", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "json": { + "name": "json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "binary": { + "name": "binary", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "key_idx": { + "name": "key_idx", + "columns": [ + "key" + ], + "isUnique": false + }, + "scope_idx": { + "name": "scope_idx", + "columns": [ + "scope" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document": { + "name": "document", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "perm": { + "name": "perm", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "name_idx": { + "name": "name_idx", + "columns": [ + "name" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image": { + "name": "image", + "columns": { + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average": { + "name": "average", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "blurhash": { + "name": "blurhash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime": { + "name": "mime", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "performance": { + "name": "performance", + "columns": { + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "search": { + "name": "search", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "time": { + "name": "time", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(STRFTIME('%s', 'now') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(current_timestamp)" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 35f9b35..4a32ec0 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1736109783318, "tag": "0008_loud_mephisto", "breakpoints": true + }, + { + "idx": 9, + "version": "6", + "when": 1736172911816, + "tag": "0009_free_robin_chapel", + "breakpoints": true } ] } \ No newline at end of file diff --git a/lib/cache.ts b/lib/cache.ts new file mode 100644 index 0000000..34fb677 --- /dev/null +++ b/lib/cache.ts @@ -0,0 +1,90 @@ +interface CreateCacheOptions { + expires?: number; // Default expiration time for all cache entries +} + +interface SetCacheOptions { + expires?: number; // Override expiration for individual cache entries +} + +export function createCache( + createOpts: CreateCacheOptions = {}, +) { + const cache = new Map(); + + return { + get(key: string): T | undefined { + const entry = cache.get(key); + if (!entry) return undefined; + + const now = Date.now(); + if (entry.expiresAt && entry.expiresAt <= now) { + cache.delete(key); // Remove expired entry + return undefined; + } + + return entry.value; // Return value if not expired + }, + + set(key: string, value: T | unknown, opts: SetCacheOptions = {}) { + console.log("Setting cache", key); + const now = Date.now(); + const expiresIn = opts.expires ?? createOpts.expires; + const expiresAt = expiresIn ? now + expiresIn : undefined; + + cache.set(key, { value: value as T, expiresAt }); + }, + + cleanup() { + const now = Date.now(); + for (const [key, entry] of cache.entries()) { + if (entry.expiresAt && entry.expiresAt <= now) { + cache.delete(key); + } + } + }, + + info(): { count: number; sizeInKB: number } { + // Cleanup expired entries before calculating info + this.cleanup(); + + // Count the number of objects in the cache + const count = cache.size; + + // Approximate the size in KB by serializing each key and value + let totalBytes = 0; + for (const [key, entry] of cache.entries()) { + const keySize = new TextEncoder().encode(key).length; + const valueSize = new TextEncoder().encode( + JSON.stringify(entry.value), + ).length; + totalBytes += keySize + valueSize; + } + + const sizeInKB = totalBytes / 1024; // Convert bytes to kilobytes + return { count, sizeInKB }; + }, + + has(key: string): boolean { + const entry = cache.get(key); + if (!entry) return false; + + const now = Date.now(); + if (entry.expiresAt && entry.expiresAt <= now) { + cache.delete(key); // Remove expired entry + return false; + } + + return true; + }, + + keys(): string[] { + this.cleanup(); // Cleanup before returning keys + return Array.from(cache.keys()); + }, + + size(): number { + this.cleanup(); // Cleanup before returning size + return cache.size; + }, + }; +} diff --git a/lib/cache/cache.ts b/lib/cache/cache.ts deleted file mode 100644 index acc6420..0000000 --- a/lib/cache/cache.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { - Bulk, - connect, - Redis, - RedisConnectOptions, - RedisValue, -} from "https://deno.land/x/redis@v0.31.0/mod.ts"; -import { createLogger } from "@lib/log.ts"; - -const REDIS_HOST = Deno.env.get("REDIS_HOST"); -const REDIS_PASS = Deno.env.get("REDIS_PASS") || ""; -const REDIS_PORT = Deno.env.get("REDIS_PORT"); - -const log = createLogger("cache"); - -async function createCache(): Promise { - if (REDIS_HOST) { - const conf: RedisConnectOptions = { - hostname: REDIS_HOST, - port: REDIS_PORT || 6379, - maxRetryCount: 2, - }; - if (REDIS_PASS) { - conf.password = REDIS_PASS; - } - try { - const client = await connect(conf); - log.info("redis connected"); - return client; - } catch (_err) { - log.info("cant connect to redis, falling back to mock"); - } - } - - const mockRedis = new Map(); - - return { - async keys() { - return mockRedis.keys(); - }, - async delete(key: string) { - mockRedis.delete(key); - return key; - }, - async set(key: string, value: RedisValue) { - mockRedis.set(key, value); - return value.toString(); - }, - async get(key: string) { - return mockRedis.get(key) as Bulk; - }, - }; -} - -const cache = await createCache(); - -export async function get(id: string, binary = false) { - if (binary && !(cache instanceof Map)) { - const cacheHit = await cache.sendCommand("GET", [id], { - returnUint8Arrays: true, - }) as T; - return cacheHit; - } - const cacheHit = await cache.get(id) as T; - return cacheHit; -} -export function clearAll() { - if ("flushall" in cache) { - return cache.flushall(); - } else { - for (const k of cache.keys()) { - cache.delete(k); - } - } -} - -export function expire(id: string, seconds: number) { - if ("expire" in cache) { - return cache.expire(id, seconds); - } -} - -type RedisOptions = { - expires?: number; - noLog?: boolean; -}; - -export function del(key: string) { - return cache.del(key); -} - -export function keys(prefix: string) { - return cache.keys(prefix); -} - -export function set( - id: string, - content: T, - options?: RedisOptions, -) { - if (options?.noLog !== true) log.debug("storing ", { id }); - return cache.set(id, content, { ex: options?.expires || undefined }); -} - -export const cacheFunction = async Promise)>( - { - fn, - id, - options = {}, - }: { - fn: T; - id: string; - options?: RedisOptions; - }, -): Promise>> => { - const cacheResult = await get(id) as string; - - if (cacheResult) { - return JSON.parse(cacheResult) as Awaited>; - } - - const result = await fn(); - - set(id, JSON.stringify(result), options); - - return result as Awaited>; -}; diff --git a/lib/crud.ts b/lib/crud.ts index eb03465..d553172 100644 --- a/lib/crud.ts +++ b/lib/crud.ts @@ -9,9 +9,10 @@ import { GenericResource } from "@lib/types.ts"; import { parseRating } from "@lib/helpers.ts"; import { isLocalImage } from "@lib/string.ts"; import { SILVERBULLET_SERVER } from "@lib/env.ts"; -import { imageTable } from "@lib/sqlite/schema.ts"; -import { db } from "@lib/sqlite/sqlite.ts"; +import { imageTable } from "@lib/db/schema.ts"; +import { db } from "@lib/db/sqlite.ts"; import { eq } from "drizzle-orm/sql"; +import { createCache } from "@lib/cache.ts"; export async function addThumbnailToResource( res: T, @@ -55,7 +56,9 @@ function sortFunction(sortType: SortType) { case "name": return a.name.localeCompare(b.name); case "author": - return a.meta?.author?.localeCompare(b.meta?.author || ""); + return a.meta?.author?.localeCompare(b.meta?.author || "") || 0; + default: + return 0; } }; } @@ -68,21 +71,30 @@ export function createCrud( parse: (doc: string, id: string) => T; }, ) { + const cache = createCache({ expires: 60 * 1000 }); + function pathFromId(id: string) { return `${prefix}${id.replaceAll(":", "")}.md`; } async function read(id: string) { const path = pathFromId(id); - const content = await getDocument(path); - const res = parse(content, id); - - if (hasThumbnails) { - return addThumbnailToResource(res); + if (cache.has(path)) { + return cache.get(path); } - return { ...res, content }; + const content = await getDocument(path); + + const parsed = parse(content, id); + + if (hasThumbnails) { + return addThumbnailToResource(parsed); + } + const doc = { ...parsed, content }; + cache.set(path, doc); + + return doc; } function create(id: string, content: string | ArrayBuffer | T) { const path = pathFromId(id); @@ -107,8 +119,11 @@ export function createCrud( } async function readAll({ sort = "rating" }: { sort?: SortType } = {}) { + if (cache.has("all")) { + return cache.get("all") as unknown as T[]; + } const allDocuments = await getDocuments(); - return (await Promise.all( + const parsed = (await Promise.all( allDocuments.filter((d) => { return d.name.startsWith(prefix) && d.contentType === "text/markdown" && @@ -118,6 +133,8 @@ export function createCrud( return read(id); }), )).sort(sortFunction(sort)); + cache.set("all", parsed); + return parsed; } return { diff --git a/lib/sqlite/schema.ts b/lib/db/schema.ts similarity index 69% rename from lib/sqlite/schema.ts rename to lib/db/schema.ts index 95222a9..034a58f 100644 --- a/lib/sqlite/schema.ts +++ b/lib/db/schema.ts @@ -1,6 +1,12 @@ -import { int, integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { + blob, + index, + int, + integer, + sqliteTable, + text, +} from "drizzle-orm/sqlite-core"; import { sql } from "drizzle-orm/sql"; -import { contentType } from "https://deno.land/std@0.216.0/media_types/content_type.ts"; export const userTable = sqliteTable("user", { id: text() @@ -52,4 +58,24 @@ export const documentTable = sqliteTable("document", { contentType: text("content_type").notNull(), size: integer().notNull(), perm: text().notNull(), +}, (table) => { + return [ + index("name_idx").on(table.name), + ]; +}); + +export const cacheTable = sqliteTable("cache", { + scope: text().notNull(), + key: text().notNull().primaryKey(), + json: text({ mode: "json" }), + binary: blob(), + createdAt: integer("created_at", { mode: "timestamp" }).default( + sql`(current_timestamp)`, + ), + expiresAt: integer("expires_at", { mode: "timestamp" }), +}, (table) => { + return [ + index("key_idx").on(table.key), + index("scope_idx").on(table.scope), + ]; }); diff --git a/lib/sqlite/sqlite.ts b/lib/db/sqlite.ts similarity index 95% rename from lib/sqlite/sqlite.ts rename to lib/db/sqlite.ts index 52ac97f..9737e40 100644 --- a/lib/sqlite/sqlite.ts +++ b/lib/db/sqlite.ts @@ -6,7 +6,6 @@ const DB_FILE = "file:" + path.resolve(DATA_DIR, "db.sqlite"); // You can specify any property from the libsql connection options export const db = drizzle({ - logger: true, connection: { url: DB_FILE, }, diff --git a/lib/documents.ts b/lib/documents.ts index e77a69b..d90b95d 100644 --- a/lib/documents.ts +++ b/lib/documents.ts @@ -11,8 +11,8 @@ import remarkFrontmatter, { import { SILVERBULLET_SERVER } from "@lib/env.ts"; import { fixRenderedMarkdown } from "@lib/helpers.ts"; import { createLogger } from "@lib/log.ts"; -import { db } from "@lib/sqlite/sqlite.ts"; -import { documentTable } from "@lib/sqlite/schema.ts"; +import { db } from "@lib/db/sqlite.ts"; +import { documentTable } from "@lib/db/schema.ts"; import { eq } from "drizzle-orm/sql"; export type Document = { diff --git a/lib/image.ts b/lib/image.ts index ca2d4f2..e1e8951 100644 --- a/lib/image.ts +++ b/lib/image.ts @@ -5,8 +5,8 @@ import { parseMediaType } from "https://deno.land/std@0.224.0/media_types/parse_ import path from "node:path"; import { ensureDir } from "https://deno.land/std@0.216.0/fs/mod.ts"; import { DATA_DIR } from "@lib/env.ts"; -import { db } from "@lib/sqlite/sqlite.ts"; -import { imageTable } from "@lib/sqlite/schema.ts"; +import { db } from "@lib/db/sqlite.ts"; +import { imageTable } from "@lib/db/schema.ts"; import { eq } from "drizzle-orm"; import sharp from "npm:sharp@next"; diff --git a/lib/openai.ts b/lib/openai.ts index fe174bc..463dcd2 100644 --- a/lib/openai.ts +++ b/lib/openai.ts @@ -1,9 +1,16 @@ import OpenAI from "https://deno.land/x/openai@v4.52.0/mod.ts"; import { OPENAI_API_KEY } from "@lib/env.ts"; -import { cacheFunction } from "@lib/cache/cache.ts"; import { hashString } from "@lib/helpers.ts"; +import { createCache } from "@lib/cache.ts"; -const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY); +const openAI = OPENAI_API_KEY && new OpenAI({ apiKey: OPENAI_API_KEY }); + +interface MovieRecommendation { + year: number; + title: string; +} + +const cache = createCache(); function extractListFromResponse(response?: string): string[] { if (!response) return []; @@ -138,52 +145,51 @@ return a list of around 20 keywords seperated by commas .map((v) => v.replaceAll(" ", "-")); } -export const getMovieRecommendations = (keywords: string, exclude: string[]) => - cacheFunction({ - fn: async () => { - if (!openAI) return; - const chatCompletion = await openAI.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: [ - { - role: "user", - content: - `Could you recommend me 10 movies based on the following attributes: +export const getMovieRecommendations = async ( + keywords: string, + exclude: string[], +) => { + if (!openAI) return; + const cacheId = `movierecs:${hashString(`${keywords}:${exclude.join()}`)}`; + if (cache.has(cacheId)) return cache.get(cacheId); + + const chatCompletion = await openAI.chat.completions.create({ + model: "gpt-3.5-turbo", + messages: [ + { + role: "user", + content: + `Could you recommend me 10 movies based on the following attributes: ${keywords} The movies should be similar to but not include ${ - exclude.join(", ") - } or remakes of that. + exclude.join(", ") + } or remakes of that. respond with a plain unordered list each item starting with the year the movie was released and then the title of the movie seperated by a -`, - }, - ], - }); - - const res = chatCompletion.choices[0].message.content?.toLowerCase(); - - if (!res) return; - - console.log("REsult:"); - console.log(res); - - const list = extractListFromResponse(res); - - console.log({ list }); - - return res.split("\n").map((entry) => { - const [year, ...title] = entry.split("-"); - - return { - year: parseInt(year.trim()), - title: title.join(" ").replaceAll('"', "").trim(), - }; - }).filter((y) => !Number.isNaN(y.year)); - }, - id: `openai:movierecs:${hashString(`${keywords}:${exclude.join()}`)}`, + }, + ], }); + const res = chatCompletion.choices[0].message.content?.toLowerCase(); + + if (!res) return; + + const recommendations = res.split("\n").map((entry) => { + const [year, ...title] = entry.split("-"); + + return { + year: parseInt(year.trim()), + title: title.join(" ").replaceAll('"', "").trim(), + }; + }).filter((y) => !Number.isNaN(y.year)); + + cache.set(cacheId, recommendations); + + return recommendations; +}; + export async function createTags(content: string) { if (!openAI) return; const chatCompletion = await openAI.chat.completions.create({ diff --git a/lib/performance.ts b/lib/performance.ts index 09d625c..b0d109c 100644 --- a/lib/performance.ts +++ b/lib/performance.ts @@ -1,5 +1,5 @@ -import { db } from "@lib/sqlite/sqlite.ts"; -import { performanceTable } from "@lib/sqlite/schema.ts"; +import { db } from "@lib/db/sqlite.ts"; +import { performanceTable } from "@lib/db/schema.ts"; import { between } from "drizzle-orm/sql"; export type PerformancePoint = { diff --git a/lib/recommendation.ts b/lib/recommendation.ts index 27d3993..7f36045 100644 --- a/lib/recommendation.ts +++ b/lib/recommendation.ts @@ -1,8 +1,8 @@ -import * as cache from "@lib/cache/cache.ts"; import * as openai from "@lib/openai.ts"; import * as tmdb from "@lib/tmdb.ts"; import { GenericResource } from "@lib/types.ts"; import { parseRating } from "@lib/helpers.ts"; +import { createCache } from "@lib/cache.ts"; type RecommendationResource = { id: string; @@ -15,12 +15,14 @@ type RecommendationResource = { year?: number; }; +const cache = createCache(); + export async function createRecommendationResource( res: GenericResource, description?: string, ) { - const cacheId = `recommendations:${res.type}:${res.id.replaceAll(":", "")}`; - const resource: RecommendationResource = await cache.get(cacheId) || { + const cacheId = `${res.type}:${res.id.replaceAll(":", "")}`; + const resource = cache.get(cacheId) || { id: res.id, type: res.type, rating: -1, @@ -58,20 +60,15 @@ export async function createRecommendationResource( cache.set(cacheId, JSON.stringify(resource)); } -export async function getRecommendation( +export function getRecommendation( id: string, type: string, -): Promise { - const res = await cache.get(`recommendations:${type}:${id}`) as string; - try { - return JSON.parse(res); - } catch (_) { - return null; - } +): RecommendationResource | undefined { + return cache.get(`recommendations:${type}:${id}`); } export async function getSimilarMovies(id: string) { - const recs = await getRecommendation(id, "movie"); + const recs = getRecommendation(id, "movie"); if (!recs?.keywords?.length) return; const recommendations = await openai.getMovieRecommendations( @@ -91,8 +88,7 @@ export async function getSimilarMovies(id: string) { export async function getAllRecommendations(): Promise< RecommendationResource[] > { - const keys = await cache.keys("recommendations:movie:*"); - return Promise.all(keys.map((k) => cache.get(k))).then((res) => - res.map((r) => JSON.parse(r)) - ); + const keys = cache.keys("recommendations:movie:*"); + const res = await Promise.all(keys.map((k) => cache.get(k))); + return res.map((r) => JSON.parse(r)); } diff --git a/lib/tmdb.ts b/lib/tmdb.ts index 4125e96..8977e47 100644 --- a/lib/tmdb.ts +++ b/lib/tmdb.ts @@ -1,74 +1,69 @@ -import * as cache from "@lib/cache/cache.ts"; -import { MovieDb } from "https://esm.sh/moviedb-promise@3.4.1"; +import { + CreditsResponse, + MovieDb, + MovieResponse, + MovieResultsResponse, + ShowResponse, + TvResultsResponse, +} from "https://esm.sh/moviedb-promise@3.4.1"; +import { createCache } from "@lib/cache.ts"; const moviedb = new MovieDb(Deno.env.get("TMDB_API_KEY") || ""); const CACHE_INTERVAL = 1000 * 60 * 24 * 30; +const cache = createCache({ expires: CACHE_INTERVAL }); -export const searchMovie = (query: string, year?: number) => - cache.cacheFunction({ - fn: () => moviedb.searchMovie({ query, year }), - id: `query:moviesearch:${query}${year ? `-${year}` : ""}`, - options: { - expires: CACHE_INTERVAL, - }, - }); +export const searchMovie = async (query: string, year?: number) => { + const id = `query:moviesearch:${query}${year ? `-${year}` : ""}`; + if (cache.has(id)) return cache.get(id) as MovieResultsResponse; + const res = await moviedb.searchMovie({ query, year }); + cache.set(id, res); + return res; +}; -export const searchTVShow = (query: string) => - cache.cacheFunction( - { - fn: () => moviedb.searchTv({ query }), - id: `query:tvshowsearch:${query}`, - options: { - expires: CACHE_INTERVAL, - }, - }, - ); +export const searchTVShow = async (query: string) => { + const id = `query:tvshowsearch:${query}`; + if (cache.has(id)) return cache.get(id) as TvResultsResponse; + const res = await moviedb.searchTv({ query }); + cache.set(id, res); + return res; +}; -export const getMovie = (id: number) => - cache.cacheFunction({ - fn: () => moviedb.movieInfo({ id }), - id: `query:movie:${id}`, - options: { - expires: CACHE_INTERVAL, - }, - }); +export const getMovie = async (id: number) => { + const cacheId = `query:movie:${id}`; + if (cache.has(cacheId)) return cache.get(cacheId) as MovieResponse; + const res = await moviedb.movieInfo({ id }); + cache.set(cacheId, res); + return res; +}; -export const getSeries = (id: number) => - cache.cacheFunction({ - fn: () => moviedb.tvInfo({ id }), - id: `query:tvshow:${id}`, - options: { - expires: CACHE_INTERVAL, - }, - }); +export const getSeries = async (id: number) => { + const cacheId = `query:tvshow:${id}`; + if (cache.has(cacheId)) return cache.get(cacheId) as ShowResponse; + const res = await moviedb.tvInfo({ id }); + cache.set(cacheId, res); + return res; +}; -export const getMovieCredits = (id: number) => - cache.cacheFunction({ - fn: () => moviedb.movieCredits(id), - id: `query:moviecredits:${id}`, - options: { - expires: CACHE_INTERVAL, - }, - }); +export const getMovieCredits = async (id: number) => { + const cacheId = `query:moviecredits:${id}`; + if (cache.has(cacheId)) return cache.get(cacheId) as CreditsResponse; + const res = await moviedb.movieCredits(id); + cache.set(cacheId, res); + return res; +}; -export const getSeriesCredits = (id: number) => - cache.cacheFunction({ - fn: () => moviedb.tvCredits(id), - id: `query:tvshowcredits:${id}`, - options: { - expires: CACHE_INTERVAL, - }, - }); +export const getSeriesCredits = async (id: number) => { + const cacheId = `query:tvshowcredits:${id}`; + if (cache.has(cacheId)) return cache.get(cacheId) as CreditsResponse; + const res = await moviedb.tvCredits(id); + cache.set(cacheId, res); + return res; +}; -export async function getMovieGenre(id: number) { - const genres = await cache.get("/genres/movies"); +export function getMovieGenre() { return moviedb.genreTvList(); } -export async function getSeriesGenre(id: number) { - const genres = await cache.get("/genres/series"); -} - export async function getMoviePoster(id: string): Promise { const posterUrl = `https://image.tmdb.org/t/p/original/${id}`; const response = await fetch(posterUrl); diff --git a/routes/api/auth/callback.ts b/routes/api/auth/callback.ts index e662e00..829ec41 100644 --- a/routes/api/auth/callback.ts +++ b/routes/api/auth/callback.ts @@ -6,8 +6,8 @@ import { codeChallengeMap } from "./login.ts"; import { GITEA_SERVER, JWT_SECRET, SESSION_DURATION } from "@lib/env.ts"; import { GiteaOauthUser } from "@lib/types.ts"; import { BadRequestError } from "@lib/errors.ts"; -import { db } from "@lib/sqlite/sqlite.ts"; -import { userTable } from "@lib/sqlite/schema.ts"; +import { db } from "@lib/db/sqlite.ts"; +import { userTable } from "@lib/db/schema.ts"; import { eq } from "drizzle-orm"; export const handler: Handlers = { diff --git a/routes/api/movies/enhance/[name].ts b/routes/api/movies/enhance/[name].ts index 6117c2c..884bf67 100644 --- a/routes/api/movies/enhance/[name].ts +++ b/routes/api/movies/enhance/[name].ts @@ -1,4 +1,4 @@ -import { HandlerContext, Handlers } from "$fresh/server.ts"; +import { FreshContext, Handlers } from "$fresh/server.ts"; import { createDocument } from "@lib/documents.ts"; import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts"; import { createMovie, getMovie } from "@lib/resource/movies.ts"; @@ -10,12 +10,11 @@ import { BadRequestError, NotFoundError, } from "@lib/errors.ts"; -import * as cache from "@lib/cache/cache.ts"; import { createRecommendationResource } from "@lib/recommendation.ts"; const POST = async ( req: Request, - ctx: HandlerContext, + ctx: FreshContext, ): Promise => { const session = ctx.state.session; if (!session) { @@ -35,25 +34,27 @@ const POST = async ( } const movieDetails = await tmdb.getMovie(tmdbId); - const movieCredits = !movie.meta.author && + const movieCredits = !movie.meta?.author && await tmdb.getMovieCredits(tmdbId); const releaseDate = movieDetails.release_date; - if (releaseDate && !movie.meta.date) { + if (releaseDate && !movie.meta?.date) { + movie.meta = movie.meta || {}; movie.meta.date = new Date(releaseDate); } const director = movieCredits?.crew?.filter?.((person) => person.job === "Director" )[0]; - if (director && !movie.meta.author) { + if (director && !movie.meta?.author) { + movie.meta = movie.meta || {}; movie.meta.author = director.name; } if (movieDetails.genres) { movie.tags = [ ...new Set([ - ...movie.tags.map((g) => g.toLowerCase()), + ...(movie.tags?.map((g) => g.toLowerCase()) || []), ...movieDetails.genres.map((g) => g.name?.toLowerCase().replaceAll(" ", "-") ), @@ -61,25 +62,24 @@ const POST = async ( ]; } - if (!movie.meta.tmdbId) { - movie.meta.tmdbId = tmdbId; + if (!movie.id) { + movie.id = tmdbId; } let finalPath = ""; const posterPath = movieDetails.poster_path; - if (posterPath && !movie.meta.image) { + if (posterPath && !movie.meta?.image) { const poster = await tmdb.getMoviePoster(posterPath); const extension = fileExtension(posterPath); finalPath = `Media/movies/images/${safeFileName(name)}_cover.${extension}`; await createDocument(finalPath, poster); + movie.meta = movie.meta || {}; movie.meta.image = finalPath; } await createMovie(movie.id, movie); - cache.del(`documents:Media:movies:${name}.md`); - createRecommendationResource(movie, movieDetails.overview); return json(movie); diff --git a/routes/api/recommendation/all.ts b/routes/api/recommendation/all.ts index 0f4d7e9..88f4255 100644 --- a/routes/api/recommendation/all.ts +++ b/routes/api/recommendation/all.ts @@ -1,6 +1,6 @@ import { Handlers } from "$fresh/server.ts"; import { createStreamResponse } from "@lib/helpers.ts"; -import { getAllMovies } from "@lib/resource/movies.ts"; +import { getAllMovies, Movie } from "@lib/resource/movies.ts"; import * as tmdb from "@lib/tmdb.ts"; import { createRecommendationResource, @@ -14,10 +14,11 @@ async function processUpdateRecommendations( const allMovies = await getAllMovies(); const movies = allMovies.filter((m) => { + if (!m?.meta) return false; if (!m.meta.rating) return false; if (!m.meta.tmdbId) return false; return true; - }); + }) as Movie[]; streamResponse.enqueue("Fetched all movies"); @@ -27,7 +28,7 @@ async function processUpdateRecommendations( await Promise.all(movies.map(async (movie) => { if (!movie.meta.tmdbId) return; if (!movie.meta.rating) return; - const recommendation = await getRecommendation(movie.id, movie.type); + const recommendation = getRecommendation(movie.id, movie.type); if (recommendation) { done++; return; diff --git a/routes/api/recommendation/movie/[id].ts b/routes/api/recommendation/movie/[id].ts index 0e1d587..eb65839 100644 --- a/routes/api/recommendation/movie/[id].ts +++ b/routes/api/recommendation/movie/[id].ts @@ -11,6 +11,7 @@ export const handler: Handlers = { } const recommendations = await getSimilarMovies(ctx.params.id); + console.log({ recommendations }); return json(recommendations); }, diff --git a/routes/api/series/enhance/[name].ts b/routes/api/series/enhance/[name].ts index fcbf712..7c90fcc 100644 --- a/routes/api/series/enhance/[name].ts +++ b/routes/api/series/enhance/[name].ts @@ -1,4 +1,4 @@ -import { HandlerContext, Handlers } from "$fresh/server.ts"; +import { FreshContext, Handlers } from "$fresh/server.ts"; import { createDocument } from "@lib/documents.ts"; import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts"; import * as tmdb from "@lib/tmdb.ts"; @@ -10,7 +10,6 @@ import { NotFoundError, } from "@lib/errors.ts"; import { createSeries, getSeries } from "@lib/resource/series.ts"; -import * as cache from "@lib/cache/cache.ts"; const isString = (input: string | undefined): input is string => { return typeof input === "string"; @@ -18,7 +17,7 @@ const isString = (input: string | undefined): input is string => { const POST = async ( req: Request, - ctx: HandlerContext, + ctx: FreshContext, ): Promise => { const session = ctx.state.session; if (!session) { @@ -38,43 +37,43 @@ const POST = async ( } const seriesDetails = await tmdb.getSeries(tmdbId); - const seriesCredits = !series.meta.author && + const seriesCredits = !series.meta?.author && await tmdb.getSeriesCredits(tmdbId); const releaseDate = seriesDetails.first_air_date; - if (releaseDate && series.meta.date) { + if (releaseDate && series.meta?.date) { series.meta.date = new Date(releaseDate); } const posterPath = seriesDetails.poster_path; const director = seriesCredits && seriesCredits.crew?.filter?.((person) => person.job === "Director")[0] || seriesDetails?.created_by?.[0]; - if (director && director.name && !series.meta.author) { + if (director && director.name && !series.meta?.author) { + series.meta = series.meta || {}; series.meta.author = director.name; } if (seriesDetails.genres) { series.tags = [ ...new Set([ - ...series.tags.map((t) => t.toLowerCase()), + ...(series.tags?.map((t) => t.toLowerCase()) || []), ...seriesDetails.genres.map((g) => g.name?.toLowerCase()), ].filter(isString)), ]; } let finalPath = ""; - if (posterPath && !series.meta.image) { + if (posterPath && !series.meta?.image) { const poster = await tmdb.getMoviePoster(posterPath); const extension = fileExtension(posterPath); finalPath = `Media/series/images/${safeFileName(name)}_cover.${extension}`; await createDocument(finalPath, poster); + series.meta = series.meta || {}; series.meta.image = finalPath; } await createSeries(series.id, series); - cache.del(`documents:Media:series:${name}.md`); - return json(series); }; diff --git a/routes/api/tmdb/[id].ts b/routes/api/tmdb/[id].ts index 2e8c255..d252ee3 100644 --- a/routes/api/tmdb/[id].ts +++ b/routes/api/tmdb/[id].ts @@ -1,7 +1,7 @@ import { FreshContext, Handlers } from "$fresh/server.ts"; import { getMovie } from "@lib/tmdb.ts"; -import * as cache from "@lib/cache/cache.ts"; import { json } from "@lib/helpers.ts"; +import { createCache } from "@lib/cache.ts"; type CachedMovieCredits = { lastUpdated: number; @@ -9,6 +9,7 @@ type CachedMovieCredits = { }; const CACHE_INTERVAL = 1000 * 60 * 24 * 30; +const cache = createCache({ expires: CACHE_INTERVAL }); const GET = async ( _req: Request, @@ -24,7 +25,7 @@ const GET = async ( const cacheId = `/movie/${id}`; - const cachedResponse = await cache.get(cacheId); + const cachedResponse = cache.get(cacheId); if ( cachedResponse && Date.now() < (cachedResponse.lastUpdated + CACHE_INTERVAL) ) { diff --git a/routes/api/tmdb/credits/[id].ts b/routes/api/tmdb/credits/[id].ts index 438abf2..674ea25 100644 --- a/routes/api/tmdb/credits/[id].ts +++ b/routes/api/tmdb/credits/[id].ts @@ -1,8 +1,8 @@ -import { HandlerContext } from "$fresh/server.ts"; +import { FreshContext } from "$fresh/server.ts"; import { getMovieCredits } from "@lib/tmdb.ts"; -import * as cache from "@lib/cache/cache.ts"; import { json } from "@lib/helpers.ts"; import { createLogger } from "@lib/log.ts"; +import { createCache } from "@lib/cache.ts"; type CachedMovieCredits = { lastUpdated: number; @@ -10,12 +10,13 @@ type CachedMovieCredits = { }; const CACHE_INTERVAL = 1000 * 60 * 24 * 30; +const cache = createCache({ expires: CACHE_INTERVAL }); const log = createLogger("api/tmdb"); export const handler = async ( _req: Request, - _ctx: HandlerContext, + _ctx: FreshContext, ) => { const id = _ctx.params.id; @@ -29,7 +30,7 @@ export const handler = async ( const cacheId = `/movie/credits/${id}`; - const cachedResponse = await cache.get(cacheId); + const cachedResponse = cache.get(cacheId); if ( cachedResponse && Date.now() < (cachedResponse.lastUpdated + CACHE_INTERVAL) ) { diff --git a/routes/api/tmdb/query.ts b/routes/api/tmdb/query.ts index e158efd..9d9a122 100644 --- a/routes/api/tmdb/query.ts +++ b/routes/api/tmdb/query.ts @@ -1,11 +1,10 @@ -import { HandlerContext, Handlers } from "$fresh/server.ts"; +import { FreshContext, Handlers } from "$fresh/server.ts"; import { searchMovie, searchTVShow } from "@lib/tmdb.ts"; -import * as cache from "@lib/cache/cache.ts"; import { AccessDeniedError, BadRequestError } from "@lib/errors.ts"; const GET = async ( req: Request, - ctx: HandlerContext, + ctx: FreshContext, ) => { const session = ctx.state.session; if (!session) {