max/memorium
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat: completely remove redis

Browse Source
This commit is contained in:
max_richter 2025-01-06 16:14:29 +01:00
parent d3009ac315
commit 53c4d5b129
24 changed files with 629 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docker-compose.yml
View File

@ -1,14 +0,0 @@
version: '3.8'
services:
redis:
image: redis:latest
ports:
- "6379:6379"
volumes:
- ./data/redis-data:/data
volumes:
redis-data:
typesense-data:

2
drizzle.config.ts
View File

@ -9,7 +9,7 @@ const DB_FILE = "file:" + path.resolve(DATA_DIR, "db.sqlite");
export default defineConfig({
out: "./drizzle",
schema: "./lib/sqlite/schema.ts",
schema: "./lib/db/schema.ts",
dialect: "turso",
dbCredentials: {
url: DB_FILE,

12
drizzle/0009_free_robin_chapel.sql Normal file
View File

@ -0,0 +1,12 @@
CREATE TABLE `cache` (
`scope` text NOT NULL,
`key` text PRIMARY KEY NOT NULL,
`json` text,
`binary` blob,
`created_at` integer DEFAULT (current_timestamp),
`expires_at` integer
);
--> statement-breakpoint
CREATE INDEX `key_idx` ON `cache` (`key`);--> statement-breakpoint
CREATE INDEX `scope_idx` ON `cache` (`scope`);--> statement-breakpoint
CREATE INDEX `name_idx` ON `document` (`name`);

309
drizzle/meta/0009_snapshot.json Normal file
View File

@ -0,0 +1,309 @@
{
"version": "6",
"dialect": "sqlite",
"id": "5694a345-e55c-4aa3-9f29-1045b28f5203",
"prevId": "11e0dfc0-1020-46a9-9b26-061a2238a2d5",
"tables": {
"cache": {
"name": "cache",
"columns": {
"scope": {
"name": "scope",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"key": {
"name": "key",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"json": {
"name": "json",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"binary": {
"name": "binary",
"type": "blob",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(current_timestamp)"
},
"expires_at": {
"name": "expires_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"key_idx": {
"name": "key_idx",
"columns": [
"key"
],
"isUnique": false
},
"scope_idx": {
"name": "scope_idx",
"columns": [
"scope"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"document": {
"name": "document",
"columns": {
"name": {
"name": "name",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"last_modified": {
"name": "last_modified",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"content": {
"name": "content",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"content_type": {
"name": "content_type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"size": {
"name": "size",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"perm": {
"name": "perm",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"name_idx": {
"name": "name_idx",
"columns": [
"name"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"image": {
"name": "image",
"columns": {
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(current_timestamp)"
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"average": {
"name": "average",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"blurhash": {
"name": "blurhash",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"mime": {
"name": "mime",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"performance": {
"name": "performance",
"columns": {
"path": {
"name": "path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"search": {
"name": "search",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"time": {
"name": "time",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(STRFTIME('%s', 'now') * 1000)"
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"session": {
"name": "session",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(current_timestamp)"
},
"expires_at": {
"name": "expires_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"user": {
"name": "user",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(current_timestamp)"
},
"email": {
"name": "email",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

7
drizzle/meta/_journal.json
View File

@ -64,6 +64,13 @@
"when": 1736109783318,
"tag": "0008_loud_mephisto",
"breakpoints": true
},
{
"idx": 9,
"version": "6",
"when": 1736172911816,
"tag": "0009_free_robin_chapel",
"breakpoints": true
}
]
}

90
lib/cache.ts Normal file
View File

@ -0,0 +1,90 @@
interface CreateCacheOptions {
expires?: number; // Default expiration time for all cache entries
}
interface SetCacheOptions {
expires?: number; // Override expiration for individual cache entries
}
export function createCache<T>(
createOpts: CreateCacheOptions = {},
) {
const cache = new Map<string, { value: T; expiresAt?: number }>();
return {
get(key: string): T | undefined {
const entry = cache.get(key);
if (!entry) return undefined;
const now = Date.now();
if (entry.expiresAt && entry.expiresAt <= now) {
cache.delete(key); // Remove expired entry
return undefined;
}
return entry.value; // Return value if not expired
},
set(key: string, value: T | unknown, opts: SetCacheOptions = {}) {
console.log("Setting cache", key);
const now = Date.now();
const expiresIn = opts.expires ?? createOpts.expires;
const expiresAt = expiresIn ? now + expiresIn : undefined;
cache.set(key, { value: value as T, expiresAt });
},
cleanup() {
const now = Date.now();
for (const [key, entry] of cache.entries()) {
if (entry.expiresAt && entry.expiresAt <= now) {
cache.delete(key);
}
}
},
info(): { count: number; sizeInKB: number } {
// Cleanup expired entries before calculating info
this.cleanup();
// Count the number of objects in the cache
const count = cache.size;
// Approximate the size in KB by serializing each key and value
let totalBytes = 0;
for (const [key, entry] of cache.entries()) {
const keySize = new TextEncoder().encode(key).length;
const valueSize = new TextEncoder().encode(
JSON.stringify(entry.value),
).length;
totalBytes += keySize + valueSize;
}
const sizeInKB = totalBytes / 1024; // Convert bytes to kilobytes
return { count, sizeInKB };
},
has(key: string): boolean {
const entry = cache.get(key);
if (!entry) return false;
const now = Date.now();
if (entry.expiresAt && entry.expiresAt <= now) {
cache.delete(key); // Remove expired entry
return false;
}
return true;
},
keys(): string[] {
this.cleanup(); // Cleanup before returning keys
return Array.from(cache.keys());
},
size(): number {
this.cleanup(); // Cleanup before returning size
return cache.size;
},
};
}

127
lib/cache/cache.ts vendored
View File

@ -1,127 +0,0 @@
import {
Bulk,
connect,
Redis,
RedisConnectOptions,
RedisValue,
} from "https://deno.land/x/redis@v0.31.0/mod.ts";
import { createLogger } from "@lib/log.ts";
const REDIS_HOST = Deno.env.get("REDIS_HOST");
const REDIS_PASS = Deno.env.get("REDIS_PASS") || "";
const REDIS_PORT = Deno.env.get("REDIS_PORT");
const log = createLogger("cache");
async function createCache<T>(): Promise<Redis> {
if (REDIS_HOST) {
const conf: RedisConnectOptions = {
hostname: REDIS_HOST,
port: REDIS_PORT || 6379,
maxRetryCount: 2,
};
if (REDIS_PASS) {
conf.password = REDIS_PASS;
}
try {
const client = await connect(conf);
log.info("redis connected");
return client;
} catch (_err) {
log.info("cant connect to redis, falling back to mock");
}
}
const mockRedis = new Map<string, RedisValue>();
return {
async keys() {
return mockRedis.keys();
},
async delete(key: string) {
mockRedis.delete(key);
return key;
},
async set(key: string, value: RedisValue) {
mockRedis.set(key, value);
return value.toString();
},
async get(key: string) {
return mockRedis.get(key) as Bulk;
},
};
}
const cache = await createCache();
export async function get<T>(id: string, binary = false) {
if (binary && !(cache instanceof Map)) {
const cacheHit = await cache.sendCommand("GET", [id], {
returnUint8Arrays: true,
}) as T;
return cacheHit;
}
const cacheHit = await cache.get(id) as T;
return cacheHit;
}
export function clearAll() {
if ("flushall" in cache) {
return cache.flushall();
} else {
for (const k of cache.keys()) {
cache.delete(k);
}
}
}
export function expire(id: string, seconds: number) {
if ("expire" in cache) {
return cache.expire(id, seconds);
}
}
type RedisOptions = {
expires?: number;
noLog?: boolean;
};
export function del(key: string) {
return cache.del(key);
}
export function keys(prefix: string) {
return cache.keys(prefix);
}
export function set<T extends RedisValue>(
id: string,
content: T,
options?: RedisOptions,
) {
if (options?.noLog !== true) log.debug("storing ", { id });
return cache.set(id, content, { ex: options?.expires || undefined });
}
export const cacheFunction = async <T extends (() => Promise<unknown>)>(
{
fn,
id,
options = {},
}: {
fn: T;
id: string;
options?: RedisOptions;
},
): Promise<Awaited<ReturnType<T>>> => {
const cacheResult = await get(id) as string;
if (cacheResult) {
return JSON.parse(cacheResult) as Awaited<ReturnType<typeof fn>>;
}
const result = await fn();
set(id, JSON.stringify(result), options);
return result as Awaited<ReturnType<typeof fn>>;
};

37
lib/crud.ts
View File

@ -9,9 +9,10 @@ import { GenericResource } from "@lib/types.ts";
import { parseRating } from "@lib/helpers.ts";
import { isLocalImage } from "@lib/string.ts";
import { SILVERBULLET_SERVER } from "@lib/env.ts";
import { imageTable } from "@lib/sqlite/schema.ts";
import { db } from "@lib/sqlite/sqlite.ts";
import { imageTable } from "@lib/db/schema.ts";
import { db } from "@lib/db/sqlite.ts";
import { eq } from "drizzle-orm/sql";
import { createCache } from "@lib/cache.ts";
export async function addThumbnailToResource<T extends GenericResource>(
res: T,
@ -55,7 +56,9 @@ function sortFunction<T extends GenericResource>(sortType: SortType) {
case "name":
return a.name.localeCompare(b.name);
case "author":
return a.meta?.author?.localeCompare(b.meta?.author || "");
return a.meta?.author?.localeCompare(b.meta?.author || "") || 0;
default:
return 0;
}
};
}
@ -68,21 +71,30 @@ export function createCrud<T extends GenericResource>(
parse: (doc: string, id: string) => T;
},
) {
const cache = createCache<T>({ expires: 60 * 1000 });
function pathFromId(id: string) {
return `${prefix}${id.replaceAll(":", "")}.md`;
}
async function read(id: string) {
const path = pathFromId(id);
const content = await getDocument(path);
const res = parse(content, id);
if (hasThumbnails) {
return addThumbnailToResource(res);
if (cache.has(path)) {
return cache.get(path);
}
return { ...res, content };
const content = await getDocument(path);
const parsed = parse(content, id);
if (hasThumbnails) {
return addThumbnailToResource(parsed);
}
const doc = { ...parsed, content };
cache.set(path, doc);
return doc;
}
function create(id: string, content: string | ArrayBuffer | T) {
const path = pathFromId(id);
@ -107,8 +119,11 @@ export function createCrud<T extends GenericResource>(
}
async function readAll({ sort = "rating" }: { sort?: SortType } = {}) {
if (cache.has("all")) {
return cache.get("all") as unknown as T[];
}
const allDocuments = await getDocuments();
return (await Promise.all(
const parsed = (await Promise.all(
allDocuments.filter((d) => {
return d.name.startsWith(prefix) &&
d.contentType === "text/markdown" &&
@ -118,6 +133,8 @@ export function createCrud<T extends GenericResource>(
return read(id);
}),
)).sort(sortFunction<T>(sort));
cache.set("all", parsed);
return parsed;
}
return {

30
lib/sqlite/schema.ts → lib/db/schema.ts
View File

@ -1,6 +1,12 @@
import { int, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
import {
blob,
index,
int,
integer,
sqliteTable,
text,
} from "drizzle-orm/sqlite-core";
import { sql } from "drizzle-orm/sql";
import { contentType } from "https://deno.land/std@0.216.0/media_types/content_type.ts";
export const userTable = sqliteTable("user", {
id: text()
@ -52,4 +58,24 @@ export const documentTable = sqliteTable("document", {
contentType: text("content_type").notNull(),
size: integer().notNull(),
perm: text().notNull(),
}, (table) => {
return [
index("name_idx").on(table.name),
];
});
export const cacheTable = sqliteTable("cache", {
scope: text().notNull(),
key: text().notNull().primaryKey(),
json: text({ mode: "json" }),
binary: blob(),
createdAt: integer("created_at", { mode: "timestamp" }).default(
sql`(current_timestamp)`,
),
expiresAt: integer("expires_at", { mode: "timestamp" }),
}, (table) => {
return [
index("key_idx").on(table.key),
index("scope_idx").on(table.scope),
];
});

1
lib/sqlite/sqlite.ts → lib/db/sqlite.ts
View File

@ -6,7 +6,6 @@ const DB_FILE = "file:" + path.resolve(DATA_DIR, "db.sqlite");
// You can specify any property from the libsql connection options
export const db = drizzle({
logger: true,
connection: {
url: DB_FILE,
},

4
lib/documents.ts
View File

@ -11,8 +11,8 @@ import remarkFrontmatter, {
import { SILVERBULLET_SERVER } from "@lib/env.ts";
import { fixRenderedMarkdown } from "@lib/helpers.ts";
import { createLogger } from "@lib/log.ts";
import { db } from "@lib/sqlite/sqlite.ts";
import { documentTable } from "@lib/sqlite/schema.ts";
import { db } from "@lib/db/sqlite.ts";
import { documentTable } from "@lib/db/schema.ts";
import { eq } from "drizzle-orm/sql";
export type Document = {

4
lib/image.ts
View File

@ -5,8 +5,8 @@ import { parseMediaType } from "https://deno.land/std@0.224.0/media_types/parse_
import path from "node:path";
import { ensureDir } from "https://deno.land/std@0.216.0/fs/mod.ts";
import { DATA_DIR } from "@lib/env.ts";
import { db } from "@lib/sqlite/sqlite.ts";
import { imageTable } from "@lib/sqlite/schema.ts";
import { db } from "@lib/db/sqlite.ts";
import { imageTable } from "@lib/db/schema.ts";
import { eq } from "drizzle-orm";
import sharp from "npm:sharp@next";

86
lib/openai.ts
View File

@ -1,9 +1,16 @@
import OpenAI from "https://deno.land/x/openai@v4.52.0/mod.ts";
import { OPENAI_API_KEY } from "@lib/env.ts";
import { cacheFunction } from "@lib/cache/cache.ts";
import { hashString } from "@lib/helpers.ts";
import { createCache } from "@lib/cache.ts";
const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY);
const openAI = OPENAI_API_KEY && new OpenAI({ apiKey: OPENAI_API_KEY });
interface MovieRecommendation {
year: number;
title: string;
}
const cache = createCache<MovieRecommendation[]>();
function extractListFromResponse(response?: string): string[] {
if (!response) return [];
@ -138,52 +145,51 @@ return a list of around 20 keywords seperated by commas
.map((v) => v.replaceAll(" ", "-"));
}
export const getMovieRecommendations = (keywords: string, exclude: string[]) =>
cacheFunction({
fn: async () => {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
messages: [
{
role: "user",
content:
`Could you recommend me 10 movies based on the following attributes:
export const getMovieRecommendations = async (
keywords: string,
exclude: string[],
) => {
if (!openAI) return;
const cacheId = `movierecs:${hashString(`${keywords}:${exclude.join()}`)}`;
if (cache.has(cacheId)) return cache.get(cacheId);
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
messages: [
{
role: "user",
content:
`Could you recommend me 10 movies based on the following attributes:
${keywords}
The movies should be similar to but not include ${
exclude.join(", ")
} or remakes of that.
exclude.join(", ")
} or remakes of that.
respond with a plain unordered list each item starting with the year the movie was released and then the title of the movie seperated by a -`,
},
],
});
const res = chatCompletion.choices[0].message.content?.toLowerCase();
if (!res) return;
console.log("REsult:");
console.log(res);
const list = extractListFromResponse(res);
console.log({ list });
return res.split("\n").map((entry) => {
const [year, ...title] = entry.split("-");
return {
year: parseInt(year.trim()),
title: title.join(" ").replaceAll('"', "").trim(),
};
}).filter((y) => !Number.isNaN(y.year));
},
id: `openai:movierecs:${hashString(`${keywords}:${exclude.join()}`)}`,
},
],
});
const res = chatCompletion.choices[0].message.content?.toLowerCase();
if (!res) return;
const recommendations = res.split("\n").map((entry) => {
const [year, ...title] = entry.split("-");
return {
year: parseInt(year.trim()),
title: title.join(" ").replaceAll('"', "").trim(),
};
}).filter((y) => !Number.isNaN(y.year));
cache.set(cacheId, recommendations);
return recommendations;
};
export async function createTags(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({

4
lib/performance.ts
View File

@ -1,5 +1,5 @@
import { db } from "@lib/sqlite/sqlite.ts";
import { performanceTable } from "@lib/sqlite/schema.ts";
import { db } from "@lib/db/sqlite.ts";
import { performanceTable } from "@lib/db/schema.ts";
import { between } from "drizzle-orm/sql";
export type PerformancePoint = {

28
lib/recommendation.ts
View File

@ -1,8 +1,8 @@
import * as cache from "@lib/cache/cache.ts";
import * as openai from "@lib/openai.ts";
import * as tmdb from "@lib/tmdb.ts";
import { GenericResource } from "@lib/types.ts";
import { parseRating } from "@lib/helpers.ts";
import { createCache } from "@lib/cache.ts";
type RecommendationResource = {
id: string;
@ -15,12 +15,14 @@ type RecommendationResource = {
year?: number;
};
const cache = createCache<RecommendationResource>();
export async function createRecommendationResource(
res: GenericResource,
description?: string,
) {
const cacheId = `recommendations:${res.type}:${res.id.replaceAll(":", "")}`;
const resource: RecommendationResource = await cache.get(cacheId) || {
const cacheId = `${res.type}:${res.id.replaceAll(":", "")}`;
const resource = cache.get(cacheId) || {
id: res.id,
type: res.type,
rating: -1,
@ -58,20 +60,15 @@ export async function createRecommendationResource(
cache.set(cacheId, JSON.stringify(resource));
}
export async function getRecommendation(
export function getRecommendation(
id: string,
type: string,
): Promise<RecommendationResource | null> {
const res = await cache.get(`recommendations:${type}:${id}`) as string;
try {
return JSON.parse(res);
} catch (_) {
return null;
}
): RecommendationResource | undefined {
return cache.get(`recommendations:${type}:${id}`);
}
export async function getSimilarMovies(id: string) {
const recs = await getRecommendation(id, "movie");
const recs = getRecommendation(id, "movie");
if (!recs?.keywords?.length) return;
const recommendations = await openai.getMovieRecommendations(
@ -91,8 +88,7 @@ export async function getSimilarMovies(id: string) {
export async function getAllRecommendations(): Promise<
RecommendationResource[]
> {
const keys = await cache.keys("recommendations:movie:*");
return Promise.all(keys.map((k) => cache.get(k))).then((res) =>
res.map((r) => JSON.parse(r))
);
const keys = cache.keys("recommendations:movie:*");
const res = await Promise.all(keys.map((k) => cache.get(k)));
return res.map((r) => JSON.parse(r));
}

111
lib/tmdb.ts
View File

@ -1,74 +1,69 @@
import * as cache from "@lib/cache/cache.ts";
import { MovieDb } from "https://esm.sh/moviedb-promise@3.4.1";
import {
CreditsResponse,
MovieDb,
MovieResponse,
MovieResultsResponse,
ShowResponse,
TvResultsResponse,
} from "https://esm.sh/moviedb-promise@3.4.1";
import { createCache } from "@lib/cache.ts";
const moviedb = new MovieDb(Deno.env.get("TMDB_API_KEY") || "");
const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
const cache = createCache({ expires: CACHE_INTERVAL });
export const searchMovie = (query: string, year?: number) =>
cache.cacheFunction({
fn: () => moviedb.searchMovie({ query, year }),
id: `query:moviesearch:${query}${year ? `-${year}` : ""}`,
options: {
expires: CACHE_INTERVAL,
},
});
export const searchMovie = async (query: string, year?: number) => {
const id = `query:moviesearch:${query}${year ? `-${year}` : ""}`;
if (cache.has(id)) return cache.get(id) as MovieResultsResponse;
const res = await moviedb.searchMovie({ query, year });
cache.set(id, res);
return res;
};
export const searchTVShow = (query: string) =>
cache.cacheFunction(
{
fn: () => moviedb.searchTv({ query }),
id: `query:tvshowsearch:${query}`,
options: {
expires: CACHE_INTERVAL,
},
},
);
export const searchTVShow = async (query: string) => {
const id = `query:tvshowsearch:${query}`;
if (cache.has(id)) return cache.get(id) as TvResultsResponse;
const res = await moviedb.searchTv({ query });
cache.set(id, res);
return res;
};
export const getMovie = (id: number) =>
cache.cacheFunction({
fn: () => moviedb.movieInfo({ id }),
id: `query:movie:${id}`,
options: {
expires: CACHE_INTERVAL,
},
});
export const getMovie = async (id: number) => {
const cacheId = `query:movie:${id}`;
if (cache.has(cacheId)) return cache.get(cacheId) as MovieResponse;
const res = await moviedb.movieInfo({ id });
cache.set(cacheId, res);
return res;
};
export const getSeries = (id: number) =>
cache.cacheFunction({
fn: () => moviedb.tvInfo({ id }),
id: `query:tvshow:${id}`,
options: {
expires: CACHE_INTERVAL,
},
});
export const getSeries = async (id: number) => {
const cacheId = `query:tvshow:${id}`;
if (cache.has(cacheId)) return cache.get(cacheId) as ShowResponse;
const res = await moviedb.tvInfo({ id });
cache.set(cacheId, res);
return res;
};
export const getMovieCredits = (id: number) =>
cache.cacheFunction({
fn: () => moviedb.movieCredits(id),
id: `query:moviecredits:${id}`,
options: {
expires: CACHE_INTERVAL,
},
});
export const getMovieCredits = async (id: number) => {
const cacheId = `query:moviecredits:${id}`;
if (cache.has(cacheId)) return cache.get(cacheId) as CreditsResponse;
const res = await moviedb.movieCredits(id);
cache.set(cacheId, res);
return res;
};
export const getSeriesCredits = (id: number) =>
cache.cacheFunction({
fn: () => moviedb.tvCredits(id),
id: `query:tvshowcredits:${id}`,
options: {
expires: CACHE_INTERVAL,
},
});
export const getSeriesCredits = async (id: number) => {
const cacheId = `query:tvshowcredits:${id}`;
if (cache.has(cacheId)) return cache.get(cacheId) as CreditsResponse;
const res = await moviedb.tvCredits(id);
cache.set(cacheId, res);
return res;
};
export async function getMovieGenre(id: number) {
const genres = await cache.get("/genres/movies");
export function getMovieGenre() {
return moviedb.genreTvList();
}
export async function getSeriesGenre(id: number) {
const genres = await cache.get("/genres/series");
}
export async function getMoviePoster(id: string): Promise<ArrayBuffer> {
const posterUrl = `https://image.tmdb.org/t/p/original/${id}`;
const response = await fetch(posterUrl);

4
routes/api/auth/callback.ts
View File

@ -6,8 +6,8 @@ import { codeChallengeMap } from "./login.ts";
import { GITEA_SERVER, JWT_SECRET, SESSION_DURATION } from "@lib/env.ts";
import { GiteaOauthUser } from "@lib/types.ts";
import { BadRequestError } from "@lib/errors.ts";
import { db } from "@lib/sqlite/sqlite.ts";
import { userTable } from "@lib/sqlite/schema.ts";
import { db } from "@lib/db/sqlite.ts";
import { userTable } from "@lib/db/schema.ts";
import { eq } from "drizzle-orm";
export const handler: Handlers = {

24
routes/api/movies/enhance/[name].ts
View File

@ -1,4 +1,4 @@
import { HandlerContext, Handlers } from "$fresh/server.ts";
import { FreshContext, Handlers } from "$fresh/server.ts";
import { createDocument } from "@lib/documents.ts";
import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
import { createMovie, getMovie } from "@lib/resource/movies.ts";
@ -10,12 +10,11 @@ import {
BadRequestError,
NotFoundError,
} from "@lib/errors.ts";
import * as cache from "@lib/cache/cache.ts";
import { createRecommendationResource } from "@lib/recommendation.ts";
const POST = async (
req: Request,
ctx: HandlerContext,
ctx: FreshContext,
): Promise<Response> => {
const session = ctx.state.session;
if (!session) {
@ -35,25 +34,27 @@ const POST = async (
}
const movieDetails = await tmdb.getMovie(tmdbId);
const movieCredits = !movie.meta.author &&
const movieCredits = !movie.meta?.author &&
await tmdb.getMovieCredits(tmdbId);
const releaseDate = movieDetails.release_date;
if (releaseDate && !movie.meta.date) {
if (releaseDate && !movie.meta?.date) {
movie.meta = movie.meta || {};
movie.meta.date = new Date(releaseDate);
}
const director = movieCredits?.crew?.filter?.((person) =>
person.job === "Director"
)[0];
if (director && !movie.meta.author) {
if (director && !movie.meta?.author) {
movie.meta = movie.meta || {};
movie.meta.author = director.name;
}
if (movieDetails.genres) {
movie.tags = [
...new Set([
...movie.tags.map((g) => g.toLowerCase()),
...(movie.tags?.map((g) => g.toLowerCase()) || []),
...movieDetails.genres.map((g) =>
g.name?.toLowerCase().replaceAll(" ", "-")
),
@ -61,25 +62,24 @@ const POST = async (
];
}
if (!movie.meta.tmdbId) {
movie.meta.tmdbId = tmdbId;
if (!movie.id) {
movie.id = tmdbId;
}
let finalPath = "";
const posterPath = movieDetails.poster_path;
if (posterPath && !movie.meta.image) {
if (posterPath && !movie.meta?.image) {
const poster = await tmdb.getMoviePoster(posterPath);
const extension = fileExtension(posterPath);
finalPath = `Media/movies/images/${safeFileName(name)}_cover.${extension}`;
await createDocument(finalPath, poster);
movie.meta = movie.meta || {};
movie.meta.image = finalPath;
}
await createMovie(movie.id, movie);
cache.del(`documents:Media:movies:${name}.md`);
createRecommendationResource(movie, movieDetails.overview);
return json(movie);

7
routes/api/recommendation/all.ts
View File

@ -1,6 +1,6 @@
import { Handlers } from "$fresh/server.ts";
import { createStreamResponse } from "@lib/helpers.ts";
import { getAllMovies } from "@lib/resource/movies.ts";
import { getAllMovies, Movie } from "@lib/resource/movies.ts";
import * as tmdb from "@lib/tmdb.ts";
import {
createRecommendationResource,
@ -14,10 +14,11 @@ async function processUpdateRecommendations(
const allMovies = await getAllMovies();
const movies = allMovies.filter((m) => {
if (!m?.meta) return false;
if (!m.meta.rating) return false;
if (!m.meta.tmdbId) return false;
return true;
});
}) as Movie[];
streamResponse.enqueue("Fetched all movies");
@ -27,7 +28,7 @@ async function processUpdateRecommendations(
await Promise.all(movies.map(async (movie) => {
if (!movie.meta.tmdbId) return;
if (!movie.meta.rating) return;
const recommendation = await getRecommendation(movie.id, movie.type);
const recommendation = getRecommendation(movie.id, movie.type);
if (recommendation) {
done++;
return;

1
routes/api/recommendation/movie/[id].ts
View File

@ -11,6 +11,7 @@ export const handler: Handlers = {
}
const recommendations = await getSimilarMovies(ctx.params.id);
console.log({ recommendations });
return json(recommendations);
},

19
routes/api/series/enhance/[name].ts
View File

@ -1,4 +1,4 @@
import { HandlerContext, Handlers } from "$fresh/server.ts";
import { FreshContext, Handlers } from "$fresh/server.ts";
import { createDocument } from "@lib/documents.ts";
import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
import * as tmdb from "@lib/tmdb.ts";
@ -10,7 +10,6 @@ import {
NotFoundError,
} from "@lib/errors.ts";
import { createSeries, getSeries } from "@lib/resource/series.ts";
import * as cache from "@lib/cache/cache.ts";
const isString = (input: string | undefined): input is string => {
return typeof input === "string";
@ -18,7 +17,7 @@ const isString = (input: string | undefined): input is string => {
const POST = async (
req: Request,
ctx: HandlerContext,
ctx: FreshContext,
): Promise<Response> => {
const session = ctx.state.session;
if (!session) {
@ -38,43 +37,43 @@ const POST = async (
}
const seriesDetails = await tmdb.getSeries(tmdbId);
const seriesCredits = !series.meta.author &&
const seriesCredits = !series.meta?.author &&
await tmdb.getSeriesCredits(tmdbId);
const releaseDate = seriesDetails.first_air_date;
if (releaseDate && series.meta.date) {
if (releaseDate && series.meta?.date) {
series.meta.date = new Date(releaseDate);
}
const posterPath = seriesDetails.poster_path;
const director = seriesCredits &&
seriesCredits.crew?.filter?.((person) => person.job === "Director")[0] ||
seriesDetails?.created_by?.[0];
if (director && director.name && !series.meta.author) {
if (director && director.name && !series.meta?.author) {
series.meta = series.meta || {};
series.meta.author = director.name;
}
if (seriesDetails.genres) {
series.tags = [
...new Set([
...series.tags.map((t) => t.toLowerCase()),
...(series.tags?.map((t) => t.toLowerCase()) || []),
...seriesDetails.genres.map((g) => g.name?.toLowerCase()),
].filter(isString)),
];
}
let finalPath = "";
if (posterPath && !series.meta.image) {
if (posterPath && !series.meta?.image) {
const poster = await tmdb.getMoviePoster(posterPath);
const extension = fileExtension(posterPath);
finalPath = `Media/series/images/${safeFileName(name)}_cover.${extension}`;
await createDocument(finalPath, poster);
series.meta = series.meta || {};
series.meta.image = finalPath;
}
await createSeries(series.id, series);
cache.del(`documents:Media:series:${name}.md`);
return json(series);
};

5
routes/api/tmdb/[id].ts
View File

@ -1,7 +1,7 @@
import { FreshContext, Handlers } from "$fresh/server.ts";
import { getMovie } from "@lib/tmdb.ts";
import * as cache from "@lib/cache/cache.ts";
import { json } from "@lib/helpers.ts";
import { createCache } from "@lib/cache.ts";
type CachedMovieCredits = {
lastUpdated: number;
@ -9,6 +9,7 @@ type CachedMovieCredits = {
};
const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
const cache = createCache<CachedMovieCredits>({ expires: CACHE_INTERVAL });
const GET = async (
_req: Request,
@ -24,7 +25,7 @@ const GET = async (
const cacheId = `/movie/${id}`;
const cachedResponse = await cache.get<CachedMovieCredits>(cacheId);
const cachedResponse = cache.get(cacheId);
if (
cachedResponse && Date.now() < (cachedResponse.lastUpdated + CACHE_INTERVAL)
) {

9
routes/api/tmdb/credits/[id].ts
View File

@ -1,8 +1,8 @@
import { HandlerContext } from "$fresh/server.ts";
import { FreshContext } from "$fresh/server.ts";
import { getMovieCredits } from "@lib/tmdb.ts";
import * as cache from "@lib/cache/cache.ts";
import { json } from "@lib/helpers.ts";
import { createLogger } from "@lib/log.ts";
import { createCache } from "@lib/cache.ts";
type CachedMovieCredits = {
lastUpdated: number;
@ -10,12 +10,13 @@ type CachedMovieCredits = {
};
const CACHE_INTERVAL = 1000 * 60 * 24 * 30;
const cache = createCache<CachedMovieCredits>({ expires: CACHE_INTERVAL });
const log = createLogger("api/tmdb");
export const handler = async (
_req: Request,
_ctx: HandlerContext,
_ctx: FreshContext,
) => {
const id = _ctx.params.id;
@ -29,7 +30,7 @@ export const handler = async (
const cacheId = `/movie/credits/${id}`;
const cachedResponse = await cache.get<CachedMovieCredits>(cacheId);
const cachedResponse = cache.get(cacheId);
if (
cachedResponse && Date.now() < (cachedResponse.lastUpdated + CACHE_INTERVAL)
) {

5
routes/api/tmdb/query.ts
View File

@ -1,11 +1,10 @@
import { HandlerContext, Handlers } from "$fresh/server.ts";
import { FreshContext, Handlers } from "$fresh/server.ts";
import { searchMovie, searchTVShow } from "@lib/tmdb.ts";
import * as cache from "@lib/cache/cache.ts";
import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
const GET = async (
req: Request,
ctx: HandlerContext,
ctx: FreshContext,
) => {
const session = ctx.state.session;
if (!session) {