feat: refactor whole bunch of stuff

lib/db/schema.ts

@@ -47,7 +47,7 @@ export const imageTable = sqliteTable("image", {
   ),
   url: text().notNull(),
   average: text().notNull(),
-  blurhash: text().notNull(),
+  thumbhash: text().notNull(),
   mime: text().notNull(),
 });
 

lib/helpers.ts

@@ -104,12 +104,13 @@ export function debounce<T extends (...args: Parameters<T>) => void>(
 export function parseRating(rating: string | number) {
   if (typeof rating === "string") {
     try {
-      return parseInt(rating);
+      const res = parseInt(rating);
+      if (!Number.isNaN(res)) return res;
     } catch (_e) {
       // This is okay
     }
 
-    return [...rating.matchAll(/⭐/g)].length;
+    return rating.length / 2;
   }
   return rating;
 }

lib/image.ts

@@ -3,7 +3,7 @@ import { createLogger } from "@lib/log/index.ts";
 import { generateThumbhash } from "@lib/thumbhash.ts";
 import { parseMediaType } from "https://deno.land/std@0.224.0/media_types/parse_media_type.ts";
 import path from "node:path";
-import { ensureDir } from "fs";
+import { mkdir } from "node:fs/promises";
 import { DATA_DIR } from "@lib/env.ts";
 import { db } from "@lib/db/sqlite.ts";
 import { imageTable } from "@lib/db/schema.ts";
@@ -13,7 +13,7 @@ import sharp from "npm:sharp@next";
 const log = createLogger("cache/image");
 
 const imageDir = path.join(DATA_DIR, "images");
-await ensureDir(imageDir);
+await mkdir(imageDir, { recursive: true });
 
 async function getRemoteImage(imageUrl: string) {
   try {
@@ -100,7 +100,7 @@ async function getLocalImagePath(
     hostname,
     pathname.split("/").filter((s) => s.length).join("-"),
   );
-  await ensureDir(imagePath);
+  await mkdir(imagePath, { recursive: true });
 
   if (width || height) {
     imagePath = path.join(imagePath, `${width ?? "-"}x${height ?? "-"}`);
@@ -313,7 +313,7 @@ export async function getImage(url: string) {
     // Store in database
     const [newImage] = await db.insert(imageTable).values({
       url: url,
-      blurhash: thumbhash.hash,
+      thumbhash: thumbhash.hash,
       average: thumbhash.average,
       mime: imageContent.mediaType,
     }).returning();

lib/log/constants.ts

@@ -1,12 +1,12 @@
 import * as env from "@lib/env.ts";
-import { ensureDir } from "fs";
 import { join } from "node:path";
 import { getLogLevel, LOG_LEVEL } from "@lib/log/types.ts";
+import { mkdir } from "node:fs/promises";
 
 export const LOG_DIR = join(env.DATA_DIR, "logs");
 
 // Ensure the log directory exists
-await ensureDir(LOG_DIR);
+await mkdir(LOG_DIR, { recursive: true });
 
 export let logLevel = getLogLevel(env.LOG_LEVEL);
 export function setLogLevel(level: LOG_LEVEL) {

lib/marka.ts

@@ -1,35 +0,0 @@
-import { MARKA_API_KEY } from "./env.ts";
-const url = `https://marka.max-richter.dev/resources`;
-//const url = "http://localhost:8080/resources";
-
-export async function fetchResource(resource: string) {
-  try {
-    const response = await fetch(
-      `${url}/${resource}`,
-    );
-    return response.json();
-  } catch (_e) {
-    return [];
-  }
-}
-
-export async function createResource(
-  path: string,
-  content: string | object | ArrayBuffer,
-) {
-  const isJson = typeof content === "object" &&
-    !(content instanceof ArrayBuffer);
-  const fetchUrl = `${url}/${path}`;
-  const response = await fetch(fetchUrl, {
-    method: "POST",
-    headers: {
-      "Content-Type": isJson ? "application/json" : "",
-      "Authentication": MARKA_API_KEY,
-    },
-    body: isJson ? JSON.stringify(content) : content,
-  });
-  if (!response.ok) {
-    throw new Error(`Failed to create resource: ${response.status}`);
-  }
-  return response.json();
-}

lib/marka/index.ts

@@ -0,0 +1,80 @@
+import { MARKA_API_KEY } from "../env.ts";
+import { getImage } from "../image.ts";
+import { GenericResource } from "./schema.ts";
+
+//const url = `https://marka.max-richter.dev`;
+const url = "http://localhost:8080";
+
+async function addImageToResource<T extends GenericResource>(
+  resource: GenericResource,
+): Promise<T> {
+  const imageUrl = resource?.content?.image;
+  if (imageUrl) {
+    try {
+      const absoluteImageUrl = (imageUrl.startsWith("https://") ||
+        imageUrl.startsWith("http://"))
+        ? imageUrl
+        : `${url}/${imageUrl}`;
+      const image = await getImage(absoluteImageUrl);
+      return { ...resource, image } as T;
+    } catch (e) {
+      console.log(`Failed to fetch image: ${imageUrl}`, e);
+    }
+  }
+  return resource as T;
+}
+
+export async function fetchResource<T extends GenericResource>(
+  resource: string,
+): Promise<T | undefined> {
+  try {
+    const response = await fetch(
+      `${url}/resources/${resource}`,
+    );
+    const res = await response.json();
+    return addImageToResource<T>(res);
+  } catch (_e) {
+    return;
+  }
+}
+
+export async function listResources<T = GenericResource>(
+  resource: string,
+): Promise<T[]> {
+  try {
+    const response = await fetch(
+      `${url}/resources/${resource}`,
+    );
+    const list = await response.json();
+    return Promise.all(
+      list?.content
+        .filter((a: GenericResource) => a?.content?._type)
+        .map((res: GenericResource) => addImageToResource(res)),
+    );
+  } catch (_e) {
+    return [];
+  }
+}
+
+export async function createResource(
+  path: string,
+  content: string | object | ArrayBuffer,
+) {
+  const isJson = typeof content === "object" &&
+    !(content instanceof ArrayBuffer);
+  const fetchUrl = `${url}/resources/${path}`;
+  const headers = new Headers();
+  headers.append("Content-Type", isJson ? "application/json" : "");
+  if (MARKA_API_KEY) {
+    headers.append("Authentication", MARKA_API_KEY);
+  }
+  const response = await fetch(fetchUrl, {
+    method: "POST",
+    headers,
+    body: isJson ? JSON.stringify(content) : content,
+  });
+  if (!response.ok) {
+    throw new Error(`Failed to create resource: ${response.status}`);
+  }
+  return response.json();
+}

lib/marka/schema.ts

@@ -0,0 +1,126 @@
+import { z } from "zod";
+import { imageTable } from "../db/schema.ts";
+
+export const PersonSchema = z.object({
+  _type: z.literal("Person"),
+  name: z.string().optional(),
+});
+
+export const ReviewRatingSchema = z.object({
+  bestRating: z.number().optional(),
+  worstRating: z.number().optional(),
+  // Accept number or string (e.g., "⭐️⭐️⭐️⭐️⭐️")
+  ratingValue: z.union([z.number(), z.string()]).optional(),
+});
+
+const WithAuthor = z.object({ author: PersonSchema.optional() });
+const WithKeywords = z.object({ keywords: z.array(z.string()).optional() });
+const WithImage = z.object({ image: z.string().optional() });
+const WithDatePublished = z.object({ datePublished: z.string().optional() });
+
+const BaseContent = WithAuthor.merge(WithKeywords)
+  .merge(WithImage)
+  .merge(WithDatePublished);
+
+export const BaseFileSchema = z.object({
+  type: z.literal("file"),
+  name: z.string(),
+  path: z.string(),
+  modTime: z.string(), // ISO timestamp string
+  mime: z.string(),
+  size: z.number().int().nonnegative(),
+});
+
+const makeContentSchema = <
+  TName extends "Article" | "Review" | "Recipe",
+  TShape extends z.ZodRawShape,
+>(
+  name: TName,
+  shape: TShape,
+) =>
+  z
+    .object({
+      _type: z.literal(name),
+      keywords: z.array(z.string()).optional(),
+    })
+    .merge(BaseContent)
+    .extend(shape);
+
+export const ArticleContentSchema = makeContentSchema("Article", {
+  headline: z.string().optional(),
+  articleBody: z.string().optional(),
+  url: z.string().optional(),
+  reviewRating: ReviewRatingSchema.optional(),
+});
+
+export const ReviewContentSchema = makeContentSchema("Review", {
+  tmdbId: z.number().optional(),
+  link: z.string().optional(),
+  reviewRating: ReviewRatingSchema.optional(),
+  reviewBody: z.string().optional(),
+  itemReviewed: z
+    .object({
+      name: z.string().optional(),
+    })
+    .optional(),
+});
+
+export const RecipeContentSchema = makeContentSchema("Recipe", {
+  description: z.string().optional(),
+  name: z.string().optional(),
+  recipeIngredient: z.array(z.string()).optional(),
+  recipeInstructions: z.array(z.string()).optional(),
+  totalTime: z.string().optional(),
+  recipeYield: z.number().optional(),
+  url: z.string().optional(),
+});
+
+export const articleMetadataSchema = z.object({
+  headline: z.union([z.null(), z.string()]).describe("Headline of the article"),
+  author: z.union([z.null(), z.string()]).describe("Author of the article"),
+  datePublished: z.union([z.null(), z.string()]).describe(
+    "Date the article was published",
+  ),
+  keywords: z.union([z.null(), z.array(z.string())]).describe(
+    "Keywords for the article",
+  ),
+});
+
+export const ArticleSchema = BaseFileSchema.extend({
+  content: ArticleContentSchema,
+});
+
+export const ReviewSchema = BaseFileSchema.extend({
+  content: ReviewContentSchema,
+});
+
+export const RecipeSchema = BaseFileSchema.extend({
+  content: RecipeContentSchema,
+});
+
+export const GenericResourceSchema = z.union([
+  ArticleSchema,
+  ReviewSchema,
+  RecipeSchema,
+]);
+
+export type Person = z.infer<typeof PersonSchema>;
+export type ReviewRating = z.infer<typeof ReviewRatingSchema>;
+
+export type BaseFile = z.infer<typeof BaseFileSchema>;
+
+export type ArticleResource = z.infer<typeof ArticleSchema> & {
+  image?: typeof imageTable.$inferSelect;
+};
+
+export type ReviewResource = z.infer<typeof ReviewSchema> & {
+  image?: typeof imageTable.$inferSelect;
+};
+
+export type RecipeResource = z.infer<typeof RecipeSchema> & {
+  image?: typeof imageTable.$inferSelect;
+};
+
+export type GenericResource = z.infer<typeof GenericResourceSchema> & {
+  image?: typeof imageTable.$inferSelect;
+};

lib/markdown.ts

@@ -44,11 +44,3 @@ export function renderMarkdown(doc: string) {
     allowMath: true,
   });
 }
-
-export function createDocument(
-  path: string,
-  entry: string,
-  mimetype = "image/jpeg",
-) {
-  console.log("creating", { path, entry, mimetype });
-}

lib/openai.ts

@@ -4,7 +4,7 @@ import { OPENAI_API_KEY } from "@lib/env.ts";
 import { hashString } from "@lib/helpers.ts";
 import { createCache } from "@lib/cache.ts";
 import { recipeResponseSchema } from "@lib/recipeSchema.ts";
-import { articleMetadataSchema } from "./resource/articles.ts";
+import { articleMetadataSchema } from "./marka/schema.ts";
 
 const openAI = OPENAI_API_KEY && new OpenAI({ apiKey: OPENAI_API_KEY });
 

lib/recipeSchema.ts

@@ -1,4 +1,5 @@
 import { z } from "npm:zod";
+import { RecipeResource } from "./marka/schema.ts";
 
 export const IngredientSchema = z.object({
   quantity: z.string().describe(
@@ -17,7 +18,6 @@ export const IngredientGroupSchema = z.object({
 export type IngredientGroup = z.infer<typeof IngredientGroupSchema>;
 
 const recipeSchema = z.object({
-  _type: z.literal("Recipe"),
   name: z.string().describe(
     "Title of the Recipe, without the name of the website or author",
   ),
@@ -29,6 +29,9 @@ const recipeSchema = z.object({
     _type: z.literal("Person"),
     name: z.string().describe("author of the Recipe (optional)"),
   }),
+  keywords: z.array(z.string()).describe(
+    "List of keywords that match the recipe",
+  ),
   recipeIngredient: z.array(z.string())
     .describe("List of ingredients"),
   recipeInstructions: z.array(z.string()).describe("List of instructions"),
@@ -48,7 +51,7 @@ export const recipeResponseSchema = z.union([recipeSchema, noRecipeSchema]);
 
 export function isValidRecipe(
   recipe:
-    | Recipe
+    | RecipeResource
     | null
     | undefined,
 ) {

lib/recommendation.ts

@@ -1,8 +1,8 @@
 import * as openai from "@lib/openai.ts";
 import * as tmdb from "@lib/tmdb.ts";
-import { GenericResource } from "@lib/types.ts";
 import { parseRating } from "@lib/helpers.ts";
 import { createCache } from "@lib/cache.ts";
+import { GenericResource, ReviewResource } from "./marka/schema.ts";
 
 type RecommendationResource = {
   id: string;
@@ -18,42 +18,48 @@ type RecommendationResource = {
 const cache = createCache<RecommendationResource>("recommendations");
 
 export async function createRecommendationResource(
-  res: GenericResource,
+  res: ReviewResource,
   description?: string,
 ) {
-  const cacheId = `${res.type}:${res.id.replaceAll(":", "")}`;
+  const cacheId = `${res.type}:${res.name.replaceAll(":", "")}`;
   const resource = cache.get(cacheId) || {
-    id: res.id,
+    id: res.name,
     type: res.type,
     rating: -1,
   };
   if (description && !resource.keywords) {
-    const keywords = await openai.createKeywords(res.type, description, res.id);
+    const keywords = await openai.createKeywords(
+      res.type,
+      description,
+      res.name,
+    );
     if (keywords?.length) {
       resource.keywords = keywords;
     }
   }
 
-  const { author, date, rating } = res.meta || {};
+  const { author, datePublished, reviewRating } = res.content;
 
-  if (res?.tags) {
-    resource.tags = res.tags;
+  if (res?.content?.keywords) {
+    resource.keywords = res.content.keywords;
   }
 
-  if (typeof rating !== "undefined") {
-    resource.rating = parseRating(rating);
+  if (typeof reviewRating?.ratingValue !== "undefined") {
+    resource.rating = parseRating(reviewRating?.ratingValue);
   }
 
-  if (author) {
-    resource.author = author;
+  if (author?.name) {
+    resource.author = author.name;
   }
 
   if (description) {
     resource.description = description;
   }
 
-  if (date) {
-    const d = typeof date === "string" ? new Date(date) : date;
+  if (datePublished) {
+    const d = typeof datePublished === "string"
+      ? new Date(datePublished)
+      : datePublished;
     resource.year = d.getFullYear();
   }
 

lib/resource/articles.ts

@@ -1,30 +0,0 @@
-import { z } from "zod";
-export type Article = {
-  _type: "Article";
-  headline?: string;
-  datePublished?: string;
-  articleBody?: string;
-  keywords?: string[];
-  image?: string;
-  url?: string;
-  reviewRating?: {
-    bestRating?: number;
-    worstRating?: number;
-    ratingValue?: number;
-  };
-  author?: {
-    _type: "Person";
-    name?: string;
-  };
-};
-
-export const articleMetadataSchema = z.object({
-  headline: z.union([z.null(), z.string()]).describe("Headline of the article"),
-  author: z.union([z.null(), z.string()]).describe("Author of the article"),
-  datePublished: z.union([z.null(), z.string()]).describe(
-    "Date the article was published",
-  ),
-  keywords: z.union([z.null(), z.array(z.string())]).describe(
-    "Keywords for the article",
-  ),
-});

lib/resource/movies.ts

@@ -1,21 +0,0 @@
-export type Movie = {
-  _type: "Review";
-  tmdbId?: number;
-  link?: string;
-  author?: {
-    _type: "Person";
-    name?: string;
-  };
-  datePublished?: string;
-  reviewRating?: {
-    bestRating?: number;
-    worstRating?: number;
-    ratingValue?: number;
-  };
-  reviewBody?: string;
-  itemReviewed?: {
-    name?: string;
-  };
-  keywords?: string[];
-  image?: string;
-};

lib/resource/recipes.ts

@@ -1,17 +0,0 @@
-export type Recipe = {
-  _type: "Recipe";
-  author?: {
-    _type: "Person";
-    name?: string;
-  };
-  description?: string;
-  image?: string;
-  name?: string;
-  recipeIngredient?: string[];
-  recipeInstructions?: string[];
-  datePublished?: string;
-  totalTime?: string;
-  recipeYield?: number;
-  url?: string;
-  keywords?: string[];
-};

lib/resource/series.ts

@@ -1,3 +0,0 @@
-import { Movie } from "./movies.ts";
-
-export type Series = Movie;

lib/search.ts

@@ -1,12 +1,8 @@
 import { resources } from "@lib/resources.ts";
 import fuzzysort from "npm:fuzzysort";
-import { GenericResource } from "@lib/types.ts";
 import { extractHashTags } from "@lib/string.ts";
-import { Movie } from "@lib/resource/movies.ts";
-import { Article } from "@lib/resource/articles.ts";
-import { Recipe } from "@lib/resource/recipes.ts";
-import { Series } from "@lib/resource/series.ts";
-import { fetchResource } from "./marka.ts";
+import { listResources } from "./marka/index.ts";
+import { GenericResource } from "./marka/schema.ts";
 
 type ResourceType = keyof typeof resources;
 
@@ -48,8 +44,8 @@ export function parseResourceUrl(_url: string | URL): SearchParams | undefined {
 }
 
 const isResource = (
-  item: Movie | Series | Article | Recipe | boolean,
-): item is Movie | Series | Article | Recipe => {
+  item: GenericResource | boolean | undefined,
+): item is GenericResource => {
   return !!item;
 };
 
@@ -57,10 +53,10 @@ export async function searchResource(
   { q, tags = [], types, rating }: SearchParams,
 ): Promise<GenericResource[]> {
   const resources = (await Promise.all([
-    (!types || types.includes("movie")) && fetchResource("movies"),
-    (!types || types.includes("series")) && fetchResource("series"),
-    (!types || types.includes("article")) && fetchResource("articles"),
-    (!types || types.includes("recipe")) && fetchResource("recipes"),
+    (!types || types.includes("movie")) && listResources("movies"),
+    (!types || types.includes("series")) && listResources("series"),
+    (!types || types.includes("article")) && listResources("articles"),
+    (!types || types.includes("recipe")) && listResources("recipes"),
   ])).flat().filter(isResource);
 
   const results: Record<string, GenericResource> = {};
@@ -68,27 +64,35 @@ export async function searchResource(
   for (const resource of resources) {
     if (
       !(resource.name in results) &&
-      tags?.length && resource.tags.length &&
-      tags.every((t) => resource.tags.includes(t))
+      tags?.length && resource.content.keywords?.length &&
+      tags.every((t) => resource.content.keywords?.includes(t))
     ) {
-      results[resource.id] = resource;
+      results[resource.name] = resource;
     }
 
     if (
-      !(resource.id in results) &&
-      rating && resource?.meta?.rating && resource.meta.rating >= rating
+      !(resource.name in results) &&
+      rating && resource?.content?.reviewRating &&
+      resource.content?.reviewRating?.ratingValue >= rating
     ) {
-      results[resource.id] = resource;
+      results[resource.name] = resource;
     }
   }
 
   if (q.length && q !== "*") {
     const fuzzyResult = fuzzysort.go(q, resources, {
-      keys: ["content", "name", "description", "meta.author"],
+      keys: [
+        "name",
+        "content.articleBody",
+        "content.reviewBody",
+        "content.name",
+        "content.description",
+        "content.author.name",
+      ],
       threshold: 0.3,
     });
     for (const result of fuzzyResult) {
-      results[result.obj.id] = result.obj;
+      results[result.obj.name] = result.obj;
     }
   }
 

lib/taskManager.ts

@@ -1,5 +1,5 @@
 import { transcribe } from "@lib/openai.ts";
-import { createDocument } from "@lib/documents.ts";
+import { createResource } from "@lib/marka/index.ts";
 import { createLogger } from "./log/index.ts";
 import { convertOggToMp3 } from "./helpers.ts";
 
@@ -48,9 +48,8 @@ export async function endTask(chatId: string): Promise<string | null> {
         finalNote += "**[Voice message could not be transcribed]**\n\n";
       }
     } else if (entry.type === "photo") {
-      const photoUrl = `${
-        task.noteName.replace(/\.md$/, "")
-      }/photo-${photoIndex++}.jpg`;
+      const photoUrl = `${task.noteName.replace(/\.md$/, "")
+        }/photo-${photoIndex++}.jpg`;
 
       finalNote += `**Photo**:\n ${photoUrl}\n\n`;
       photoTasks.push({
@@ -62,13 +61,13 @@ export async function endTask(chatId: string): Promise<string | null> {
 
   try {
     for (const entry of photoTasks) {
-      await createDocument(entry.path, entry.content, "image/jpeg");
+      await createResource(entry.path, entry.content);
     }
   } catch (err) {
     log.error("Error creating photo document:", err);
   }
   try {
-    await createDocument(task.noteName, finalNote, "text/markdown");
+    await createResource(task.noteName, finalNote);
   } catch (error) {
     log.error("Error creating document:", error);
     return error instanceof Error

lib/types.ts

@@ -1,4 +1,4 @@
-import { resources } from "@lib/resources.ts";
+import { GenericResource, GenericResourceSchema } from "./marka/schema.ts";
 
 export interface TMDBMovie {
   adult: boolean;
@@ -33,22 +33,6 @@ export interface TMDBSeries {
   vote_count: number;
 }
 
-export type GenericResource = {
-  name: string;
-  id: string;
-  tags?: string[];
-  type: keyof typeof resources;
-  content?: string;
-  meta?: {
-    image?: string;
-    author?: string;
-    rating?: number;
-    average?: string;
-    date?: Date | string;
-    thumbnail?: string;
-  };
-};
-
 export interface GiteaOauthUser {
   sub: string;
   name: string;
@@ -61,7 +45,7 @@ export interface GiteaOauthUser {
 export type SearchResult = {
   id: string;
   name: string;
-  type: keyof typeof resources;
+  type: GenericResource["content"]["_type"];
   date?: string;
   rating: number;
   tags: string[];