refactor: split telegram.ts into seperate files

Dockerfile

@@ -1,7 +1,7 @@
-FROM denoland/deno:2.1.4 AS build
+FROM denoland/deno:2.3.1 AS build
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl && \
+    curl ffmpeg && \
     deno run -A npm:playwright install --with-deps firefox &&\
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*

components/MetaTags.tsx

@@ -23,7 +23,11 @@ function generateJsonLd(resource: GenericResource): string {
   }
 
   if (resource.meta?.date) {
+    try {
       baseSchema.datePublished = new Date(resource.meta.date).toISOString();
+    } catch (_) {
+      // Ignore invalid date
+    }
   }
 
   if (resource.meta?.rating) {

deno.json

@@ -6,7 +6,8 @@
   ],
   "tasks": {
     "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx",
-    "start": "deno run --env-file -A --watch=static/,routes/ dev.ts",
+    "dev": "deno run --env-file -A --watch=static/,routes/ dev.ts",
+    "start": "deno run --env-file -A main.ts",
     "db": "deno run --env-file -A npm:drizzle-kit",
     "build": "deno run -A dev.ts build",
     "preview": "deno run -A main.ts",

islands/Link.tsx

@@ -27,9 +27,11 @@ export function Link(
       clearTimeout(globalThis.loadingTimeout);
       delete globalThis.loadingTimeout;
       setTimeout(() => {
+        globalThis.dispatchEvent(new CustomEvent("loading-finished"));
         document.querySelector("main")?.classList.remove("loading");
       }, 100);
     } else {
+      globalThis.dispatchEvent(new CustomEvent("loading-finished"));
       document.querySelector("main")?.classList.remove("loading");
     }
   });

lib/cache.ts

@@ -6,9 +6,12 @@ interface SetCacheOptions {
   expires?: number; // Override expiration for individual cache entries
 }
 
-const caches = new Map<
+export const caches = new Map<
   string,
-  { info: () => { name: string; count: number; sizeInKB: number } }
+  {
+    info: () => { name: string; count: number; sizeInKB: number };
+    clear: () => void;
+  }
 >();
 
 export function createCache<T>(
@@ -31,6 +34,10 @@ export function createCache<T>(
       return entry.value; // Return value if not expired
     },
 
+    clear() {
+      cache.clear();
+    },
+
     set(key: string, value: T | unknown, opts: SetCacheOptions = {}) {
       const now = Date.now();
       const expiresIn = opts.expires ?? createOpts.expires;
@@ -94,6 +101,7 @@ export function createCache<T>(
   };
 
   caches.set(cacheName, {
+    clear: api.clear.bind(api),
     info: api.info.bind(api),
   });
 

lib/crud.ts

@@ -80,10 +80,6 @@ export function createCrud<T extends GenericResource>(
   async function read(id: string) {
     const path = pathFromId(id);
 
-    if (cache.has(path)) {
-      return cache.get(path);
-    }
-
     const content = await getDocument(path);
     if (!content) {
       return;
@@ -95,7 +91,6 @@ export function createCrud<T extends GenericResource>(
       parsed = await addThumbnailToResource(parsed);
     }
     const doc = { ...parsed, content };
-    cache.set(path, doc, { expires: 10 * 1000 });
 
     return doc;
   }
@@ -110,7 +105,6 @@ export function createCrud<T extends GenericResource>(
 
     if (render) {
       const rendered = render(content);
-      cache.set(path, content);
       return createDocument(path, rendered);
     }
 
@@ -143,7 +137,6 @@ export function createCrud<T extends GenericResource>(
       }),
     )).sort(sortFunction<T>(sort)).filter((v) => !!v);
 
-    cache.set("all", parsed);
     return parsed;
   }
 

lib/documents.ts

@@ -132,8 +132,35 @@ export function parseDocument(doc: string) {
     .parse(doc);
 }
 
+function removeFrontmatter(doc: string) {
+  if (doc.trim().startsWith("---")) {
+    return doc.trim().split("---").filter((s) => s.length).slice(1).join("---");
+  }
+  return doc;
+}
+
+export function removeImage(doc: string, imageUrl?: string) {
+  if (!imageUrl) {
+    return doc;
+  }
+  // Remove image from content
+  const first = doc.slice(0, 500);
+  const second = doc.slice(500);
+
+  // Regex pattern to match the image Markdown syntax with the specific URL
+  const pattern = new RegExp(
+    `!\\[.*?\\]\\(${imageUrl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\)`,
+    "g",
+  );
+
+  // Remove the matched image
+  const updatedMarkdown = first.replace(pattern, "");
+  return updatedMarkdown + second;
+}
+
 export function renderMarkdown(doc: string) {
-  return render(doc, {
+  return render(removeFrontmatter(doc), {
+    baseUrl: SILVERBULLET_SERVER,
     allowMath: true,
   });
 }
@@ -172,4 +199,3 @@ export function getTextOfChild(child: DocumentChild): string | undefined {
   }
   return;
 }
-

lib/env.ts

@@ -8,6 +8,7 @@ export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER");
 export const TMDB_API_KEY = Deno.env.get("TMDB_API_KEY");
 export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY");
 export const YOUTUBE_API_KEY = Deno.env.get("YOUTUBE_API_KEY");
+export const TELEGRAM_API_KEY = Deno.env.get("TELEGRAM_API_KEY")!;
 
 export const GITEA_SERVER = Deno.env.get("GITEA_SERVER");
 export const GITEA_CLIENT_ID = Deno.env.get("GITEA_CLIENT_ID")!;

lib/helpers.ts

@@ -107,3 +107,24 @@ export function parseRating(rating: string | number) {
   }
   return rating;
 }
+
+export async function convertOggToMp3(
+  oggData: ArrayBuffer,
+): Promise<Uint8Array> {
+  const ffmpeg = new Deno.Command("ffmpeg", {
+    args: ["-f", "ogg", "-i", "pipe:0", "-f", "mp3", "pipe:1"],
+    stdin: "piped",
+    stdout: "piped",
+    stderr: "null",
+  });
+
+  const process = ffmpeg.spawn();
+  const writer = process.stdin.getWriter();
+  await writer.write(new Uint8Array(oggData));
+  await writer.close();
+
+  const output = await process.output();
+  const { code } = await process.status;
+  if (code !== 0) throw new Error(`FFmpeg exited with code ${code}`);
+  return output.stdout;
+}

lib/openai.ts

@@ -228,3 +228,21 @@ export async function extractRecipe(content: string) {
 
   return recipeResponseSchema.parse(completion.choices[0].message.parsed);
 }
+
+export async function transcribe(
+  mp3Data: Uint8Array,
+): Promise<string | undefined> {
+  if (!openAI) return;
+
+  const file = new File([mp3Data], "audio.mp3", {
+    type: "audio/mpeg",
+  });
+
+  const result = await openAI.audio.transcriptions.create({
+    file,
+    model: "whisper-1",
+    response_format: "text",
+  });
+
+  return result;
+}

lib/recipeSchema.ts

@@ -50,7 +50,8 @@ export function isValidRecipe(
     | null
     | undefined,
 ) {
-  return recipe?.ingredients?.length && recipe?.instructions?.length &&
+  return recipe?.ingredients?.length && recipe.ingredients.length > 1 &&
+    recipe?.instructions?.length &&
     recipe.name?.length;
 }
 

lib/taskManager.ts

@@ -0,0 +1,110 @@
+import { transcribe } from "@lib/openai.ts";
+import { createDocument } from "@lib/documents.ts";
+import { createLogger } from "./log/index.ts";
+import { convertOggToMp3 } from "./helpers.ts";
+
+const log = createLogger("taskManager");
+
+// In-memory task state
+const activeTasks: Record<
+  string,
+  {
+    noteName: string;
+    entries: Array<
+      { type: string; content: string | ArrayBufferLike; fileName?: string }
+    >;
+  }
+> = {};
+
+export function startTask(chatId: string, noteName: string) {
+  activeTasks[chatId] = { noteName, entries: [] };
+  log.info(`Started note: ${noteName}`);
+}
+
+export async function endTask(chatId: string): Promise<string | null> {
+  const task = activeTasks[chatId];
+  if (!task) return null;
+
+  log.info("Ending note", task.noteName);
+
+  let finalNote = `# ${task.noteName}\n\n`;
+
+  const photoTasks: { content: ArrayBuffer; path: string }[] = [];
+
+  let photoIndex = 0;
+  for (const entry of task.entries) {
+    if (entry.type === "text") {
+      finalNote += entry.content + "\n\n";
+    } else if (entry.type === "voice") {
+      try {
+        log.info("Converting OGG to MP3");
+        const mp3Data = await convertOggToMp3(entry.content as ArrayBuffer);
+        log.info("Finished converting OGG to MP3, transcribing...");
+        const transcript = await transcribe(mp3Data);
+        finalNote += `**Voice Transcript:**\n${transcript}\n\n`;
+        log.info("Finished transcribing");
+      } catch (error) {
+        log.error(error);
+        finalNote += "**[Voice message could not be transcribed]**\n\n";
+      }
+    } else if (entry.type === "photo") {
+      const photoUrl = `${
+        task.noteName.replace(/\.md$/, "")
+      }/photo-${photoIndex++}.jpg`;
+
+      finalNote += `**Photo**:\n ${photoUrl}\n\n`;
+      photoTasks.push({
+        content: entry.content as ArrayBuffer,
+        path: photoUrl,
+      });
+    }
+  }
+
+  try {
+    for (const entry of photoTasks) {
+      await createDocument(entry.path, entry.content, "image/jpeg");
+    }
+  } catch (err) {
+    log.error("Error creating photo document:", err);
+  }
+  try {
+    await createDocument(task.noteName, finalNote, "text/markdown");
+  } catch (error) {
+    log.error("Error creating document:", error);
+    return error instanceof Error
+      ? error.toString()
+      : "Error creating document";
+  }
+
+  delete activeTasks[chatId];
+  log.debug({ finalNote });
+  return finalNote;
+}
+
+export function addTextEntry(chatId: string, text: string) {
+  const task = activeTasks[chatId];
+  if (!task) return;
+  const entry = { type: "text", content: text };
+  log.debug("New Entry", entry);
+  task.entries.push(entry);
+}
+
+export function addVoiceEntry(chatId: string, buffer: ArrayBufferLike) {
+  const task = activeTasks[chatId];
+  if (!task) return;
+  const entry = { type: "voice", content: buffer };
+  log.debug("New Entry", entry);
+  task.entries.push(entry);
+}
+
+export function addPhotoEntry(
+  chatId: string,
+  buffer: ArrayBufferLike,
+  fileName: string,
+) {
+  const task = activeTasks[chatId];
+  if (!task) return;
+  const entry = { type: "photo", content: buffer, fileName };
+  log.debug("New Entry", entry);
+  task.entries.push(entry);
+}

lib/telegram.ts

@@ -0,0 +1,65 @@
+import { Bot } from "https://deno.land/x/grammy@v1.36.1/mod.ts";
+import { TELEGRAM_API_KEY } from "@lib/env.ts";
+import { createLogger } from "./log/index.ts";
+
+const bot = new Bot(TELEGRAM_API_KEY);
+const log = createLogger("telegram");
+
+import * as manager from "./taskManager.ts";
+
+async function downloadFile(filePath: string): Promise<Uint8Array> {
+  log.info(`Downloading file from path: ${filePath}`);
+  const url = `https://api.telegram.org/file/bot${bot.token}/${filePath}`;
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error("Failed to download file: " + response.statusText);
+  }
+  log.info("File downloaded successfully");
+  const buffer = await response.arrayBuffer();
+  return new Uint8Array(buffer);
+}
+
+bot.command("start", async (ctx) => {
+  log.info("Received /start command");
+  const [_, noteName] = ctx.message?.text?.split(" ") || [];
+  if (!noteName) {
+    return ctx.reply("Please provide a note name. Usage: /start NoteName");
+  }
+  manager.startTask(ctx.chat.id.toString(), noteName);
+  await ctx.reply(`Started note: ${noteName}`);
+});
+
+bot.command("end", async (ctx) => {
+  log.info("Received /end command");
+  const finalNote = await manager.endTask(ctx.chat.id.toString());
+  if (!finalNote) return ctx.reply("No active note found.");
+
+  try {
+    await ctx.reply("Note complete. Here is your markdown:");
+    await ctx.reply(finalNote, { parse_mode: "MarkdownV2" });
+  } catch (err) {
+    console.error("Error sending final note:", err);
+  }
+});
+
+bot.on("message:text", (ctx) => {
+  log.info("Received text message");
+  manager.addTextEntry(ctx.chat.id.toString(), ctx.message.text);
+});
+
+bot.on("message:voice", async (ctx) => {
+  log.info("Received photo message");
+  log.info("Received voice message");
+  const file = await ctx.getFile();
+  const buffer = await downloadFile(file.file_path!);
+  manager.addVoiceEntry(ctx.chat.id.toString(), buffer.buffer);
+});
+
+bot.on("message:photo", async (ctx) => {
+  const file = await ctx.getFile();
+  const buffer = await downloadFile(file.file_path!);
+  const fileName = file.file_path!.split("/").pop()!;
+  manager.addPhotoEntry(ctx.chat.id.toString(), buffer.buffer, fileName);
+});
+
+bot.start();

main.ts

@@ -7,5 +7,6 @@
 import { start } from "$fresh/server.ts";
 import manifest from "./fresh.gen.ts";
 import config from "./fresh.config.ts";
+import "@lib/telegram.ts";
 
 await start(manifest, config);

routes/api/cache.ts

@@ -2,11 +2,14 @@ import { Handlers } from "$fresh/server.ts";
 import { documentTable } from "@lib/db/schema.ts";
 import { db } from "@lib/db/sqlite.ts";
 import { json } from "@lib/helpers.ts";
+import { caches } from "@lib/cache.ts";
 
 export const handler: Handlers = {
   async DELETE() {
+    for (const cache of caches.values()) {
+      cache.clear();
+    }
     await db.delete(documentTable).run();
     return json({ status: "ok" });
   },
 };
-

routes/articles/[name].tsx

@@ -5,7 +5,7 @@ import { KMenu } from "@islands/KMenu.tsx";
 import { YoutubePlayer } from "@components/Youtube.tsx";
 import { HashTags } from "@components/HashTags.tsx";
 import { isYoutubeLink } from "@lib/string.ts";
-import { renderMarkdown } from "@lib/documents.ts";
+import { removeImage, renderMarkdown } from "@lib/documents.ts";
 import { RedirectSearchHandler } from "@islands/Search.tsx";
 import PageHero from "@components/PageHero.tsx";
 import { Star } from "@components/Stars.tsx";
@@ -14,6 +14,9 @@ import { MetaTags } from "@components/MetaTags.tsx";
 export const handler: Handlers<{ article: Article; session: unknown }> = {
   async GET(_, ctx) {
     const article = await getArticle(ctx.params.name);
+    if (!article) {
+      return ctx.renderNotFound();
+    }
     return ctx.render({ article, session: ctx.state.session });
   },
 };
@@ -25,7 +28,9 @@ export default function Greet(
 
   const { author = "", date = "" } = article.meta;
 
-  const content = renderMarkdown(article.content);
+  const content = renderMarkdown(
+    removeImage(article.content, article.meta.image),
+  );
 
   return (
     <MainLayout

routes/index.tsx

@@ -4,6 +4,7 @@ import { PageProps } from "$fresh/server.ts";
 import { resources } from "@lib/resources.ts";
 import { RedirectSearchHandler } from "@islands/Search.tsx";
 import { KMenu } from "@islands/KMenu.tsx";
+import "@lib/telegram.ts";
 
 export default function Home(props: PageProps) {
   return (

routes/movies/[name].tsx

@@ -2,7 +2,7 @@ import { PageProps, RouteContext } from "$fresh/server.ts";
 import { MainLayout } from "@components/layouts/main.tsx";
 import { getMovie, Movie } from "@lib/resource/movies.ts";
 import { HashTags } from "@components/HashTags.tsx";
-import { renderMarkdown } from "@lib/documents.ts";
+import { removeImage, renderMarkdown } from "@lib/documents.ts";
 import { KMenu } from "@islands/KMenu.tsx";
 import { RedirectSearchHandler } from "@islands/Search.tsx";
 import { Recommendations } from "@islands/Recommendations.tsx";
@@ -17,9 +17,15 @@ export default async function Greet(
   const movie = await getMovie(ctx.params.name);
   const session = ctx.state.session;
 
+  if (!movie) {
+    return ctx.renderNotFound();
+  }
+
   const { author = "", date = "" } = movie.meta;
 
-  const content = renderMarkdown(movie.description || "");
+  const content = renderMarkdown(
+    removeImage(movie.description || "", movie.meta.image),
+  );
 
   return (
     <MainLayout url={props.url} title={`Movie > ${movie.name}`} context={movie}>

routes/recipes/[name].tsx

@@ -60,7 +60,7 @@ function ValidRecipe({
   );
 }
 
-export default function Greet(
+export default function Page(
   props: PageProps<{ recipe: Recipe; session: Record<string, string> }>,
 ) {
   const { recipe, session } = props.data;
@@ -114,6 +114,7 @@ export default function Greet(
           )
           : (
             <div
+              class="whitespace-break-spaces markdown-body"
               dangerouslySetInnerHTML={{
                 __html: renderMarkdown(recipe?.markdown || ""),
               }}

routes/series/[name].tsx

@@ -1,7 +1,7 @@
 import { Handlers, PageProps } from "$fresh/server.ts";
 import { MainLayout } from "@components/layouts/main.tsx";
 import { HashTags } from "@components/HashTags.tsx";
-import { renderMarkdown } from "@lib/documents.ts";
+import { removeImage, renderMarkdown } from "@lib/documents.ts";
 import { getSeries, Series } from "@lib/resource/series.ts";
 import { RedirectSearchHandler } from "@islands/Search.tsx";
 import { KMenu } from "@islands/KMenu.tsx";
@@ -12,6 +12,10 @@ import { MetaTags } from "@components/MetaTags.tsx";
 export const handler: Handlers<{ serie: Series; session: unknown }> = {
   async GET(_, ctx) {
     const serie = await getSeries(ctx.params.name);
+
+    if (!serie) {
+      return ctx.renderNotFound();
+    }
     return ctx.render({ serie, session: ctx.state.session });
   },
 };
@@ -23,7 +27,9 @@ export default function Greet(
 
   const { author = "", date = "" } = serie.meta;
 
-  const content = renderMarkdown(serie.description || "");
+  const content = renderMarkdown(
+    removeImage(serie.description || "", serie.meta.image),
+  );
 
   return (
     <MainLayout url={props.url} title={`Serie > ${serie.name}`} context={serie}>

static/global.css

@@ -128,3 +128,34 @@ main {
 main.loading {
   opacity: 0;
 }
+
+.markdown-body>h1 {
+  font-size: 2.25rem;
+  font-weight: 600;
+}
+
+.markdown-body>h2 {
+  font-size: 1.75rem;
+  font-weight: 600;
+}
+
+.markdown-body>h3 {
+  font-size: 1.5rem;
+  font-weight: 600;
+}
+
+.markdown-body>h1>.anchor,
+.markdown-body>h2>.anchor,
+.markdown-body>h3>.anchor {
+  display: inline-flex;
+  margin-left: -26px;
+  margin-right: 12px;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.markdown-body>h1:hover .anchor,
+.markdown-body>h2:hover .anchor,
+.markdown-body>h3:hover .anchor {
+  opacity: 1;
+}

static/thumbnails.js

@@ -226,7 +226,8 @@ function rgbaToDataURL(w, h, rgba) {
   return "data:image/png;base64," + btoa(String.fromCharCode(...bytes));
 }
 
-document.querySelectorAll("[data-thumb]").forEach((entry) => {
+function updateThumbnailImages() {
+  document.querySelectorAll("[data-thumb]").forEach((entry) => {
     const hash = entry.getAttribute("data-thumb");
 
     if (!hash) return;
@@ -261,4 +262,8 @@ document.querySelectorAll("[data-thumb]").forEach((entry) => {
         });
       }
     }, 50);
-});
+  });
+}
+
+globalThis.addEventListener("load", updateThumbnailImages);
+globalThis.addEventListener("loading-finished", updateThumbnailImages);