fix: remove unused packages

deno.json

@@ -45,9 +45,7 @@
     "tsx": "npm:tsx@^4.19.2",
     "yaml": "https://deno.land/std@0.197.0/yaml/mod.ts",
     "zod": "npm:zod@^3.24.1",
-    "domparser": "https://deno.land/x/deno_dom@v0.1.56/deno-dom-wasm.ts",
     "fs": "https://deno.land/std/fs/mod.ts",
-    "imagemagick": "https://deno.land/x/imagemagick_deno@0.0.31/mod.ts"
   },
   "compilerOptions": { "jsx": "react-jsx", "jsxImportSource": "preact" },
   "exclude": ["**/_fresh/*"]

lib/openai.ts

@@ -25,10 +25,12 @@ function extractListFromResponse(response?: string): string[] {
     .filter((line) => line.length > 2);
 }
 
+const model = "gpt-4.1-mini";
+
 export async function summarize(content: string) {
   if (!openAI) return;
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         role: "user",
@@ -44,7 +46,7 @@ export async function summarize(content: string) {
 export async function shortenTitle(content: string) {
   if (!openAI) return;
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         role: "system",
@@ -64,7 +66,7 @@ export async function shortenTitle(content: string) {
 export async function extractAuthorName(content: string) {
   if (!openAI) return;
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         role: "system",
@@ -95,7 +97,7 @@ export async function createGenres(
 ) {
   if (!openAI) return;
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         role: "system",
@@ -123,7 +125,7 @@ export async function createKeywords(
 ) {
   if (!openAI) return;
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         "role": "system",
@@ -155,7 +157,7 @@ export const getMovieRecommendations = async (
   if (cache.has(cacheId)) return cache.get(cacheId);
 
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         role: "user",
@@ -193,7 +195,7 @@ respond with a plain unordered list each item starting with the year the movie w
 export async function createTags(content: string) {
   if (!openAI) return;
   const chatCompletion = await openAI.chat.completions.create({
-    model: "gpt-3.5-turbo",
+    model: model,
     messages: [
       {
         role: "system",
@@ -212,7 +214,7 @@ export async function createTags(content: string) {
 export async function extractRecipe(content: string) {
   if (!openAI) return;
   const completion = await openAI.beta.chat.completions.parse({
-    model: "gpt-4o-2024-08-06",
+    model: model,
     temperature: 0.1,
     messages: [
       {
@@ -230,7 +232,7 @@ export async function extractRecipe(content: string) {
 export async function extractArticleMetadata(content: string) {
   if (!openAI) return;
   const completion = await openAI.beta.chat.completions.parse({
-    model: "gpt-4o-2024-08-06",
+    model: model,
     temperature: 0.1,
     messages: [
       {

lib/recipeSchema.ts

@@ -33,8 +33,7 @@ const recipeSchema = z.object({
     .describe("List of ingredients"),
   recipeInstructions: z.array(z.string()).describe("List of instructions"),
   recipeYield: z.number().describe("Amount of Portions"),
-  prepTime: z.number().describe("Preparation time in minutes"),
+  totalTime: z.number().describe("Preparation time in minutes"),
-  cookTime: z.number().describe("Cooking time in minutes"),
 });
 
 export type Recipe = z.infer<typeof recipeSchema>;

lib/resource/recipes.ts

@@ -1,21 +1,17 @@
 export type Recipe = {
-  type: "recipe";
+  _type: "Recipe";
-  id: string;
+  author?: {
-  name: string;
+    _type: "Person";
-  description?: string;
+    name?: string;
-  markdown?: string;
-  ingredients: (Ingredient | IngredientGroup)[];
-  instructions?: string[];
-  notes?: string[];
-  tags: string[];
-  meta?: {
-    time?: string;
-    link?: string;
-    image?: string;
-    rating?: number;
-    portion?: number;
-    author?: string;
-    average?: string;
-    thumbnail?: string;
   };
+  description?: string;
+  image?: string;
+  name?: string;
+  recipeIngredient?: string[];
+  recipeInstructions?: string[];
+  datePublished?: string;
+  totalTime?: string;
+  recipeYield?: number;
+  url?: string;
+  keywords?: string[];
 };

routes/api/movies/[name].ts

@@ -5,7 +5,7 @@ import * as tmdb from "@lib/tmdb.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
 import { isString, safeFileName } from "@lib/string.ts";
 import { AccessDeniedError } from "@lib/errors.ts";
-import { fetchResource } from "@lib/marka.ts";
+import { createResource, fetchResource } from "@lib/marka.ts";
 
 export const handler: Handlers = {
   async GET(_, ctx) {
@@ -40,7 +40,7 @@ export const handler: Handlers = {
 
       finalPath = `Media/movies/images/${safeFileName(name)
         }_cover.${extension}`;
-      await createDocument(finalPath, poster);
+      await createResource(finalPath, poster);
     }
 
     const metadata = {
@@ -74,7 +74,7 @@ export const handler: Handlers = {
       meta: metadata,
     };
 
-    await createMovie(name, movie);
+    await createResource(`movies/${safeFileName(name)}.md`, movie);
 
     return json(movie);
   },

routes/api/movies/enhance/[name].ts

@@ -9,7 +9,7 @@ import {
   NotFoundError,
 } from "@lib/errors.ts";
 import { createRecommendationResource } from "@lib/recommendation.ts";
-import { fetchResource } from "@lib/marka.ts";
+import { createResource, fetchResource } from "@lib/marka.ts";
 
 const POST = async (
   req: Request,
@@ -71,12 +71,12 @@ const POST = async (
     const poster = await tmdb.getMoviePoster(posterPath);
     const extension = fileExtension(posterPath);
     finalPath = `Media/movies/images/${safeFileName(name)}_cover.${extension}`;
-    // await createDocument(finalPath, poster);
+    await createResource(finalPath, poster);
     movie.meta = movie.meta || {};
     movie.meta.image = finalPath;
   }
 
-  // await createMovie(movie.id, movie);
+  await createResource(`movies/${safeFileName(movie.id)}.md`, movie);
 
   createRecommendationResource(movie, movieDetails.overview);
 

routes/api/recipes/create/index.ts

@@ -1,9 +1,7 @@
 import { Handlers } from "$fresh/server.ts";
-import { Readability } from "https://cdn.skypack.dev/@mozilla/readability";
 import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
 import { createStreamResponse, isValidUrl } from "@lib/helpers.ts";
 import * as openai from "@lib/openai.ts";
-import tds from "https://cdn.skypack.dev/turndown@7.2.0";
 import { createLogger } from "@lib/log/index.ts";
 import { Recipe } from "@lib/resource/recipes.ts";
 import recipeSchema from "@lib/recipeSchema.ts";
@@ -11,92 +9,12 @@ import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts"
 import { safeFileName } from "@lib/string.ts";
 import { parseJsonLdToRecipeSchema } from "./parseJsonLd.ts";
 import z from "zod";
-import { fetchHtmlWithPlaywright } from "@lib/playwright.ts";
+import { createResource } from "@lib/marka.ts";
+import { webScrape } from "@lib/webScraper.ts";
+import { Defuddle } from "defuddle/node";
 
 const log = createLogger("api/article");
 
-function makeUrlAbsolute(url: URL, src: string) {
-  if (src.startsWith("/")) {
-    return `${url.origin}${src.replace(/$\//, "")}`;
-  }
-
-  if (!src.startsWith("https://") && !src.startsWith("http://")) {
-    return `${url.origin.replace(/\/$/, "")}/${src.replace(/^\//, "")})`;
-  }
-
-  return src;
-}
-
-async function extractUsingAI(
-  url: URL,
-  document: Parameters<typeof Readability>[0] | null,
-  streamResponse: ReturnType<typeof createStreamResponse>,
-) {
-  const readable = new Readability(document);
-
-  const result = readable.parse();
-
-  const service = new tds({
-    headingStyle: "atx",
-    codeBlockStyle: "fenced",
-    hr: "---",
-    bulletListMarker: "-",
-  });
-
-  service.addRule("fix image links", {
-    filter: ["img"],
-    replacement: function(_: string, node: HTMLImageElement) {
-      const src = node.getAttribute("src");
-      const alt = node.getAttribute("alt") || "";
-      if (!src || src.startsWith("data:image")) return "";
-
-      return `![${alt}](${makeUrlAbsolute(url, src)})`;
-    },
-  });
-  service.addRule("fix normal links", {
-    filter: ["a"],
-    replacement: function(content: string, node: HTMLImageElement) {
-      const href = node.getAttribute("href");
-      if (!href) return content;
-
-      if (href.startsWith("/")) {
-        return `[${content}](${url.origin}${href.replace(/$\//, "")})`;
-      }
-
-      if (href.startsWith("#")) {
-        if (content.length < 2) return "";
-        return `[${content}](${url.href}#${href})`.replace("##", "#");
-      }
-
-      if (!href.startsWith("https://") && !href.startsWith("http://")) {
-        return `[${content}](${url.origin.replace(/\/$/, "")}/${href.replace(/^\//, "")
-          })`;
-      }
-
-      return `[${content}](${href})`;
-    },
-  });
-
-  const cleanDocument = parser.parseFromString(
-    result.content,
-    "text/html",
-  );
-
-  const markdown = service.turndown(cleanDocument);
-
-  streamResponse.enqueue("extracting recipe with openai");
-
-  const recipe = await openai.extractRecipe(markdown);
-
-  if (recipe) {
-    if ("errorMessages" in recipe) {
-      throw new Error("Failed to extract recipe: " + recipe.errorMessages[0]);
-    } else {
-      return recipe;
-    }
-  }
-}
-
 async function processCreateRecipeFromUrl(
   { fetchUrl, streamResponse }: {
     fetchUrl: string;
@@ -104,32 +22,19 @@ async function processCreateRecipeFromUrl(
   },
 ) {
   log.info("create article from url", { url: fetchUrl });
-  const url = new URL(fetchUrl);
 
   streamResponse.enqueue("downloading article");
 
-  const html = await fetchHtmlWithPlaywright(fetchUrl, streamResponse);
+  const doc = await webScrape(fetchUrl, streamResponse);
 
-  streamResponse.enqueue("download success");
+  const result = await Defuddle(doc, fetchUrl, {
-  Deno.writeTextFile("article.html", html);
+    markdown: true,
-
-  const document = parser.parseFromString(html, "text/html");
-
-  const title = document?.querySelector("title")?.innerText;
-
-  const images: HTMLImageElement[] = [];
-  document?.querySelectorAll("img").forEach((img) => {
-    images.push(img as unknown as HTMLImageElement);
   });
 
-  const metaAuthor =
+  streamResponse.enqueue("download success");
-    document?.querySelector('meta[name="twitter:creator"]')?.getAttribute(
-      "content",
-    ) ||
-    document?.querySelector('meta[name="author"]')?.getAttribute("content");
 
   const jsonLds = Array.from(
-    document?.querySelectorAll(
+    doc?.querySelectorAll(
       "script[type='application/ld+json']",
     ),
   ) as unknown as HTMLScriptElement[];
@@ -143,78 +48,58 @@ async function processCreateRecipeFromUrl(
   }
 
   if (!recipe) {
-    recipe = await extractUsingAI(url, document, streamResponse);
+    recipe = await openai.extractRecipe(result.content);
   }
 
-  const id = (recipe?.title || title || "").replace(/--+/, "-");
+  const id = safeFileName(recipe?.title || "");
 
   if (!recipe) {
     streamResponse.enqueue("failed to parse recipe");
     streamResponse.cancel();
     return;
   }
-  if (!recipe.image) {
-    const largestImage = images.filter((img) => {
-      const src = img.getAttribute("src");
-      return !!src && !src.startsWith("data:");
-    }).sort((a, b) => {
-      const aSize = +(a.getAttribute("width") || 0) +
-        +(a.getAttribute("height") || 0);
-      const bSize = +(b.getAttribute("width") || 0) +
-        +(b.getAttribute("height") || 0);
-      return aSize > bSize ? -1 : 1;
-    })[0];
-    const src = largestImage.getAttribute("src");
-    if (src) {
-      recipe.image = makeUrlAbsolute(url, src);
-    }
-  }
 
   const newRecipe: Recipe = {
-    type: "recipe",
+    _type: "Recipe",
-    id,
+    name: recipe?.title,
-    name: recipe?.title || title || "",
     description: recipe?.description,
-    ingredients: recipe?.ingredients || [],
+    recipeIngredient: recipe?.ingredients || [],
-    instructions: recipe?.instructions || [],
+    recipeInstructions: recipe?.instructions || [],
-    notes: recipe?.notes,
+    keywords: recipe.tags || [],
-    tags: recipe.tags || [],
-    meta: {
     image: recipe?.image,
-      time: recipe?.totalTime
+    totalTime: recipe?.totalTime
       ? `${recipe?.totalTime?.toString()} minutes`
       : undefined,
-      link: fetchUrl,
+    url: fetchUrl,
-      portion: recipe?.servings,
+    author: {
-      author: metaAuthor ?? recipe?.author,
+      _type: "Person",
+      name: recipe?.author,
     },
+    recipeYield: recipe?.servings,
   };
 
-  if (newRecipe.meta?.image) {
+  if (newRecipe?.image && newRecipe.image.length > 5) {
-    const src = makeUrlAbsolute(url, newRecipe.meta.image);
+    const extension = fileExtension(new URL(newRecipe.image).pathname);
-    if (src?.length > 5) {
+    const finalPath = `resources/recipes/images/${safeFileName(id)
-      const extension = fileExtension(new URL(src).pathname);
-      const finalPath = `Media/articles/images/${safeFileName(id)
       }_cover.${extension}`;
     streamResponse.enqueue("downloading image");
     try {
       streamResponse.enqueue("downloading image");
-        // const res = await fetch(src);
+      const res = await fetch(newRecipe.image);
       streamResponse.enqueue("saving image");
-        // const buffer = await res.arrayBuffer();
+      const buffer = await res.arrayBuffer();
-        // await createDocument(finalPath, buffer);
+      await createResource(finalPath, buffer);
-        newRecipe.meta.image = finalPath;
+      newRecipe.image = finalPath;
     } catch (err) {
       console.log("Failed to save image", err);
     }
   }
-  }
 
   streamResponse.enqueue("finished processing, creating file");
 
-  // await createRecipe(newRecipe.id, newRecipe);
+  await createResource(`recipes/${id}.md`, newRecipe);
 
-  streamResponse.enqueue("id: " + newRecipe.id);
+  streamResponse.enqueue("id: " + id);
 }
 
 export const handler: Handlers = {

routes/api/recipes/create/parseJsonLd.ts

@@ -49,6 +49,7 @@ export function parseJsonLdToRecipeSchema(jsonLdContent: string) {
 
     // Build the recipe object
     const recipe = {
+      _type: "Recipe",
       title: data.name || "Unnamed Recipe",
       image: pickImage(image || data.image || ""),
       author: Array.isArray(data.author)

routes/api/series/[name].ts

@@ -3,10 +3,9 @@ import { json } from "@lib/helpers.ts";
 import * as tmdb from "@lib/tmdb.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
 import { isString, safeFileName } from "@lib/string.ts";
-import { createDocument } from "@lib/documents.ts";
 import { AccessDeniedError } from "@lib/errors.ts";
 import { Series } from "@lib/resource/series.ts";
-import { fetchResource } from "@lib/marka.ts";
+import { createResource, fetchResource } from "@lib/marka.ts";
 
 export const handler: Handlers = {
   async GET(_, ctx) {
@@ -39,7 +38,7 @@ export const handler: Handlers = {
 
       finalPath = `Media/series/images/${safeFileName(name)
         }_cover.${extension}`;
-      await createDocument(finalPath, poster);
+      await createResource(finalPath, poster);
     }
 
     const metadata = { tmdbId } as Series["meta"];
@@ -71,7 +70,7 @@ export const handler: Handlers = {
       meta: metadata,
     };
 
-    await createSeries(name, series);
+    await createResource(`series/${safeFileName(name)}.md`, series);
 
     return json(series);
   },

routes/api/series/enhance/[name].ts

@@ -1,5 +1,5 @@
 import { FreshContext, Handlers } from "$fresh/server.ts";
-import { createDocument } from "@lib/documents.ts";
+import { FreshContext, Handlers } from "$fresh/server.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
 import * as tmdb from "@lib/tmdb.ts";
 import { safeFileName } from "@lib/string.ts";
@@ -9,6 +9,7 @@ import {
   BadRequestError,
   NotFoundError,
 } from "@lib/errors.ts";
+import { createResource, fetchResource } from "@lib/marka.ts";
 
 const isString = (input: string | undefined): input is string => {
   return typeof input === "string";
@@ -30,7 +31,7 @@ const POST = async (
     throw new BadRequestError();
   }
 
-  const series = await getSeries(ctx.params.name);
+  const series = await fetchResource(`series/${ctx.params.name}`);
   if (!series) {
     throw new NotFoundError();
   }
@@ -63,15 +64,15 @@ const POST = async (
 
   let finalPath = "";
   if (posterPath && !series.meta?.image) {
-    // const poster = await tmdb.getMoviePoster(posterPath);
+    const poster = await tmdb.getMoviePoster(posterPath);
     const extension = fileExtension(posterPath);
 
     finalPath = `Media/series/images/${safeFileName(name)}_cover.${extension}`;
-    // await createDocument(finalPath, poster);
+    await createResource(finalPath, poster);
     series.meta = series.meta || {};
     series.meta.image = finalPath;
   }
-  // await createSeries(series.id, series);
+  await createResource(`series/${safeFileName(series.id)}.md`, series);
 
   return json(series);
 };

routes/api/series/index.ts

@@ -1,6 +1,10 @@
 import { Handlers } from "$fresh/server.ts";
+import { json } from "@lib/helpers.ts";
+import { fetchResource } from "@lib/marka.ts";
 
 export const handler: Handlers = {
   async GET() {
+    const series = await fetchResource("series");
+    return json(series);
   },
 };