max/memorium
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat: use marka api in all apis

Browse Source
This commit is contained in:
Max Richter
2025-10-31 16:23:20 +01:00
parent 79d692c2c6
commit 1f67f8af34
10 changed files with 85 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/openai.ts
View File

@@ -25,10 +25,12 @@ function extractListFromResponse(response?: string): string[] {
.filter((line) => line.length > 2);
}
const model = "gpt-4.1-mini";
export async function summarize(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
role: "user",
@@ -44,7 +46,7 @@ export async function summarize(content: string) {
export async function shortenTitle(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
role: "system",
@@ -64,7 +66,7 @@ export async function shortenTitle(content: string) {
export async function extractAuthorName(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
role: "system",
@@ -95,7 +97,7 @@ export async function createGenres(
) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
role: "system",
@@ -123,7 +125,7 @@ export async function createKeywords(
) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
"role": "system",
@@ -155,7 +157,7 @@ export const getMovieRecommendations = async (
if (cache.has(cacheId)) return cache.get(cacheId);
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
role: "user",
@@ -193,7 +195,7 @@ respond with a plain unordered list each item starting with the year the movie w
export async function createTags(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.chat.completions.create({
model: "gpt-3.5-turbo",
model: model,
messages: [
{
role: "system",
@@ -212,7 +214,7 @@ export async function createTags(content: string) {
export async function extractRecipe(content: string) {
if (!openAI) return;
const completion = await openAI.beta.chat.completions.parse({
model: "gpt-4o-2024-08-06",
model: model,
temperature: 0.1,
messages: [
{
@@ -230,7 +232,7 @@ export async function extractRecipe(content: string) {
export async function extractArticleMetadata(content: string) {
if (!openAI) return;
const completion = await openAI.beta.chat.completions.parse({
model: "gpt-4o-2024-08-06",
model: model,
temperature: 0.1,
messages: [
{

3
lib/recipeSchema.ts
View File

@@ -33,8 +33,7 @@ const recipeSchema = z.object({
.describe("List of ingredients"),
recipeInstructions: z.array(z.string()).describe("List of instructions"),
recipeYield: z.number().describe("Amount of Portions"),
prepTime: z.number().describe("Preparation time in minutes"),
cookTime: z.number().describe("Cooking time in minutes"),
totalTime: z.number().describe("Preparation time in minutes"),
});
export type Recipe = z.infer<typeof recipeSchema>;

32
lib/resource/recipes.ts
View File

@@ -1,21 +1,17 @@
export type Recipe = {
type: "recipe";
id: string;
name: string;
description?: string;
markdown?: string;
ingredients: (Ingredient | IngredientGroup)[];
instructions?: string[];
notes?: string[];
tags: string[];
meta?: {
time?: string;
link?: string;
image?: string;
rating?: number;
portion?: number;
author?: string;
average?: string;
thumbnail?: string;
_type: "Recipe";
author?: {
_type: "Person";
name?: string;
};
description?: string;
image?: string;
name?: string;
recipeIngredient?: string[];
recipeInstructions?: string[];
datePublished?: string;
totalTime?: string;
recipeYield?: number;
url?: string;
keywords?: string[];
};

6
routes/api/movies/[name].ts
View File

@@ -5,7 +5,7 @@ import * as tmdb from "@lib/tmdb.ts";
import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
import { isString, safeFileName } from "@lib/string.ts";
import { AccessDeniedError } from "@lib/errors.ts";
import { fetchResource } from "@lib/marka.ts";
import { createResource, fetchResource } from "@lib/marka.ts";
export const handler: Handlers = {
async GET(_, ctx) {
@@ -40,7 +40,7 @@ export const handler: Handlers = {
finalPath = `Media/movies/images/${safeFileName(name)
}_cover.${extension}`;
await createDocument(finalPath, poster);
await createResource(finalPath, poster);
}
const metadata = {
@@ -74,7 +74,7 @@ export const handler: Handlers = {
meta: metadata,
};
await createMovie(name, movie);
await createResource(`movies/${safeFileName(name)}.md`, movie);
return json(movie);
},

6
routes/api/movies/enhance/[name].ts
View File

@@ -9,7 +9,7 @@ import {
NotFoundError,
} from "@lib/errors.ts";
import { createRecommendationResource } from "@lib/recommendation.ts";
import { fetchResource } from "@lib/marka.ts";
import { createResource, fetchResource } from "@lib/marka.ts";
const POST = async (
req: Request,
@@ -71,12 +71,12 @@ const POST = async (
const poster = await tmdb.getMoviePoster(posterPath);
const extension = fileExtension(posterPath);
finalPath = `Media/movies/images/${safeFileName(name)}_cover.${extension}`;
// await createDocument(finalPath, poster);
await createResource(finalPath, poster);
movie.meta = movie.meta || {};
movie.meta.image = finalPath;
}
// await createMovie(movie.id, movie);
await createResource(`movies/${safeFileName(movie.id)}.md`, movie);
createRecommendationResource(movie, movieDetails.overview);

193
routes/api/recipes/create/index.ts
View File

@@ -1,9 +1,7 @@
import { Handlers } from "$fresh/server.ts";
import { Readability } from "https://cdn.skypack.dev/@mozilla/readability";
import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
import { createStreamResponse, isValidUrl } from "@lib/helpers.ts";
import * as openai from "@lib/openai.ts";
import tds from "https://cdn.skypack.dev/turndown@7.2.0";
import { createLogger } from "@lib/log/index.ts";
import { Recipe } from "@lib/resource/recipes.ts";
import recipeSchema from "@lib/recipeSchema.ts";
@@ -11,92 +9,12 @@ import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts"
import { safeFileName } from "@lib/string.ts";
import { parseJsonLdToRecipeSchema } from "./parseJsonLd.ts";
import z from "zod";
import { fetchHtmlWithPlaywright } from "@lib/playwright.ts";
import { createResource } from "@lib/marka.ts";
import { webScrape } from "@lib/webScraper.ts";
import { Defuddle } from "defuddle/node";
const log = createLogger("api/article");
function makeUrlAbsolute(url: URL, src: string) {
if (src.startsWith("/")) {
return `${url.origin}${src.replace(/$\//, "")}`;
}
if (!src.startsWith("https://") && !src.startsWith("http://")) {
return `${url.origin.replace(/\/$/, "")}/${src.replace(/^\//, "")})`;
}
return src;
}
async function extractUsingAI(
url: URL,
document: Parameters<typeof Readability>[0] | null,
streamResponse: ReturnType<typeof createStreamResponse>,
) {
const readable = new Readability(document);
const result = readable.parse();
const service = new tds({
headingStyle: "atx",
codeBlockStyle: "fenced",
hr: "---",
bulletListMarker: "-",
});
service.addRule("fix image links", {
filter: ["img"],
replacement: function(_: string, node: HTMLImageElement) {
const src = node.getAttribute("src");
const alt = node.getAttribute("alt") || "";
if (!src || src.startsWith("data:image")) return "";
return `![${alt}](${makeUrlAbsolute(url, src)})`;
},
});
service.addRule("fix normal links", {
filter: ["a"],
replacement: function(content: string, node: HTMLImageElement) {
const href = node.getAttribute("href");
if (!href) return content;
if (href.startsWith("/")) {
return `[${content}](${url.origin}${href.replace(/$\//, "")})`;
}
if (href.startsWith("#")) {
if (content.length < 2) return "";
return `[${content}](${url.href}#${href})`.replace("##", "#");
}
if (!href.startsWith("https://") && !href.startsWith("http://")) {
return `[${content}](${url.origin.replace(/\/$/, "")}/${href.replace(/^\//, "")
})`;
}
return `[${content}](${href})`;
},
});
const cleanDocument = parser.parseFromString(
result.content,
"text/html",
);
const markdown = service.turndown(cleanDocument);
streamResponse.enqueue("extracting recipe with openai");
const recipe = await openai.extractRecipe(markdown);
if (recipe) {
if ("errorMessages" in recipe) {
throw new Error("Failed to extract recipe: " + recipe.errorMessages[0]);
} else {
return recipe;
}
}
}
async function processCreateRecipeFromUrl(
{ fetchUrl, streamResponse }: {
fetchUrl: string;
@@ -104,32 +22,19 @@ async function processCreateRecipeFromUrl(
},
) {
log.info("create article from url", { url: fetchUrl });
const url = new URL(fetchUrl);
streamResponse.enqueue("downloading article");
const html = await fetchHtmlWithPlaywright(fetchUrl, streamResponse);
const doc = await webScrape(fetchUrl, streamResponse);
streamResponse.enqueue("download success");
Deno.writeTextFile("article.html", html);
const document = parser.parseFromString(html, "text/html");
const title = document?.querySelector("title")?.innerText;
const images: HTMLImageElement[] = [];
document?.querySelectorAll("img").forEach((img) => {
images.push(img as unknown as HTMLImageElement);
const result = await Defuddle(doc, fetchUrl, {
markdown: true,
});
const metaAuthor =
document?.querySelector('meta[name="twitter:creator"]')?.getAttribute(
"content",
) ||
document?.querySelector('meta[name="author"]')?.getAttribute("content");
streamResponse.enqueue("download success");
const jsonLds = Array.from(
document?.querySelectorAll(
doc?.querySelectorAll(
"script[type='application/ld+json']",
),
) as unknown as HTMLScriptElement[];
@@ -143,78 +48,58 @@ async function processCreateRecipeFromUrl(
}
if (!recipe) {
recipe = await extractUsingAI(url, document, streamResponse);
recipe = await openai.extractRecipe(result.content);
}
const id = (recipe?.title || title || "").replace(/--+/, "-");
const id = safeFileName(recipe?.title || "");
if (!recipe) {
streamResponse.enqueue("failed to parse recipe");
streamResponse.cancel();
return;
}
if (!recipe.image) {
const largestImage = images.filter((img) => {
const src = img.getAttribute("src");
return !!src && !src.startsWith("data:");
}).sort((a, b) => {
const aSize = +(a.getAttribute("width") || 0) +
+(a.getAttribute("height") || 0);
const bSize = +(b.getAttribute("width") || 0) +
+(b.getAttribute("height") || 0);
return aSize > bSize ? -1 : 1;
})[0];
const src = largestImage.getAttribute("src");
if (src) {
recipe.image = makeUrlAbsolute(url, src);
}
}
const newRecipe: Recipe = {
type: "recipe",
id,
name: recipe?.title || title || "",
_type: "Recipe",
name: recipe?.title,
description: recipe?.description,
ingredients: recipe?.ingredients || [],
instructions: recipe?.instructions || [],
notes: recipe?.notes,
tags: recipe.tags || [],
meta: {
image: recipe?.image,
time: recipe?.totalTime
? `${recipe?.totalTime?.toString()} minutes`
: undefined,
link: fetchUrl,
portion: recipe?.servings,
author: metaAuthor ?? recipe?.author,
recipeIngredient: recipe?.ingredients || [],
recipeInstructions: recipe?.instructions || [],
keywords: recipe.tags || [],
image: recipe?.image,
totalTime: recipe?.totalTime
? `${recipe?.totalTime?.toString()} minutes`
: undefined,
url: fetchUrl,
author: {
_type: "Person",
name: recipe?.author,
},
recipeYield: recipe?.servings,
};
if (newRecipe.meta?.image) {
const src = makeUrlAbsolute(url, newRecipe.meta.image);
if (src?.length > 5) {
const extension = fileExtension(new URL(src).pathname);
const finalPath = `Media/articles/images/${safeFileName(id)
}_cover.${extension}`;
if (newRecipe?.image && newRecipe.image.length > 5) {
const extension = fileExtension(new URL(newRecipe.image).pathname);
const finalPath = `resources/recipes/images/${safeFileName(id)
}_cover.${extension}`;
streamResponse.enqueue("downloading image");
try {
streamResponse.enqueue("downloading image");
try {
streamResponse.enqueue("downloading image");
// const res = await fetch(src);
streamResponse.enqueue("saving image");
// const buffer = await res.arrayBuffer();
// await createDocument(finalPath, buffer);
newRecipe.meta.image = finalPath;
} catch (err) {
console.log("Failed to save image", err);
}
const res = await fetch(newRecipe.image);
streamResponse.enqueue("saving image");
const buffer = await res.arrayBuffer();
await createResource(finalPath, buffer);
newRecipe.image = finalPath;
} catch (err) {
console.log("Failed to save image", err);
}
}
streamResponse.enqueue("finished processing, creating file");
// await createRecipe(newRecipe.id, newRecipe);
await createResource(`recipes/${id}.md`, newRecipe);
streamResponse.enqueue("id: " + newRecipe.id);
streamResponse.enqueue("id: " + id);
}
export const handler: Handlers = {

1
routes/api/recipes/create/parseJsonLd.ts
View File

@@ -49,6 +49,7 @@ export function parseJsonLdToRecipeSchema(jsonLdContent: string) {
// Build the recipe object
const recipe = {
_type: "Recipe",
title: data.name || "Unnamed Recipe",
image: pickImage(image || data.image || ""),
author: Array.isArray(data.author)

7
routes/api/series/[name].ts
View File

@@ -3,10 +3,9 @@ import { json } from "@lib/helpers.ts";
import * as tmdb from "@lib/tmdb.ts";
import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
import { isString, safeFileName } from "@lib/string.ts";
import { createDocument } from "@lib/documents.ts";
import { AccessDeniedError } from "@lib/errors.ts";
import { Series } from "@lib/resource/series.ts";
import { fetchResource } from "@lib/marka.ts";
import { createResource, fetchResource } from "@lib/marka.ts";
export const handler: Handlers = {
async GET(_, ctx) {
@@ -39,7 +38,7 @@ export const handler: Handlers = {
finalPath = `Media/series/images/${safeFileName(name)
}_cover.${extension}`;
await createDocument(finalPath, poster);
await createResource(finalPath, poster);
}
const metadata = { tmdbId } as Series["meta"];
@@ -71,7 +70,7 @@ export const handler: Handlers = {
meta: metadata,
};
await createSeries(name, series);
await createResource(`series/${safeFileName(name)}.md`, series);
return json(series);
},

11
routes/api/series/enhance/[name].ts
View File

@@ -1,5 +1,5 @@
import { FreshContext, Handlers } from "$fresh/server.ts";
import { createDocument } from "@lib/documents.ts";
import { FreshContext, Handlers } from "$fresh/server.ts";
import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
import * as tmdb from "@lib/tmdb.ts";
import { safeFileName } from "@lib/string.ts";
@@ -9,6 +9,7 @@ import {
BadRequestError,
NotFoundError,
} from "@lib/errors.ts";
import { createResource, fetchResource } from "@lib/marka.ts";
const isString = (input: string | undefined): input is string => {
return typeof input === "string";
@@ -30,7 +31,7 @@ const POST = async (
throw new BadRequestError();
}
const series = await getSeries(ctx.params.name);
const series = await fetchResource(`series/${ctx.params.name}`);
if (!series) {
throw new NotFoundError();
}
@@ -63,15 +64,15 @@ const POST = async (
let finalPath = "";
if (posterPath && !series.meta?.image) {
// const poster = await tmdb.getMoviePoster(posterPath);
const poster = await tmdb.getMoviePoster(posterPath);
const extension = fileExtension(posterPath);
finalPath = `Media/series/images/${safeFileName(name)}_cover.${extension}`;
// await createDocument(finalPath, poster);
await createResource(finalPath, poster);
series.meta = series.meta || {};
series.meta.image = finalPath;
}
// await createSeries(series.id, series);
await createResource(`series/${safeFileName(series.id)}.md`, series);
return json(series);
};

4
routes/api/series/index.ts
View File

@@ -1,6 +1,10 @@
import { Handlers } from "$fresh/server.ts";
import { json } from "@lib/helpers.ts";
import { fetchResource } from "@lib/marka.ts";
export const handler: Handlers = {
async GET() {
const series = await fetchResource("series");
return json(series);
},
};