feat: cache marka api responses

2025-11-04 12:09:17 +01:00
parent bb4e895770
commit fea9b69d4d
9 changed files with 163 additions and 98 deletions
--- a/components/MetaTags.tsx
+++ b/components/MetaTags.tsx
@@ -26,7 +26,7 @@ function generateJsonLd(resource: GenericResource): string {
  if (resource.content?.datePublished) {
    try {
      baseSchema.datePublished = formatDate(
-        new Date(resource.content.datePublished),
+        resource.content.datePublished,
      );
    } catch (_) {
      // Ignore invalid date
--- a/islands/KMenu/commands/create_series.ts
+++ b/islands/KMenu/commands/create_series.ts
@@ -50,6 +50,7 @@ export const createNewSeries: MenuEntry = {
          return {
            title: `${r.name} - ${r.first_air_date}`,
            cb: async () => {
+              try {
                state.activeState.value = "loading";
                const response = await fetch("/api/series/" + r.id, {
                  method: "POST",
@@ -57,6 +58,9 @@ export const createNewSeries: MenuEntry = {
                const series = await response.json() as ReviewResource;
                unsub();
                globalThis.location.href = "/series/" + series.name;
+              } catch (_e) {
+                state.activeState.value = "normal";
+              }
            },
          };
        }),
--- a/lib/marka/index.ts
+++ b/lib/marka/index.ts
@@ -1,3 +1,4 @@
+import { createCache } from "../cache.ts";
 import { MARKA_API_KEY } from "../env.ts";
 import { getImage } from "../image.ts";
 import { GenericResource } from "./schema.ts";
@@ -24,32 +25,62 @@ async function addImageToResource<T extends GenericResource>(
  return resource as T;
 }

-export async function fetchResource<T extends GenericResource>(
+type Resource = GenericResource & {
+  content: GenericResource["content"] & Array<GenericResource>;
+};
+
+const fetchCache = createCache<Resource>("marka");
+const cacheLock = new Map<string, Promise<Resource>>();
+async function cachedFetch(
+  url: string,
+): Promise<Resource | undefined> {
+  if (fetchCache.has(url)) {
+    return fetchCache.get(url);
+  }
+  if (cacheLock.has(url)) {
+    return cacheLock.get(url);
+  }
+  const response = (async () => {
+    const response = await fetch(url);
+    const res = await response.json();
+    fetchCache.set(url, res);
+    return res;
+  })();
+  cacheLock.set(
+    url,
+    response,
+  );
+  const res = await response;
+  if (!res) {
+    cacheLock.delete(url);
+  }
+  return res;
+}
+
+export async function fetchResource<T extends Resource>(
  resource: string,
 ): Promise<T | undefined> {
  try {
-    const response = await fetch(
-      `${url}/resources/${resource}`,
-    );
-    const res = await response.json();
+    const d = `${url}/resources/${resource}`;
+    const res = await cachedFetch(d);
+    if (!res) return;
    return addImageToResource<T>(res);
  } catch (_e) {
    return;
  }
 }

-export async function listResources<T = GenericResource>(
+export async function listResources<T extends GenericResource>(
  resource: string,
 ): Promise<T[]> {
  try {
-    const response = await fetch(
-      `${url}/resources/${resource}`,
-    );
-    const list = await response.json();
+    const d = `${url}/resources/${resource}`;
+    const list = await cachedFetch(d);
+    if (!list) return [];
    return Promise.all(
      list?.content
-        .filter((a: GenericResource) => a?.content?._type)
-        .map((res: GenericResource) => addImageToResource(res)),
+        .filter((a) => a?.content?._type)
+        .map((res) => addImageToResource(res) as Promise<T>),
    );
  } catch (_e) {
    return [];
--- a/lib/string.ts
+++ b/lib/string.ts
@@ -1,4 +1,13 @@
-export function formatDate(date: Date): string {
+export function formatDate(date?: string | Date): string {
+  if (!date) return "";
+  if (typeof date === "string") {
+    try {
+      const d = new Date(date);
+      return formatDate(d);
+    } catch (_e) {
+      return "";
+    }
+  }
  const options = { year: "numeric", month: "long", day: "numeric" } as const;
  return new Intl.DateTimeFormat("en-US", options).format(date);
 }
--- a/routes/api/articles/create/index.ts
+++ b/routes/api/articles/create/index.ts
@@ -6,13 +6,15 @@ import * as openai from "@lib/openai.ts";
 import { getYoutubeVideoDetails } from "@lib/youtube.ts";
 import {
  extractYoutubeId,
+  formatDate,
  isYoutubeLink,
-  toUrlSafeString,
+  safeFileName,
 } from "@lib/string.ts";
 import { createLogger } from "@lib/log/index.ts";
 import { createResource } from "@lib/marka/index.ts";
 import { webScrape } from "@lib/webScraper.ts";
 import { ArticleResource } from "@lib/marka/schema.ts";
+import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";

 const log = createLogger("api/article");

@@ -44,16 +46,34 @@ async function processCreateArticle(
  streamResponse.enqueue("postprocessing article");

  const title = result?.title || aiMeta?.headline || "";
-  const id = toUrlSafeString(title);
+
+  let finalPath = result.image;
+  if (result?.image) {
+    const extension = fileExtension(result?.image);
+    const imagePath = `resources/articles/images/${
+      safeFileName(title)
+    }_cover.${extension}`;
+    try {
+      streamResponse.enqueue("downloading image");
+      const res = await fetch(result.image);
+      streamResponse.enqueue("saving image");
+      const buffer = await res.arrayBuffer();
+      await createResource(imagePath, buffer);
+      finalPath = imagePath;
+    } catch (err) {
+      console.log("Failed to save image", err);
+    }
+  }

  const newArticle: ArticleResource["content"] = {
    _type: "Article",
    headline: title,
    articleBody: result.content,
    url: fetchUrl,
-    datePublished: result?.published || aiMeta?.datePublished ||
-      new Date().toISOString(),
-    image: result?.image,
+    datePublished: formatDate(
+      result?.published || aiMeta?.datePublished || undefined,
+    ),
+    image: finalPath,
    author: {
      _type: "Person",
      name: (result.schemaOrgData?.author?.name || aiMeta?.author || "")
@@ -66,9 +86,9 @@ async function processCreateArticle(

  streamResponse.enqueue("writing to disk");

-  await createResource(`articles/${id}.md`, newArticle);
+  await createResource(`articles/${title}.md`, newArticle);

-  streamResponse.enqueue("id: " + id);
+  streamResponse.enqueue("id: " + title);
 }

 async function processCreateYoutubeVideo(
--- a/routes/api/movies/[name].ts
+++ b/routes/api/movies/[name].ts
@@ -2,8 +2,8 @@ import { Handlers } from "$fresh/server.ts";
 import { json } from "@lib/helpers.ts";
 import * as tmdb from "@lib/tmdb.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
-import { isString, safeFileName } from "@lib/string.ts";
-import { AccessDeniedError } from "@lib/errors.ts";
+import { formatDate, isString, safeFileName } from "@lib/string.ts";
+import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
 import { createResource, fetchResource } from "@lib/marka/index.ts";
 import { ReviewResource } from "@lib/marka/schema.ts";

@@ -14,26 +14,22 @@ export const handler: Handlers = {
  },
  async POST(_, ctx) {
    const session = ctx.state.session;
-    if (!session) {
-      throw new AccessDeniedError();
-    }
+    if (!session) throw new AccessDeniedError();

    const tmdbId = parseInt(ctx.params.name);
+    if (Number.isNaN(tmdbId)) throw new BadRequestError();

-    const movieDetails = await tmdb.getMovie(tmdbId);
-    const movieCredits = await tmdb.getMovieCredits(tmdbId);
+    const [movieDetails, movieCredits] = await Promise.all([
+      tmdb.getMovie(tmdbId),
+      tmdb.getMovieCredits(tmdbId),
+    ]);

-    const releaseDate = movieDetails.release_date;
-    const posterPath = movieDetails.poster_path;
-    const director = movieCredits?.crew?.filter?.((person) =>
-      person.job === "Director"
-    )[0];
-
-    movieDetails.overview;
-
-    let finalPath = "";
-    const name = movieDetails.title || movieDetails.original_title ||
+    const name = movieDetails.title ||
+      movieDetails.original_title ||
      ctx.params.name;
+
+    const posterPath = movieDetails.poster_path;
+    let finalPath = "";
    if (posterPath) {
      const poster = await tmdb.getMoviePoster(posterPath);
      const extension = fileExtension(posterPath);
@@ -41,33 +37,32 @@ export const handler: Handlers = {
      await createResource(finalPath, poster);
    }

-    const tags: string[] = [];
-    if (movieDetails.genres) {
-      tags.push(
-        ...movieDetails.genres.map((g) => g.name?.toLowerCase()).filter(
-          isString,
-        ),
-      );
-    }
+    const keywords = movieDetails.genres
+      ?.map((g) => g.name?.toLowerCase())
+      .filter(isString) || [];

    const movie: ReviewResource["content"] = {
      _type: "Review",
      image: `resources/${finalPath}`,
-      datePublished: releaseDate,
+      datePublished: formatDate(movieDetails.release_date),
      tmdbId,
      author: {
        _type: "Person",
-        name: director?.name,
+        name: movieCredits.crew?.filter?.((person) =>
+          person.job === "Director"
+        )[0]?.name,
      },
      itemReviewed: {
-        name: name,
+        name,
      },
      reviewBody: "",
-      keywords: tags,
+      keywords,
    };

-    await createResource(`movies/${safeFileName(name)}.md`, movie);
+    const fileName = `${safeFileName(name)}.md`;

-    return json(movie);
+    await createResource(`movies/${fileName}`, movie);
+
+    return json({ name: fileName });
  },
 };
--- a/routes/api/movies/enhance/[name].ts
+++ b/routes/api/movies/enhance/[name].ts
@@ -39,15 +39,16 @@ const POST = async (
  const movieCredits = !movie.content?.author &&
    await tmdb.getMovieCredits(tmdbId);

-  const releaseDate = movieDetails.release_date;
-  if (releaseDate && !movie.content?.datePublished) {
-    movie.content = movie.content || {};
-    movie.content.datePublished = formatDate(new Date(releaseDate));
-  }
  const director = movieCredits &&
    movieCredits?.crew?.filter?.((person) => person.job === "Director")[0];
+
+  movie.content ??= {
+    _type: "Review",
+  };
+
+  movie.content.datePublished ??= formatDate(movieDetails.release_date);
+
  if (director && !movie.content?.author) {
-    movie.content = movie.content || {};
    movie.content.author = {
      _type: "Person",
      name: director.name,
@@ -65,18 +66,15 @@ const POST = async (
    ];
  }

-  if (!movie.name) {
-    movie.name = tmdbId;
-  }
+  movie.content.tmdbId ??= tmdbId;

  let finalPath = "";
  const posterPath = movieDetails.poster_path;
-  if (posterPath && !movie.content?.image) {
+  if (posterPath && !movie.content.image) {
    const poster = await tmdb.getMoviePoster(posterPath);
    const extension = fileExtension(posterPath);
    finalPath = `movies/images/${safeFileName(name)}_cover.${extension}`;
    await createResource(finalPath, poster);
-    movie.content = movie.content || {};
    movie.content.image = finalPath;
  }

--- a/routes/api/recipes/create/index.ts
+++ b/routes/api/recipes/create/index.ts
@@ -76,7 +76,7 @@ async function processCreateRecipeFromUrl(
  };

  if (newRecipe?.image && newRecipe.image.length > 5) {
-    const extension = fileExtension(new URL(newRecipe.image).pathname);
+    const extension = fileExtension(newRecipe.image);
    const finalPath = `resources/recipes/images/${
      safeFileName(id)
    }_cover.${extension}`;
--- a/routes/api/series/[name].ts
+++ b/routes/api/series/[name].ts
@@ -2,11 +2,19 @@ import { Handlers } from "$fresh/server.ts";
 import { json } from "@lib/helpers.ts";
 import * as tmdb from "@lib/tmdb.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
-import { isString, safeFileName } from "@lib/string.ts";
-import { AccessDeniedError } from "@lib/errors.ts";
+import { formatDate, isString, safeFileName } from "@lib/string.ts";
+import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
 import { createResource, fetchResource } from "@lib/marka/index.ts";
 import { ReviewResource } from "@lib/marka/schema.ts";

+function pickDirector(
+  credits: Awaited<ReturnType<typeof tmdb.getSeriesCredits>>,
+  createdBy?: { name?: string }[],
+): string | undefined {
+  const crewDirector = credits?.crew?.find?.((p) => p.job === "Director");
+  return crewDirector?.name ?? createdBy?.[0]?.name;
+}
+
 export const handler: Handlers = {
  async GET(_, ctx) {
    const series = await fetchResource(`series/${ctx.params.name}`);
@@ -19,54 +27,54 @@ export const handler: Handlers = {
    }

    const tmdbId = parseInt(ctx.params.name);
+    if (Number.isNaN(tmdbId)) throw new BadRequestError();

-    const seriesDetails = await tmdb.getSeries(tmdbId);
-    const seriesCredits = await tmdb.getSeriesCredits(tmdbId);
+    const [seriesDetails, seriesCredits] = await Promise.all([
+      tmdb.getSeries(tmdbId),
+      tmdb.getSeriesCredits(tmdbId),
+    ]);

-    const releaseDate = seriesDetails.first_air_date;
-    const posterPath = seriesDetails.poster_path;
-    const director =
-      seriesCredits?.crew?.filter?.((person) => person.job === "Director")[0] ||
-      seriesDetails.created_by?.[0];
+    const name = seriesDetails.name ||
+      seriesDetails.original_name ||
+      ctx.params.name;

    let finalPath = "";
-    const name = seriesDetails.name || seriesDetails.original_name ||
-      ctx.params.name;
+    const posterPath = seriesDetails.poster_path;
    if (posterPath) {
      const poster = await tmdb.getMoviePoster(posterPath);
      const extension = fileExtension(posterPath);
-
-      finalPath = `series/images/${safeFileName(name)}_cover.${extension}`;
-      await createResource(finalPath, poster);
+      const imagePath = `series/images/${
+        safeFileName(name)
+      }_cover.${extension}`;
+      await createResource(imagePath, poster);
+      finalPath = imagePath;
    }

-    const tags: string[] = [];
-    if (seriesDetails.genres) {
-      tags.push(
-        ...seriesDetails.genres.map((g) => g.name?.toLowerCase()).filter(
-          isString,
-        ),
-      );
-    }
+    const keywords = seriesDetails.genres
+      ?.map((g) => g.name?.toLowerCase())
+      .filter(isString) ??
+      [];

    const series: ReviewResource["content"] = {
      _type: "Review",
      image: `resources/${finalPath}`,
-      datePublished: releaseDate,
+      datePublished: formatDate(seriesDetails.first_air_date),
      tmdbId,
      author: {
        _type: "Person",
-        name: director?.name,
+        name: pickDirector(seriesCredits, seriesDetails?.created_by),
      },
      itemReviewed: {
        name: name,
      },
      reviewBody: "",
-      keywords: tags,
+      keywords: keywords,
    };

-    await createResource(`series/${safeFileName(name)}.md`, series);
+    const fileName = `${safeFileName(name)}.md`;

-    return json(series);
+    await createResource(`series/${fileName}`, series);
+
+    return json({ name: fileName });
  },
 };