feat: cache marka api responses

components/MetaTags.tsx

@@ -26,7 +26,7 @@ function generateJsonLd(resource: GenericResource): string {
   if (resource.content?.datePublished) {
     try {
       baseSchema.datePublished = formatDate(
-        new Date(resource.content.datePublished),
+        resource.content.datePublished,
       );
     } catch (_) {
       // Ignore invalid date

islands/KMenu/commands/create_series.ts

@@ -50,6 +50,7 @@ export const createNewSeries: MenuEntry = {
           return {
             title: `${r.name} - ${r.first_air_date}`,
             cb: async () => {
+              try {
                 state.activeState.value = "loading";
                 const response = await fetch("/api/series/" + r.id, {
                   method: "POST",
@@ -57,6 +58,9 @@ export const createNewSeries: MenuEntry = {
                 const series = await response.json() as ReviewResource;
                 unsub();
                 globalThis.location.href = "/series/" + series.name;
+              } catch (_e) {
+                state.activeState.value = "normal";
+              }
             },
           };
         }),

lib/marka/index.ts

@@ -1,3 +1,4 @@
+import { createCache } from "../cache.ts";
 import { MARKA_API_KEY } from "../env.ts";
 import { getImage } from "../image.ts";
 import { GenericResource } from "./schema.ts";
@@ -24,32 +25,62 @@ async function addImageToResource<T extends GenericResource>(
   return resource as T;
 }
 
-export async function fetchResource<T extends GenericResource>(
+type Resource = GenericResource & {
+  content: GenericResource["content"] & Array<GenericResource>;
+};
+
+const fetchCache = createCache<Resource>("marka");
+const cacheLock = new Map<string, Promise<Resource>>();
+async function cachedFetch(
+  url: string,
+): Promise<Resource | undefined> {
+  if (fetchCache.has(url)) {
+    return fetchCache.get(url);
+  }
+  if (cacheLock.has(url)) {
+    return cacheLock.get(url);
+  }
+  const response = (async () => {
+    const response = await fetch(url);
+    const res = await response.json();
+    fetchCache.set(url, res);
+    return res;
+  })();
+  cacheLock.set(
+    url,
+    response,
+  );
+  const res = await response;
+  if (!res) {
+    cacheLock.delete(url);
+  }
+  return res;
+}
+
+export async function fetchResource<T extends Resource>(
   resource: string,
 ): Promise<T | undefined> {
   try {
-    const response = await fetch(
+    const d = `${url}/resources/${resource}`;
-      `${url}/resources/${resource}`,
+    const res = await cachedFetch(d);
-    );
+    if (!res) return;
-    const res = await response.json();
     return addImageToResource<T>(res);
   } catch (_e) {
     return;
   }
 }
 
-export async function listResources<T = GenericResource>(
+export async function listResources<T extends GenericResource>(
   resource: string,
 ): Promise<T[]> {
   try {
-    const response = await fetch(
+    const d = `${url}/resources/${resource}`;
-      `${url}/resources/${resource}`,
+    const list = await cachedFetch(d);
-    );
+    if (!list) return [];
-    const list = await response.json();
     return Promise.all(
       list?.content
-        .filter((a: GenericResource) => a?.content?._type)
+        .filter((a) => a?.content?._type)
-        .map((res: GenericResource) => addImageToResource(res)),
+        .map((res) => addImageToResource(res) as Promise<T>),
     );
   } catch (_e) {
     return [];

lib/string.ts

@@ -1,4 +1,13 @@
-export function formatDate(date: Date): string {
+export function formatDate(date?: string | Date): string {
+  if (!date) return "";
+  if (typeof date === "string") {
+    try {
+      const d = new Date(date);
+      return formatDate(d);
+    } catch (_e) {
+      return "";
+    }
+  }
   const options = { year: "numeric", month: "long", day: "numeric" } as const;
   return new Intl.DateTimeFormat("en-US", options).format(date);
 }

routes/api/articles/create/index.ts

@@ -6,13 +6,15 @@ import * as openai from "@lib/openai.ts";
 import { getYoutubeVideoDetails } from "@lib/youtube.ts";
 import {
   extractYoutubeId,
+  formatDate,
   isYoutubeLink,
-  toUrlSafeString,
+  safeFileName,
 } from "@lib/string.ts";
 import { createLogger } from "@lib/log/index.ts";
 import { createResource } from "@lib/marka/index.ts";
 import { webScrape } from "@lib/webScraper.ts";
 import { ArticleResource } from "@lib/marka/schema.ts";
+import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
 
 const log = createLogger("api/article");
 
@@ -44,16 +46,34 @@ async function processCreateArticle(
   streamResponse.enqueue("postprocessing article");
 
   const title = result?.title || aiMeta?.headline || "";
-  const id = toUrlSafeString(title);
+
+  let finalPath = result.image;
+  if (result?.image) {
+    const extension = fileExtension(result?.image);
+    const imagePath = `resources/articles/images/${
+      safeFileName(title)
+    }_cover.${extension}`;
+    try {
+      streamResponse.enqueue("downloading image");
+      const res = await fetch(result.image);
+      streamResponse.enqueue("saving image");
+      const buffer = await res.arrayBuffer();
+      await createResource(imagePath, buffer);
+      finalPath = imagePath;
+    } catch (err) {
+      console.log("Failed to save image", err);
+    }
+  }
 
   const newArticle: ArticleResource["content"] = {
     _type: "Article",
     headline: title,
     articleBody: result.content,
     url: fetchUrl,
-    datePublished: result?.published || aiMeta?.datePublished ||
+    datePublished: formatDate(
-      new Date().toISOString(),
+      result?.published || aiMeta?.datePublished || undefined,
-    image: result?.image,
+    ),
+    image: finalPath,
     author: {
       _type: "Person",
       name: (result.schemaOrgData?.author?.name || aiMeta?.author || "")
@@ -66,9 +86,9 @@ async function processCreateArticle(
 
   streamResponse.enqueue("writing to disk");
 
-  await createResource(`articles/${id}.md`, newArticle);
+  await createResource(`articles/${title}.md`, newArticle);
 
-  streamResponse.enqueue("id: " + id);
+  streamResponse.enqueue("id: " + title);
 }
 
 async function processCreateYoutubeVideo(

routes/api/movies/[name].ts

@@ -2,8 +2,8 @@ import { Handlers } from "$fresh/server.ts";
 import { json } from "@lib/helpers.ts";
 import * as tmdb from "@lib/tmdb.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
-import { isString, safeFileName } from "@lib/string.ts";
+import { formatDate, isString, safeFileName } from "@lib/string.ts";
-import { AccessDeniedError } from "@lib/errors.ts";
+import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
 import { createResource, fetchResource } from "@lib/marka/index.ts";
 import { ReviewResource } from "@lib/marka/schema.ts";
 
@@ -14,26 +14,22 @@ export const handler: Handlers = {
   },
   async POST(_, ctx) {
     const session = ctx.state.session;
-    if (!session) {
+    if (!session) throw new AccessDeniedError();
-      throw new AccessDeniedError();
-    }
 
     const tmdbId = parseInt(ctx.params.name);
+    if (Number.isNaN(tmdbId)) throw new BadRequestError();
 
-    const movieDetails = await tmdb.getMovie(tmdbId);
+    const [movieDetails, movieCredits] = await Promise.all([
-    const movieCredits = await tmdb.getMovieCredits(tmdbId);
+      tmdb.getMovie(tmdbId),
+      tmdb.getMovieCredits(tmdbId),
+    ]);
 
-    const releaseDate = movieDetails.release_date;
+    const name = movieDetails.title ||
-    const posterPath = movieDetails.poster_path;
+      movieDetails.original_title ||
-    const director = movieCredits?.crew?.filter?.((person) =>
-      person.job === "Director"
-    )[0];
-
-    movieDetails.overview;
-
-    let finalPath = "";
-    const name = movieDetails.title || movieDetails.original_title ||
       ctx.params.name;
+
+    const posterPath = movieDetails.poster_path;
+    let finalPath = "";
     if (posterPath) {
       const poster = await tmdb.getMoviePoster(posterPath);
       const extension = fileExtension(posterPath);
@@ -41,33 +37,32 @@ export const handler: Handlers = {
       await createResource(finalPath, poster);
     }
 
-    const tags: string[] = [];
+    const keywords = movieDetails.genres
-    if (movieDetails.genres) {
+      ?.map((g) => g.name?.toLowerCase())
-      tags.push(
+      .filter(isString) || [];
-        ...movieDetails.genres.map((g) => g.name?.toLowerCase()).filter(
-          isString,
-        ),
-      );
-    }
 
     const movie: ReviewResource["content"] = {
       _type: "Review",
       image: `resources/${finalPath}`,
-      datePublished: releaseDate,
+      datePublished: formatDate(movieDetails.release_date),
       tmdbId,
       author: {
         _type: "Person",
-        name: director?.name,
+        name: movieCredits.crew?.filter?.((person) =>
+          person.job === "Director"
+        )[0]?.name,
       },
       itemReviewed: {
-        name: name,
+        name,
       },
       reviewBody: "",
-      keywords: tags,
+      keywords,
     };
 
-    await createResource(`movies/${safeFileName(name)}.md`, movie);
+    const fileName = `${safeFileName(name)}.md`;
 
-    return json(movie);
+    await createResource(`movies/${fileName}`, movie);
+
+    return json({ name: fileName });
   },
 };

routes/api/movies/enhance/[name].ts

@@ -39,15 +39,16 @@ const POST = async (
   const movieCredits = !movie.content?.author &&
     await tmdb.getMovieCredits(tmdbId);
 
-  const releaseDate = movieDetails.release_date;
-  if (releaseDate && !movie.content?.datePublished) {
-    movie.content = movie.content || {};
-    movie.content.datePublished = formatDate(new Date(releaseDate));
-  }
   const director = movieCredits &&
     movieCredits?.crew?.filter?.((person) => person.job === "Director")[0];
+
+  movie.content ??= {
+    _type: "Review",
+  };
+
+  movie.content.datePublished ??= formatDate(movieDetails.release_date);
+
   if (director && !movie.content?.author) {
-    movie.content = movie.content || {};
     movie.content.author = {
       _type: "Person",
       name: director.name,
@@ -65,18 +66,15 @@ const POST = async (
     ];
   }
 
-  if (!movie.name) {
+  movie.content.tmdbId ??= tmdbId;
-    movie.name = tmdbId;
-  }
 
   let finalPath = "";
   const posterPath = movieDetails.poster_path;
-  if (posterPath && !movie.content?.image) {
+  if (posterPath && !movie.content.image) {
     const poster = await tmdb.getMoviePoster(posterPath);
     const extension = fileExtension(posterPath);
     finalPath = `movies/images/${safeFileName(name)}_cover.${extension}`;
     await createResource(finalPath, poster);
-    movie.content = movie.content || {};
     movie.content.image = finalPath;
   }
 

routes/api/recipes/create/index.ts

@@ -76,7 +76,7 @@ async function processCreateRecipeFromUrl(
   };
 
   if (newRecipe?.image && newRecipe.image.length > 5) {
-    const extension = fileExtension(new URL(newRecipe.image).pathname);
+    const extension = fileExtension(newRecipe.image);
     const finalPath = `resources/recipes/images/${
       safeFileName(id)
     }_cover.${extension}`;

routes/api/series/[name].ts

@@ -2,11 +2,19 @@ import { Handlers } from "$fresh/server.ts";
 import { json } from "@lib/helpers.ts";
 import * as tmdb from "@lib/tmdb.ts";
 import { fileExtension } from "https://deno.land/x/file_extension@v2.1.0/mod.ts";
-import { isString, safeFileName } from "@lib/string.ts";
+import { formatDate, isString, safeFileName } from "@lib/string.ts";
-import { AccessDeniedError } from "@lib/errors.ts";
+import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
 import { createResource, fetchResource } from "@lib/marka/index.ts";
 import { ReviewResource } from "@lib/marka/schema.ts";
 
+function pickDirector(
+  credits: Awaited<ReturnType<typeof tmdb.getSeriesCredits>>,
+  createdBy?: { name?: string }[],
+): string | undefined {
+  const crewDirector = credits?.crew?.find?.((p) => p.job === "Director");
+  return crewDirector?.name ?? createdBy?.[0]?.name;
+}
+
 export const handler: Handlers = {
   async GET(_, ctx) {
     const series = await fetchResource(`series/${ctx.params.name}`);
@@ -19,54 +27,54 @@ export const handler: Handlers = {
     }
 
     const tmdbId = parseInt(ctx.params.name);
+    if (Number.isNaN(tmdbId)) throw new BadRequestError();
 
-    const seriesDetails = await tmdb.getSeries(tmdbId);
+    const [seriesDetails, seriesCredits] = await Promise.all([
-    const seriesCredits = await tmdb.getSeriesCredits(tmdbId);
+      tmdb.getSeries(tmdbId),
+      tmdb.getSeriesCredits(tmdbId),
+    ]);
 
-    const releaseDate = seriesDetails.first_air_date;
+    const name = seriesDetails.name ||
-    const posterPath = seriesDetails.poster_path;
+      seriesDetails.original_name ||
-    const director =
+      ctx.params.name;
-      seriesCredits?.crew?.filter?.((person) => person.job === "Director")[0] ||
-      seriesDetails.created_by?.[0];
 
     let finalPath = "";
-    const name = seriesDetails.name || seriesDetails.original_name ||
+    const posterPath = seriesDetails.poster_path;
-      ctx.params.name;
     if (posterPath) {
       const poster = await tmdb.getMoviePoster(posterPath);
       const extension = fileExtension(posterPath);
-
+      const imagePath = `series/images/${
-      finalPath = `series/images/${safeFileName(name)}_cover.${extension}`;
+        safeFileName(name)
-      await createResource(finalPath, poster);
+      }_cover.${extension}`;
+      await createResource(imagePath, poster);
+      finalPath = imagePath;
     }
 
-    const tags: string[] = [];
+    const keywords = seriesDetails.genres
-    if (seriesDetails.genres) {
+      ?.map((g) => g.name?.toLowerCase())
-      tags.push(
+      .filter(isString) ??
-        ...seriesDetails.genres.map((g) => g.name?.toLowerCase()).filter(
+      [];
-          isString,
-        ),
-      );
-    }
 
     const series: ReviewResource["content"] = {
       _type: "Review",
       image: `resources/${finalPath}`,
-      datePublished: releaseDate,
+      datePublished: formatDate(seriesDetails.first_air_date),
       tmdbId,
       author: {
         _type: "Person",
-        name: director?.name,
+        name: pickDirector(seriesCredits, seriesDetails?.created_by),
       },
       itemReviewed: {
         name: name,
       },
       reviewBody: "",
-      keywords: tags,
+      keywords: keywords,
     };
 
-    await createResource(`series/${safeFileName(name)}.md`, series);
+    const fileName = `${safeFileName(name)}.md`;
 
-    return json(series);
+    await createResource(`series/${fileName}`, series);
+
+    return json({ name: fileName });
   },
 };