feat: add update all recommendations command

fresh.gen.ts

@@ -25,23 +25,24 @@ import * as $19 from "./routes/api/query/index.ts";
 import * as $20 from "./routes/api/query/sync.ts";
 import * as $21 from "./routes/api/recipes/[name].ts";
 import * as $22 from "./routes/api/recipes/index.ts";
-import * as $23 from "./routes/api/recommendation/index.ts";
+import * as $23 from "./routes/api/recommendation/all.ts";
-import * as $24 from "./routes/api/resources.ts";
+import * as $24 from "./routes/api/recommendation/index.ts";
-import * as $25 from "./routes/api/series/[name].ts";
+import * as $25 from "./routes/api/resources.ts";
-import * as $26 from "./routes/api/series/enhance/[name].ts";
+import * as $26 from "./routes/api/series/[name].ts";
-import * as $27 from "./routes/api/series/index.ts";
+import * as $27 from "./routes/api/series/enhance/[name].ts";
-import * as $28 from "./routes/api/tmdb/[id].ts";
+import * as $28 from "./routes/api/series/index.ts";
-import * as $29 from "./routes/api/tmdb/credits/[id].ts";
+import * as $29 from "./routes/api/tmdb/[id].ts";
-import * as $30 from "./routes/api/tmdb/query.ts";
+import * as $30 from "./routes/api/tmdb/credits/[id].ts";
-import * as $31 from "./routes/articles/[name].tsx";
+import * as $31 from "./routes/api/tmdb/query.ts";
-import * as $32 from "./routes/articles/index.tsx";
+import * as $32 from "./routes/articles/[name].tsx";
-import * as $33 from "./routes/index.tsx";
+import * as $33 from "./routes/articles/index.tsx";
-import * as $34 from "./routes/movies/[name].tsx";
+import * as $34 from "./routes/index.tsx";
-import * as $35 from "./routes/movies/index.tsx";
+import * as $35 from "./routes/movies/[name].tsx";
-import * as $36 from "./routes/recipes/[name].tsx";
+import * as $36 from "./routes/movies/index.tsx";
-import * as $37 from "./routes/recipes/index.tsx";
+import * as $37 from "./routes/recipes/[name].tsx";
-import * as $38 from "./routes/series/[name].tsx";
+import * as $38 from "./routes/recipes/index.tsx";
-import * as $39 from "./routes/series/index.tsx";
+import * as $39 from "./routes/series/[name].tsx";
+import * as $40 from "./routes/series/index.tsx";
 import * as $$0 from "./islands/Counter.tsx";
 import * as $$1 from "./islands/IngredientsList.tsx";
 import * as $$2 from "./islands/KMenu.tsx";
@@ -50,9 +51,10 @@ import * as $$4 from "./islands/KMenu/commands/add_movie_infos.ts";
 import * as $$5 from "./islands/KMenu/commands/add_series_infos.ts";
 import * as $$6 from "./islands/KMenu/commands/create_article.ts";
 import * as $$7 from "./islands/KMenu/commands/create_movie.ts";
-import * as $$8 from "./islands/KMenu/commands/create_series.ts";
+import * as $$8 from "./islands/KMenu/commands/create_recommendations.ts";
-import * as $$9 from "./islands/KMenu/types.ts";
+import * as $$9 from "./islands/KMenu/commands/create_series.ts";
-import * as $$10 from "./islands/Search.tsx";
+import * as $$10 from "./islands/KMenu/types.ts";
+import * as $$11 from "./islands/Search.tsx";
 
 const manifest = {
   routes: {
@@ -79,23 +81,24 @@ const manifest = {
     "./routes/api/query/sync.ts": $20,
     "./routes/api/recipes/[name].ts": $21,
     "./routes/api/recipes/index.ts": $22,
-    "./routes/api/recommendation/index.ts": $23,
+    "./routes/api/recommendation/all.ts": $23,
-    "./routes/api/resources.ts": $24,
+    "./routes/api/recommendation/index.ts": $24,
-    "./routes/api/series/[name].ts": $25,
+    "./routes/api/resources.ts": $25,
-    "./routes/api/series/enhance/[name].ts": $26,
+    "./routes/api/series/[name].ts": $26,
-    "./routes/api/series/index.ts": $27,
+    "./routes/api/series/enhance/[name].ts": $27,
-    "./routes/api/tmdb/[id].ts": $28,
+    "./routes/api/series/index.ts": $28,
-    "./routes/api/tmdb/credits/[id].ts": $29,
+    "./routes/api/tmdb/[id].ts": $29,
-    "./routes/api/tmdb/query.ts": $30,
+    "./routes/api/tmdb/credits/[id].ts": $30,
-    "./routes/articles/[name].tsx": $31,
+    "./routes/api/tmdb/query.ts": $31,
-    "./routes/articles/index.tsx": $32,
+    "./routes/articles/[name].tsx": $32,
-    "./routes/index.tsx": $33,
+    "./routes/articles/index.tsx": $33,
-    "./routes/movies/[name].tsx": $34,
+    "./routes/index.tsx": $34,
-    "./routes/movies/index.tsx": $35,
+    "./routes/movies/[name].tsx": $35,
-    "./routes/recipes/[name].tsx": $36,
+    "./routes/movies/index.tsx": $36,
-    "./routes/recipes/index.tsx": $37,
+    "./routes/recipes/[name].tsx": $37,
-    "./routes/series/[name].tsx": $38,
+    "./routes/recipes/index.tsx": $38,
-    "./routes/series/index.tsx": $39,
+    "./routes/series/[name].tsx": $39,
+    "./routes/series/index.tsx": $40,
   },
   islands: {
     "./islands/Counter.tsx": $$0,
@@ -106,9 +109,10 @@ const manifest = {
     "./islands/KMenu/commands/add_series_infos.ts": $$5,
     "./islands/KMenu/commands/create_article.ts": $$6,
     "./islands/KMenu/commands/create_movie.ts": $$7,
-    "./islands/KMenu/commands/create_series.ts": $$8,
+    "./islands/KMenu/commands/create_recommendations.ts": $$8,
-    "./islands/KMenu/types.ts": $$9,
+    "./islands/KMenu/commands/create_series.ts": $$9,
-    "./islands/Search.tsx": $$10,
+    "./islands/KMenu/types.ts": $$10,
+    "./islands/Search.tsx": $$11,
   },
   baseUrl: import.meta.url,
 };

islands/KMenu/commands.ts

@@ -5,6 +5,7 @@ import { createNewArticle } from "@islands/KMenu/commands/create_article.ts";
 import { getCookie } from "@lib/string.ts";
 import { addSeriesInfo } from "@islands/KMenu/commands/add_series_infos.ts";
 import { createNewSeries } from "@islands/KMenu/commands/create_series.ts";
+import { updateAllRecommendations } from "@islands/KMenu/commands/create_recommendations.ts";
 
 export const menus: Record<string, Menu> = {
   main: {
@@ -85,6 +86,7 @@ export const menus: Record<string, Menu> = {
       createNewMovie,
       createNewSeries,
       addMovieInfos,
+      updateAllRecommendations,
     ],
   },
 };

islands/KMenu/commands/create_recommendations.ts

@@ -0,0 +1,29 @@
+import { MenuEntry } from "@islands/KMenu/types.ts";
+import { fetchStream } from "@lib/helpers.ts";
+import { getCookie } from "@lib/string.ts";
+
+export const updateAllRecommendations: MenuEntry = {
+  title: "Update all recommendations",
+  meta: "",
+  icon: "IconSquareRoundedPlus",
+  cb: (state) => {
+    state.activeState.value = "loading";
+
+    fetchStream("/api/recommendation/all", (chunk) => {
+      if (chunk.toLowerCase().includes("finish")) {
+        setTimeout(() => {
+          window.location.reload();
+        }, 500);
+      } else {
+        state.loadingText.value = chunk;
+      }
+    });
+  },
+  visible: () => {
+    if (!getCookie("session_cookie")) return false;
+    if (
+      !globalThis?.location?.pathname?.includes("movies")
+    ) return false;
+    return true;
+  },
+};

lib/openai.ts

@@ -3,6 +3,16 @@ import { OPENAI_API_KEY } from "@lib/env.ts";
 
 const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY);
 
+function extractListFromResponse(response?: string): string[] {
+  if (!response) return [];
+  return response
+    .split(/[\n,]/)
+    .map((line) => line.trim())
+    .filter((line) => !line.endsWith(":"))
+    .map((line) => line.replace(/^[^(a-zA-Z)]*/, "").trim())
+    .filter((line) => line.length > 0);
+}
+
 export async function summarize(content: string) {
   if (!openAI) return;
   const chatCompletion = await openAI.createChatCompletion({
@@ -63,7 +73,11 @@ export async function extractAuthorName(content: string) {
   return author;
 }
 
-export async function createKeywords(type: string, description: string) {
+export async function createKeywords(
+  type: string,
+  description: string,
+  title = "unknown",
+) {
   if (!openAI) return;
   const chatCompletion = await openAI.createChatCompletion({
     model: "gpt-3.5-turbo",
@@ -71,18 +85,21 @@ export async function createKeywords(type: string, description: string) {
       {
         "role": "system",
         "content":
-          `you create some keywords that can be used in a recommendation system. The keywords are based on a ${type} description. Create a range of keywords from very specific ones that describe the general vibe. Also include some that describe the genre`,
+          `you create some keywords that can be used in a recommendation system. The keywords are based on a ${type} description. Create a range of keywords from very specific ones that describe the general vibe. Also include some that describe the genre. ${
+            title ? `The name of the ${type} is ${title}` : ""
+          }`,
       },
       { "role": "user", "content": description.slice(0, 2000) },
       {
         "role": "user",
-        "content":
+        "content": "return a list of around 20 keywords seperated by commas",
-          "only return a list of around 20 keywords seperated by commas",
       },
     ],
   });
 
-  return chatCompletion.choices[0].message.content?.toLowerCase().split(", ")
+  const res = chatCompletion.choices[0].message.content?.toLowerCase();
+
+  return extractListFromResponse(res)
     .map((v) => v.replaceAll(" ", "-"));
 }
 
@@ -101,6 +118,7 @@ export async function createTags(content: string) {
     ],
   });
 
-  return chatCompletion.choices[0].message.content?.toLowerCase().split(", ")
+  const res = chatCompletion.choices[0].message.content?.toLowerCase();
-    .map((v) => v.replaceAll(" ", "-"));
+
+  return extractListFromResponse(res).map((v) => v.replaceAll(" ", "-"));
 }

lib/recommendation.ts

@@ -8,6 +8,7 @@ type RecommendationResource = {
   type: string;
   rating: number;
   tags?: string[];
+  description?: string;
   keywords?: string[];
   author?: string;
   year?: number;
@@ -17,14 +18,14 @@ export async function createRecommendationResource(
   res: GenericResource,
   description?: string,
 ) {
-  const cacheId = `recommendations:${res.type}:${res.id}`;
+  const cacheId = `recommendations:${res.type}:${res.id.replaceAll(":", "")}`;
   const resource: RecommendationResource = await cache.get(cacheId) || {
     id: res.id,
     type: res.type,
     rating: -1,
   };
   if (description && !resource.keywords) {
-    const keywords = await openai.createKeywords(res.type, description);
+    const keywords = await openai.createKeywords(res.type, description, res.id);
     if (keywords?.length) {
       resource.keywords = keywords;
     }
@@ -44,6 +45,10 @@ export async function createRecommendationResource(
     resource.author = author;
   }
 
+  if (description) {
+    resource.description = description;
+  }
+
   if (date) {
     const d = typeof date === "string" ? new Date(date) : date;
     resource.year = d.getFullYear();
@@ -52,6 +57,10 @@ export async function createRecommendationResource(
   cache.set(cacheId, JSON.stringify(resource));
 }
 
+export function getRecommendation(id: string, type: string) {
+  return cache.get(`recommendations:${type}:${id}`);
+}
+
 export async function getAllRecommendations() {
   const keys = await cache.keys("recommendations:movie:*");
   return Promise.all(keys.map((k) => cache.get(k))).then((res) =>

routes/api/articles/create/index.ts

@@ -21,6 +21,8 @@ async function processCreateArticle(
     streamResponse: ReturnType<typeof createStreamResponse>;
   },
 ) {
+
+
   log.info("create article from url", { url: fetchUrl });
 
   streamResponse.enqueue("downloading article");

routes/api/recommendation/all.ts

@@ -0,0 +1,64 @@
+import { Handlers } from "$fresh/server.ts";
+import { createStreamResponse } from "@lib/helpers.ts";
+import { getAllMovies } from "@lib/resource/movies.ts";
+import * as tmdb from "@lib/tmdb.ts";
+import {
+  createRecommendationResource,
+  getRecommendation,
+} from "@lib/recommendation.ts";
+import { AccessDeniedError } from "@lib/errors.ts";
+
+async function processUpdateRecommendations(
+  streamResponse: ReturnType<typeof createStreamResponse>,
+) {
+  const allMovies = await getAllMovies();
+
+  const movies = allMovies.filter((m) => {
+    if (!m.meta.rating) return false;
+    if (!m.meta.tmdbId) return false;
+    return true;
+  });
+
+  streamResponse.enqueue("Fetched all movies");
+
+  let done = 0;
+  const total = movies.length;
+
+  await Promise.all(movies.map(async (movie) => {
+    if (!movie.meta.tmdbId) return;
+    if (!movie.meta.rating) return;
+    const recommendation = await getRecommendation(movie.id, movie.type);
+    if (recommendation) {
+      done++;
+      return;
+    }
+    try {
+      const movieDetails = await tmdb.getMovie(movie.meta.tmdbId);
+      await createRecommendationResource(movie, movieDetails.overview);
+    } catch (err) {
+      console.log(err);
+    }
+    done++;
+    streamResponse.enqueue(
+      `${Math.floor((done / total) * 100)}% [${done + 1}/${total}] ${movie.id}`,
+    );
+  })).catch((err) => {
+    console.log(err);
+  });
+
+  streamResponse.enqueue("100% Finished");
+}
+
+export const handler: Handlers = {
+  GET(_, ctx) {
+    const session = ctx.state.session;
+    if (!session) {
+      throw new AccessDeniedError();
+    }
+
+    const streamResponse = createStreamResponse();
+    processUpdateRecommendations(streamResponse);
+
+    return streamResponse.response;
+  },
+};

routes/api/recommendation/index.ts

@@ -5,7 +5,6 @@ import { getAllRecommendations } from "@lib/recommendation.ts";
 export const handler: Handlers = {
   async GET() {
     const recommendations = await getAllRecommendations();
-    console.log({ recommendations });
     return json(recommendations);
   },
 };