diff --git a/fresh.gen.ts b/fresh.gen.ts index 1a33313..79c3d1e 100644 --- a/fresh.gen.ts +++ b/fresh.gen.ts @@ -25,23 +25,24 @@ import * as $19 from "./routes/api/query/index.ts"; import * as $20 from "./routes/api/query/sync.ts"; import * as $21 from "./routes/api/recipes/[name].ts"; import * as $22 from "./routes/api/recipes/index.ts"; -import * as $23 from "./routes/api/recommendation/index.ts"; -import * as $24 from "./routes/api/resources.ts"; -import * as $25 from "./routes/api/series/[name].ts"; -import * as $26 from "./routes/api/series/enhance/[name].ts"; -import * as $27 from "./routes/api/series/index.ts"; -import * as $28 from "./routes/api/tmdb/[id].ts"; -import * as $29 from "./routes/api/tmdb/credits/[id].ts"; -import * as $30 from "./routes/api/tmdb/query.ts"; -import * as $31 from "./routes/articles/[name].tsx"; -import * as $32 from "./routes/articles/index.tsx"; -import * as $33 from "./routes/index.tsx"; -import * as $34 from "./routes/movies/[name].tsx"; -import * as $35 from "./routes/movies/index.tsx"; -import * as $36 from "./routes/recipes/[name].tsx"; -import * as $37 from "./routes/recipes/index.tsx"; -import * as $38 from "./routes/series/[name].tsx"; -import * as $39 from "./routes/series/index.tsx"; +import * as $23 from "./routes/api/recommendation/all.ts"; +import * as $24 from "./routes/api/recommendation/index.ts"; +import * as $25 from "./routes/api/resources.ts"; +import * as $26 from "./routes/api/series/[name].ts"; +import * as $27 from "./routes/api/series/enhance/[name].ts"; +import * as $28 from "./routes/api/series/index.ts"; +import * as $29 from "./routes/api/tmdb/[id].ts"; +import * as $30 from "./routes/api/tmdb/credits/[id].ts"; +import * as $31 from "./routes/api/tmdb/query.ts"; +import * as $32 from "./routes/articles/[name].tsx"; +import * as $33 from "./routes/articles/index.tsx"; +import * as $34 from "./routes/index.tsx"; +import * as $35 from "./routes/movies/[name].tsx"; +import * as $36 from "./routes/movies/index.tsx"; +import * as $37 from "./routes/recipes/[name].tsx"; +import * as $38 from "./routes/recipes/index.tsx"; +import * as $39 from "./routes/series/[name].tsx"; +import * as $40 from "./routes/series/index.tsx"; import * as $$0 from "./islands/Counter.tsx"; import * as $$1 from "./islands/IngredientsList.tsx"; import * as $$2 from "./islands/KMenu.tsx"; @@ -50,9 +51,10 @@ import * as $$4 from "./islands/KMenu/commands/add_movie_infos.ts"; import * as $$5 from "./islands/KMenu/commands/add_series_infos.ts"; import * as $$6 from "./islands/KMenu/commands/create_article.ts"; import * as $$7 from "./islands/KMenu/commands/create_movie.ts"; -import * as $$8 from "./islands/KMenu/commands/create_series.ts"; -import * as $$9 from "./islands/KMenu/types.ts"; -import * as $$10 from "./islands/Search.tsx"; +import * as $$8 from "./islands/KMenu/commands/create_recommendations.ts"; +import * as $$9 from "./islands/KMenu/commands/create_series.ts"; +import * as $$10 from "./islands/KMenu/types.ts"; +import * as $$11 from "./islands/Search.tsx"; const manifest = { routes: { @@ -79,23 +81,24 @@ const manifest = { "./routes/api/query/sync.ts": $20, "./routes/api/recipes/[name].ts": $21, "./routes/api/recipes/index.ts": $22, - "./routes/api/recommendation/index.ts": $23, - "./routes/api/resources.ts": $24, - "./routes/api/series/[name].ts": $25, - "./routes/api/series/enhance/[name].ts": $26, - "./routes/api/series/index.ts": $27, - "./routes/api/tmdb/[id].ts": $28, - "./routes/api/tmdb/credits/[id].ts": $29, - "./routes/api/tmdb/query.ts": $30, - "./routes/articles/[name].tsx": $31, - "./routes/articles/index.tsx": $32, - "./routes/index.tsx": $33, - "./routes/movies/[name].tsx": $34, - "./routes/movies/index.tsx": $35, - "./routes/recipes/[name].tsx": $36, - "./routes/recipes/index.tsx": $37, - "./routes/series/[name].tsx": $38, - "./routes/series/index.tsx": $39, + "./routes/api/recommendation/all.ts": $23, + "./routes/api/recommendation/index.ts": $24, + "./routes/api/resources.ts": $25, + "./routes/api/series/[name].ts": $26, + "./routes/api/series/enhance/[name].ts": $27, + "./routes/api/series/index.ts": $28, + "./routes/api/tmdb/[id].ts": $29, + "./routes/api/tmdb/credits/[id].ts": $30, + "./routes/api/tmdb/query.ts": $31, + "./routes/articles/[name].tsx": $32, + "./routes/articles/index.tsx": $33, + "./routes/index.tsx": $34, + "./routes/movies/[name].tsx": $35, + "./routes/movies/index.tsx": $36, + "./routes/recipes/[name].tsx": $37, + "./routes/recipes/index.tsx": $38, + "./routes/series/[name].tsx": $39, + "./routes/series/index.tsx": $40, }, islands: { "./islands/Counter.tsx": $$0, @@ -106,9 +109,10 @@ const manifest = { "./islands/KMenu/commands/add_series_infos.ts": $$5, "./islands/KMenu/commands/create_article.ts": $$6, "./islands/KMenu/commands/create_movie.ts": $$7, - "./islands/KMenu/commands/create_series.ts": $$8, - "./islands/KMenu/types.ts": $$9, - "./islands/Search.tsx": $$10, + "./islands/KMenu/commands/create_recommendations.ts": $$8, + "./islands/KMenu/commands/create_series.ts": $$9, + "./islands/KMenu/types.ts": $$10, + "./islands/Search.tsx": $$11, }, baseUrl: import.meta.url, }; diff --git a/islands/KMenu/commands.ts b/islands/KMenu/commands.ts index 31363bb..caf293f 100644 --- a/islands/KMenu/commands.ts +++ b/islands/KMenu/commands.ts @@ -5,6 +5,7 @@ import { createNewArticle } from "@islands/KMenu/commands/create_article.ts"; import { getCookie } from "@lib/string.ts"; import { addSeriesInfo } from "@islands/KMenu/commands/add_series_infos.ts"; import { createNewSeries } from "@islands/KMenu/commands/create_series.ts"; +import { updateAllRecommendations } from "@islands/KMenu/commands/create_recommendations.ts"; export const menus: Record = { main: { @@ -85,6 +86,7 @@ export const menus: Record = { createNewMovie, createNewSeries, addMovieInfos, + updateAllRecommendations, ], }, }; diff --git a/islands/KMenu/commands/create_recommendations.ts b/islands/KMenu/commands/create_recommendations.ts new file mode 100644 index 0000000..698ee57 --- /dev/null +++ b/islands/KMenu/commands/create_recommendations.ts @@ -0,0 +1,29 @@ +import { MenuEntry } from "@islands/KMenu/types.ts"; +import { fetchStream } from "@lib/helpers.ts"; +import { getCookie } from "@lib/string.ts"; + +export const updateAllRecommendations: MenuEntry = { + title: "Update all recommendations", + meta: "", + icon: "IconSquareRoundedPlus", + cb: (state) => { + state.activeState.value = "loading"; + + fetchStream("/api/recommendation/all", (chunk) => { + if (chunk.toLowerCase().includes("finish")) { + setTimeout(() => { + window.location.reload(); + }, 500); + } else { + state.loadingText.value = chunk; + } + }); + }, + visible: () => { + if (!getCookie("session_cookie")) return false; + if ( + !globalThis?.location?.pathname?.includes("movies") + ) return false; + return true; + }, +}; diff --git a/lib/openai.ts b/lib/openai.ts index 318a95f..0c233ad 100644 --- a/lib/openai.ts +++ b/lib/openai.ts @@ -3,6 +3,16 @@ import { OPENAI_API_KEY } from "@lib/env.ts"; const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY); +function extractListFromResponse(response?: string): string[] { + if (!response) return []; + return response + .split(/[\n,]/) + .map((line) => line.trim()) + .filter((line) => !line.endsWith(":")) + .map((line) => line.replace(/^[^(a-zA-Z)]*/, "").trim()) + .filter((line) => line.length > 0); +} + export async function summarize(content: string) { if (!openAI) return; const chatCompletion = await openAI.createChatCompletion({ @@ -63,7 +73,11 @@ export async function extractAuthorName(content: string) { return author; } -export async function createKeywords(type: string, description: string) { +export async function createKeywords( + type: string, + description: string, + title = "unknown", +) { if (!openAI) return; const chatCompletion = await openAI.createChatCompletion({ model: "gpt-3.5-turbo", @@ -71,18 +85,21 @@ export async function createKeywords(type: string, description: string) { { "role": "system", "content": - `you create some keywords that can be used in a recommendation system. The keywords are based on a ${type} description. Create a range of keywords from very specific ones that describe the general vibe. Also include some that describe the genre`, + `you create some keywords that can be used in a recommendation system. The keywords are based on a ${type} description. Create a range of keywords from very specific ones that describe the general vibe. Also include some that describe the genre. ${ + title ? `The name of the ${type} is ${title}` : "" + }`, }, { "role": "user", "content": description.slice(0, 2000) }, { "role": "user", - "content": - "only return a list of around 20 keywords seperated by commas", + "content": "return a list of around 20 keywords seperated by commas", }, ], }); - return chatCompletion.choices[0].message.content?.toLowerCase().split(", ") + const res = chatCompletion.choices[0].message.content?.toLowerCase(); + + return extractListFromResponse(res) .map((v) => v.replaceAll(" ", "-")); } @@ -101,6 +118,7 @@ export async function createTags(content: string) { ], }); - return chatCompletion.choices[0].message.content?.toLowerCase().split(", ") - .map((v) => v.replaceAll(" ", "-")); + const res = chatCompletion.choices[0].message.content?.toLowerCase(); + + return extractListFromResponse(res).map((v) => v.replaceAll(" ", "-")); } diff --git a/lib/recommendation.ts b/lib/recommendation.ts index dd52993..dea95f2 100644 --- a/lib/recommendation.ts +++ b/lib/recommendation.ts @@ -8,6 +8,7 @@ type RecommendationResource = { type: string; rating: number; tags?: string[]; + description?: string; keywords?: string[]; author?: string; year?: number; @@ -17,14 +18,14 @@ export async function createRecommendationResource( res: GenericResource, description?: string, ) { - const cacheId = `recommendations:${res.type}:${res.id}`; + const cacheId = `recommendations:${res.type}:${res.id.replaceAll(":", "")}`; const resource: RecommendationResource = await cache.get(cacheId) || { id: res.id, type: res.type, rating: -1, }; if (description && !resource.keywords) { - const keywords = await openai.createKeywords(res.type, description); + const keywords = await openai.createKeywords(res.type, description, res.id); if (keywords?.length) { resource.keywords = keywords; } @@ -44,6 +45,10 @@ export async function createRecommendationResource( resource.author = author; } + if (description) { + resource.description = description; + } + if (date) { const d = typeof date === "string" ? new Date(date) : date; resource.year = d.getFullYear(); @@ -52,6 +57,10 @@ export async function createRecommendationResource( cache.set(cacheId, JSON.stringify(resource)); } +export function getRecommendation(id: string, type: string) { + return cache.get(`recommendations:${type}:${id}`); +} + export async function getAllRecommendations() { const keys = await cache.keys("recommendations:movie:*"); return Promise.all(keys.map((k) => cache.get(k))).then((res) => diff --git a/routes/api/articles/create/index.ts b/routes/api/articles/create/index.ts index 0b9e459..3354ab0 100644 --- a/routes/api/articles/create/index.ts +++ b/routes/api/articles/create/index.ts @@ -21,6 +21,8 @@ async function processCreateArticle( streamResponse: ReturnType; }, ) { + + log.info("create article from url", { url: fetchUrl }); streamResponse.enqueue("downloading article"); diff --git a/routes/api/recommendation/all.ts b/routes/api/recommendation/all.ts new file mode 100644 index 0000000..0f4d7e9 --- /dev/null +++ b/routes/api/recommendation/all.ts @@ -0,0 +1,64 @@ +import { Handlers } from "$fresh/server.ts"; +import { createStreamResponse } from "@lib/helpers.ts"; +import { getAllMovies } from "@lib/resource/movies.ts"; +import * as tmdb from "@lib/tmdb.ts"; +import { + createRecommendationResource, + getRecommendation, +} from "@lib/recommendation.ts"; +import { AccessDeniedError } from "@lib/errors.ts"; + +async function processUpdateRecommendations( + streamResponse: ReturnType, +) { + const allMovies = await getAllMovies(); + + const movies = allMovies.filter((m) => { + if (!m.meta.rating) return false; + if (!m.meta.tmdbId) return false; + return true; + }); + + streamResponse.enqueue("Fetched all movies"); + + let done = 0; + const total = movies.length; + + await Promise.all(movies.map(async (movie) => { + if (!movie.meta.tmdbId) return; + if (!movie.meta.rating) return; + const recommendation = await getRecommendation(movie.id, movie.type); + if (recommendation) { + done++; + return; + } + try { + const movieDetails = await tmdb.getMovie(movie.meta.tmdbId); + await createRecommendationResource(movie, movieDetails.overview); + } catch (err) { + console.log(err); + } + done++; + streamResponse.enqueue( + `${Math.floor((done / total) * 100)}% [${done + 1}/${total}] ${movie.id}`, + ); + })).catch((err) => { + console.log(err); + }); + + streamResponse.enqueue("100% Finished"); +} + +export const handler: Handlers = { + GET(_, ctx) { + const session = ctx.state.session; + if (!session) { + throw new AccessDeniedError(); + } + + const streamResponse = createStreamResponse(); + processUpdateRecommendations(streamResponse); + + return streamResponse.response; + }, +}; diff --git a/routes/api/recommendation/index.ts b/routes/api/recommendation/index.ts index d8e33ac..b3e495a 100644 --- a/routes/api/recommendation/index.ts +++ b/routes/api/recommendation/index.ts @@ -5,7 +5,6 @@ import { getAllRecommendations } from "@lib/recommendation.ts"; export const handler: Handlers = { async GET() { const recommendations = await getAllRecommendations(); - console.log({ recommendations }); return json(recommendations); }, };