max/memorium
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat: add update all recommendations command

Browse Source
This commit is contained in:
max_richter 2023-09-08 15:15:36 +02:00
parent 297dab97cd
commit 6d5a3a1a0c
8 changed files with 177 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
fresh.gen.ts
View File

@ -25,23 +25,24 @@ import * as $19 from "./routes/api/query/index.ts";
import * as $20 from "./routes/api/query/sync.ts";
import * as $21 from "./routes/api/recipes/[name].ts";
import * as $22 from "./routes/api/recipes/index.ts";
import * as $23 from "./routes/api/recommendation/index.ts";
import * as $24 from "./routes/api/resources.ts";
import * as $25 from "./routes/api/series/[name].ts";
import * as $26 from "./routes/api/series/enhance/[name].ts";
import * as $27 from "./routes/api/series/index.ts";
import * as $28 from "./routes/api/tmdb/[id].ts";
import * as $29 from "./routes/api/tmdb/credits/[id].ts";
import * as $30 from "./routes/api/tmdb/query.ts";
import * as $31 from "./routes/articles/[name].tsx";
import * as $32 from "./routes/articles/index.tsx";
import * as $33 from "./routes/index.tsx";
import * as $34 from "./routes/movies/[name].tsx";
import * as $35 from "./routes/movies/index.tsx";
import * as $36 from "./routes/recipes/[name].tsx";
import * as $37 from "./routes/recipes/index.tsx";
import * as $38 from "./routes/series/[name].tsx";
import * as $39 from "./routes/series/index.tsx";
import * as $23 from "./routes/api/recommendation/all.ts";
import * as $24 from "./routes/api/recommendation/index.ts";
import * as $25 from "./routes/api/resources.ts";
import * as $26 from "./routes/api/series/[name].ts";
import * as $27 from "./routes/api/series/enhance/[name].ts";
import * as $28 from "./routes/api/series/index.ts";
import * as $29 from "./routes/api/tmdb/[id].ts";
import * as $30 from "./routes/api/tmdb/credits/[id].ts";
import * as $31 from "./routes/api/tmdb/query.ts";
import * as $32 from "./routes/articles/[name].tsx";
import * as $33 from "./routes/articles/index.tsx";
import * as $34 from "./routes/index.tsx";
import * as $35 from "./routes/movies/[name].tsx";
import * as $36 from "./routes/movies/index.tsx";
import * as $37 from "./routes/recipes/[name].tsx";
import * as $38 from "./routes/recipes/index.tsx";
import * as $39 from "./routes/series/[name].tsx";
import * as $40 from "./routes/series/index.tsx";
import * as $$0 from "./islands/Counter.tsx";
import * as $$1 from "./islands/IngredientsList.tsx";
import * as $$2 from "./islands/KMenu.tsx";
@ -50,9 +51,10 @@ import * as $$4 from "./islands/KMenu/commands/add_movie_infos.ts";
import * as $$5 from "./islands/KMenu/commands/add_series_infos.ts";
import * as $$6 from "./islands/KMenu/commands/create_article.ts";
import * as $$7 from "./islands/KMenu/commands/create_movie.ts";
import * as $$8 from "./islands/KMenu/commands/create_series.ts";
import * as $$9 from "./islands/KMenu/types.ts";
import * as $$10 from "./islands/Search.tsx";
import * as $$8 from "./islands/KMenu/commands/create_recommendations.ts";
import * as $$9 from "./islands/KMenu/commands/create_series.ts";
import * as $$10 from "./islands/KMenu/types.ts";
import * as $$11 from "./islands/Search.tsx";
const manifest = {
routes: {
@ -79,23 +81,24 @@ const manifest = {
"./routes/api/query/sync.ts": $20,
"./routes/api/recipes/[name].ts": $21,
"./routes/api/recipes/index.ts": $22,
"./routes/api/recommendation/index.ts": $23,
"./routes/api/resources.ts": $24,
"./routes/api/series/[name].ts": $25,
"./routes/api/series/enhance/[name].ts": $26,
"./routes/api/series/index.ts": $27,
"./routes/api/tmdb/[id].ts": $28,
"./routes/api/tmdb/credits/[id].ts": $29,
"./routes/api/tmdb/query.ts": $30,
"./routes/articles/[name].tsx": $31,
"./routes/articles/index.tsx": $32,
"./routes/index.tsx": $33,
"./routes/movies/[name].tsx": $34,
"./routes/movies/index.tsx": $35,
"./routes/recipes/[name].tsx": $36,
"./routes/recipes/index.tsx": $37,
"./routes/series/[name].tsx": $38,
"./routes/series/index.tsx": $39,
"./routes/api/recommendation/all.ts": $23,
"./routes/api/recommendation/index.ts": $24,
"./routes/api/resources.ts": $25,
"./routes/api/series/[name].ts": $26,
"./routes/api/series/enhance/[name].ts": $27,
"./routes/api/series/index.ts": $28,
"./routes/api/tmdb/[id].ts": $29,
"./routes/api/tmdb/credits/[id].ts": $30,
"./routes/api/tmdb/query.ts": $31,
"./routes/articles/[name].tsx": $32,
"./routes/articles/index.tsx": $33,
"./routes/index.tsx": $34,
"./routes/movies/[name].tsx": $35,
"./routes/movies/index.tsx": $36,
"./routes/recipes/[name].tsx": $37,
"./routes/recipes/index.tsx": $38,
"./routes/series/[name].tsx": $39,
"./routes/series/index.tsx": $40,
},
islands: {
"./islands/Counter.tsx": $$0,
@ -106,9 +109,10 @@ const manifest = {
"./islands/KMenu/commands/add_series_infos.ts": $$5,
"./islands/KMenu/commands/create_article.ts": $$6,
"./islands/KMenu/commands/create_movie.ts": $$7,
"./islands/KMenu/commands/create_series.ts": $$8,
"./islands/KMenu/types.ts": $$9,
"./islands/Search.tsx": $$10,
"./islands/KMenu/commands/create_recommendations.ts": $$8,
"./islands/KMenu/commands/create_series.ts": $$9,
"./islands/KMenu/types.ts": $$10,
"./islands/Search.tsx": $$11,
},
baseUrl: import.meta.url,
};

2
islands/KMenu/commands.ts
View File

@ -5,6 +5,7 @@ import { createNewArticle } from "@islands/KMenu/commands/create_article.ts";
import { getCookie } from "@lib/string.ts";
import { addSeriesInfo } from "@islands/KMenu/commands/add_series_infos.ts";
import { createNewSeries } from "@islands/KMenu/commands/create_series.ts";
import { updateAllRecommendations } from "@islands/KMenu/commands/create_recommendations.ts";
export const menus: Record<string, Menu> = {
main: {
@ -85,6 +86,7 @@ export const menus: Record<string, Menu> = {
createNewMovie,
createNewSeries,
addMovieInfos,
updateAllRecommendations,
],
},
};

29
islands/KMenu/commands/create_recommendations.ts Normal file
View File

@ -0,0 +1,29 @@
import { MenuEntry } from "@islands/KMenu/types.ts";
import { fetchStream } from "@lib/helpers.ts";
import { getCookie } from "@lib/string.ts";
export const updateAllRecommendations: MenuEntry = {
title: "Update all recommendations",
meta: "",
icon: "IconSquareRoundedPlus",
cb: (state) => {
state.activeState.value = "loading";
fetchStream("/api/recommendation/all", (chunk) => {
if (chunk.toLowerCase().includes("finish")) {
setTimeout(() => {
window.location.reload();
}, 500);
} else {
state.loadingText.value = chunk;
}
});
},
visible: () => {
if (!getCookie("session_cookie")) return false;
if (
!globalThis?.location?.pathname?.includes("movies")
) return false;
return true;
},
};

32
lib/openai.ts
View File

@ -3,6 +3,16 @@ import { OPENAI_API_KEY } from "@lib/env.ts";
const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY);
function extractListFromResponse(response?: string): string[] {
if (!response) return [];
return response
.split(/[\n,]/)
.map((line) => line.trim())
.filter((line) => !line.endsWith(":"))
.map((line) => line.replace(/^[^(a-zA-Z)]*/, "").trim())
.filter((line) => line.length > 0);
}
export async function summarize(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.createChatCompletion({
@ -63,7 +73,11 @@ export async function extractAuthorName(content: string) {
return author;
}
export async function createKeywords(type: string, description: string) {
export async function createKeywords(
type: string,
description: string,
title = "unknown",
) {
if (!openAI) return;
const chatCompletion = await openAI.createChatCompletion({
model: "gpt-3.5-turbo",
@ -71,18 +85,21 @@ export async function createKeywords(type: string, description: string) {
{
"role": "system",
"content":
`you create some keywords that can be used in a recommendation system. The keywords are based on a ${type} description. Create a range of keywords from very specific ones that describe the general vibe. Also include some that describe the genre`,
`you create some keywords that can be used in a recommendation system. The keywords are based on a ${type} description. Create a range of keywords from very specific ones that describe the general vibe. Also include some that describe the genre. ${
title ? `The name of the ${type} is ${title}` : ""
}`,
},
{ "role": "user", "content": description.slice(0, 2000) },
{
"role": "user",
"content":
"only return a list of around 20 keywords seperated by commas",
"content": "return a list of around 20 keywords seperated by commas",
},
],
});
return chatCompletion.choices[0].message.content?.toLowerCase().split(", ")
const res = chatCompletion.choices[0].message.content?.toLowerCase();
return extractListFromResponse(res)
.map((v) => v.replaceAll(" ", "-"));
}
@ -101,6 +118,7 @@ export async function createTags(content: string) {
],
});
return chatCompletion.choices[0].message.content?.toLowerCase().split(", ")
.map((v) => v.replaceAll(" ", "-"));
const res = chatCompletion.choices[0].message.content?.toLowerCase();
return extractListFromResponse(res).map((v) => v.replaceAll(" ", "-"));
}

13
lib/recommendation.ts
View File

@ -8,6 +8,7 @@ type RecommendationResource = {
type: string;
rating: number;
tags?: string[];
description?: string;
keywords?: string[];
author?: string;
year?: number;
@ -17,14 +18,14 @@ export async function createRecommendationResource(
res: GenericResource,
description?: string,
) {
const cacheId = `recommendations:${res.type}:${res.id}`;
const cacheId = `recommendations:${res.type}:${res.id.replaceAll(":", "")}`;
const resource: RecommendationResource = await cache.get(cacheId) || {
id: res.id,
type: res.type,
rating: -1,
};
if (description && !resource.keywords) {
const keywords = await openai.createKeywords(res.type, description);
const keywords = await openai.createKeywords(res.type, description, res.id);
if (keywords?.length) {
resource.keywords = keywords;
}
@ -44,6 +45,10 @@ export async function createRecommendationResource(
resource.author = author;
}
if (description) {
resource.description = description;
}
if (date) {
const d = typeof date === "string" ? new Date(date) : date;
resource.year = d.getFullYear();
@ -52,6 +57,10 @@ export async function createRecommendationResource(
cache.set(cacheId, JSON.stringify(resource));
}
export function getRecommendation(id: string, type: string) {
return cache.get(`recommendations:${type}:${id}`);
}
export async function getAllRecommendations() {
const keys = await cache.keys("recommendations:movie:*");
return Promise.all(keys.map((k) => cache.get(k))).then((res) =>

2
routes/api/articles/create/index.ts
View File

@ -21,6 +21,8 @@ async function processCreateArticle(
streamResponse: ReturnType<typeof createStreamResponse>;
},
) {
log.info("create article from url", { url: fetchUrl });
streamResponse.enqueue("downloading article");

64
routes/api/recommendation/all.ts Normal file
View File

@ -0,0 +1,64 @@
import { Handlers } from "$fresh/server.ts";
import { createStreamResponse } from "@lib/helpers.ts";
import { getAllMovies } from "@lib/resource/movies.ts";
import * as tmdb from "@lib/tmdb.ts";
import {
createRecommendationResource,
getRecommendation,
} from "@lib/recommendation.ts";
import { AccessDeniedError } from "@lib/errors.ts";
async function processUpdateRecommendations(
streamResponse: ReturnType<typeof createStreamResponse>,
) {
const allMovies = await getAllMovies();
const movies = allMovies.filter((m) => {
if (!m.meta.rating) return false;
if (!m.meta.tmdbId) return false;
return true;
});
streamResponse.enqueue("Fetched all movies");
let done = 0;
const total = movies.length;
await Promise.all(movies.map(async (movie) => {
if (!movie.meta.tmdbId) return;
if (!movie.meta.rating) return;
const recommendation = await getRecommendation(movie.id, movie.type);
if (recommendation) {
done++;
return;
}
try {
const movieDetails = await tmdb.getMovie(movie.meta.tmdbId);
await createRecommendationResource(movie, movieDetails.overview);
} catch (err) {
console.log(err);
}
done++;
streamResponse.enqueue(
`${Math.floor((done / total) * 100)}% [${done + 1}/${total}] ${movie.id}`,
);
})).catch((err) => {
console.log(err);
});
streamResponse.enqueue("100% Finished");
}
export const handler: Handlers = {
GET(_, ctx) {
const session = ctx.state.session;
if (!session) {
throw new AccessDeniedError();
}
const streamResponse = createStreamResponse();
processUpdateRecommendations(streamResponse);
return streamResponse.response;
},
};

1
routes/api/recommendation/index.ts
View File

@ -5,7 +5,6 @@ import { getAllRecommendations } from "@lib/recommendation.ts";
export const handler: Handlers = {
async GET() {
const recommendations = await getAllRecommendations();
console.log({ recommendations });
return json(recommendations);
},
};