diff --git a/components/RecipeHero.tsx b/components/RecipeHero.tsx index da73fce..754d01a 100644 --- a/components/RecipeHero.tsx +++ b/components/RecipeHero.tsx @@ -52,11 +52,15 @@ export function RecipeHero( )} -
+

- {subline.map((s) => { + {subline.filter((s) => s && s?.length > 1).map((s) => { return {s}; })}

diff --git a/components/layouts/main.tsx b/components/layouts/main.tsx index 1decd96..87925c0 100644 --- a/components/layouts/main.tsx +++ b/components/layouts/main.tsx @@ -22,6 +22,10 @@ export const MainLayout = ({ children, url }: Props) => { name: "🍿 Movies", link: "/movies", }, + { + name: "📝 Articles", + link: "/articles", + }, ]; return ( diff --git a/fresh.gen.ts b/fresh.gen.ts index 5067d08..b8d413d 100644 --- a/fresh.gen.ts +++ b/fresh.gen.ts @@ -5,22 +5,27 @@ import * as $0 from "./routes/_404.tsx"; import * as $1 from "./routes/_app.tsx"; import * as $2 from "./routes/_middleware.ts"; -import * as $3 from "./routes/api/cache/index.ts"; -import * as $4 from "./routes/api/images/index.ts"; -import * as $5 from "./routes/api/index.ts"; -import * as $6 from "./routes/api/movies/[name].ts"; -import * as $7 from "./routes/api/movies/enhance/[name].ts"; -import * as $8 from "./routes/api/movies/index.ts"; -import * as $9 from "./routes/api/recipes/[name].ts"; -import * as $10 from "./routes/api/recipes/index.ts"; -import * as $11 from "./routes/api/tmdb/[id].ts"; -import * as $12 from "./routes/api/tmdb/credits/[id].ts"; -import * as $13 from "./routes/api/tmdb/query.ts"; -import * as $14 from "./routes/index.tsx"; -import * as $15 from "./routes/movies/[name].tsx"; -import * as $16 from "./routes/movies/index.tsx"; -import * as $17 from "./routes/recipes/[name].tsx"; -import * as $18 from "./routes/recipes/index.tsx"; +import * as $3 from "./routes/api/articles/[name].ts"; +import * as $4 from "./routes/api/articles/create/index.ts"; +import * as $5 from "./routes/api/articles/index.ts"; +import * as $6 from "./routes/api/cache/index.ts"; +import * as $7 from "./routes/api/images/index.ts"; +import * as $8 from "./routes/api/index.ts"; +import * as $9 from "./routes/api/movies/[name].ts"; +import * as $10 from "./routes/api/movies/enhance/[name].ts"; +import * as $11 from "./routes/api/movies/index.ts"; +import * as $12 from "./routes/api/recipes/[name].ts"; +import * as $13 from "./routes/api/recipes/index.ts"; +import * as $14 from "./routes/api/tmdb/[id].ts"; +import * as $15 from "./routes/api/tmdb/credits/[id].ts"; +import * as $16 from "./routes/api/tmdb/query.ts"; +import * as $17 from "./routes/articles/[name].tsx"; +import * as $18 from "./routes/articles/index.tsx"; +import * as $19 from "./routes/index.tsx"; +import * as $20 from "./routes/movies/[name].tsx"; +import * as $21 from "./routes/movies/index.tsx"; +import * as $22 from "./routes/recipes/[name].tsx"; +import * as $23 from "./routes/recipes/index.tsx"; import * as $$0 from "./islands/Counter.tsx"; import * as $$1 from "./islands/IngredientsList.tsx"; import * as $$2 from "./islands/KMenu.tsx"; @@ -32,22 +37,27 @@ const manifest = { "./routes/_404.tsx": $0, "./routes/_app.tsx": $1, "./routes/_middleware.ts": $2, - "./routes/api/cache/index.ts": $3, - "./routes/api/images/index.ts": $4, - "./routes/api/index.ts": $5, - "./routes/api/movies/[name].ts": $6, - "./routes/api/movies/enhance/[name].ts": $7, - "./routes/api/movies/index.ts": $8, - "./routes/api/recipes/[name].ts": $9, - "./routes/api/recipes/index.ts": $10, - "./routes/api/tmdb/[id].ts": $11, - "./routes/api/tmdb/credits/[id].ts": $12, - "./routes/api/tmdb/query.ts": $13, - "./routes/index.tsx": $14, - "./routes/movies/[name].tsx": $15, - "./routes/movies/index.tsx": $16, - "./routes/recipes/[name].tsx": $17, - "./routes/recipes/index.tsx": $18, + "./routes/api/articles/[name].ts": $3, + "./routes/api/articles/create/index.ts": $4, + "./routes/api/articles/index.ts": $5, + "./routes/api/cache/index.ts": $6, + "./routes/api/images/index.ts": $7, + "./routes/api/index.ts": $8, + "./routes/api/movies/[name].ts": $9, + "./routes/api/movies/enhance/[name].ts": $10, + "./routes/api/movies/index.ts": $11, + "./routes/api/recipes/[name].ts": $12, + "./routes/api/recipes/index.ts": $13, + "./routes/api/tmdb/[id].ts": $14, + "./routes/api/tmdb/credits/[id].ts": $15, + "./routes/api/tmdb/query.ts": $16, + "./routes/articles/[name].tsx": $17, + "./routes/articles/index.tsx": $18, + "./routes/index.tsx": $19, + "./routes/movies/[name].tsx": $20, + "./routes/movies/index.tsx": $21, + "./routes/recipes/[name].tsx": $22, + "./routes/recipes/index.tsx": $23, }, islands: { "./islands/Counter.tsx": $$0, diff --git a/islands/KMenu.tsx b/islands/KMenu.tsx index 146e154..09462fe 100644 --- a/islands/KMenu.tsx +++ b/islands/KMenu.tsx @@ -4,10 +4,6 @@ import { useEventListener } from "@lib/hooks/useEventListener.ts"; import { menus } from "@islands/KMenu/commands.ts"; import { MenuEntry } from "@islands/KMenu/types.ts"; -function filterMenu(menu: MenuEntry[], activeMenu?: string) { - return menu; -} - const KMenuEntry = ( { entry, activeIndex, index }: { entry: MenuEntry; @@ -34,15 +30,42 @@ export const KMenu = ( ) => { const activeMenuType = useSignal(type); const activeMenu = menus[activeMenuType.value || "main"]; - const activeState = useSignal<"normal" | "loading" | "error">("normal"); - const activeIndex = useSignal(-1); - const visible = useSignal(false); const input = useRef(null); const commandInput = useSignal(""); + const visible = useSignal(false); + if (visible.value === false) { + setTimeout(() => { + activeMenuType.value = "main"; + activeState.value = "normal"; + if (input.current) { + input.current.value = ""; + } + commandInput.value = ""; + }, 100); + } + + const entries = activeMenu?.entries?.filter((entry) => { + const entryVisible = typeof entry["visible"] === "boolean" + ? entry.visible + : typeof entry["visible"] === "function" + ? entry.visible?.(context) + : true; + + const search = commandInput?.value?.toLowerCase(); + const searchMatches = entry.title.toLowerCase().includes(search) || + entry.meta?.toLowerCase()?.includes(search); + + return entryVisible && searchMatches; + }); + + if (entries.length === 1) { + activeIndex.value = 0; + } + function activateEntry(menuEntry?: MenuEntry) { if (!menuEntry) return; @@ -50,6 +73,7 @@ export const KMenu = ( activeMenu: activeMenuType, menus, activeState, + commandInput, visible, }, context); } @@ -60,7 +84,6 @@ export const KMenu = ( } if (ev.key === "ArrowDown") { - const entries = filterMenu(activeMenu.entries); const index = activeIndex.value; if (index + 1 >= entries.length) { activeIndex.value = 0; @@ -70,12 +93,10 @@ export const KMenu = ( } if (ev.key === "Enter") { - const entries = filterMenu(activeMenu.entries); activateEntry(entries[activeIndex.value]); } if (ev.key === "ArrowUp") { - const entries = filterMenu(activeMenu.entries); const index = activeIndex.value; if (index - 1 < 0) { activeIndex.value = entries.length - 1; @@ -140,7 +161,7 @@ export const KMenu = ( {activeState.value === "normal" && (
- {filterMenu(activeMenu.entries, input.current?.value).map( + {entries.map( (k, index) => { return ( = { main: { @@ -14,6 +15,35 @@ export const menus: Record = { }, visible: () => false, }, + { + title: "Create new article", + meta: "", + cb: (state) => { + state.menus["input_link"] = { + title: "Link:", + entries: [], + }; + state.activeMenu.value = "input_link"; + + const unsub = state.commandInput.subscribe(async (value) => { + if (isValidUrl(value)) { + unsub(); + + state.activeState.value = "loading"; + + const response = await fetch("/api/articles/create?url=" + value); + const newArticle = await response.json(); + + if (newArticle?.id) { + window.location.href = "/articles/" + newArticle.id; + } + + state.visible.value = false; + } + }); + }, + visible: () => true, + }, { title: "Clear Cache", cb: async (state) => { @@ -59,13 +89,14 @@ export const menus: Record = { })), }; - console.log({ state }); - state.activeMenu.value = menuID; state.activeState.value = "normal"; }, - visible: () => false, + visible: () => { + const loc = globalThis["location"]; + return loc?.pathname?.includes("movie"); + }, }, { title: "Reload Page", diff --git a/islands/KMenu/types.ts b/islands/KMenu/types.ts index 91b26fc..348fff7 100644 --- a/islands/KMenu/types.ts +++ b/islands/KMenu/types.ts @@ -3,6 +3,7 @@ import { Signal } from "@preact/signals"; export type MenuState = { activeMenu: Signal; activeState: Signal<"error" | "normal" | "loading">; + commandInput: Signal; visible: Signal; menus: Record; }; diff --git a/lib/documents.ts b/lib/documents.ts index 7132b28..ef39d4c 100644 --- a/lib/documents.ts +++ b/lib/documents.ts @@ -9,6 +9,7 @@ import rehypeSanitize from "https://esm.sh/rehype-sanitize@5.0.1"; import rehypeStringify from "https://esm.sh/rehype-stringify@9.0.3"; import * as cache from "@lib/cache/documents.ts"; import { SILVERBULLET_SERVER } from "@lib/env.ts"; +import { fixRenderedMarkdown } from "@lib/helpers.ts"; export type Document = { name: string; @@ -75,26 +76,13 @@ export function transformDocument(input: string, cb: (r: Root) => Root) { .use(remarkStringify) .processSync(input); - return String(out) - .replace("***\n", "---") - .replace("----------------", "---") - .replace("\n---", "---") - .replace(/^(date:[^'\n]*)'|'/gm, (match, p1, p2) => { - if (p1) { - // This is a line starting with date: followed by single quotes - return p1.replace(/'/gm, ""); - } else if (p2) { - return ""; - } else { - // This is a line with single quotes, but not starting with date: - return match; - } - }); + return fixRenderedMarkdown(String(out)); } export function parseDocument(doc: string) { return unified() - .use(remarkParse).use(remarkFrontmatter, ["yaml", "toml"]) + .use(remarkParse) + .use(remarkFrontmatter, ["yaml", "toml"]) .parse(doc); } diff --git a/lib/env.ts b/lib/env.ts index e01ca15..6dd04b2 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -2,3 +2,4 @@ export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER"); export const REDIS_HOST = Deno.env.get("REDIS_HOST"); export const REDIS_PASS = Deno.env.get("REDIS_PASS"); export const TMDB_API_KEY = Deno.env.get("TMDB_API_KEY"); +export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY"); diff --git a/lib/helpers.ts b/lib/helpers.ts index df709b9..d725e02 100644 --- a/lib/helpers.ts +++ b/lib/helpers.ts @@ -5,3 +5,28 @@ export function json(content: unknown) { headers, }); } + +export const isValidUrl = (urlString: string) => { + try { + return Boolean(new URL(urlString)); + } catch (e) { + return false; + } +}; + +export const fixRenderedMarkdown = (content: string) => { + return content.replace("***\n", "---") + .replace("----------------", "---") + .replace("\n---", "---") + .replace(/^(date:[^'\n]*)'|'/gm, (match, p1, p2) => { + if (p1) { + // This is a line starting with date: followed by single quotes + return p1.replace(/'/gm, ""); + } else if (p2) { + return ""; + } else { + // This is a line with single quotes, but not starting with date: + return match; + } + }); +}; diff --git a/lib/openai.ts b/lib/openai.ts new file mode 100644 index 0000000..36842f9 --- /dev/null +++ b/lib/openai.ts @@ -0,0 +1,79 @@ +import { OpenAI } from "https://deno.land/x/openai/mod.ts"; +import { OPENAI_API_KEY } from "@lib/env.ts"; + +const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY); + +export async function summarize(content: string) { + if (!openAI) return; + const chatCompletion = await openAI.createChatCompletion({ + model: "gpt-3.5-turbo", + messages: [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": content.slice(0, 2000) }, + { + "role": "user", + "content": + "Please summarize the article in one sentence as short as possible", + }, + ], + }); + + return chatCompletion.choices[0].message.content; +} +export async function shortenTitle(content: string) { + if (!openAI) return; + const chatCompletion = await openAI.createChatCompletion({ + model: "gpt-3.5-turbo", + messages: [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": content.slice(0, 2000) }, + { + "role": "user", + "content": + "Please shorten the provided website title as much as possible, don't rewrite it, just remove unneccesary informations. Please remove for example any mention of the name of the website.", + }, + ], + }); + + return chatCompletion.choices[0].message.content; +} + +export async function extractAuthorName(content: string) { + if (!openAI) return; + const chatCompletion = await openAI.createChatCompletion({ + model: "gpt-3.5-turbo", + messages: [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": content.slice(0, 2000) }, + { + "role": "user", + "content": + "If you are able to extract the name of the author from the text please respond with the name, otherwise respond with 'not found'", + }, + ], + }); + + const author = chatCompletion.choices[0].message.content; + + if (author !== "not found") return author; + return ""; +} + +export async function createTags(content: string) { + if (!openAI) return; + const chatCompletion = await openAI.createChatCompletion({ + model: "gpt-3.5-turbo", + messages: [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": content.slice(0, 2000) }, + { + "role": "user", + "content": + "Please respond with a list of genres the corresponding article falls into, dont include any other informations, just a comma seperated list of top 5 categories. Please respond only with tags that make sense even if there are less than five.", + }, + ], + }); + + return chatCompletion.choices[0].message.content?.toLowerCase().split(", ") + .map((v) => v.replaceAll(" ", "-")); +} diff --git a/lib/resource/articles.ts b/lib/resource/articles.ts new file mode 100644 index 0000000..2dcf7c0 --- /dev/null +++ b/lib/resource/articles.ts @@ -0,0 +1,104 @@ +import { parseDocument, renderMarkdown } from "@lib/documents.ts"; +import { parse } from "yaml"; +import { createCrud } from "@lib/crud.ts"; +import { stringify } from "https://deno.land/std@0.194.0/yaml/stringify.ts"; +import { formatDate } from "@lib/string.ts"; +import { fixRenderedMarkdown } from "@lib/helpers.ts"; + +export type Article = { + id: string; + content: string; + name: string; + tags: string[]; + meta: { + status: "finished" | "not-finished"; + date: Date; + link: string; + author?: string; + rating?: number; + }; +}; + +const crud = createCrud
({ + prefix: "Media/articles/", + parse: parseArticle, +}); + +function renderArticle(article: Article) { + const meta = article.meta; + if ("date" in meta) { + meta.date = formatDate(meta.date); + } + + return fixRenderedMarkdown(`${ + meta + ? `--- +${stringify(meta)} +---` + : `--- +---` + } +# ${article.name} +${article.tags.map((t) => `#${t}`).join(" ")} +${article.content} +`); +} + +function parseArticle(original: string, id: string): Article { + const doc = parseDocument(original); + + let meta = {} as Article["meta"]; + let name = ""; + + const range = [Infinity, -Infinity]; + + for (const child of doc.children) { + if (child.type === "yaml") { + meta = parse(child.value) as Article["meta"]; + + if (meta["rating"] && typeof meta["rating"] === "string") { + meta.rating = [...meta.rating?.matchAll("⭐")].length; + } + + continue; + } + + if ( + child.type === "heading" && child.depth === 1 && !name && + child.children.length === 1 && child.children[0].type === "text" + ) { + name = child.children[0].value; + continue; + } + + if (name) { + const start = child.position?.start.offset || Infinity; + const end = child.position?.end.offset || -Infinity; + if (start < range[0]) range[0] = start; + if (end > range[1]) range[1] = end; + } + } + + let content = original.slice(range[0], range[1]); + const tags = []; + for (const [hashtag] of original.matchAll(/\B(\#[a-zA-Z\-]+\b)(?!;)/g)) { + tags.push(hashtag.replace(/\#/g, "")); + content = content.replace(hashtag, ""); + } + + return { + id, + name, + tags, + content: renderMarkdown(content), + meta, + }; +} + +export const getAllArticles = crud.readAll; +export const getArticle = crud.read; +export const createArticle = (article: Article) => { + console.log("creating article", { article }); + const content = renderArticle(article); + return crud.create(article.id, content); +}; diff --git a/routes/api/articles/[name].ts b/routes/api/articles/[name].ts new file mode 100644 index 0000000..8616d12 --- /dev/null +++ b/routes/api/articles/[name].ts @@ -0,0 +1,10 @@ +import { Handlers } from "$fresh/server.ts"; +import { getArticle } from "@lib/resource/articles.ts"; +import { json } from "@lib/helpers.ts"; + +export const handler: Handlers = { + async GET(_, ctx) { + const article = await getArticle(ctx.params.name); + return json(article); + }, +}; diff --git a/routes/api/articles/create/index.ts b/routes/api/articles/create/index.ts new file mode 100644 index 0000000..c3ad850 --- /dev/null +++ b/routes/api/articles/create/index.ts @@ -0,0 +1,90 @@ +import { Handlers } from "$fresh/server.ts"; +import { Readability } from "https://cdn.skypack.dev/@mozilla/readability"; +import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.38/deno-dom-wasm.ts"; +import { BadRequestError } from "@lib/errors.ts"; +import { isValidUrl, json } from "@lib/helpers.ts"; +import * as openai from "@lib/openai.ts"; + +import tds from "https://cdn.skypack.dev/turndown@7.1.1"; +//import { gfm } from "https://cdn.skypack.dev/@guyplusplus/turndown-plugin-gfm@1.0.7"; +import { createArticle } from "@lib/resource/articles.ts"; + +const service = new tds({ + headingStyle: "atx", + codeBlockStyle: "fenced", + hr: "---", + bulletListMarker: "-", +}); +const parser = new DOMParser(); + +//service.use(gfm); + +export const handler: Handlers = { + async GET(req) { + const url = new URL(req.url); + const fetchUrl = url.searchParams.get("url"); + + if (!fetchUrl || !isValidUrl(fetchUrl)) { + throw new BadRequestError(); + } + + console.log("[api/article] create article from url", { url: fetchUrl }); + + const request = await fetch(fetchUrl); + const html = await request.text(); + + const document = parser.parseFromString(html, "text/html"); + + const title = document?.querySelector("title")?.innerText; + + const metaAuthor = + document?.querySelector('meta[name="twitter:creator"]')?.getAttribute( + "content", + ) || + document?.querySelector('meta[name="author"]')?.getAttribute("content"); + + console.log({ metaAuthor }); + + const readable = new Readability(document); + + const result = readable.parse(); + + console.log("[api/article] parsed ", { + url: fetchUrl, + content: result.textContent, + }); + + const cleanDocument = parser.parseFromString( + result.content, + "text/html", + ); + + const [tags, summary, shortTitle, author] = await Promise.all([ + openai.createTags(result.textContent), + openai.summarize(result.textContent), + title && openai.shortenTitle(title), + metaAuthor || openai.extractAuthorName(result.textContent), + ]); + + const markdown = service.turndown(cleanDocument); + + const id = shortTitle || title || ""; + + const newArticle = { + id, + name: title || "", + content: markdown, + tags: tags || [], + meta: { + author: author || "", + link: fetchUrl, + status: "not-finished", + date: new Date(), + }, + } as const; + + await createArticle(newArticle); + + return json(newArticle); + }, +}; diff --git a/routes/api/articles/index.ts b/routes/api/articles/index.ts new file mode 100644 index 0000000..513c24b --- /dev/null +++ b/routes/api/articles/index.ts @@ -0,0 +1,10 @@ +import { Handlers } from "$fresh/server.ts"; +import { getAllArticles } from "@lib/resource/articles.ts"; +import { json } from "@lib/helpers.ts"; + +export const handler: Handlers = { + async GET() { + const movies = await getAllArticles(); + return json(movies); + }, +}; diff --git a/routes/api/readable/index.ts b/routes/api/readable/index.ts deleted file mode 100644 index e69de29..0000000 diff --git a/routes/articles/[name].tsx b/routes/articles/[name].tsx new file mode 100644 index 0000000..a8bec31 --- /dev/null +++ b/routes/articles/[name].tsx @@ -0,0 +1,51 @@ +import { Handlers, PageProps } from "$fresh/server.ts"; +import { MainLayout } from "@components/layouts/main.tsx"; +import { Article, getArticle } from "@lib/resource/articles.ts"; +import { RecipeHero } from "@components/RecipeHero.tsx"; +import { KMenu } from "@islands/KMenu.tsx"; + +export const handler: Handlers
= { + async GET(_, ctx) { + const movie = await getArticle(ctx.params.name); + return ctx.render(movie); + }, +}; + +export default function Greet(props: PageProps
) { + const article = props.data; + + const { author = "", date = "" } = article.meta; + + console.log({ tags: article.tags }); + + return ( + + + + {article.tags.length && + ( +
+ {article.tags.map((t) => { + return ( + + #{t} + + ); + })} +
+ )} +
+ 
+          {article.content}
+
+
+ + ); +} diff --git a/routes/articles/index.tsx b/routes/articles/index.tsx new file mode 100644 index 0000000..d151384 --- /dev/null +++ b/routes/articles/index.tsx @@ -0,0 +1,38 @@ +import { Handlers, PageProps } from "$fresh/server.ts"; +import { MainLayout } from "@components/layouts/main.tsx"; +import IconArrowLeft from "https://deno.land/x/tabler_icons_tsx@0.0.3/tsx/arrow-left.tsx"; +import { Article, getAllArticles } from "@lib/resource/articles.ts"; +import { Card } from "@components/Card.tsx"; +import { KMenu } from "@islands/KMenu.tsx"; + +export const handler: Handlers = { + async GET(_, ctx) { + const movies = await getAllArticles(); + return ctx.render(movies); + }, +}; + +export default function Greet(props: PageProps) { + return ( + +
+ + + Back + + +

📝 Articles

+
+ + +
+ {props.data?.map((doc) => { + return ; + })} +
+ + ); +}