feat: add initial add article command

This commit is contained in:
max_richter 2023-08-01 21:35:21 +02:00
parent e51667bbac
commit ea11495d1a
17 changed files with 534 additions and 67 deletions

View File

@ -52,11 +52,15 @@ export function RecipeHero(
</div>
)}
<div class="absolute inset-x-0 bottom-0 py-4 px-8 py-8 ">
<div
class={`absolute inset-x-0 bottom-0 py-4 px-8 ${
imageUrl ? "py-8" : ""
} `}
>
<div
class={`${
imageUrl ? "noisy-gradient" : ""
} flex gap-4 items-center pt-12`}
class={`${imageUrl ? "noisy-gradient" : ""} flex gap-4 items-center ${
imageUrl ? "pt-12" : ""
}`}
>
<h2
class="relative text-4xl font-bold z-10"
@ -73,7 +77,7 @@ export function RecipeHero(
class="relative z-50 flex gap-5 font-sm text-light mt-3"
style={{ color: imageUrl ? "#1F1F1F" : "white" }}
>
{subline.map((s) => {
{subline.filter((s) => s && s?.length > 1).map((s) => {
return <span>{s}</span>;
})}
</div>

View File

@ -22,6 +22,10 @@ export const MainLayout = ({ children, url }: Props) => {
name: "🍿 Movies",
link: "/movies",
},
{
name: "📝 Articles",
link: "/articles",
},
];
return (

View File

@ -5,22 +5,27 @@
import * as $0 from "./routes/_404.tsx";
import * as $1 from "./routes/_app.tsx";
import * as $2 from "./routes/_middleware.ts";
import * as $3 from "./routes/api/cache/index.ts";
import * as $4 from "./routes/api/images/index.ts";
import * as $5 from "./routes/api/index.ts";
import * as $6 from "./routes/api/movies/[name].ts";
import * as $7 from "./routes/api/movies/enhance/[name].ts";
import * as $8 from "./routes/api/movies/index.ts";
import * as $9 from "./routes/api/recipes/[name].ts";
import * as $10 from "./routes/api/recipes/index.ts";
import * as $11 from "./routes/api/tmdb/[id].ts";
import * as $12 from "./routes/api/tmdb/credits/[id].ts";
import * as $13 from "./routes/api/tmdb/query.ts";
import * as $14 from "./routes/index.tsx";
import * as $15 from "./routes/movies/[name].tsx";
import * as $16 from "./routes/movies/index.tsx";
import * as $17 from "./routes/recipes/[name].tsx";
import * as $18 from "./routes/recipes/index.tsx";
import * as $3 from "./routes/api/articles/[name].ts";
import * as $4 from "./routes/api/articles/create/index.ts";
import * as $5 from "./routes/api/articles/index.ts";
import * as $6 from "./routes/api/cache/index.ts";
import * as $7 from "./routes/api/images/index.ts";
import * as $8 from "./routes/api/index.ts";
import * as $9 from "./routes/api/movies/[name].ts";
import * as $10 from "./routes/api/movies/enhance/[name].ts";
import * as $11 from "./routes/api/movies/index.ts";
import * as $12 from "./routes/api/recipes/[name].ts";
import * as $13 from "./routes/api/recipes/index.ts";
import * as $14 from "./routes/api/tmdb/[id].ts";
import * as $15 from "./routes/api/tmdb/credits/[id].ts";
import * as $16 from "./routes/api/tmdb/query.ts";
import * as $17 from "./routes/articles/[name].tsx";
import * as $18 from "./routes/articles/index.tsx";
import * as $19 from "./routes/index.tsx";
import * as $20 from "./routes/movies/[name].tsx";
import * as $21 from "./routes/movies/index.tsx";
import * as $22 from "./routes/recipes/[name].tsx";
import * as $23 from "./routes/recipes/index.tsx";
import * as $$0 from "./islands/Counter.tsx";
import * as $$1 from "./islands/IngredientsList.tsx";
import * as $$2 from "./islands/KMenu.tsx";
@ -32,22 +37,27 @@ const manifest = {
"./routes/_404.tsx": $0,
"./routes/_app.tsx": $1,
"./routes/_middleware.ts": $2,
"./routes/api/cache/index.ts": $3,
"./routes/api/images/index.ts": $4,
"./routes/api/index.ts": $5,
"./routes/api/movies/[name].ts": $6,
"./routes/api/movies/enhance/[name].ts": $7,
"./routes/api/movies/index.ts": $8,
"./routes/api/recipes/[name].ts": $9,
"./routes/api/recipes/index.ts": $10,
"./routes/api/tmdb/[id].ts": $11,
"./routes/api/tmdb/credits/[id].ts": $12,
"./routes/api/tmdb/query.ts": $13,
"./routes/index.tsx": $14,
"./routes/movies/[name].tsx": $15,
"./routes/movies/index.tsx": $16,
"./routes/recipes/[name].tsx": $17,
"./routes/recipes/index.tsx": $18,
"./routes/api/articles/[name].ts": $3,
"./routes/api/articles/create/index.ts": $4,
"./routes/api/articles/index.ts": $5,
"./routes/api/cache/index.ts": $6,
"./routes/api/images/index.ts": $7,
"./routes/api/index.ts": $8,
"./routes/api/movies/[name].ts": $9,
"./routes/api/movies/enhance/[name].ts": $10,
"./routes/api/movies/index.ts": $11,
"./routes/api/recipes/[name].ts": $12,
"./routes/api/recipes/index.ts": $13,
"./routes/api/tmdb/[id].ts": $14,
"./routes/api/tmdb/credits/[id].ts": $15,
"./routes/api/tmdb/query.ts": $16,
"./routes/articles/[name].tsx": $17,
"./routes/articles/index.tsx": $18,
"./routes/index.tsx": $19,
"./routes/movies/[name].tsx": $20,
"./routes/movies/index.tsx": $21,
"./routes/recipes/[name].tsx": $22,
"./routes/recipes/index.tsx": $23,
},
islands: {
"./islands/Counter.tsx": $$0,

View File

@ -4,10 +4,6 @@ import { useEventListener } from "@lib/hooks/useEventListener.ts";
import { menus } from "@islands/KMenu/commands.ts";
import { MenuEntry } from "@islands/KMenu/types.ts";
function filterMenu(menu: MenuEntry[], activeMenu?: string) {
return menu;
}
const KMenuEntry = (
{ entry, activeIndex, index }: {
entry: MenuEntry;
@ -34,15 +30,42 @@ export const KMenu = (
) => {
const activeMenuType = useSignal(type);
const activeMenu = menus[activeMenuType.value || "main"];
const activeState = useSignal<"normal" | "loading" | "error">("normal");
const activeIndex = useSignal(-1);
const visible = useSignal(false);
const input = useRef<HTMLInputElement>(null);
const commandInput = useSignal("");
const visible = useSignal(false);
if (visible.value === false) {
setTimeout(() => {
activeMenuType.value = "main";
activeState.value = "normal";
if (input.current) {
input.current.value = "";
}
commandInput.value = "";
}, 100);
}
const entries = activeMenu?.entries?.filter((entry) => {
const entryVisible = typeof entry["visible"] === "boolean"
? entry.visible
: typeof entry["visible"] === "function"
? entry.visible?.(context)
: true;
const search = commandInput?.value?.toLowerCase();
const searchMatches = entry.title.toLowerCase().includes(search) ||
entry.meta?.toLowerCase()?.includes(search);
return entryVisible && searchMatches;
});
if (entries.length === 1) {
activeIndex.value = 0;
}
function activateEntry(menuEntry?: MenuEntry) {
if (!menuEntry) return;
@ -50,6 +73,7 @@ export const KMenu = (
activeMenu: activeMenuType,
menus,
activeState,
commandInput,
visible,
}, context);
}
@ -60,7 +84,6 @@ export const KMenu = (
}
if (ev.key === "ArrowDown") {
const entries = filterMenu(activeMenu.entries);
const index = activeIndex.value;
if (index + 1 >= entries.length) {
activeIndex.value = 0;
@ -70,12 +93,10 @@ export const KMenu = (
}
if (ev.key === "Enter") {
const entries = filterMenu(activeMenu.entries);
activateEntry(entries[activeIndex.value]);
}
if (ev.key === "ArrowUp") {
const entries = filterMenu(activeMenu.entries);
const index = activeIndex.value;
if (index - 1 < 0) {
activeIndex.value = entries.length - 1;
@ -140,7 +161,7 @@ export const KMenu = (
{activeState.value === "normal" &&
(
<div class="" style={{ maxHeight: "12rem", overflowY: "auto" }}>
{filterMenu(activeMenu.entries, input.current?.value).map(
{entries.map(
(k, index) => {
return (
<KMenuEntry

View File

@ -1,6 +1,7 @@
import { Menu } from "@islands/KMenu/types.ts";
import { Movie } from "@lib/resource/movies.ts";
import { TMDBMovie } from "@lib/types.ts";
import { isValidUrl } from "@lib/helpers.ts";
export const menus: Record<string, Menu> = {
main: {
@ -14,6 +15,35 @@ export const menus: Record<string, Menu> = {
},
visible: () => false,
},
{
title: "Create new article",
meta: "",
cb: (state) => {
state.menus["input_link"] = {
title: "Link:",
entries: [],
};
state.activeMenu.value = "input_link";
const unsub = state.commandInput.subscribe(async (value) => {
if (isValidUrl(value)) {
unsub();
state.activeState.value = "loading";
const response = await fetch("/api/articles/create?url=" + value);
const newArticle = await response.json();
if (newArticle?.id) {
window.location.href = "/articles/" + newArticle.id;
}
state.visible.value = false;
}
});
},
visible: () => true,
},
{
title: "Clear Cache",
cb: async (state) => {
@ -59,13 +89,14 @@ export const menus: Record<string, Menu> = {
})),
};
console.log({ state });
state.activeMenu.value = menuID;
state.activeState.value = "normal";
},
visible: () => false,
visible: () => {
const loc = globalThis["location"];
return loc?.pathname?.includes("movie");
},
},
{
title: "Reload Page",

View File

@ -3,6 +3,7 @@ import { Signal } from "@preact/signals";
export type MenuState = {
activeMenu: Signal<string>;
activeState: Signal<"error" | "normal" | "loading">;
commandInput: Signal<string>;
visible: Signal<boolean>;
menus: Record<string, Menu>;
};

View File

@ -9,6 +9,7 @@ import rehypeSanitize from "https://esm.sh/rehype-sanitize@5.0.1";
import rehypeStringify from "https://esm.sh/rehype-stringify@9.0.3";
import * as cache from "@lib/cache/documents.ts";
import { SILVERBULLET_SERVER } from "@lib/env.ts";
import { fixRenderedMarkdown } from "@lib/helpers.ts";
export type Document = {
name: string;
@ -75,26 +76,13 @@ export function transformDocument(input: string, cb: (r: Root) => Root) {
.use(remarkStringify)
.processSync(input);
return String(out)
.replace("***\n", "---")
.replace("----------------", "---")
.replace("\n---", "---")
.replace(/^(date:[^'\n]*)'|'/gm, (match, p1, p2) => {
if (p1) {
// This is a line starting with date: followed by single quotes
return p1.replace(/'/gm, "");
} else if (p2) {
return "";
} else {
// This is a line with single quotes, but not starting with date:
return match;
}
});
return fixRenderedMarkdown(String(out));
}
export function parseDocument(doc: string) {
return unified()
.use(remarkParse).use(remarkFrontmatter, ["yaml", "toml"])
.use(remarkParse)
.use(remarkFrontmatter, ["yaml", "toml"])
.parse(doc);
}

View File

@ -2,3 +2,4 @@ export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER");
export const REDIS_HOST = Deno.env.get("REDIS_HOST");
export const REDIS_PASS = Deno.env.get("REDIS_PASS");
export const TMDB_API_KEY = Deno.env.get("TMDB_API_KEY");
export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY");

View File

@ -5,3 +5,28 @@ export function json(content: unknown) {
headers,
});
}
export const isValidUrl = (urlString: string) => {
try {
return Boolean(new URL(urlString));
} catch (e) {
return false;
}
};
export const fixRenderedMarkdown = (content: string) => {
return content.replace("***\n", "---")
.replace("----------------", "---")
.replace("\n---", "---")
.replace(/^(date:[^'\n]*)'|'/gm, (match, p1, p2) => {
if (p1) {
// This is a line starting with date: followed by single quotes
return p1.replace(/'/gm, "");
} else if (p2) {
return "";
} else {
// This is a line with single quotes, but not starting with date:
return match;
}
});
};

79
lib/openai.ts Normal file
View File

@ -0,0 +1,79 @@
import { OpenAI } from "https://deno.land/x/openai/mod.ts";
import { OPENAI_API_KEY } from "@lib/env.ts";
const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY);
export async function summarize(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.createChatCompletion({
model: "gpt-3.5-turbo",
messages: [
{ "role": "system", "content": "You are a helpful assistant." },
{ "role": "user", "content": content.slice(0, 2000) },
{
"role": "user",
"content":
"Please summarize the article in one sentence as short as possible",
},
],
});
return chatCompletion.choices[0].message.content;
}
export async function shortenTitle(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.createChatCompletion({
model: "gpt-3.5-turbo",
messages: [
{ "role": "system", "content": "You are a helpful assistant." },
{ "role": "user", "content": content.slice(0, 2000) },
{
"role": "user",
"content":
"Please shorten the provided website title as much as possible, don't rewrite it, just remove unneccesary informations. Please remove for example any mention of the name of the website.",
},
],
});
return chatCompletion.choices[0].message.content;
}
export async function extractAuthorName(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.createChatCompletion({
model: "gpt-3.5-turbo",
messages: [
{ "role": "system", "content": "You are a helpful assistant." },
{ "role": "user", "content": content.slice(0, 2000) },
{
"role": "user",
"content":
"If you are able to extract the name of the author from the text please respond with the name, otherwise respond with 'not found'",
},
],
});
const author = chatCompletion.choices[0].message.content;
if (author !== "not found") return author;
return "";
}
export async function createTags(content: string) {
if (!openAI) return;
const chatCompletion = await openAI.createChatCompletion({
model: "gpt-3.5-turbo",
messages: [
{ "role": "system", "content": "You are a helpful assistant." },
{ "role": "user", "content": content.slice(0, 2000) },
{
"role": "user",
"content":
"Please respond with a list of genres the corresponding article falls into, dont include any other informations, just a comma seperated list of top 5 categories. Please respond only with tags that make sense even if there are less than five.",
},
],
});
return chatCompletion.choices[0].message.content?.toLowerCase().split(", ")
.map((v) => v.replaceAll(" ", "-"));
}

104
lib/resource/articles.ts Normal file
View File

@ -0,0 +1,104 @@
import { parseDocument, renderMarkdown } from "@lib/documents.ts";
import { parse } from "yaml";
import { createCrud } from "@lib/crud.ts";
import { stringify } from "https://deno.land/std@0.194.0/yaml/stringify.ts";
import { formatDate } from "@lib/string.ts";
import { fixRenderedMarkdown } from "@lib/helpers.ts";
export type Article = {
id: string;
content: string;
name: string;
tags: string[];
meta: {
status: "finished" | "not-finished";
date: Date;
link: string;
author?: string;
rating?: number;
};
};
const crud = createCrud<Article>({
prefix: "Media/articles/",
parse: parseArticle,
});
function renderArticle(article: Article) {
const meta = article.meta;
if ("date" in meta) {
meta.date = formatDate(meta.date);
}
return fixRenderedMarkdown(`${
meta
? `---
${stringify(meta)}
---`
: `---
---`
}
# ${article.name}
${article.tags.map((t) => `#${t}`).join(" ")}
${article.content}
`);
}
function parseArticle(original: string, id: string): Article {
const doc = parseDocument(original);
let meta = {} as Article["meta"];
let name = "";
const range = [Infinity, -Infinity];
for (const child of doc.children) {
if (child.type === "yaml") {
meta = parse(child.value) as Article["meta"];
if (meta["rating"] && typeof meta["rating"] === "string") {
meta.rating = [...meta.rating?.matchAll("⭐")].length;
}
continue;
}
if (
child.type === "heading" && child.depth === 1 && !name &&
child.children.length === 1 && child.children[0].type === "text"
) {
name = child.children[0].value;
continue;
}
if (name) {
const start = child.position?.start.offset || Infinity;
const end = child.position?.end.offset || -Infinity;
if (start < range[0]) range[0] = start;
if (end > range[1]) range[1] = end;
}
}
let content = original.slice(range[0], range[1]);
const tags = [];
for (const [hashtag] of original.matchAll(/\B(\#[a-zA-Z\-]+\b)(?!;)/g)) {
tags.push(hashtag.replace(/\#/g, ""));
content = content.replace(hashtag, "");
}
return {
id,
name,
tags,
content: renderMarkdown(content),
meta,
};
}
export const getAllArticles = crud.readAll;
export const getArticle = crud.read;
export const createArticle = (article: Article) => {
console.log("creating article", { article });
const content = renderArticle(article);
return crud.create(article.id, content);
};

View File

@ -0,0 +1,10 @@
import { Handlers } from "$fresh/server.ts";
import { getArticle } from "@lib/resource/articles.ts";
import { json } from "@lib/helpers.ts";
export const handler: Handlers = {
async GET(_, ctx) {
const article = await getArticle(ctx.params.name);
return json(article);
},
};

View File

@ -0,0 +1,90 @@
import { Handlers } from "$fresh/server.ts";
import { Readability } from "https://cdn.skypack.dev/@mozilla/readability";
import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.38/deno-dom-wasm.ts";
import { BadRequestError } from "@lib/errors.ts";
import { isValidUrl, json } from "@lib/helpers.ts";
import * as openai from "@lib/openai.ts";
import tds from "https://cdn.skypack.dev/turndown@7.1.1";
//import { gfm } from "https://cdn.skypack.dev/@guyplusplus/turndown-plugin-gfm@1.0.7";
import { createArticle } from "@lib/resource/articles.ts";
const service = new tds({
headingStyle: "atx",
codeBlockStyle: "fenced",
hr: "---",
bulletListMarker: "-",
});
const parser = new DOMParser();
//service.use(gfm);
export const handler: Handlers = {
async GET(req) {
const url = new URL(req.url);
const fetchUrl = url.searchParams.get("url");
if (!fetchUrl || !isValidUrl(fetchUrl)) {
throw new BadRequestError();
}
console.log("[api/article] create article from url", { url: fetchUrl });
const request = await fetch(fetchUrl);
const html = await request.text();
const document = parser.parseFromString(html, "text/html");
const title = document?.querySelector("title")?.innerText;
const metaAuthor =
document?.querySelector('meta[name="twitter:creator"]')?.getAttribute(
"content",
) ||
document?.querySelector('meta[name="author"]')?.getAttribute("content");
console.log({ metaAuthor });
const readable = new Readability(document);
const result = readable.parse();
console.log("[api/article] parsed ", {
url: fetchUrl,
content: result.textContent,
});
const cleanDocument = parser.parseFromString(
result.content,
"text/html",
);
const [tags, summary, shortTitle, author] = await Promise.all([
openai.createTags(result.textContent),
openai.summarize(result.textContent),
title && openai.shortenTitle(title),
metaAuthor || openai.extractAuthorName(result.textContent),
]);
const markdown = service.turndown(cleanDocument);
const id = shortTitle || title || "";
const newArticle = {
id,
name: title || "",
content: markdown,
tags: tags || [],
meta: {
author: author || "",
link: fetchUrl,
status: "not-finished",
date: new Date(),
},
} as const;
await createArticle(newArticle);
return json(newArticle);
},
};

View File

@ -0,0 +1,10 @@
import { Handlers } from "$fresh/server.ts";
import { getAllArticles } from "@lib/resource/articles.ts";
import { json } from "@lib/helpers.ts";
export const handler: Handlers = {
async GET() {
const movies = await getAllArticles();
return json(movies);
},
};

View File

@ -0,0 +1,51 @@
import { Handlers, PageProps } from "$fresh/server.ts";
import { MainLayout } from "@components/layouts/main.tsx";
import { Article, getArticle } from "@lib/resource/articles.ts";
import { RecipeHero } from "@components/RecipeHero.tsx";
import { KMenu } from "@islands/KMenu.tsx";
export const handler: Handlers<Article | null> = {
async GET(_, ctx) {
const movie = await getArticle(ctx.params.name);
return ctx.render(movie);
},
};
export default function Greet(props: PageProps<Article>) {
const article = props.data;
const { author = "", date = "" } = article.meta;
console.log({ tags: article.tags });
return (
<MainLayout url={props.url}>
<RecipeHero
data={article}
subline={[author, date.toString()]}
backlink="/articles"
/>
<KMenu type="main" context={article} />
{article.tags.length &&
(
<div class="flex gap-2 px-8">
{article.tags.map((t) => {
return (
<span class="bg-gray-700 text-white p-2 rounded-xl text-sm">
#{t}
</span>
);
})}
</div>
)}
<div class="px-8 text-white mt-10">
<pre
class="whitespace-break-spaces"
dangerouslySetInnerHTML={{ __html: article.content || "" }}
>
{article.content}
</pre>
</div>
</MainLayout>
);
}

38
routes/articles/index.tsx Normal file
View File

@ -0,0 +1,38 @@
import { Handlers, PageProps } from "$fresh/server.ts";
import { MainLayout } from "@components/layouts/main.tsx";
import IconArrowLeft from "https://deno.land/x/tabler_icons_tsx@0.0.3/tsx/arrow-left.tsx";
import { Article, getAllArticles } from "@lib/resource/articles.ts";
import { Card } from "@components/Card.tsx";
import { KMenu } from "@islands/KMenu.tsx";
export const handler: Handlers<Article[] | null> = {
async GET(_, ctx) {
const movies = await getAllArticles();
return ctx.render(movies);
},
};
export default function Greet(props: PageProps<Article[] | null>) {
return (
<MainLayout url={props.url}>
<header class="flex gap-4 items-center mb-5 md:hidden">
<a
class="px-4 ml-4 py-2 bg-gray-300 text-gray-800 rounded-lg flex items-center gap-1"
href="/"
>
<IconArrowLeft class="w-5 h-5" />
Back
</a>
<h3 class="text-2xl text-white font-light">📝 Articles</h3>
</header>
<KMenu type="main" context={false} />
<div class="flex flex-wrap items-center gap-4 px-4">
{props.data?.map((doc) => {
return <Card link={`/articles/${doc.id}`} title={doc.name} />;
})}
</div>
</MainLayout>
);
}