feat: add initial add article command

lib/documents.ts

@@ -9,6 +9,7 @@ import rehypeSanitize from "https://esm.sh/rehype-sanitize@5.0.1";
 import rehypeStringify from "https://esm.sh/rehype-stringify@9.0.3";
 import * as cache from "@lib/cache/documents.ts";
 import { SILVERBULLET_SERVER } from "@lib/env.ts";
+import { fixRenderedMarkdown } from "@lib/helpers.ts";
 
 export type Document = {
   name: string;
@@ -75,26 +76,13 @@ export function transformDocument(input: string, cb: (r: Root) => Root) {
     .use(remarkStringify)
     .processSync(input);
 
-  return String(out)
-    .replace("***\n", "---")
-    .replace("----------------", "---")
-    .replace("\n---", "---")
-    .replace(/^(date:[^'\n]*)'|'/gm, (match, p1, p2) => {
-      if (p1) {
-        // This is a line starting with date: followed by single quotes
-        return p1.replace(/'/gm, "");
-      } else if (p2) {
-        return "";
-      } else {
-        // This is a line with single quotes, but not starting with date:
-        return match;
-      }
-    });
+  return fixRenderedMarkdown(String(out));
 }
 
 export function parseDocument(doc: string) {
   return unified()
-    .use(remarkParse).use(remarkFrontmatter, ["yaml", "toml"])
+    .use(remarkParse)
+    .use(remarkFrontmatter, ["yaml", "toml"])
     .parse(doc);
 }
 

lib/env.ts

@@ -2,3 +2,4 @@ export const SILVERBULLET_SERVER = Deno.env.get("SILVERBULLET_SERVER");
 export const REDIS_HOST = Deno.env.get("REDIS_HOST");
 export const REDIS_PASS = Deno.env.get("REDIS_PASS");
 export const TMDB_API_KEY = Deno.env.get("TMDB_API_KEY");
+export const OPENAI_API_KEY = Deno.env.get("OPENAI_API_KEY");

lib/helpers.ts

@@ -5,3 +5,28 @@ export function json(content: unknown) {
     headers,
   });
 }
+
+export const isValidUrl = (urlString: string) => {
+  try {
+    return Boolean(new URL(urlString));
+  } catch (e) {
+    return false;
+  }
+};
+
+export const fixRenderedMarkdown = (content: string) => {
+  return content.replace("***\n", "---")
+    .replace("----------------", "---")
+    .replace("\n---", "---")
+    .replace(/^(date:[^'\n]*)'|'/gm, (match, p1, p2) => {
+      if (p1) {
+        // This is a line starting with date: followed by single quotes
+        return p1.replace(/'/gm, "");
+      } else if (p2) {
+        return "";
+      } else {
+        // This is a line with single quotes, but not starting with date:
+        return match;
+      }
+    });
+};

lib/openai.ts

@@ -0,0 +1,79 @@
+import { OpenAI } from "https://deno.land/x/openai/mod.ts";
+import { OPENAI_API_KEY } from "@lib/env.ts";
+
+const openAI = OPENAI_API_KEY && new OpenAI(OPENAI_API_KEY);
+
+export async function summarize(content: string) {
+  if (!openAI) return;
+  const chatCompletion = await openAI.createChatCompletion({
+    model: "gpt-3.5-turbo",
+    messages: [
+      { "role": "system", "content": "You are a helpful assistant." },
+      { "role": "user", "content": content.slice(0, 2000) },
+      {
+        "role": "user",
+        "content":
+          "Please summarize the article in one sentence as short as possible",
+      },
+    ],
+  });
+
+  return chatCompletion.choices[0].message.content;
+}
+export async function shortenTitle(content: string) {
+  if (!openAI) return;
+  const chatCompletion = await openAI.createChatCompletion({
+    model: "gpt-3.5-turbo",
+    messages: [
+      { "role": "system", "content": "You are a helpful assistant." },
+      { "role": "user", "content": content.slice(0, 2000) },
+      {
+        "role": "user",
+        "content":
+          "Please shorten the provided website title as much as possible, don't rewrite it, just remove unneccesary informations. Please remove for example any mention of the name of the website.",
+      },
+    ],
+  });
+
+  return chatCompletion.choices[0].message.content;
+}
+
+export async function extractAuthorName(content: string) {
+  if (!openAI) return;
+  const chatCompletion = await openAI.createChatCompletion({
+    model: "gpt-3.5-turbo",
+    messages: [
+      { "role": "system", "content": "You are a helpful assistant." },
+      { "role": "user", "content": content.slice(0, 2000) },
+      {
+        "role": "user",
+        "content":
+          "If you are able to extract the name of the author from the text please respond with the name, otherwise respond with 'not found'",
+      },
+    ],
+  });
+
+  const author = chatCompletion.choices[0].message.content;
+
+  if (author !== "not found") return author;
+  return "";
+}
+
+export async function createTags(content: string) {
+  if (!openAI) return;
+  const chatCompletion = await openAI.createChatCompletion({
+    model: "gpt-3.5-turbo",
+    messages: [
+      { "role": "system", "content": "You are a helpful assistant." },
+      { "role": "user", "content": content.slice(0, 2000) },
+      {
+        "role": "user",
+        "content":
+          "Please respond with a list of genres the corresponding article falls into, dont include any other informations, just a comma seperated list of top 5 categories. Please respond only with tags that make sense even if there are less than five.",
+      },
+    ],
+  });
+
+  return chatCompletion.choices[0].message.content?.toLowerCase().split(", ")
+    .map((v) => v.replaceAll(" ", "-"));
+}

lib/resource/articles.ts

@@ -0,0 +1,104 @@
+import { parseDocument, renderMarkdown } from "@lib/documents.ts";
+import { parse } from "yaml";
+import { createCrud } from "@lib/crud.ts";
+import { stringify } from "https://deno.land/std@0.194.0/yaml/stringify.ts";
+import { formatDate } from "@lib/string.ts";
+import { fixRenderedMarkdown } from "@lib/helpers.ts";
+
+export type Article = {
+  id: string;
+  content: string;
+  name: string;
+  tags: string[];
+  meta: {
+    status: "finished" | "not-finished";
+    date: Date;
+    link: string;
+    author?: string;
+    rating?: number;
+  };
+};
+
+const crud = createCrud<Article>({
+  prefix: "Media/articles/",
+  parse: parseArticle,
+});
+
+function renderArticle(article: Article) {
+  const meta = article.meta;
+  if ("date" in meta) {
+    meta.date = formatDate(meta.date);
+  }
+
+  return fixRenderedMarkdown(`${
+    meta
+      ? `---
+${stringify(meta)}
+---`
+      : `---
+---`
+  }
+# ${article.name}
+${article.tags.map((t) => `#${t}`).join(" ")}
+${article.content}
+`);
+}
+
+function parseArticle(original: string, id: string): Article {
+  const doc = parseDocument(original);
+
+  let meta = {} as Article["meta"];
+  let name = "";
+
+  const range = [Infinity, -Infinity];
+
+  for (const child of doc.children) {
+    if (child.type === "yaml") {
+      meta = parse(child.value) as Article["meta"];
+
+      if (meta["rating"] && typeof meta["rating"] === "string") {
+        meta.rating = [...meta.rating?.matchAll("⭐")].length;
+      }
+
+      continue;
+    }
+
+    if (
+      child.type === "heading" && child.depth === 1 && !name &&
+      child.children.length === 1 && child.children[0].type === "text"
+    ) {
+      name = child.children[0].value;
+      continue;
+    }
+
+    if (name) {
+      const start = child.position?.start.offset || Infinity;
+      const end = child.position?.end.offset || -Infinity;
+      if (start < range[0]) range[0] = start;
+      if (end > range[1]) range[1] = end;
+    }
+  }
+
+  let content = original.slice(range[0], range[1]);
+  const tags = [];
+  for (const [hashtag] of original.matchAll(/\B(\#[a-zA-Z\-]+\b)(?!;)/g)) {
+    tags.push(hashtag.replace(/\#/g, ""));
+    content = content.replace(hashtag, "");
+  }
+
+  return {
+    id,
+    name,
+    tags,
+    content: renderMarkdown(content),
+    meta,
+  };
+}
+
+export const getAllArticles = crud.readAll;
+export const getArticle = crud.read;
+export const createArticle = (article: Article) => {
+  console.log("creating article", { article });
+  const content = renderArticle(article);
+  return crud.create(article.id, content);
+};