feat: url scraper to recipe

lib/documents.ts

@@ -58,6 +58,10 @@ export function createDocument(
 
   log.info("creating document", { name });
 
+  if (typeof content === "string") {
+    updateDocument(name, content).catch(log.error);
+  }
+
   return fetch(SILVERBULLET_SERVER + "/" + name, {
     body: content,
     method: "PUT",
@@ -65,25 +69,49 @@ export function createDocument(
   });
 }
 
-export async function getDocument(name: string): Promise<string> {
-  const documents = await db.select().from(documentTable).where(
-    eq(documentTable.name, name),
-  ).limit(1);
-  if (documents[0]?.content) return documents[0].content;
-
+async function fetchDocument(name: string) {
   log.debug("fetching document", { name });
   const headers = new Headers();
   headers.append("X-Sync-Mode", "true");
   const response = await fetch(SILVERBULLET_SERVER + "/" + name, { headers });
-  const text = await response.text();
+  if (response.status === 404) {
+    return;
+  }
+  return response.text();
+}
 
-  await db.update(documentTable).set({
-    content: text,
-  }).where(eq(documentTable.name, name));
+export async function getDocument(name: string): Promise<string | undefined> {
+  const documents = await db.select().from(documentTable).where(
+    eq(documentTable.name, name),
+  ).limit(1);
+  // This updates the document in the background
+  fetchDocument(name).then((content) => {
+    if (content) {
+      updateDocument(name, content);
+    } else {
+      db.delete(documentTable).where(eq(documentTable.name, name));
+    }
+  }).catch(
+    log.error,
+  );
+  if (documents[0]?.content) return documents[0].content;
+
+  const text = await fetchDocument(name);
+  if (!text) {
+    db.delete(documentTable).where(eq(documentTable.name, name));
+    return;
+  }
+  await updateDocument(name, text);
 
   return text;
 }
 
+export function updateDocument(name: string, content: string) {
+  return db.update(documentTable).set({
+    content,
+  }).where(eq(documentTable.name, name));
+}
+
 export function transformDocument(input: string, cb: (r: Root) => Root) {
   const out = unified()
     .use(remarkParse)