import { Handlers } from "$fresh/server.ts"; import { Readability } from "https://cdn.skypack.dev/@mozilla/readability"; import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.38/deno-dom-wasm.ts"; import { BadRequestError } from "@lib/errors.ts"; import { isValidUrl, json } from "@lib/helpers.ts"; import * as openai from "@lib/openai.ts"; import tds from "https://cdn.skypack.dev/turndown@7.1.1"; //import { gfm } from "https://cdn.skypack.dev/@guyplusplus/turndown-plugin-gfm@1.0.7"; import { createArticle } from "@lib/resource/articles.ts"; const service = new tds({ headingStyle: "atx", codeBlockStyle: "fenced", hr: "---", bulletListMarker: "-", }); const parser = new DOMParser(); //service.use(gfm); export const handler: Handlers = { async GET(req) { const url = new URL(req.url); const fetchUrl = url.searchParams.get("url"); if (!fetchUrl || !isValidUrl(fetchUrl)) { throw new BadRequestError(); } console.log("[api/article] create article from url", { url: fetchUrl }); const request = await fetch(fetchUrl); const html = await request.text(); const document = parser.parseFromString(html, "text/html"); const title = document?.querySelector("title")?.innerText; const metaAuthor = document?.querySelector('meta[name="twitter:creator"]')?.getAttribute( "content", ) || document?.querySelector('meta[name="author"]')?.getAttribute("content"); console.log({ metaAuthor }); const readable = new Readability(document); const result = readable.parse(); console.log("[api/article] parsed ", { url: fetchUrl, content: result.textContent, }); const cleanDocument = parser.parseFromString( result.content, "text/html", ); const [tags, summary, shortTitle, author] = await Promise.all([ openai.createTags(result.textContent), openai.summarize(result.textContent), title && openai.shortenTitle(title), metaAuthor || openai.extractAuthorName(result.textContent), ]); const markdown = service.turndown(cleanDocument); const id = shortTitle || title || ""; const newArticle = { id, name: title || "", content: markdown, tags: tags || [], meta: { author: author || "", link: fetchUrl, status: "not-finished", date: new Date(), }, } as const; await createArticle(newArticle); return json(newArticle); }, };