91 lines
2.4 KiB
TypeScript
91 lines
2.4 KiB
TypeScript
|
import { Handlers } from "$fresh/server.ts";
|
||
|
|
import { Readability } from "https://cdn.skypack.dev/@mozilla/readability";
|
||
|
|
import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.38/deno-dom-wasm.ts";
|
||
|
|
import { BadRequestError } from "@lib/errors.ts";
|
||
|
|
import { isValidUrl, json } from "@lib/helpers.ts";
|
||
|
|
import * as openai from "@lib/openai.ts";
|
||
|
|
|
||
|
|
import tds from "https://cdn.skypack.dev/turndown@7.1.1";
|
||
|
|
//import { gfm } from "https://cdn.skypack.dev/@guyplusplus/turndown-plugin-gfm@1.0.7";
|
||
|
|
import { createArticle } from "@lib/resource/articles.ts";
|
||
|
|
|
||
|
|
const service = new tds({
|
||
|
|
headingStyle: "atx",
|
||
|
|
codeBlockStyle: "fenced",
|
||
|
|
hr: "---",
|
||
|
|
bulletListMarker: "-",
|
||
|
|
});
|
||
|
|
const parser = new DOMParser();
|
||
|
|
|
||
|
|
//service.use(gfm);
|
||
|
|
|
||
|
|
export const handler: Handlers = {
|
||
|
|
async GET(req) {
|
||
|
|
const url = new URL(req.url);
|
||
|
|
const fetchUrl = url.searchParams.get("url");
|
||
|
|
|
||
|
|
if (!fetchUrl || !isValidUrl(fetchUrl)) {
|
||
|
|
throw new BadRequestError();
|
||
|
|
}
|
||
|
|
|
||
|
|
console.log("[api/article] create article from url", { url: fetchUrl });
|
||
|
|
|
||
|
|
const request = await fetch(fetchUrl);
|
||
|
|
const html = await request.text();
|
||
|
|
|
||
|
|
const document = parser.parseFromString(html, "text/html");
|
||
|
|
|
||
|
|
const title = document?.querySelector("title")?.innerText;
|
||
|
|
|
||
|
|
const metaAuthor =
|
||
|
|
document?.querySelector('meta[name="twitter:creator"]')?.getAttribute(
|
||
|
|
"content",
|
||
|
|
) ||
|
||
|
|
document?.querySelector('meta[name="author"]')?.getAttribute("content");
|
||
|
|
|
||
|
|
console.log({ metaAuthor });
|
||
|
|
|
||
|
|
const readable = new Readability(document);
|
||
|
|
|
||
|
|
const result = readable.parse();
|
||
|
|
|
||
|
|
console.log("[api/article] parsed ", {
|
||
|
|
url: fetchUrl,
|
||
|
|
content: result.textContent,
|
||
|
|
});
|
||
|
|
|
||
|
|
const cleanDocument = parser.parseFromString(
|
||
|
|
result.content,
|
||
|
|
"text/html",
|
||
|
|
);
|
||
|
|
|
||
|
|
const [tags, summary, shortTitle, author] = await Promise.all([
|
||
|
|
openai.createTags(result.textContent),
|
||
|
|
openai.summarize(result.textContent),
|
||
|
|
title && openai.shortenTitle(title),
|
||
|
|
metaAuthor || openai.extractAuthorName(result.textContent),
|
||
|
|
]);
|
||
|
|
|
||
|
|
const markdown = service.turndown(cleanDocument);
|
||
|
|
|
||
|
|
const id = shortTitle || title || "";
|
||
|
|
|
||
|
|
const newArticle = {
|
||
|
|
id,
|
||
|
|
name: title || "",
|
||
|
|
content: markdown,
|
||
|
|
tags: tags || [],
|
||
|
|
meta: {
|
||
|
|
author: author || "",
|
||
|
|
link: fetchUrl,
|
||
|
|
status: "not-finished",
|
||
|
|
date: new Date(),
|
||
|
|
},
|
||
|
|
} as const;
|
||
|
|
|
||
|
|
await createArticle(newArticle);
|
||
|
|
|
||
|
|
return json(newArticle);
|
||
|
|
},
|
||
|
|
};
|