feat: allow creating articles with marka

lib/webScraper.ts

@@ -1,2 +1,174 @@
-export function webScrape(url: URL) {
+import { JSDOM } from "jsdom";
+import { fetchHtmlWithPlaywright } from "./playwright.ts";
+import { createStreamResponse } from "./helpers.ts";
+
+/**
+ * Mutates the given JSDOM instance: rewrites all relevant URL-bearing attributes
+ * to absolute URLs, resolving against the provided domain (e.g., "https://example.com").
+ */
+export function absolutizeDomUrls(dom: JSDOM, domain: string): void {
+  const { document } = dom.window;
+  const base = toBase(domain);
+
+  const rewrite = (selector: string, attr: string) => {
+    document.querySelectorAll<HTMLElement>(selector).forEach((el) => {
+      const v = el.getAttribute(attr);
+      if (!v) return;
+      const abs = toAbsolute(v, base);
+      if (abs !== v) el.setAttribute(attr, abs);
+    });
+  };
+
+  // Common URL attributes
+  rewrite("a[href]", "href");
+  rewrite("area[href]", "href");
+  rewrite("link[href]", "href");
+  rewrite("use[href]", "href"); // SVG 2
+  rewrite("use[xlink\\:href]", "xlink:href"); // legacy SVG
+  rewrite("image[href]", "href"); // SVG
+  rewrite("image[xlink\\:href]", "xlink:href"); // legacy SVG
+
+  rewrite("script[src]", "src");
+  rewrite("img[src]", "src");
+  rewrite("source[src]", "src");
+  rewrite("track[src]", "src");
+  rewrite("iframe[src]", "src");
+  rewrite("embed[src]", "src");
+  rewrite("audio[src]", "src");
+  rewrite("video[src]", "src");
+  rewrite("object[data]", "data");
+  rewrite("input[src]", "src");
+  rewrite("form[action]", "action");
+  rewrite("video[poster]", "poster");
+
+  // srcset (img, source)
+  document
+    .querySelectorAll<HTMLElement>("img[srcset], source[srcset]")
+    .forEach((el) => {
+      const v = el.getAttribute("srcset");
+      if (!v) return;
+      const abs = absolutizeSrcset(v, base);
+      if (abs !== v) el.setAttribute("srcset", abs);
+    });
+
+  // Inline CSS in style attributes: url(...)
+  document.querySelectorAll<HTMLElement>("[style]").forEach((el) => {
+    const v = el.getAttribute("style");
+    if (!v) return;
+    const abs = absolutizeCssUrls(v, base);
+    if (abs !== v) el.setAttribute("style", abs);
+  });
+
+  // <style> blocks (inline CSS): url(...)
+  document.querySelectorAll<HTMLStyleElement>("style").forEach((styleEl) => {
+    const css = styleEl.textContent ?? "";
+    const abs = absolutizeCssUrls(css, base);
+    if (abs !== css) styleEl.textContent = abs;
+  });
+
+  // <meta http-equiv="refresh" content="5; url=/path">
+  document
+    .querySelectorAll<HTMLMetaElement>('meta[http-equiv="refresh" i][content]')
+    .forEach((meta) => {
+      const content = meta.getAttribute("content") || "";
+      const abs = absolutizeMetaRefresh(content, base);
+      if (abs !== content) meta.setAttribute("content", abs);
+    });
+}
+
+/** Normalize the base to a valid absolute URL root. */
+function toBase(domain: string): string {
+  // Allow callers to pass "example.com" or "//example.com"
+  let d = domain.trim();
+  if (!/^[a-zA-Z][a-zA-Z0-9+\-.]*:/.test(d)) {
+    d = d.startsWith("//") ? `https:${d}` : `https://${d}`;
+  }
+  // Ensure trailing slash does not matter for URL resolution
+  try {
+    // new URL('/', base) works whether base ends with slash or not
+    return new URL("/", d).toString();
+  } catch {
+    // Fallback: if domain is irreparably bad, throw early
+    throw new Error(`Invalid base domain: ${domain}`);
+  }
+}
+
+/** Convert a possibly-relative URL to absolute, using the provided base. */
+function toAbsolute(url: string, base: string): string {
+  const trimmed = url.trim();
+
+  // Leave already absolute or special schemes untouched by just parsing directly.
+  // If it's not a valid absolute URL, resolve against base.
+  try {
+    // If parse succeeds without base and includes a scheme, keep as-is
+    const abs = new URL(trimmed);
+    return abs.toString();
+  } catch {
+    // Not absolute, resolve relative to base (handles #hash, ?q, //host, etc.)
+    try {
+      return new URL(trimmed, base).toString();
+    } catch {
+      // If still invalid (e.g., badly formed), return original
+      return url;
+    }
+  }
+}
+
+/** Absolutize a srcset list. */
+function absolutizeSrcset(srcset: string, base: string): string {
+  // Split by commas but keep descriptors (1x, 2x, 100w, etc.)
+  // Each candidate: <url> [<descriptor>]
+  return srcset
+    .split(",")
+    .map((part) => {
+      const s = part.trim();
+      if (!s) return s;
+      // First whitespace separates URL and descriptor
+      const spaceIdx = s.search(/\s/);
+      if (spaceIdx === -1) {
+        return toAbsolute(s, base);
+      }
+      const url = s.slice(0, spaceIdx);
+      const desc = s.slice(spaceIdx).trim();
+      return `${toAbsolute(url, base)} ${desc}`;
+    })
+    .join(", ");
+}
+
+/** Replace url(...) in CSS text with absolute URLs. */
+function absolutizeCssUrls(cssText: string, base: string): string {
+  // Matches url("..."), url('...'), url(...)
+  return cssText.replace(
+    /url\(\s*(['"]?)([^'")]+)\1\s*\)/g,
+    (_m, _q, rawUrl) => {
+      const abs = toAbsolute(rawUrl, base);
+      // Preserve quoting if present; browsers accept unquoted if safe, but keep simple.
+      return `url(${abs})`;
+    },
+  );
+}
+
+/** Rewrite the URL in a meta refresh content value if present. */
+function absolutizeMetaRefresh(content: string, base: string): string {
+  // Format examples:
+  // "5; url=/path", "0;URL='page.html'"
+  const match = content.match(
+    /^\s*([^;]+)\s*;\s*(url|URL)\s*=\s*('?)([^']+)\3\s*$/,
+  );
+  if (!match) return content;
+  const delay = match[1].trim();
+  const url = match[4].trim();
+  const abs = toAbsolute(url, base);
+  return `${delay}; url=${abs}`;
+}
+
+export async function webScrape(
+  url: string,
+  streamResponse: ReturnType<typeof createStreamResponse>,
+): JSDOM {
+  const u = new URL(url);
+  const html = await fetchHtmlWithPlaywright(url, streamResponse);
+  const dom = new JSDOM(html);
+  absolutizeDomUrls(dom, u.origin);
+  return dom;
 }