v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, é and ’ included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions
--- a/api/test/remote/html.test.ts
+++ b/api/test/remote/html.test.ts
@@ -1,68 +1,122 @@
 import "../setup.ts";
 import { assertEquals } from "@std/assert";
 import {
-  extractBestIcon,
-  extractFirstContentImage,
-  extractJsonLd,
-  extractLargeImage,
-  extractMaskIconColor,
-  extractMetaName,
-  extractOgTag,
-  extractPageTitle,
-  extractThemeColor,
+  absoluteHttpUrl,
+  decodeHtmlEntities,
  normalizeCssColor,
+  parsePage,
 } from "../../remote/html.ts";

 const BASE = "https://example.com/articles/1";

-Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => {
-  const page = `
+const read = (html: string) => parsePage(html, BASE);
+
+Deno.test("parsePage meta: either attribute, either order, entities decoded", () => {
+  const page = read(`
    <meta property="og:title" content="Tom &amp; Jerry &#8212; &#x2764;">
    <meta content='Desc' property='og:description'>
    <meta name="twitter:title" content="TW">
-    <meta name="og:title" content="wrong attribute">`;
-  assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤");
-  assertEquals(extractOgTag(page, "description"), "Desc");
-  assertEquals(extractOgTag(page, "image"), undefined);
-  assertEquals(extractMetaName(page, "twitter:title"), "TW");
+    <meta name="og:image" content="https://example.com/by-name.jpg">
+    <meta property="og:title" content="second, ignored">`);
+  assertEquals(page.meta("og:title"), "Tom & Jerry — ❤");
+  assertEquals(page.meta("og:description"), "Desc");
+  assertEquals(page.meta("twitter:title"), "TW");
+  // Sites write og: tags under `name` too.
+  assertEquals(page.meta("og:image"), "https://example.com/by-name.jpg");
+  assertEquals(page.meta("OG:Title"), "Tom & Jerry — ❤");
+  assertEquals(page.meta("og:url"), undefined);
 });

-Deno.test("extractOgTag: content never spans into the next tag", () => {
-  const page = `<meta property="og:title"><meta name="x" content="leak">`;
-  assertEquals(extractOgTag(page, "title"), undefined);
+Deno.test("parsePage meta: attributes read as a browser reads them", () => {
+  const page = read(`
+    <meta property=og:title content=Unquoted>
+    <meta property="og:description" content="a > b, still the tag">
+    <meta data-property="og:image" content="https://example.com/no.jpg">
+    <meta property="og:site_name"><meta name="x" content="leak">
+    <meta property="og:type" content="  ">`);
+  assertEquals(page.meta("og:title"), "Unquoted");
+  assertEquals(page.meta("og:description"), "a > b, still the tag");
+  assertEquals(page.meta("og:image"), undefined);
+  assertEquals(page.meta("og:site_name"), undefined);
+  // A blank value is no value, so a fallback can take over.
+  assertEquals(page.meta("og:type"), undefined);
 });

-Deno.test("extractPageTitle", () => {
+Deno.test("parsePage: tags in comments and scripts aren't read", () => {
+  const page = read(`
+    <!-- <meta property="og:title" content="commented"> -->
+    <script>document.write('<meta property="og:title" content="scripted">')</script>
+    <meta property="og:title" content="Real">`);
+  assertEquals(page.meta("og:title"), "Real");
+  // An unclosed comment runs to the end of the page.
  assertEquals(
-    extractPageTitle("<title lang=en>  A &lt;b&gt;  </title>"),
-    "A <b>",
+    read(`<!-- <meta property="og:title" content="x">`).meta("og:title"),
+    undefined,
  );
-  assertEquals(extractPageTitle("<p>none</p>"), undefined);
 });

-Deno.test("extractThemeColor: unscoped, then light, then any", () => {
+Deno.test("parsePage title", () => {
+  assertEquals(read("<title lang=en>  A &lt;b&gt;  </title>").title, "A <b>");
+  assertEquals(read("<title>  </title>").title, undefined);
+  assertEquals(read("<p>none</p>").title, undefined);
+});
+
+Deno.test("decodeHtmlEntities: one pass, the names pages use, nothing thrown", () => {
  assertEquals(
-    extractThemeColor(`
+    decodeHtmlEntities("Caf&eacute; &Agrave; l&rsquo;&eacute;t&eacute;"),
+    "Café À l’été",
+  );
+  assertEquals(
+    decodeHtmlEntities("&laquo;&nbsp;Oui&hellip;&raquo;"),
+    "« Oui…»",
+  );
+  assertEquals(decodeHtmlEntities("&ccedil;a &AMP; &Ntilde;"), "ça & Ñ");
+  // `&amp;quot;` is a literal `&quot;` on the page.
+  assertEquals(decodeHtmlEntities("&amp;quot;"), "&quot;");
+  // No such code point or name: left as written.
+  assertEquals(decodeHtmlEntities("&#99999999;"), "&#99999999;");
+  assertEquals(decodeHtmlEntities("&#xD800;"), "&#xD800;");
+  assertEquals(decodeHtmlEntities("&bogus;"), "&bogus;");
+});
+
+Deno.test("absoluteHttpUrl", () => {
+  assertEquals(absoluteHttpUrl("/og.png", BASE), "https://example.com/og.png");
+  assertEquals(
+    absoluteHttpUrl("//cdn.test/a.jpg", BASE),
+    "https://cdn.test/a.jpg",
+  );
+  assertEquals(
+    absoluteHttpUrl(" og.png ", BASE),
+    "https://example.com/articles/og.png",
+  );
+  assertEquals(absoluteHttpUrl("javascript:alert(1)", BASE), undefined);
+  assertEquals(absoluteHttpUrl("data:image/png;base64,xx", BASE), undefined);
+  assertEquals(absoluteHttpUrl("http://[bad", BASE), undefined);
+  assertEquals(absoluteHttpUrl("", BASE), undefined);
+});
+
+Deno.test("parsePage themeColor: unscoped, then light, then any", () => {
+  assertEquals(
+    read(`
      <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
-      <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`),
+      <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`)
+      .themeColor,
    "#fff",
  );
  assertEquals(
-    extractThemeColor(`
+    read(`
      <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
-      <meta name="theme-color" content="#123">`),
+      <meta name="theme-color" content="#123">`).themeColor,
    "#123",
  );
  assertEquals(
-    extractThemeColor(
+    read(
      `<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
-    ),
+    ).themeColor,
    "#000",
  );
  assertEquals(
-    extractMaskIconColor(
-      `<link rel="mask-icon" href="/m.svg" color="#5bbad5">`,
-    ),
+    read(`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`).maskIconColor,
    "#5bbad5",
  );
 });
@@ -82,78 +136,106 @@ Deno.test("normalizeCssColor", () => {
  assertEquals(normalizeCssColor(undefined), undefined);
 });

-Deno.test("extractJsonLd: @graph, arrays and image shapes", () => {
+Deno.test("parsePage jsonLd: @graph, arrays and image shapes", () => {
  const ld = (data: unknown) =>
    `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
  assertEquals(
-    extractJsonLd(
-      ld({
-        "@graph": [{ "@type": "WebSite" }, {
-          headline: "Head",
-          image: { url: "https://i/1.jpg" },
-        }],
-      }),
-    ),
+    read(ld({
+      "@graph": [{ "@type": "WebSite" }, {
+        headline: "Head",
+        image: { url: "https://i/1.jpg" },
+      }],
+    })).jsonLd,
    { title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
  );
  assertEquals(
-    extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl,
+    read(ld({ name: ["N"], image: ["https://i/2.jpg"] })).jsonLd.thumbnailUrl,
    "https://i/2.jpg",
  );
  assertEquals(
-    extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl,
-    undefined,
+    read(ld({ name: "N", image: "/relative.jpg" })).jsonLd.thumbnailUrl,
+    "https://example.com/relative.jpg",
  );
  // Invalid JSON is skipped for the next block.
  assertEquals(
-    extractJsonLd(
+    read(
      `<script type="application/ld+json">{oops</script>${
        ld({ name: "Second" })
      }`,
-    ).title,
+    ).jsonLd.title,
    "Second",
  );
-  assertEquals(extractJsonLd("<p></p>"), {});
+  assertEquals(read("<p></p>").jsonLd, {});
 });

-Deno.test("image extraction: large images, content images, relative URLs", () => {
-  const page = `
+Deno.test("parsePage jsonLd: the page's node beats the site's", () => {
+  const ld = (data: unknown) =>
+    `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
+  const page = read(
+    ld({ "@type": "Organization", name: "Acme", logo: "https://a/l.png" }) +
+      ld({
+        "@graph": [
+          { "@type": ["WebSite"], name: "Acme site" },
+          { "@type": "NewsArticle", headline: "The news" },
+        ],
+      }),
+  );
+  assertEquals(page.jsonLd.title, "The news");
+  // With nothing else, the site's node still names the page.
+  assertEquals(
+    read(ld({ "@type": "Organization", name: "Acme" })).jsonLd.title,
+    "Acme",
+  );
+});
+
+Deno.test("parsePage images: large images, content images, relative URLs", () => {
+  const page = read(`
    <img src="data:image/png;base64,xx" width="500" height="500">
-    <img src="/small.jpg" width="50" height="50">
+    <img src="/pixel.gif" width="1" height="1">
+    <img src="/small.jpg" width="60" height="60">
    <img src="/img/site-logo.png">
    <img src="/icons/sprite.svg">
-    <img src="/photos/cat.jpg" width="640" height="480">`;
+    <img src="/photos/cat.jpg" width="640" height="480">`);
+  assertEquals(page.largeImage, "https://example.com/photos/cat.jpg");
+  assertEquals(page.firstContentImage, "https://example.com/small.jpg");
  assertEquals(
-    extractLargeImage(page, BASE),
-    "https://example.com/photos/cat.jpg",
-  );
-  assertEquals(
-    extractFirstContentImage(page, BASE),
-    "https://example.com/small.jpg",
-  );
-  assertEquals(
-    extractFirstContentImage(`<img src="//cdn.test/a.jpg">`, BASE),
+    read(`<img src="//cdn.test/a.jpg">`).firstContentImage,
    "https://cdn.test/a.jpg",
  );
 });

-Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
+Deno.test("parsePage images: lazy images by their data-src", () => {
+  const page = read(`
+    <img data-src="/wrong.jpg" src="/img/lazy-placeholder.gif">
+    <img src="data:image/gif;base64,R0lGOD" data-src="/real.jpg" width="800" height="600">`);
+  assertEquals(page.largeImage, "https://example.com/real.jpg");
+  assertEquals(page.firstContentImage, "https://example.com/real.jpg");
+});
+
+Deno.test("parsePage bestIcon: SVG beats everything, else the largest", () => {
  assertEquals(
-    extractBestIcon(
-      `<link rel="icon" href="/16.png" sizes="16x16">
+    read(`<link rel="icon" href="/16.png" sizes="16x16">
       <link rel="apple-touch-icon" href="/180.png" sizes="180x180">
-       <link rel="stylesheet" href="/x.css">`,
-      BASE,
-    ),
+       <link rel="stylesheet" href="/x.css">`).bestIcon,
    "https://example.com/180.png",
  );
  assertEquals(
-    extractBestIcon(
-      `<link rel="icon" href="/180.png" sizes="180x180">
-       <link rel="icon" type="image/svg+xml" href="/icon">`,
-      BASE,
-    ),
+    read(`<link rel="icon" href="/180.png" sizes="180x180">
+       <link rel="icon" type="image/svg+xml" href="/icon">`).bestIcon,
    "https://example.com/icon",
  );
-  assertEquals(extractBestIcon("<p></p>", BASE), undefined);
+  assertEquals(
+    read(`<link rel="shortcut icon" href="/f.ico" sizes="16x16 48x48">
+       <link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
+    "https://example.com/f.ico",
+  );
+  assertEquals(read("<p></p>").bestIcon, undefined);
+});
+
+Deno.test("parsePage bestIcon: a mask icon is never the icon", () => {
+  assertEquals(
+    read(`<link rel="mask-icon" href="/pinned.svg" color="#000">
+       <link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
+    "https://example.com/32.png",
+  );
 });