v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

api/test/remote/fetch.test.ts

@@ -1,6 +1,7 @@
-import { stubNetwork } from "../helpers.ts";
+import { html, stubNetwork } from "../helpers.ts";
 import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert";
 import {
+  fetchPage,
   fetchWithTimeout,
   isValidHttpUrl,
   readTextCapped,
@@ -110,3 +111,33 @@ Deno.test("readTextCapped: decodes the charset the header or a <meta> declares",
     "caf<61>",
   );
 });
+
+Deno.test("fetchPage: the page and where it was served from", async () => {
+  using _net = stubNetwork((url) => {
+    if (url.pathname === "/moved") {
+      return new Response(null, {
+        status: 301,
+        headers: { location: "https://www.example.com/here" },
+      });
+    }
+    if (url.pathname === "/gone") return html("<title>Gone</title>", 410);
+    if (url.pathname === "/data") {
+      return new Response("{}", {
+        headers: { "content-type": "application/json" },
+      });
+    }
+    return html("<title>Here</title>");
+  });
+  assertEquals(await fetchPage("https://example.com/moved"), {
+    url: "https://www.example.com/here",
+    html: "<title>Here</title>",
+  });
+  assertEquals(await fetchPage("https://example.com/data"), {
+    url: "https://example.com/data",
+  });
+  await assertRejects(
+    () => fetchPage("https://example.com/gone"),
+    Error,
+    "410",
+  );
+});

api/test/remote/html.test.ts

@@ -1,68 +1,122 @@
 import "../setup.ts";
 import { assertEquals } from "@std/assert";
 import {
-  extractBestIcon,
-  extractFirstContentImage,
-  extractJsonLd,
-  extractLargeImage,
-  extractMaskIconColor,
-  extractMetaName,
-  extractOgTag,
-  extractPageTitle,
-  extractThemeColor,
+  absoluteHttpUrl,
+  decodeHtmlEntities,
   normalizeCssColor,
+  parsePage,
 } from "../../remote/html.ts";
 
 const BASE = "https://example.com/articles/1";
 
-Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => {
-  const page = `
+const read = (html: string) => parsePage(html, BASE);
+
+Deno.test("parsePage meta: either attribute, either order, entities decoded", () => {
+  const page = read(`
     <meta property="og:title" content="Tom &amp; Jerry &#8212; &#x2764;">
     <meta content='Desc' property='og:description'>
     <meta name="twitter:title" content="TW">
-    <meta name="og:title" content="wrong attribute">`;
-  assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤");
-  assertEquals(extractOgTag(page, "description"), "Desc");
-  assertEquals(extractOgTag(page, "image"), undefined);
-  assertEquals(extractMetaName(page, "twitter:title"), "TW");
+    <meta name="og:image" content="https://example.com/by-name.jpg">
+    <meta property="og:title" content="second, ignored">`);
+  assertEquals(page.meta("og:title"), "Tom & Jerry — ❤");
+  assertEquals(page.meta("og:description"), "Desc");
+  assertEquals(page.meta("twitter:title"), "TW");
+  // Sites write og: tags under `name` too.
+  assertEquals(page.meta("og:image"), "https://example.com/by-name.jpg");
+  assertEquals(page.meta("OG:Title"), "Tom & Jerry — ❤");
+  assertEquals(page.meta("og:url"), undefined);
 });
 
-Deno.test("extractOgTag: content never spans into the next tag", () => {
-  const page = `<meta property="og:title"><meta name="x" content="leak">`;
-  assertEquals(extractOgTag(page, "title"), undefined);
+Deno.test("parsePage meta: attributes read as a browser reads them", () => {
+  const page = read(`
+    <meta property=og:title content=Unquoted>
+    <meta property="og:description" content="a > b, still the tag">
+    <meta data-property="og:image" content="https://example.com/no.jpg">
+    <meta property="og:site_name"><meta name="x" content="leak">
+    <meta property="og:type" content="  ">`);
+  assertEquals(page.meta("og:title"), "Unquoted");
+  assertEquals(page.meta("og:description"), "a > b, still the tag");
+  assertEquals(page.meta("og:image"), undefined);
+  assertEquals(page.meta("og:site_name"), undefined);
+  // A blank value is no value, so a fallback can take over.
+  assertEquals(page.meta("og:type"), undefined);
 });
 
-Deno.test("extractPageTitle", () => {
+Deno.test("parsePage: tags in comments and scripts aren't read", () => {
+  const page = read(`
+    <!-- <meta property="og:title" content="commented"> -->
+    <script>document.write('<meta property="og:title" content="scripted">')</script>
+    <meta property="og:title" content="Real">`);
+  assertEquals(page.meta("og:title"), "Real");
+  // An unclosed comment runs to the end of the page.
   assertEquals(
-    extractPageTitle("<title lang=en>  A &lt;b&gt;  </title>"),
-    "A <b>",
+    read(`<!-- <meta property="og:title" content="x">`).meta("og:title"),
+    undefined,
   );
-  assertEquals(extractPageTitle("<p>none</p>"), undefined);
 });
 
-Deno.test("extractThemeColor: unscoped, then light, then any", () => {
+Deno.test("parsePage title", () => {
+  assertEquals(read("<title lang=en>  A &lt;b&gt;  </title>").title, "A <b>");
+  assertEquals(read("<title>  </title>").title, undefined);
+  assertEquals(read("<p>none</p>").title, undefined);
+});
+
+Deno.test("decodeHtmlEntities: one pass, the names pages use, nothing thrown", () => {
   assertEquals(
-    extractThemeColor(`
+    decodeHtmlEntities("Caf&eacute; &Agrave; l&rsquo;&eacute;t&eacute;"),
+    "Café À l’été",
+  );
+  assertEquals(
+    decodeHtmlEntities("&laquo;&nbsp;Oui&hellip;&raquo;"),
+    "« Oui…»",
+  );
+  assertEquals(decodeHtmlEntities("&ccedil;a &AMP; &Ntilde;"), "ça & Ñ");
+  // `&amp;quot;` is a literal `&quot;` on the page.
+  assertEquals(decodeHtmlEntities("&amp;quot;"), "&quot;");
+  // No such code point or name: left as written.
+  assertEquals(decodeHtmlEntities("&#99999999;"), "&#99999999;");
+  assertEquals(decodeHtmlEntities("&#xD800;"), "&#xD800;");
+  assertEquals(decodeHtmlEntities("&bogus;"), "&bogus;");
+});
+
+Deno.test("absoluteHttpUrl", () => {
+  assertEquals(absoluteHttpUrl("/og.png", BASE), "https://example.com/og.png");
+  assertEquals(
+    absoluteHttpUrl("//cdn.test/a.jpg", BASE),
+    "https://cdn.test/a.jpg",
+  );
+  assertEquals(
+    absoluteHttpUrl(" og.png ", BASE),
+    "https://example.com/articles/og.png",
+  );
+  assertEquals(absoluteHttpUrl("javascript:alert(1)", BASE), undefined);
+  assertEquals(absoluteHttpUrl("data:image/png;base64,xx", BASE), undefined);
+  assertEquals(absoluteHttpUrl("http://[bad", BASE), undefined);
+  assertEquals(absoluteHttpUrl("", BASE), undefined);
+});
+
+Deno.test("parsePage themeColor: unscoped, then light, then any", () => {
+  assertEquals(
+    read(`
       <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
-      <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`),
+      <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`)
+      .themeColor,
     "#fff",
   );
   assertEquals(
-    extractThemeColor(`
+    read(`
       <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
-      <meta name="theme-color" content="#123">`),
+      <meta name="theme-color" content="#123">`).themeColor,
     "#123",
   );
   assertEquals(
-    extractThemeColor(
+    read(
       `<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
-    ),
+    ).themeColor,
     "#000",
   );
   assertEquals(
-    extractMaskIconColor(
-      `<link rel="mask-icon" href="/m.svg" color="#5bbad5">`,
-    ),
+    read(`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`).maskIconColor,
     "#5bbad5",
   );
 });
@@ -82,78 +136,106 @@ Deno.test("normalizeCssColor", () => {
   assertEquals(normalizeCssColor(undefined), undefined);
 });
 
-Deno.test("extractJsonLd: @graph, arrays and image shapes", () => {
+Deno.test("parsePage jsonLd: @graph, arrays and image shapes", () => {
   const ld = (data: unknown) =>
     `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
   assertEquals(
-    extractJsonLd(
-      ld({
-        "@graph": [{ "@type": "WebSite" }, {
-          headline: "Head",
-          image: { url: "https://i/1.jpg" },
-        }],
-      }),
-    ),
+    read(ld({
+      "@graph": [{ "@type": "WebSite" }, {
+        headline: "Head",
+        image: { url: "https://i/1.jpg" },
+      }],
+    })).jsonLd,
     { title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
   );
   assertEquals(
-    extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl,
+    read(ld({ name: ["N"], image: ["https://i/2.jpg"] })).jsonLd.thumbnailUrl,
     "https://i/2.jpg",
   );
   assertEquals(
-    extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl,
-    undefined,
+    read(ld({ name: "N", image: "/relative.jpg" })).jsonLd.thumbnailUrl,
+    "https://example.com/relative.jpg",
   );
   // Invalid JSON is skipped for the next block.
   assertEquals(
-    extractJsonLd(
+    read(
       `<script type="application/ld+json">{oops</script>${
         ld({ name: "Second" })
       }`,
-    ).title,
+    ).jsonLd.title,
     "Second",
   );
-  assertEquals(extractJsonLd("<p></p>"), {});
+  assertEquals(read("<p></p>").jsonLd, {});
 });
 
-Deno.test("image extraction: large images, content images, relative URLs", () => {
-  const page = `
+Deno.test("parsePage jsonLd: the page's node beats the site's", () => {
+  const ld = (data: unknown) =>
+    `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
+  const page = read(
+    ld({ "@type": "Organization", name: "Acme", logo: "https://a/l.png" }) +
+      ld({
+        "@graph": [
+          { "@type": ["WebSite"], name: "Acme site" },
+          { "@type": "NewsArticle", headline: "The news" },
+        ],
+      }),
+  );
+  assertEquals(page.jsonLd.title, "The news");
+  // With nothing else, the site's node still names the page.
+  assertEquals(
+    read(ld({ "@type": "Organization", name: "Acme" })).jsonLd.title,
+    "Acme",
+  );
+});
+
+Deno.test("parsePage images: large images, content images, relative URLs", () => {
+  const page = read(`
     <img src="data:image/png;base64,xx" width="500" height="500">
-    <img src="/small.jpg" width="50" height="50">
+    <img src="/pixel.gif" width="1" height="1">
+    <img src="/small.jpg" width="60" height="60">
     <img src="/img/site-logo.png">
     <img src="/icons/sprite.svg">
-    <img src="/photos/cat.jpg" width="640" height="480">`;
+    <img src="/photos/cat.jpg" width="640" height="480">`);
+  assertEquals(page.largeImage, "https://example.com/photos/cat.jpg");
+  assertEquals(page.firstContentImage, "https://example.com/small.jpg");
   assertEquals(
-    extractLargeImage(page, BASE),
-    "https://example.com/photos/cat.jpg",
-  );
-  assertEquals(
-    extractFirstContentImage(page, BASE),
-    "https://example.com/small.jpg",
-  );
-  assertEquals(
-    extractFirstContentImage(`<img src="//cdn.test/a.jpg">`, BASE),
+    read(`<img src="//cdn.test/a.jpg">`).firstContentImage,
     "https://cdn.test/a.jpg",
   );
 });
 
-Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
+Deno.test("parsePage images: lazy images by their data-src", () => {
+  const page = read(`
+    <img data-src="/wrong.jpg" src="/img/lazy-placeholder.gif">
+    <img src="data:image/gif;base64,R0lGOD" data-src="/real.jpg" width="800" height="600">`);
+  assertEquals(page.largeImage, "https://example.com/real.jpg");
+  assertEquals(page.firstContentImage, "https://example.com/real.jpg");
+});
+
+Deno.test("parsePage bestIcon: SVG beats everything, else the largest", () => {
   assertEquals(
-    extractBestIcon(
-      `<link rel="icon" href="/16.png" sizes="16x16">
+    read(`<link rel="icon" href="/16.png" sizes="16x16">
        <link rel="apple-touch-icon" href="/180.png" sizes="180x180">
-       <link rel="stylesheet" href="/x.css">`,
-      BASE,
-    ),
+       <link rel="stylesheet" href="/x.css">`).bestIcon,
     "https://example.com/180.png",
   );
   assertEquals(
-    extractBestIcon(
-      `<link rel="icon" href="/180.png" sizes="180x180">
-       <link rel="icon" type="image/svg+xml" href="/icon">`,
-      BASE,
-    ),
+    read(`<link rel="icon" href="/180.png" sizes="180x180">
+       <link rel="icon" type="image/svg+xml" href="/icon">`).bestIcon,
     "https://example.com/icon",
   );
-  assertEquals(extractBestIcon("<p></p>", BASE), undefined);
+  assertEquals(
+    read(`<link rel="shortcut icon" href="/f.ico" sizes="16x16 48x48">
+       <link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
+    "https://example.com/f.ico",
+  );
+  assertEquals(read("<p></p>").bestIcon, undefined);
+});
+
+Deno.test("parsePage bestIcon: a mask icon is never the icon", () => {
+  assertEquals(
+    read(`<link rel="mask-icon" href="/pinned.svg" color="#000">
+       <link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
+    "https://example.com/32.png",
+  );
 });

api/test/remote/providers/bandcamp.test.ts

@@ -20,3 +20,15 @@ Deno.test("bandcamp provider: bandcamp.com and its subdomains only", () => {
     bandcampProvider.matches("https://bandcamp.com.evil.example/album/x"),
   );
 });
+
+Deno.test("bandcamp provider: falls back like any page, embeds only Bandcamp's player", async () => {
+  using _ = stubNetwork(() =>
+    html(`<meta property="og:title" content="Album">
+          <meta name="description" content="An album by Artist">
+          <meta property="og:video" content="https://evil.example/EmbeddedPlayer/x">`)
+  );
+  const bc = await fetchRichContent("https://artist.bandcamp.com/album/b");
+  assertEquals(bc?.description, "An album by Artist");
+  assertEquals(bc?.siteName, "Bandcamp");
+  assertEquals(bc?.embedUrl, undefined);
+});

api/test/remote/providers/generic.test.ts

@@ -1,5 +1,5 @@
 import { html, stubNetwork } from "../../helpers.ts";
-import { assertEquals } from "@std/assert";
+import { assert, assertEquals } from "@std/assert";
 import { fetchRichContent } from "../../../services/rich-content-service.ts";
 
 const BASE = "https://example.com/articles/1";
@@ -51,3 +51,67 @@ Deno.test("generic provider: non-HTML responses carry just the URL", async () =>
   );
   assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
 });
+
+Deno.test("generic provider: a canonical og:url with a slug keeps the OG tags", async () => {
+  using _ = stubNetwork(() =>
+    html(`
+      <meta property="og:url" content="https://example.com/q/123/how-to-x">
+      <meta property="og:title" content="How to X">
+      <meta property="og:image" content="https://example.com/x.jpg">
+      <title>How to X - Example</title>`)
+  );
+  const content = await fetchRichContent("https://example.com/q/123");
+  assertEquals(content?.title, "How to X");
+  assertEquals(content?.thumbnailUrl, "https://example.com/x.jpg");
+});
+
+Deno.test("generic provider: relative URLs resolve against the page after redirects", async () => {
+  using _ = stubNetwork((url) =>
+    url.hostname === "old.example"
+      ? new Response(null, {
+        status: 301,
+        headers: { location: "https://new.example/b/page" },
+      })
+      : html(`<meta property="og:image" content="/og.png">
+              <meta name="twitter:image" content="https://new.example/tw.png">`)
+  );
+  const content = await fetchRichContent("https://old.example/a");
+  assertEquals(content?.url, "https://old.example/a");
+  assertEquals(content?.thumbnailUrl, "https://new.example/og.png");
+  assertEquals(content?.faviconUrl, "https://new.example/favicon.ico");
+});
+
+Deno.test("generic provider: an og:image that isn't http(s) falls through", async () => {
+  using _ = stubNetwork(() =>
+    html(`<meta property="og:image" content="javascript:alert(1)">
+          <meta name="twitter:image" content="//cdn.example/tw.png">`)
+  );
+  assertEquals(
+    (await fetchRichContent(BASE))?.thumbnailUrl,
+    "https://cdn.example/tw.png",
+  );
+});
+
+Deno.test("generic provider: a <title> loses its site suffix", async () => {
+  using _ = stubNetwork(() =>
+    html(`<meta property="og:site_name" content="Le Monde">
+          <title>Un article &eacute;crit — Le Monde</title>`)
+  );
+  const content = await fetchRichContent(BASE);
+  assertEquals(content?.title, "Un article écrit");
+  assertEquals(content?.siteName, "Le Monde");
+});
+
+Deno.test("generic provider: text on one line, and of bounded length", async () => {
+  using _ = stubNetwork(() =>
+    html(`<title>
+            A title
+            on two lines
+          </title>
+          <meta name="description" content="${"word ".repeat(400)}">`)
+  );
+  const content = await fetchRichContent(BASE);
+  assertEquals(content?.title, "A title on two lines");
+  assertEquals([...content!.description!].length, 1000);
+  assert(content!.description!.endsWith("word…"));
+});

api/test/remote/providers/soundcloud.test.ts

@@ -1,16 +1,69 @@
 import { html, stubNetwork } from "../../helpers.ts";
-import { assertEquals, assertStringIncludes } from "@std/assert";
+import { assert, assertEquals, assertFalse } from "@std/assert";
 import { fetchRichContent } from "../../../services/rich-content-service.ts";
+import { soundcloudProvider } from "../../../remote/providers/soundcloud.ts";
+
+const embedOf = (url: string) =>
+  `https://w.soundcloud.com/player/?url=${
+    encodeURIComponent(url)
+  }&visual=true&auto_play=false`;
 
 Deno.test("soundcloud provider", async () => {
   using _ = stubNetwork(() =>
-    html(`<meta property="og:title" content="Album">
-          <meta property="og:video" content="https://bandcamp.com/EmbeddedPlayer/album=1">`)
+    html(`<meta property="og:title" content="Track">
+          <meta property="og:image" content="https://i1.sndcdn.com/a.jpg">
+          <meta name="description" content="Listen to Track">`)
   );
   const sc = await fetchRichContent("https://soundcloud.com/artist/track");
   assertEquals(sc?.type, "soundcloud");
-  assertStringIncludes(
-    sc?.embedUrl ?? "",
-    encodeURIComponent("https://soundcloud.com/artist/track"),
+  assertEquals(sc?.siteName, "SoundCloud");
+  assertEquals(sc?.title, "Track");
+  assertEquals(sc?.description, "Listen to Track");
+  assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/track"));
+});
+
+Deno.test("soundcloud provider: a short link embeds the page it leads to", async () => {
+  using _ = stubNetwork((url) =>
+    url.hostname === "on.soundcloud.com"
+      ? new Response(null, {
+        status: 302,
+        headers: {
+          location: "https://m.soundcloud.com/artist/track?si=abc&utm_source=x",
+        },
+      })
+      : html(`<meta property="og:title" content="Track">`)
+  );
+  const sc = await fetchRichContent("https://on.soundcloud.com/AbCd");
+  assertEquals(sc?.type, "soundcloud");
+  assertEquals(sc?.url, "https://on.soundcloud.com/AbCd");
+  assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/track"));
+});
+
+Deno.test("soundcloud provider: the page's og:url names what is embedded", async () => {
+  using _ = stubNetwork(() =>
+    html(
+      `<meta property="og:url" content="https://soundcloud.com/artist/real">`,
+    )
+  );
+  const sc = await fetchRichContent("https://soundcloud.com/artist/real?in=x");
+  assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/real"));
+});
+
+Deno.test("soundcloud provider: SoundCloud's hosts only", () => {
+  for (
+    const url of [
+      "https://soundcloud.com/a/t",
+      "https://www.soundcloud.com/a/t",
+      "https://m.soundcloud.com/a/t",
+      "https://on.soundcloud.com/AbCd",
+    ]
+  ) {
+    assert(soundcloudProvider.matches(url), url);
+  }
+  assertFalse(
+    soundcloudProvider.matches("https://soundcloud.com.evil.example/"),
+  );
+  assertFalse(
+    soundcloudProvider.matches("https://api.soundcloud.com/tracks/1"),
   );
 });

api/test/remote/providers/youtube.test.ts

@@ -13,7 +13,6 @@ Deno.test("youtube provider: video with start time and playlist", async () => {
     "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL123&t=1m30s",
   );
   assertEquals(content?.type, "youtube");
-  assertEquals(content?.videoId, "dQw4w9WgXcQ");
   assertEquals(content?.title, "Never Gonna");
   assertEquals(
     content?.thumbnailUrl,
@@ -41,8 +40,8 @@ Deno.test("youtube provider: youtu.be, shorts, playlists and channels", async ()
     "https://www.youtube.com/embed/abc123?rel=0&start=42",
   );
   assertEquals(
-    (await fetchRichContent("https://youtube.com/shorts/xyz"))?.videoId,
-    "xyz",
+    (await fetchRichContent("https://youtube.com/shorts/xyz"))?.embedUrl,
+    "https://www.youtube.com/embed/xyz?rel=0",
   );
   const list = await fetchRichContent(
     "https://www.youtube.com/playlist?list=PLabc",