v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions

View File

@@ -1,5 +1,5 @@
import { html, stubNetwork } from "../../helpers.ts";
import { assertEquals } from "@std/assert";
import { assert, assertEquals } from "@std/assert";
import { fetchRichContent } from "../../../services/rich-content-service.ts";
const BASE = "https://example.com/articles/1";
@@ -51,3 +51,67 @@ Deno.test("generic provider: non-HTML responses carry just the URL", async () =>
);
assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
});
Deno.test("generic provider: a canonical og:url with a slug keeps the OG tags", async () => {
using _ = stubNetwork(() =>
html(`
<meta property="og:url" content="https://example.com/q/123/how-to-x">
<meta property="og:title" content="How to X">
<meta property="og:image" content="https://example.com/x.jpg">
<title>How to X - Example</title>`)
);
const content = await fetchRichContent("https://example.com/q/123");
assertEquals(content?.title, "How to X");
assertEquals(content?.thumbnailUrl, "https://example.com/x.jpg");
});
Deno.test("generic provider: relative URLs resolve against the page after redirects", async () => {
using _ = stubNetwork((url) =>
url.hostname === "old.example"
? new Response(null, {
status: 301,
headers: { location: "https://new.example/b/page" },
})
: html(`<meta property="og:image" content="/og.png">
<meta name="twitter:image" content="https://new.example/tw.png">`)
);
const content = await fetchRichContent("https://old.example/a");
assertEquals(content?.url, "https://old.example/a");
assertEquals(content?.thumbnailUrl, "https://new.example/og.png");
assertEquals(content?.faviconUrl, "https://new.example/favicon.ico");
});
Deno.test("generic provider: an og:image that isn't http(s) falls through", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:image" content="javascript:alert(1)">
<meta name="twitter:image" content="//cdn.example/tw.png">`)
);
assertEquals(
(await fetchRichContent(BASE))?.thumbnailUrl,
"https://cdn.example/tw.png",
);
});
Deno.test("generic provider: a <title> loses its site suffix", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:site_name" content="Le Monde">
<title>Un article &eacute;crit — Le Monde</title>`)
);
const content = await fetchRichContent(BASE);
assertEquals(content?.title, "Un article écrit");
assertEquals(content?.siteName, "Le Monde");
});
Deno.test("generic provider: text on one line, and of bounded length", async () => {
using _ = stubNetwork(() =>
html(`<title>
A title
on two lines
</title>
<meta name="description" content="${"word ".repeat(400)}">`)
);
const content = await fetchRichContent(BASE);
assertEquals(content?.title, "A title on two lines");
assertEquals([...content!.description!].length, 1000);
assert(content!.description!.endsWith("word…"));
});