v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions

View File

@@ -1,6 +1,7 @@
import { stubNetwork } from "../helpers.ts";
import { html, stubNetwork } from "../helpers.ts";
import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert";
import {
fetchPage,
fetchWithTimeout,
isValidHttpUrl,
readTextCapped,
@@ -110,3 +111,33 @@ Deno.test("readTextCapped: decodes the charset the header or a <meta> declares",
"caf<61>",
);
});
Deno.test("fetchPage: the page and where it was served from", async () => {
using _net = stubNetwork((url) => {
if (url.pathname === "/moved") {
return new Response(null, {
status: 301,
headers: { location: "https://www.example.com/here" },
});
}
if (url.pathname === "/gone") return html("<title>Gone</title>", 410);
if (url.pathname === "/data") {
return new Response("{}", {
headers: { "content-type": "application/json" },
});
}
return html("<title>Here</title>");
});
assertEquals(await fetchPage("https://example.com/moved"), {
url: "https://www.example.com/here",
html: "<title>Here</title>",
});
assertEquals(await fetchPage("https://example.com/data"), {
url: "https://example.com/data",
});
await assertRejects(
() => fetchPage("https://example.com/gone"),
Error,
"410",
);
});

View File

@@ -1,68 +1,122 @@
import "../setup.ts";
import { assertEquals } from "@std/assert";
import {
extractBestIcon,
extractFirstContentImage,
extractJsonLd,
extractLargeImage,
extractMaskIconColor,
extractMetaName,
extractOgTag,
extractPageTitle,
extractThemeColor,
absoluteHttpUrl,
decodeHtmlEntities,
normalizeCssColor,
parsePage,
} from "../../remote/html.ts";
const BASE = "https://example.com/articles/1";
Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => {
const page = `
const read = (html: string) => parsePage(html, BASE);
Deno.test("parsePage meta: either attribute, either order, entities decoded", () => {
const page = read(`
<meta property="og:title" content="Tom &amp; Jerry &#8212; &#x2764;">
<meta content='Desc' property='og:description'>
<meta name="twitter:title" content="TW">
<meta name="og:title" content="wrong attribute">`;
assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤");
assertEquals(extractOgTag(page, "description"), "Desc");
assertEquals(extractOgTag(page, "image"), undefined);
assertEquals(extractMetaName(page, "twitter:title"), "TW");
<meta name="og:image" content="https://example.com/by-name.jpg">
<meta property="og:title" content="second, ignored">`);
assertEquals(page.meta("og:title"), "Tom & Jerry — ❤");
assertEquals(page.meta("og:description"), "Desc");
assertEquals(page.meta("twitter:title"), "TW");
// Sites write og: tags under `name` too.
assertEquals(page.meta("og:image"), "https://example.com/by-name.jpg");
assertEquals(page.meta("OG:Title"), "Tom & Jerry — ❤");
assertEquals(page.meta("og:url"), undefined);
});
Deno.test("extractOgTag: content never spans into the next tag", () => {
const page = `<meta property="og:title"><meta name="x" content="leak">`;
assertEquals(extractOgTag(page, "title"), undefined);
Deno.test("parsePage meta: attributes read as a browser reads them", () => {
const page = read(`
<meta property=og:title content=Unquoted>
<meta property="og:description" content="a > b, still the tag">
<meta data-property="og:image" content="https://example.com/no.jpg">
<meta property="og:site_name"><meta name="x" content="leak">
<meta property="og:type" content=" ">`);
assertEquals(page.meta("og:title"), "Unquoted");
assertEquals(page.meta("og:description"), "a > b, still the tag");
assertEquals(page.meta("og:image"), undefined);
assertEquals(page.meta("og:site_name"), undefined);
// A blank value is no value, so a fallback can take over.
assertEquals(page.meta("og:type"), undefined);
});
Deno.test("extractPageTitle", () => {
Deno.test("parsePage: tags in comments and scripts aren't read", () => {
const page = read(`
<!-- <meta property="og:title" content="commented"> -->
<script>document.write('<meta property="og:title" content="scripted">')</script>
<meta property="og:title" content="Real">`);
assertEquals(page.meta("og:title"), "Real");
// An unclosed comment runs to the end of the page.
assertEquals(
extractPageTitle("<title lang=en> A &lt;b&gt; </title>"),
"A <b>",
read(`<!-- <meta property="og:title" content="x">`).meta("og:title"),
undefined,
);
assertEquals(extractPageTitle("<p>none</p>"), undefined);
});
Deno.test("extractThemeColor: unscoped, then light, then any", () => {
Deno.test("parsePage title", () => {
assertEquals(read("<title lang=en> A &lt;b&gt; </title>").title, "A <b>");
assertEquals(read("<title> </title>").title, undefined);
assertEquals(read("<p>none</p>").title, undefined);
});
Deno.test("decodeHtmlEntities: one pass, the names pages use, nothing thrown", () => {
assertEquals(
extractThemeColor(`
decodeHtmlEntities("Caf&eacute; &Agrave; l&rsquo;&eacute;t&eacute;"),
"Café À l’été",
);
assertEquals(
decodeHtmlEntities("&laquo;&nbsp;Oui&hellip;&raquo;"),
"« Oui…»",
);
assertEquals(decodeHtmlEntities("&ccedil;a &AMP; &Ntilde;"), "ça & Ñ");
// `&amp;quot;` is a literal `&quot;` on the page.
assertEquals(decodeHtmlEntities("&amp;quot;"), "&quot;");
// No such code point or name: left as written.
assertEquals(decodeHtmlEntities("&#99999999;"), "&#99999999;");
assertEquals(decodeHtmlEntities("&#xD800;"), "&#xD800;");
assertEquals(decodeHtmlEntities("&bogus;"), "&bogus;");
});
Deno.test("absoluteHttpUrl", () => {
assertEquals(absoluteHttpUrl("/og.png", BASE), "https://example.com/og.png");
assertEquals(
absoluteHttpUrl("//cdn.test/a.jpg", BASE),
"https://cdn.test/a.jpg",
);
assertEquals(
absoluteHttpUrl(" og.png ", BASE),
"https://example.com/articles/og.png",
);
assertEquals(absoluteHttpUrl("javascript:alert(1)", BASE), undefined);
assertEquals(absoluteHttpUrl("data:image/png;base64,xx", BASE), undefined);
assertEquals(absoluteHttpUrl("http://[bad", BASE), undefined);
assertEquals(absoluteHttpUrl("", BASE), undefined);
});
Deno.test("parsePage themeColor: unscoped, then light, then any", () => {
assertEquals(
read(`
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
<meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`),
<meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`)
.themeColor,
"#fff",
);
assertEquals(
extractThemeColor(`
read(`
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
<meta name="theme-color" content="#123">`),
<meta name="theme-color" content="#123">`).themeColor,
"#123",
);
assertEquals(
extractThemeColor(
read(
`<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
),
).themeColor,
"#000",
);
assertEquals(
extractMaskIconColor(
`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`,
),
read(`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`).maskIconColor,
"#5bbad5",
);
});
@@ -82,78 +136,106 @@ Deno.test("normalizeCssColor", () => {
assertEquals(normalizeCssColor(undefined), undefined);
});
Deno.test("extractJsonLd: @graph, arrays and image shapes", () => {
Deno.test("parsePage jsonLd: @graph, arrays and image shapes", () => {
const ld = (data: unknown) =>
`<script type="application/ld+json">${JSON.stringify(data)}</script>`;
assertEquals(
extractJsonLd(
ld({
"@graph": [{ "@type": "WebSite" }, {
headline: "Head",
image: { url: "https://i/1.jpg" },
}],
}),
),
read(ld({
"@graph": [{ "@type": "WebSite" }, {
headline: "Head",
image: { url: "https://i/1.jpg" },
}],
})).jsonLd,
{ title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
);
assertEquals(
extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl,
read(ld({ name: ["N"], image: ["https://i/2.jpg"] })).jsonLd.thumbnailUrl,
"https://i/2.jpg",
);
assertEquals(
extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl,
undefined,
read(ld({ name: "N", image: "/relative.jpg" })).jsonLd.thumbnailUrl,
"https://example.com/relative.jpg",
);
// Invalid JSON is skipped for the next block.
assertEquals(
extractJsonLd(
read(
`<script type="application/ld+json">{oops</script>${
ld({ name: "Second" })
}`,
).title,
).jsonLd.title,
"Second",
);
assertEquals(extractJsonLd("<p></p>"), {});
assertEquals(read("<p></p>").jsonLd, {});
});
Deno.test("image extraction: large images, content images, relative URLs", () => {
const page = `
Deno.test("parsePage jsonLd: the page's node beats the site's", () => {
const ld = (data: unknown) =>
`<script type="application/ld+json">${JSON.stringify(data)}</script>`;
const page = read(
ld({ "@type": "Organization", name: "Acme", logo: "https://a/l.png" }) +
ld({
"@graph": [
{ "@type": ["WebSite"], name: "Acme site" },
{ "@type": "NewsArticle", headline: "The news" },
],
}),
);
assertEquals(page.jsonLd.title, "The news");
// With nothing else, the site's node still names the page.
assertEquals(
read(ld({ "@type": "Organization", name: "Acme" })).jsonLd.title,
"Acme",
);
});
Deno.test("parsePage images: large images, content images, relative URLs", () => {
const page = read(`
<img src="data:image/png;base64,xx" width="500" height="500">
<img src="/small.jpg" width="50" height="50">
<img src="/pixel.gif" width="1" height="1">
<img src="/small.jpg" width="60" height="60">
<img src="/img/site-logo.png">
<img src="/icons/sprite.svg">
<img src="/photos/cat.jpg" width="640" height="480">`;
<img src="/photos/cat.jpg" width="640" height="480">`);
assertEquals(page.largeImage, "https://example.com/photos/cat.jpg");
assertEquals(page.firstContentImage, "https://example.com/small.jpg");
assertEquals(
extractLargeImage(page, BASE),
"https://example.com/photos/cat.jpg",
);
assertEquals(
extractFirstContentImage(page, BASE),
"https://example.com/small.jpg",
);
assertEquals(
extractFirstContentImage(`<img src="//cdn.test/a.jpg">`, BASE),
read(`<img src="//cdn.test/a.jpg">`).firstContentImage,
"https://cdn.test/a.jpg",
);
});
Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
Deno.test("parsePage images: lazy images by their data-src", () => {
const page = read(`
<img data-src="/wrong.jpg" src="/img/lazy-placeholder.gif">
<img src="data:image/gif;base64,R0lGOD" data-src="/real.jpg" width="800" height="600">`);
assertEquals(page.largeImage, "https://example.com/real.jpg");
assertEquals(page.firstContentImage, "https://example.com/real.jpg");
});
Deno.test("parsePage bestIcon: SVG beats everything, else the largest", () => {
assertEquals(
extractBestIcon(
`<link rel="icon" href="/16.png" sizes="16x16">
read(`<link rel="icon" href="/16.png" sizes="16x16">
<link rel="apple-touch-icon" href="/180.png" sizes="180x180">
<link rel="stylesheet" href="/x.css">`,
BASE,
),
<link rel="stylesheet" href="/x.css">`).bestIcon,
"https://example.com/180.png",
);
assertEquals(
extractBestIcon(
`<link rel="icon" href="/180.png" sizes="180x180">
<link rel="icon" type="image/svg+xml" href="/icon">`,
BASE,
),
read(`<link rel="icon" href="/180.png" sizes="180x180">
<link rel="icon" type="image/svg+xml" href="/icon">`).bestIcon,
"https://example.com/icon",
);
assertEquals(extractBestIcon("<p></p>", BASE), undefined);
assertEquals(
read(`<link rel="shortcut icon" href="/f.ico" sizes="16x16 48x48">
<link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
"https://example.com/f.ico",
);
assertEquals(read("<p></p>").bestIcon, undefined);
});
Deno.test("parsePage bestIcon: a mask icon is never the icon", () => {
assertEquals(
read(`<link rel="mask-icon" href="/pinned.svg" color="#000">
<link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
"https://example.com/32.png",
);
});

View File

@@ -20,3 +20,15 @@ Deno.test("bandcamp provider: bandcamp.com and its subdomains only", () => {
bandcampProvider.matches("https://bandcamp.com.evil.example/album/x"),
);
});
Deno.test("bandcamp provider: falls back like any page, embeds only Bandcamp's player", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:title" content="Album">
<meta name="description" content="An album by Artist">
<meta property="og:video" content="https://evil.example/EmbeddedPlayer/x">`)
);
const bc = await fetchRichContent("https://artist.bandcamp.com/album/b");
assertEquals(bc?.description, "An album by Artist");
assertEquals(bc?.siteName, "Bandcamp");
assertEquals(bc?.embedUrl, undefined);
});

View File

@@ -1,5 +1,5 @@
import { html, stubNetwork } from "../../helpers.ts";
import { assertEquals } from "@std/assert";
import { assert, assertEquals } from "@std/assert";
import { fetchRichContent } from "../../../services/rich-content-service.ts";
const BASE = "https://example.com/articles/1";
@@ -51,3 +51,67 @@ Deno.test("generic provider: non-HTML responses carry just the URL", async () =>
);
assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
});
Deno.test("generic provider: a canonical og:url with a slug keeps the OG tags", async () => {
using _ = stubNetwork(() =>
html(`
<meta property="og:url" content="https://example.com/q/123/how-to-x">
<meta property="og:title" content="How to X">
<meta property="og:image" content="https://example.com/x.jpg">
<title>How to X - Example</title>`)
);
const content = await fetchRichContent("https://example.com/q/123");
assertEquals(content?.title, "How to X");
assertEquals(content?.thumbnailUrl, "https://example.com/x.jpg");
});
Deno.test("generic provider: relative URLs resolve against the page after redirects", async () => {
using _ = stubNetwork((url) =>
url.hostname === "old.example"
? new Response(null, {
status: 301,
headers: { location: "https://new.example/b/page" },
})
: html(`<meta property="og:image" content="/og.png">
<meta name="twitter:image" content="https://new.example/tw.png">`)
);
const content = await fetchRichContent("https://old.example/a");
assertEquals(content?.url, "https://old.example/a");
assertEquals(content?.thumbnailUrl, "https://new.example/og.png");
assertEquals(content?.faviconUrl, "https://new.example/favicon.ico");
});
Deno.test("generic provider: an og:image that isn't http(s) falls through", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:image" content="javascript:alert(1)">
<meta name="twitter:image" content="//cdn.example/tw.png">`)
);
assertEquals(
(await fetchRichContent(BASE))?.thumbnailUrl,
"https://cdn.example/tw.png",
);
});
Deno.test("generic provider: a <title> loses its site suffix", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:site_name" content="Le Monde">
<title>Un article &eacute;crit — Le Monde</title>`)
);
const content = await fetchRichContent(BASE);
assertEquals(content?.title, "Un article écrit");
assertEquals(content?.siteName, "Le Monde");
});
Deno.test("generic provider: text on one line, and of bounded length", async () => {
using _ = stubNetwork(() =>
html(`<title>
A title
on two lines
</title>
<meta name="description" content="${"word ".repeat(400)}">`)
);
const content = await fetchRichContent(BASE);
assertEquals(content?.title, "A title on two lines");
assertEquals([...content!.description!].length, 1000);
assert(content!.description!.endsWith("word…"));
});

View File

@@ -1,16 +1,69 @@
import { html, stubNetwork } from "../../helpers.ts";
import { assertEquals, assertStringIncludes } from "@std/assert";
import { assert, assertEquals, assertFalse } from "@std/assert";
import { fetchRichContent } from "../../../services/rich-content-service.ts";
import { soundcloudProvider } from "../../../remote/providers/soundcloud.ts";
const embedOf = (url: string) =>
`https://w.soundcloud.com/player/?url=${
encodeURIComponent(url)
}&visual=true&auto_play=false`;
Deno.test("soundcloud provider", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:title" content="Album">
<meta property="og:video" content="https://bandcamp.com/EmbeddedPlayer/album=1">`)
html(`<meta property="og:title" content="Track">
<meta property="og:image" content="https://i1.sndcdn.com/a.jpg">
<meta name="description" content="Listen to Track">`)
);
const sc = await fetchRichContent("https://soundcloud.com/artist/track");
assertEquals(sc?.type, "soundcloud");
assertStringIncludes(
sc?.embedUrl ?? "",
encodeURIComponent("https://soundcloud.com/artist/track"),
assertEquals(sc?.siteName, "SoundCloud");
assertEquals(sc?.title, "Track");
assertEquals(sc?.description, "Listen to Track");
assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/track"));
});
Deno.test("soundcloud provider: a short link embeds the page it leads to", async () => {
using _ = stubNetwork((url) =>
url.hostname === "on.soundcloud.com"
? new Response(null, {
status: 302,
headers: {
location: "https://m.soundcloud.com/artist/track?si=abc&utm_source=x",
},
})
: html(`<meta property="og:title" content="Track">`)
);
const sc = await fetchRichContent("https://on.soundcloud.com/AbCd");
assertEquals(sc?.type, "soundcloud");
assertEquals(sc?.url, "https://on.soundcloud.com/AbCd");
assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/track"));
});
Deno.test("soundcloud provider: the page's og:url names what is embedded", async () => {
using _ = stubNetwork(() =>
html(
`<meta property="og:url" content="https://soundcloud.com/artist/real">`,
)
);
const sc = await fetchRichContent("https://soundcloud.com/artist/real?in=x");
assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/real"));
});
Deno.test("soundcloud provider: SoundCloud's hosts only", () => {
for (
const url of [
"https://soundcloud.com/a/t",
"https://www.soundcloud.com/a/t",
"https://m.soundcloud.com/a/t",
"https://on.soundcloud.com/AbCd",
]
) {
assert(soundcloudProvider.matches(url), url);
}
assertFalse(
soundcloudProvider.matches("https://soundcloud.com.evil.example/"),
);
assertFalse(
soundcloudProvider.matches("https://api.soundcloud.com/tracks/1"),
);
});

View File

@@ -13,7 +13,6 @@ Deno.test("youtube provider: video with start time and playlist", async () => {
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL123&t=1m30s",
);
assertEquals(content?.type, "youtube");
assertEquals(content?.videoId, "dQw4w9WgXcQ");
assertEquals(content?.title, "Never Gonna");
assertEquals(
content?.thumbnailUrl,
@@ -41,8 +40,8 @@ Deno.test("youtube provider: youtu.be, shorts, playlists and channels", async ()
"https://www.youtube.com/embed/abc123?rel=0&start=42",
);
assertEquals(
(await fetchRichContent("https://youtube.com/shorts/xyz"))?.videoId,
"xyz",
(await fetchRichContent("https://youtube.com/shorts/xyz"))?.embedUrl,
"https://www.youtube.com/embed/xyz?rel=0",
);
const list = await fetchRichContent(
"https://www.youtube.com/playlist?list=PLabc",

View File

@@ -41,3 +41,15 @@ Deno.test("tryFetchRichContent: an unreachable page is ok:false with a stub", as
},
});
});
Deno.test("tryFetchRichContent: an error page is unreachable, not a preview", async () => {
using net = stubNetwork(() => html(`<title>404 Not Found</title>`, 404));
const url = "https://example.com/gone";
assertEquals(await tryFetchRichContent(url), {
ok: false,
content: { type: "generic", url, siteName: "example.com" },
});
// Not kept: the page may be back next time.
await tryFetchRichContent(url);
assertEquals(net.calls.length, 2);
});