// Link previews: the HTML extractors, the providers, and fetchWithTimeout's // guard on every hop. All network access is stubbed. import { createUser, html, insertDump, stubNetwork, test } from "./helpers.ts"; import { assert, assertEquals, assertFalse, assertRejects, assertStringIncludes, } from "@std/assert"; import { extractBestIcon, extractFirstContentImage, extractJsonLd, extractLargeImage, extractLinkHref, extractMaskIconColor, extractMetaName, extractOgTag, extractPageTitle, extractThemeColor, fetchRichContent, fetchWithTimeout, isValidHttpUrl, normalizeCssColor, tryFetchRichContent, } from "../services/rich-content-service.ts"; const BASE = "https://example.com/articles/1"; Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => { const page = ` `; assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤"); assertEquals(extractOgTag(page, "description"), "Desc"); assertEquals(extractOgTag(page, "image"), undefined); assertEquals(extractMetaName(page, "twitter:title"), "TW"); }); Deno.test("extractOgTag: content never spans into the next tag", () => { const page = ``; assertEquals(extractOgTag(page, "title"), undefined); }); Deno.test("extractPageTitle", () => { assertEquals( extractPageTitle("
none
"), undefined); }); Deno.test("extractThemeColor: unscoped, then light, then any", () => { assertEquals( extractThemeColor(` `), "#fff", ); assertEquals( extractThemeColor(` `), "#123", ); assertEquals( extractThemeColor( ``, ), "#000", ); assertEquals( extractMaskIconColor( ``, ), "#5bbad5", ); }); Deno.test("normalizeCssColor", () => { assertEquals(normalizeCssColor("#ABC"), "#aabbcc"); assertEquals(normalizeCssColor("#abcd"), "#aabbcc"); assertEquals(normalizeCssColor("#112233"), "#112233"); assertEquals(normalizeCssColor("#11223380"), "#112233"); assertEquals(normalizeCssColor(" Navy "), "#000080"); assertEquals(normalizeCssColor("rgb(255, 0, 128)"), "#ff0080"); assertEquals(normalizeCssColor("rgba(300 -5 50% / .5)"), "#ff0080"); assertEquals(normalizeCssColor("#12345"), undefined); assertEquals(normalizeCssColor("hsl(0 100% 50%)"), undefined); assertEquals(normalizeCssColor("rgb(1,2)"), undefined); assertEquals(normalizeCssColor(undefined), undefined); }); Deno.test("extractJsonLd: @graph, arrays and image shapes", () => { const ld = (data: unknown) => ``; assertEquals( extractJsonLd( ld({ "@graph": [{ "@type": "WebSite" }, { headline: "Head", image: { url: "https://i/1.jpg" }, }], }), ), { title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" }, ); assertEquals( extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl, "https://i/2.jpg", ); assertEquals( extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl, undefined, ); // Invalid JSON is skipped for the next block. assertEquals( extractJsonLd( `${ ld({ name: "Second" }) }`, ).title, "Second", ); assertEquals(extractJsonLd(""), {}); }); Deno.test("image extraction: large images, content images, relative URLs", () => { const page = `
`;
assertEquals(
extractLargeImage(page, BASE),
"https://example.com/photos/cat.jpg",
);
assertEquals(
extractFirstContentImage(page, BASE),
"https://example.com/small.jpg",
);
assertEquals(
extractFirstContentImage(`
`, BASE),
"https://cdn.test/a.jpg",
);
});
Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
assertEquals(
extractBestIcon(
`
`,
BASE,
),
"https://example.com/180.png",
);
assertEquals(
extractBestIcon(
`
`,
BASE,
),
"https://example.com/icon",
);
assertEquals(extractBestIcon("", BASE), undefined);
assertEquals(
extractLinkHref(``, "alternate", BASE),
"https://example.com/feed",
);
});
Deno.test("isValidHttpUrl: http(s) to public hosts, or our own pages", () => {
assert(isValidHttpUrl("https://example.com/x"));
assert(isValidHttpUrl("http://gerbeur.test/dumps/abc"));
assertFalse(isValidHttpUrl("http://localhost:8000/"));
assertFalse(isValidHttpUrl("http://10.0.0.1/"));
assertFalse(isValidHttpUrl("file:///etc/passwd"));
assertFalse(isValidHttpUrl("nope"));
});
Deno.test("fetchWithTimeout: follows redirects, checking every hop", async () => {
using net = stubNetwork((url) => {
if (url.pathname === "/start") {
return new Response(null, { status: 302, headers: { location: "/end" } });
}
if (url.pathname === "/to-private") {
return new Response(null, {
status: 301,
headers: { location: "http://169.254.169.254/latest/meta-data" },
});
}
if (url.pathname === "/loop") {
return new Response(null, {
status: 302,
headers: { location: "/loop" },
});
}
return new Response("done");
});
const res = await fetchWithTimeout("https://example.com/start");
assertEquals(await res.text(), "done");
assertEquals(net.calls.map((u) => u.pathname), ["/start", "/end"]);
await assertRejects(
() => fetchWithTimeout("https://example.com/to-private"),
Error,
"private",
);
await assertRejects(
() => fetchWithTimeout("https://example.com/loop"),
TypeError,
"Too many redirects",
);
await assertRejects(
() => fetchWithTimeout("ftp://example.com/"),
TypeError,
"Unsupported protocol",
);
});
Deno.test("generic provider: the full fallback chain", async () => {
using _ = stubNetwork(() =>
html(`
`)
);
const content = await fetchRichContent("https://www.example.com/articles/1");
assertEquals(content?.title, "Article title");
assertEquals(content?.thumbnailUrl, "https://www.example.com/photo.jpg");
assertEquals(content?.siteName, "example.com");
assertEquals(content?.faviconUrl, "https://www.example.com/favicon.ico");
});
Deno.test("generic provider: non-HTML responses carry just the URL", async () => {
using _ = stubNetwork(() =>
new Response("{}", { headers: { "content-type": "application/json" } })
);
assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
});
Deno.test("tryFetchRichContent: an unreachable page is ok:false with a stub", async () => {
using _ = stubNetwork(() => {
throw new TypeError("connection refused");
});
assertEquals(await tryFetchRichContent("https://www.example.com/x"), {
ok: false,
content: {
type: "generic",
url: "https://www.example.com/x",
siteName: "example.com",
},
});
});
Deno.test("youtube provider: video with start time and playlist", async () => {
using net = stubNetwork((url) =>
url.pathname === "/oembed"
? Response.json({ title: "Never Gonna", thumbnail_url: "ignored" })
: new Response(null, { status: 404 })
);
const content = await fetchRichContent(
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL123&t=1m30s",
);
assertEquals(content?.type, "youtube");
assertEquals(content?.videoId, "dQw4w9WgXcQ");
assertEquals(content?.title, "Never Gonna");
assertEquals(
content?.thumbnailUrl,
"https://img.youtube.com/vi/dQw4w9WgXcQ/hqdefault.jpg",
);
assertEquals(
content?.embedUrl,
"https://www.youtube.com/embed/dQw4w9WgXcQ?rel=0&list=PL123&start=90",
);
assertEquals(net.calls[0].hostname, "www.youtube.com");
});
Deno.test("youtube provider: youtu.be, shorts, playlists and channels", async () => {
using _ = stubNetwork((url) =>
url.pathname === "/oembed"
? Response.json({
title: "T",
thumbnail_url: "https://i.ytimg.com/x.jpg",
})
: html(`
`)
);
assertEquals(
(await fetchRichContent("https://youtu.be/abc123?t=42"))?.embedUrl,
"https://www.youtube.com/embed/abc123?rel=0&start=42",
);
assertEquals(
(await fetchRichContent("https://youtube.com/shorts/xyz"))?.videoId,
"xyz",
);
const list = await fetchRichContent(
"https://www.youtube.com/playlist?list=PLabc",
);
assertEquals(
list?.embedUrl,
"https://www.youtube.com/embed/videoseries?list=PLabc&rel=0",
);
assertEquals(list?.thumbnailUrl, "https://i.ytimg.com/x.jpg");
const channel = await fetchRichContent("https://www.youtube.com/@someone");
assertEquals(channel?.title, "Channel");
assertEquals(channel?.embedUrl, undefined);
});
Deno.test("bandcamp and soundcloud providers", async () => {
using _ = stubNetwork(() =>
html(`
`)
);
const bc = await fetchRichContent("https://artist.bandcamp.com/album/a");
assertEquals(bc?.type, "bandcamp");
assertEquals(bc?.title, "Album");
assertEquals(bc?.embedUrl, "https://bandcamp.com/EmbeddedPlayer/album=1");
const sc = await fetchRichContent("https://soundcloud.com/artist/track");
assertEquals(sc?.type, "soundcloud");
assertStringIncludes(
sc?.embedUrl ?? "",
encodeURIComponent("https://soundcloud.com/artist/track"),
);
});
test("self provider: our own dump, user and playlist pages come from the DB", async () => {
const user = await createUser();
const dump = insertDump({
userId: user.id,
title: "Local dump",
comment: "About it",
richContent: { type: "generic", url: "x", thumbnailUrl: "https://t/x.jpg" },
});
using net = stubNetwork(() => {
throw new Error("no network for self links");
});
const content = await fetchRichContent(
`http://gerbeur.test/dumps/${dump.slug}`,
);
assertEquals(content?.title, "Local dump");
assertEquals(content?.description, "About it");
assertEquals(content?.thumbnailUrl, "https://t/x.jpg");
assertEquals(content?.siteName, "gerbeur");
const profile = await fetchRichContent(
`http://gerbeur.test/users/${user.username}`,
);
assertEquals(profile?.title, user.username);
assertEquals(net.calls.length, 0);
});