v3: the backend gets a test suite, and the bugs it turned up are fixed — deno task test runs 186 tests over api/ in about 18s, against a throwaway database and uploads directory (GERBEUR_DB_PATH and GERBEUR_UPLOADS_DIR now override the defaults; api/test/setup.ts sets them before any app module loads), with requests sent through the whole middleware stack in-process via app.handle, outbound fetches and DNS stubbed, and the socket tests on a real Oak listener since an upgraded request has no response for handle() to return. It covers auth (token purposes, revocation after a password change, live roles, login rate limiting, invites claimed once even under parallel registrations, password resets), dumps end to end (create, privacy, by-url matching on the canonical key, backlinks, categories, edits, file replacement, thumbnails, refresh, delete), file serving headers and ranges, attachments, avatars, comments and likes, playlists, follows and feeds, notifications and mentions, categories, search, chat cursors and read markers, link previews and the SSRF guard on every redirect hop, the Bandcamp and YouTube resolvers, migrations (a fresh schema is left untouched; the data rewrites in 0004/0005 and 0010–0013 do what they say), CORS, OG tags, static caching, and the WebSocket protocol. Fixed along the way: an unknown category id on a new dump returned 400 but kept the dump row, and for a file dump deleted the file while keeping the row; an edit or file replacement with one was half-applied before the 400 — categories are now checked before anything is written. A suffix range (bytes=-3) was read as an end offset and got a 416 instead of the last three bytes. Reordering a playlist accepted a list naming a member twice. verifyPassword threw on a stored hash without a salt instead of answering false. And a request with no Accept-Encoding, which RFC 9110 lets a server treat as accepting anything, was sent the brotli sibling; curl and bots now get the plain file.
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 46s
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 46s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
382
api/test/rich-content.test.ts
Normal file
382
api/test/rich-content.test.ts
Normal file
@@ -0,0 +1,382 @@
|
||||
// Link previews: the HTML extractors, the providers, and fetchWithTimeout's
|
||||
// guard on every hop. All network access is stubbed.
|
||||
import { createUser, html, insertDump, stubNetwork, test } from "./helpers.ts";
|
||||
import {
|
||||
assert,
|
||||
assertEquals,
|
||||
assertFalse,
|
||||
assertRejects,
|
||||
assertStringIncludes,
|
||||
} from "@std/assert";
|
||||
import {
|
||||
extractBestIcon,
|
||||
extractFirstContentImage,
|
||||
extractJsonLd,
|
||||
extractLargeImage,
|
||||
extractLinkHref,
|
||||
extractMaskIconColor,
|
||||
extractMetaName,
|
||||
extractOgTag,
|
||||
extractPageTitle,
|
||||
extractThemeColor,
|
||||
fetchRichContent,
|
||||
fetchWithTimeout,
|
||||
isValidHttpUrl,
|
||||
normalizeCssColor,
|
||||
tryFetchRichContent,
|
||||
} from "../services/rich-content-service.ts";
|
||||
|
||||
const BASE = "https://example.com/articles/1";
|
||||
|
||||
Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => {
|
||||
const page = `
|
||||
<meta property="og:title" content="Tom & Jerry — ❤">
|
||||
<meta content='Desc' property='og:description'>
|
||||
<meta name="twitter:title" content="TW">
|
||||
<meta name="og:title" content="wrong attribute">`;
|
||||
assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤");
|
||||
assertEquals(extractOgTag(page, "description"), "Desc");
|
||||
assertEquals(extractOgTag(page, "image"), undefined);
|
||||
assertEquals(extractMetaName(page, "twitter:title"), "TW");
|
||||
});
|
||||
|
||||
Deno.test("extractOgTag: content never spans into the next tag", () => {
|
||||
const page = `<meta property="og:title"><meta name="x" content="leak">`;
|
||||
assertEquals(extractOgTag(page, "title"), undefined);
|
||||
});
|
||||
|
||||
Deno.test("extractPageTitle", () => {
|
||||
assertEquals(
|
||||
extractPageTitle("<title lang=en> A <b> </title>"),
|
||||
"A <b>",
|
||||
);
|
||||
assertEquals(extractPageTitle("<p>none</p>"), undefined);
|
||||
});
|
||||
|
||||
Deno.test("extractThemeColor: unscoped, then light, then any", () => {
|
||||
assertEquals(
|
||||
extractThemeColor(`
|
||||
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
|
||||
<meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`),
|
||||
"#fff",
|
||||
);
|
||||
assertEquals(
|
||||
extractThemeColor(`
|
||||
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
|
||||
<meta name="theme-color" content="#123">`),
|
||||
"#123",
|
||||
);
|
||||
assertEquals(
|
||||
extractThemeColor(
|
||||
`<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
|
||||
),
|
||||
"#000",
|
||||
);
|
||||
assertEquals(
|
||||
extractMaskIconColor(
|
||||
`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`,
|
||||
),
|
||||
"#5bbad5",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("normalizeCssColor", () => {
|
||||
assertEquals(normalizeCssColor("#ABC"), "#aabbcc");
|
||||
assertEquals(normalizeCssColor("#abcd"), "#aabbcc");
|
||||
assertEquals(normalizeCssColor("#112233"), "#112233");
|
||||
assertEquals(normalizeCssColor("#11223380"), "#112233");
|
||||
assertEquals(normalizeCssColor(" Navy "), "#000080");
|
||||
assertEquals(normalizeCssColor("rgb(255, 0, 128)"), "#ff0080");
|
||||
assertEquals(normalizeCssColor("rgba(300 -5 50% / .5)"), "#ff0080");
|
||||
assertEquals(normalizeCssColor("#12345"), undefined);
|
||||
assertEquals(normalizeCssColor("hsl(0 100% 50%)"), undefined);
|
||||
assertEquals(normalizeCssColor("rgb(1,2)"), undefined);
|
||||
assertEquals(normalizeCssColor(undefined), undefined);
|
||||
});
|
||||
|
||||
Deno.test("extractJsonLd: @graph, arrays and image shapes", () => {
|
||||
const ld = (data: unknown) =>
|
||||
`<script type="application/ld+json">${JSON.stringify(data)}</script>`;
|
||||
assertEquals(
|
||||
extractJsonLd(
|
||||
ld({
|
||||
"@graph": [{ "@type": "WebSite" }, {
|
||||
headline: "Head",
|
||||
image: { url: "https://i/1.jpg" },
|
||||
}],
|
||||
}),
|
||||
),
|
||||
{ title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
|
||||
);
|
||||
assertEquals(
|
||||
extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl,
|
||||
"https://i/2.jpg",
|
||||
);
|
||||
assertEquals(
|
||||
extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl,
|
||||
undefined,
|
||||
);
|
||||
// Invalid JSON is skipped for the next block.
|
||||
assertEquals(
|
||||
extractJsonLd(
|
||||
`<script type="application/ld+json">{oops</script>${
|
||||
ld({ name: "Second" })
|
||||
}`,
|
||||
).title,
|
||||
"Second",
|
||||
);
|
||||
assertEquals(extractJsonLd("<p></p>"), {});
|
||||
});
|
||||
|
||||
Deno.test("image extraction: large images, content images, relative URLs", () => {
|
||||
const page = `
|
||||
<img src="data:image/png;base64,xx" width="500" height="500">
|
||||
<img src="/small.jpg" width="50" height="50">
|
||||
<img src="/img/site-logo.png">
|
||||
<img src="/icons/sprite.svg">
|
||||
<img src="/photos/cat.jpg" width="640" height="480">`;
|
||||
assertEquals(
|
||||
extractLargeImage(page, BASE),
|
||||
"https://example.com/photos/cat.jpg",
|
||||
);
|
||||
assertEquals(
|
||||
extractFirstContentImage(page, BASE),
|
||||
"https://example.com/small.jpg",
|
||||
);
|
||||
assertEquals(
|
||||
extractFirstContentImage(`<img src="//cdn.test/a.jpg">`, BASE),
|
||||
"https://cdn.test/a.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
|
||||
assertEquals(
|
||||
extractBestIcon(
|
||||
`<link rel="icon" href="/16.png" sizes="16x16">
|
||||
<link rel="apple-touch-icon" href="/180.png" sizes="180x180">
|
||||
<link rel="stylesheet" href="/x.css">`,
|
||||
BASE,
|
||||
),
|
||||
"https://example.com/180.png",
|
||||
);
|
||||
assertEquals(
|
||||
extractBestIcon(
|
||||
`<link rel="icon" href="/180.png" sizes="180x180">
|
||||
<link rel="icon" type="image/svg+xml" href="/icon">`,
|
||||
BASE,
|
||||
),
|
||||
"https://example.com/icon",
|
||||
);
|
||||
assertEquals(extractBestIcon("<p></p>", BASE), undefined);
|
||||
assertEquals(
|
||||
extractLinkHref(`<link href="/feed" rel="alternate">`, "alternate", BASE),
|
||||
"https://example.com/feed",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("isValidHttpUrl: http(s) to public hosts, or our own pages", () => {
|
||||
assert(isValidHttpUrl("https://example.com/x"));
|
||||
assert(isValidHttpUrl("http://gerbeur.test/dumps/abc"));
|
||||
assertFalse(isValidHttpUrl("http://localhost:8000/"));
|
||||
assertFalse(isValidHttpUrl("http://10.0.0.1/"));
|
||||
assertFalse(isValidHttpUrl("file:///etc/passwd"));
|
||||
assertFalse(isValidHttpUrl("nope"));
|
||||
});
|
||||
|
||||
Deno.test("fetchWithTimeout: follows redirects, checking every hop", async () => {
|
||||
using net = stubNetwork((url) => {
|
||||
if (url.pathname === "/start") {
|
||||
return new Response(null, { status: 302, headers: { location: "/end" } });
|
||||
}
|
||||
if (url.pathname === "/to-private") {
|
||||
return new Response(null, {
|
||||
status: 301,
|
||||
headers: { location: "http://169.254.169.254/latest/meta-data" },
|
||||
});
|
||||
}
|
||||
if (url.pathname === "/loop") {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "/loop" },
|
||||
});
|
||||
}
|
||||
return new Response("done");
|
||||
});
|
||||
const res = await fetchWithTimeout("https://example.com/start");
|
||||
assertEquals(await res.text(), "done");
|
||||
assertEquals(net.calls.map((u) => u.pathname), ["/start", "/end"]);
|
||||
|
||||
await assertRejects(
|
||||
() => fetchWithTimeout("https://example.com/to-private"),
|
||||
Error,
|
||||
"private",
|
||||
);
|
||||
await assertRejects(
|
||||
() => fetchWithTimeout("https://example.com/loop"),
|
||||
TypeError,
|
||||
"Too many redirects",
|
||||
);
|
||||
await assertRejects(
|
||||
() => fetchWithTimeout("ftp://example.com/"),
|
||||
TypeError,
|
||||
"Unsupported protocol",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("generic provider: the full fallback chain", async () => {
|
||||
using _ = stubNetwork(() =>
|
||||
html(`<html><head>
|
||||
<title>Page title</title>
|
||||
<meta property="og:site_name" content="Example">
|
||||
<meta property="og:title" content="OG title">
|
||||
<meta property="og:image" content="https://example.com/og.jpg">
|
||||
<meta name="description" content="Plain description">
|
||||
<meta name="theme-color" content="rgb(0, 128, 255)">
|
||||
<link rel="icon" href="/favicon-32.png" sizes="32x32">
|
||||
</head></html>`)
|
||||
);
|
||||
const content = await fetchRichContent(BASE);
|
||||
assertEquals(content, {
|
||||
type: "generic",
|
||||
url: BASE,
|
||||
title: "OG title",
|
||||
description: "Plain description",
|
||||
thumbnailUrl: "https://example.com/og.jpg",
|
||||
faviconUrl: "https://example.com/favicon-32.png",
|
||||
accentColor: "#0080ff",
|
||||
siteName: "Example",
|
||||
});
|
||||
});
|
||||
|
||||
Deno.test("generic provider: a site-wide og:url means og:title/image are ignored", async () => {
|
||||
using _ = stubNetwork(() =>
|
||||
html(`
|
||||
<meta property="og:url" content="https://example.com/">
|
||||
<meta property="og:title" content="Site name">
|
||||
<meta property="og:image" content="https://example.com/site.jpg">
|
||||
<meta name="twitter:title" content="Article title">
|
||||
<img src="/photo.jpg">`)
|
||||
);
|
||||
const content = await fetchRichContent("https://www.example.com/articles/1");
|
||||
assertEquals(content?.title, "Article title");
|
||||
assertEquals(content?.thumbnailUrl, "https://www.example.com/photo.jpg");
|
||||
assertEquals(content?.siteName, "example.com");
|
||||
assertEquals(content?.faviconUrl, "https://www.example.com/favicon.ico");
|
||||
});
|
||||
|
||||
Deno.test("generic provider: non-HTML responses carry just the URL", async () => {
|
||||
using _ = stubNetwork(() =>
|
||||
new Response("{}", { headers: { "content-type": "application/json" } })
|
||||
);
|
||||
assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
|
||||
});
|
||||
|
||||
Deno.test("tryFetchRichContent: an unreachable page is ok:false with a stub", async () => {
|
||||
using _ = stubNetwork(() => {
|
||||
throw new TypeError("connection refused");
|
||||
});
|
||||
assertEquals(await tryFetchRichContent("https://www.example.com/x"), {
|
||||
ok: false,
|
||||
content: {
|
||||
type: "generic",
|
||||
url: "https://www.example.com/x",
|
||||
siteName: "example.com",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
Deno.test("youtube provider: video with start time and playlist", async () => {
|
||||
using net = stubNetwork((url) =>
|
||||
url.pathname === "/oembed"
|
||||
? Response.json({ title: "Never Gonna", thumbnail_url: "ignored" })
|
||||
: new Response(null, { status: 404 })
|
||||
);
|
||||
const content = await fetchRichContent(
|
||||
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL123&t=1m30s",
|
||||
);
|
||||
assertEquals(content?.type, "youtube");
|
||||
assertEquals(content?.videoId, "dQw4w9WgXcQ");
|
||||
assertEquals(content?.title, "Never Gonna");
|
||||
assertEquals(
|
||||
content?.thumbnailUrl,
|
||||
"https://img.youtube.com/vi/dQw4w9WgXcQ/hqdefault.jpg",
|
||||
);
|
||||
assertEquals(
|
||||
content?.embedUrl,
|
||||
"https://www.youtube.com/embed/dQw4w9WgXcQ?rel=0&list=PL123&start=90",
|
||||
);
|
||||
assertEquals(net.calls[0].hostname, "www.youtube.com");
|
||||
});
|
||||
|
||||
Deno.test("youtube provider: youtu.be, shorts, playlists and channels", async () => {
|
||||
using _ = stubNetwork((url) =>
|
||||
url.pathname === "/oembed"
|
||||
? Response.json({
|
||||
title: "T",
|
||||
thumbnail_url: "https://i.ytimg.com/x.jpg",
|
||||
})
|
||||
: html(`<meta property="og:title" content="Channel">
|
||||
<meta property="og:image" content="https://yt3/ch.jpg">`)
|
||||
);
|
||||
assertEquals(
|
||||
(await fetchRichContent("https://youtu.be/abc123?t=42"))?.embedUrl,
|
||||
"https://www.youtube.com/embed/abc123?rel=0&start=42",
|
||||
);
|
||||
assertEquals(
|
||||
(await fetchRichContent("https://youtube.com/shorts/xyz"))?.videoId,
|
||||
"xyz",
|
||||
);
|
||||
const list = await fetchRichContent(
|
||||
"https://www.youtube.com/playlist?list=PLabc",
|
||||
);
|
||||
assertEquals(
|
||||
list?.embedUrl,
|
||||
"https://www.youtube.com/embed/videoseries?list=PLabc&rel=0",
|
||||
);
|
||||
assertEquals(list?.thumbnailUrl, "https://i.ytimg.com/x.jpg");
|
||||
const channel = await fetchRichContent("https://www.youtube.com/@someone");
|
||||
assertEquals(channel?.title, "Channel");
|
||||
assertEquals(channel?.embedUrl, undefined);
|
||||
});
|
||||
|
||||
Deno.test("bandcamp and soundcloud providers", async () => {
|
||||
using _ = stubNetwork(() =>
|
||||
html(`<meta property="og:title" content="Album">
|
||||
<meta property="og:video" content="https://bandcamp.com/EmbeddedPlayer/album=1">`)
|
||||
);
|
||||
const bc = await fetchRichContent("https://artist.bandcamp.com/album/a");
|
||||
assertEquals(bc?.type, "bandcamp");
|
||||
assertEquals(bc?.title, "Album");
|
||||
assertEquals(bc?.embedUrl, "https://bandcamp.com/EmbeddedPlayer/album=1");
|
||||
const sc = await fetchRichContent("https://soundcloud.com/artist/track");
|
||||
assertEquals(sc?.type, "soundcloud");
|
||||
assertStringIncludes(
|
||||
sc?.embedUrl ?? "",
|
||||
encodeURIComponent("https://soundcloud.com/artist/track"),
|
||||
);
|
||||
});
|
||||
|
||||
test("self provider: our own dump, user and playlist pages come from the DB", async () => {
|
||||
const user = await createUser();
|
||||
const dump = insertDump({
|
||||
userId: user.id,
|
||||
title: "Local dump",
|
||||
comment: "About it",
|
||||
richContent: { type: "generic", url: "x", thumbnailUrl: "https://t/x.jpg" },
|
||||
});
|
||||
using net = stubNetwork(() => {
|
||||
throw new Error("no network for self links");
|
||||
});
|
||||
const content = await fetchRichContent(
|
||||
`http://gerbeur.test/dumps/${dump.slug}`,
|
||||
);
|
||||
assertEquals(content?.title, "Local dump");
|
||||
assertEquals(content?.description, "About it");
|
||||
assertEquals(content?.thumbnailUrl, "https://t/x.jpg");
|
||||
assertEquals(content?.siteName, "gerbeur");
|
||||
const profile = await fetchRichContent(
|
||||
`http://gerbeur.test/users/${user.username}`,
|
||||
);
|
||||
assertEquals(profile?.title, user.username);
|
||||
assertEquals(net.calls.length, 0);
|
||||
});
|
||||
Reference in New Issue
Block a user