vincent/gerbeur
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
api/remote/bandcamp-tralbum.ts
View File

@@ -8,6 +8,7 @@
*/
import type { Tralbum, TralbumTrack } from "../../shared/player.ts";
import { decodeHtmlEntities } from "./html.ts";
/**
* Matches the hostnames `bandcamp-streams.ts` accepts: bandcamp.com and its
@@ -16,30 +17,6 @@ import type { Tralbum, TralbumTrack } from "../../shared/player.ts";
*/
export const BANDCAMP_HOST_RE = /(?:^|\.)bandcamp\.com$/;
/**
* Decodes HTML entities in a single pass. `html.ts` decodes `&amp;` before
* `&quot;`, which turns a title's literal `&quot;` (escaped as `&amp;quot;`)
* into a bare `"` that breaks the JSON.
*/
function decodeEntitiesOnce(input: string): string {
const named: Record<string, string> = {
quot: '"',
apos: "'",
amp: "&",
lt: "<",
gt: ">",
nbsp: " ",
};
return input.replace(
/&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|([a-zA-Z]+));/g,
(match, dec: string, hex: string, name: string) => {
if (dec) return String.fromCodePoint(Number(dec));
if (hex) return String.fromCodePoint(parseInt(hex, 16));
return named[name.toLowerCase()] ?? match;
},
);
}
/** Returns the URL of the artwork `artId`; undefined when it isn't numeric. */
function artworkUrl(artId: unknown): string | undefined {
if (typeof artId !== "number" && typeof artId !== "string") return undefined;
@@ -60,7 +37,7 @@ export function parseTralbum(html: string, sourceUrl: string): Tralbum | null {
let raw: Record<string, unknown>;
try {
raw = JSON.parse(decodeEntitiesOnce(match[2]));
raw = JSON.parse(decodeHtmlEntities(match[2]));
} catch {
return null;
}

72
api/remote/fetch.ts
View File

@@ -175,7 +175,7 @@ function declaredCharset(contentType: string, bytes: Uint8Array): string {
/**
* Reads the body of `res` as text, up to `maxBytes`, in the charset the page
* declares; the rest is cancelled, never downloaded. The providers and the
* declares; the rest is cancelled, never downloaded. {@link fetchPage} and the
* Bandcamp and YouTube resolvers read pages through it.
*/
export async function readTextCapped(
@@ -235,6 +235,33 @@ async function fetchOnce(
});
}
/**
* Fetches `url` as {@link fetchWithTimeout} does, and also returns the URL the
* last hop answered from, which {@link fetchPage} resolves the page's links
* against.
*/
async function fetchFollowing(
url: string,
timeoutMs: number,
): Promise<{ res: Response; url: URL }> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) {
return { res, url: current };
}
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
}
}
/**
* Fetches a user-supplied URL, following up to five redirects, each of which
* must resolve to a public address too. `timeoutMs` bounds each hop, body
@@ -249,20 +276,41 @@ export async function fetchWithTimeout(
url: string,
timeoutMs = 5000,
): Promise<Response> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) return res;
return (await fetchFollowing(url, timeoutMs)).res;
}
/**
* Returned by {@link fetchPage} to the link preview providers.
*/
export interface FetchedPage {
/**
* The URL the page was served from, after redirects: the base its relative
* links resolve against.
*/
url: string;
/** The page's text, up to {@link MAX_PAGE_BYTES}; missing when it isn't HTML. */
html?: string;
}
/**
* Fetches a page for its link preview, through {@link fetchWithTimeout}. A
* response that isn't HTML is cancelled unread.
*
* @throws {Error} When the page answers with an error status, so that an error
* page never becomes a preview; and as {@link fetchWithTimeout} does.
*/
export async function fetchPage(url: string): Promise<FetchedPage> {
const { res, url: finalUrl } = await fetchFollowing(url, 5000);
if (!res.ok) {
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
throw new Error(`${finalUrl} answered ${res.status}`);
}
const contentType = res.headers.get("content-type") ?? "";
if (!/^(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
await res.body?.cancel();
return { url: finalUrl.href };
}
return { url: finalUrl.href, html: await readTextCapped(res) };
}
/**

683
api/remote/html.ts
View File

@@ -1,8 +1,9 @@
/**
* Reads a link preview's metadata out of a page's HTML with regular
* expressions, without parsing the document. The generic provider uses most of
* it; the Bandcamp, SoundCloud and YouTube providers read only OG tags. Each
* function returns undefined when the page doesn't say.
* Reads a link preview's metadata out of a page's HTML. {@link parsePage}
* scans the page once for the tags that carry it (`<meta>`, `<link>`, `<img>`,
* `<title>` and JSON-LD scripts) without building a document, and returns a
* {@link Page} that the providers' `genericPreview` picks from. Every field is
* undefined when the page doesn't say.
*
* @module
*/
@@ -10,71 +11,220 @@
import { hexOf, parseCssColor } from "../lib/css-color.ts";
/**
* Decodes the five XML entities and numeric ones in `str`. `&amp;` goes first,
* so `&amp;quot;` decodes twice; `bandcamp-tralbum.ts`, which decodes JSON,
* has its own single-pass decoder.
* The named entities decoded besides numeric ones: the XML five, and those
* page titles and descriptions actually use, Latin-1 letters and typography.
*/
function decodeHtmlEntities(str: string): string {
return str
.replace(/&amp;/gi, "&")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">")
.replace(/&quot;/gi, '"')
.replace(/&apos;/gi, "'")
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
.replace(
/&#x([0-9a-f]+);/gi,
(_, hex) => String.fromCodePoint(parseInt(hex, 16)),
);
}
const NAMED_ENTITIES: Record<string, string> = {
amp: "&",
lt: "<",
gt: ">",
quot: '"',
apos: "'",
nbsp: " ",
aelig: "æ",
AElig: "Æ",
oelig: "œ",
OElig: "Œ",
szlig: "ß",
oslash: "ø",
Oslash: "Ø",
eth: "ð",
ETH: "Ð",
thorn: "þ",
THORN: "Þ",
hellip: "…",
lsquo: "‘",
rsquo: "’",
sbquo: "‚",
ldquo: "“",
rdquo: "”",
bdquo: "„",
laquo: "«",
raquo: "»",
lsaquo: "‹",
rsaquo: "›",
ndash: "–",
mdash: "—",
bull: "•",
middot: "·",
dagger: "†",
Dagger: "‡",
prime: "′",
Prime: "″",
euro: "€",
pound: "£",
yen: "¥",
cent: "¢",
copy: "©",
reg: "®",
trade: "™",
deg: "°",
plusmn: "±",
times: "×",
divide: "÷",
micro: "µ",
para: "¶",
sect: "§",
iexcl: "¡",
iquest: "¿",
ordf: "ª",
ordm: "º",
sup2: "²",
sup3: "³",
frac14: "¼",
frac12: "½",
frac34: "¾",
shy: "­",
ensp: " ",
emsp: " ",
thinsp: " ",
zwnj: "‌",
zwj: "‍",
};
/**
* Returns the `content` of the first `<meta>` tag whose `attr` equals `value`,
* in whichever order the attributes come. Each tag is matched on its own, so
* the capture can't run into the next tag.
*/
function extractMetaContent(
html: string,
attr: string,
value: string,
): string | undefined {
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
const tagRe = /<meta\b[^>]*>/gi;
let m: RegExpExecArray | null;
while ((m = tagRe.exec(html)) !== null) {
const tag = m[0];
if (!keyRe.test(tag)) continue;
const content = contentRe.exec(tag);
if (content) return decodeHtmlEntities(content[2]);
// The accented letters, `&eacute;` and the like, composed from their letter
// and accent rather than listed one by one.
for (
const [accent, mark, letters] of [
["grave", "̀", "aeiouAEIOU"],
["acute", "́", "aeiouyAEIOUY"],
["circ", "̂", "aeiouAEIOU"],
["tilde", "̃", "anoANO"],
["uml", "̈", "aeiouyAEIOUY"],
["ring", "̊", "aA"],
["cedil", "̧", "cC"],
]
) {
for (const letter of letters) {
NAMED_ENTITIES[letter + accent] = (letter + mark).normalize("NFC");
}
return undefined;
}
/** Returns the content of `<meta property="og:<tag>">`, entities decoded. */
export function extractOgTag(
html: string,
tag: string,
): string | undefined {
return extractMetaContent(html, "property", `og:${tag}`);
/** Returns the character of code point `n`, or undefined when there is none. */
function fromCodePoint(n: number): string | undefined {
if (n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) {
return undefined;
}
return String.fromCodePoint(n);
}
/**
* Returns the content of `<meta name="<name>">`, entities decoded. Twitter
* card tags use `name`, not `property`.
* Decodes the entities in `str` in a single pass, so `&amp;quot;` reads
* `&quot;`. An entity it doesn't know, or a code point that doesn't exist, is
* left as written. Also used by `bandcamp-tralbum.ts` on the JSON a page
* carries in an attribute.
*/
export function extractMetaName(
html: string,
name: string,
): string | undefined {
return extractMetaContent(html, "name", name);
export function decodeHtmlEntities(str: string): string {
return str.replace(
/&(?:#(\d{1,8})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z][a-zA-Z0-9]{1,31}));/g,
(match, dec?: string, hex?: string, name?: string) => {
if (dec) return fromCodePoint(Number(dec)) ?? match;
if (hex) return fromCodePoint(parseInt(hex, 16)) ?? match;
return NAMED_ENTITIES[name!] ?? NAMED_ENTITIES[name!.toLowerCase()] ??
match;
},
);
}
/** Returns the text of the `<title>` element, trimmed and entities decoded. */
export function extractPageTitle(html: string): string | undefined {
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
return match ? decodeHtmlEntities(match[1].trim()) : undefined;
/**
* Returns `raw` resolved against `base` as an absolute http(s) URL; undefined
* for a `data:` URI, another scheme or a URL that doesn't parse. Every URL a
* page names goes through it, so `/og.png` and `//cdn/og.png` work.
*/
export function absoluteHttpUrl(
raw: string | undefined,
base: string,
): string | undefined {
if (!raw?.trim()) return undefined;
try {
const url = new URL(raw.trim(), base);
return url.protocol === "http:" || url.protocol === "https:"
? url.href
: undefined;
} catch {
return undefined;
}
}
/** A tag's attributes by lowercase name, values decoded; the first one wins. */
type Attrs = Map<string, string>;
/**
* Matches the start of a tag the scan reads, or of a comment. Quoted values
* may hold `>`, but no tag spans a `<`, so each attempt stops at the next one
* and the scan stays linear in the page's size.
*/
const TAG_RE =
/<(meta|link|img|title|script)\b((?:[^<>"']|"[^"<]*"|'[^'<]*')*)>|<!--/gi;
/** Matches one attribute: unquoted, single- or double-quoted, or bare. */
const ATTR_RE =
/([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
function parseAttrs(source: string): Attrs {
const attrs: Attrs = new Map();
for (const m of source.matchAll(ATTR_RE)) {
const name = m[1].toLowerCase();
if (attrs.has(name)) continue;
attrs.set(name, decodeHtmlEntities(m[2] ?? m[3] ?? m[4] ?? ""));
}
return attrs;
}
/** What {@link scanTags} found, in page order. */
interface Scan {
metas: Attrs[];
links: Attrs[];
imgs: Attrs[];
/** The raw text of the first `<title>`. */
title?: string;
/** The bodies of the `application/ld+json` scripts. */
jsonLd: string[];
}
/**
* Collects the tags {@link parsePage} reads in one pass. Comments and the
* bodies of scripts and the title are skipped over, so a tag written inside
* them isn't read; one left open runs to the end of the page, as in a browser.
*/
function scanTags(html: string): Scan {
const scan: Scan = { metas: [], links: [], imgs: [], jsonLd: [] };
const re = new RegExp(TAG_RE);
let m: RegExpExecArray | null;
while ((m = re.exec(html)) !== null) {
const tag = m[1]?.toLowerCase();
if (!tag) {
const end = html.indexOf("-->", re.lastIndex);
if (end < 0) break;
re.lastIndex = end + 3;
continue;
}
const attrs = parseAttrs(m[2]);
if (tag === "meta") scan.metas.push(attrs);
else if (tag === "link") scan.links.push(attrs);
else if (tag === "img") scan.imgs.push(attrs);
else {
const close = new RegExp(`</${tag}\\s*>`, "gi");
close.lastIndex = re.lastIndex;
const end = close.exec(html);
if (!end) break;
const body = html.slice(re.lastIndex, end.index);
if (tag === "title") scan.title ??= body;
else if (
attrs.get("type")?.trim().toLowerCase() === "application/ld+json"
) {
scan.jsonLd.push(body);
}
re.lastIndex = close.lastIndex;
}
}
return scan;
}
/** Returns `value` trimmed, or undefined when nothing is left. */
function nonBlank(value: string | undefined): string | undefined {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
}
/**
@@ -82,21 +232,15 @@ export function extractPageTitle(html: string): string | undefined {
* {@link normalizeCssColor}. Sites often ship one per color scheme with the
* dark one first, so the unscoped tag wins, then the light one, then any.
*/
export function extractThemeColor(html: string): string | undefined {
const tagRe = /<meta\b[^>]*>/gi;
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
function themeColorOf(metas: Attrs[]): string | undefined {
let unscoped: string | undefined;
let light: string | undefined;
let any: string | undefined;
let m: RegExpExecArray | null;
while ((m = tagRe.exec(html)) !== null) {
const tag = m[0];
if (!/\bname=["']theme-color["']/i.test(tag)) continue;
const content = contentRe.exec(tag)?.[2];
for (const meta of metas) {
if (meta.get("name")?.toLowerCase() !== "theme-color") continue;
const content = nonBlank(meta.get("content"));
if (!content) continue;
const media = mediaRe.exec(tag)?.[2];
const media = meta.get("media");
if (!media) unscoped ??= content;
else if (/light/i.test(media)) light ??= content;
any ??= content;
@@ -104,180 +248,179 @@ export function extractThemeColor(html: string): string | undefined {
return unscoped ?? light ?? any;
}
/** Returns the lowercase tokens of a `<link>`'s `rel`. */
function relTokens(link: Attrs): string[] {
return (link.get("rel") ?? "").toLowerCase().split(/\s+/);
}
/**
* Returns the `color` of `<link rel="mask-icon">`, Safari's pinned-tab icon,
* as written. The generic provider's last resort for a brand color.
* as written. The generic preview's last resort for a brand color.
*/
export function extractMaskIconColor(html: string): string | undefined {
const linkRe = /<link[^>]+>/gi;
let m: RegExpExecArray | null;
while ((m = linkRe.exec(html)) !== null) {
const tag = m[0];
if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
function maskIconColorOf(links: Attrs[]): string | undefined {
for (const link of links) {
if (!relTokens(link).includes("mask-icon")) continue;
const color = nonBlank(link.get("color"));
if (color) return color;
}
return undefined;
}
/**
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
* common named color; undefined for any other form. The generic provider
* stores it as the preview's `accentColor`, a tint over the app's own surface,
* so alpha is dropped, and a fully transparent one is no color at all.
*/
export function normalizeCssColor(raw: string | undefined): string | undefined {
const color = raw ? parseCssColor(raw) : undefined;
return color && color.a > 0 ? hexOf(color) : undefined;
}
/**
* Returned to the generic provider by {@link extractJsonLd}, which ranks each
* field below the page's OG and Twitter tags.
*/
export interface JsonLdResult {
/** From `name`, or else `headline`. */
title?: string;
description?: string;
/**
* From `image`, or else `thumbnailUrl` or `thumbnail`, and only when it is
* an absolute URL.
*/
thumbnailUrl?: string;
}
function ldString(v: unknown): string | undefined {
if (typeof v === "string" && v.trim()) return v.trim();
if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
return v[0].trim();
}
return undefined;
}
function ldImage(v: unknown): string | undefined {
if (
typeof v === "string" &&
(v.startsWith("http://") || v.startsWith("https://"))
) return v;
if (Array.isArray(v)) return ldImage(v[0]);
if (v && typeof v === "object") {
const o = v as Record<string, unknown>;
return ldImage(o.url ?? o.contentUrl);
}
return undefined;
}
function ldExtractNode(data: unknown): JsonLdResult {
if (Array.isArray(data)) {
for (const item of data) {
const r = ldExtractNode(item);
if (r.title || r.thumbnailUrl) return r;
}
return {};
}
if (!data || typeof data !== "object") return {};
const o = data as Record<string, unknown>;
if (o["@graph"]) return ldExtractNode(o["@graph"]);
return {
title: ldString(o.name ?? o.headline),
description: ldString(o.description),
thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
};
}
/**
* Returns the first JSON-LD node in `html` that has a title or an image,
* looking through `@graph` and arrays; empty when there is none. An image may
* be a URL, a list of them or an `ImageObject`. Blocks that aren't valid JSON
* are skipped.
*/
export function extractJsonLd(html: string): JsonLdResult {
const pattern =
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
let match: RegExpExecArray | null;
while ((match = pattern.exec(html)) !== null) {
try {
const result = ldExtractNode(JSON.parse(match[1]));
if (result.title || result.thumbnailUrl) return result;
} catch { /* invalid JSON: try the next block */ }
}
return {};
}
/**
* Returns the absolute URL of the first `<img>` whose declared width and height
* are both at least `minSize` pixels; an image without them doesn't count.
* Data URIs are skipped.
*/
export function extractLargeImage(
html: string,
baseUrl: string,
minSize = 200,
): string | undefined {
const imgPattern = /<img[^>]+>/gi;
let match: RegExpExecArray | null;
while ((match = imgPattern.exec(html)) !== null) {
const tag = match[0];
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
if (!src || src.startsWith("data:")) continue;
const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
if (w >= minSize && h >= minSize) {
try {
return new URL(src, baseUrl).toString();
} catch {
continue;
}
}
}
return undefined;
}
/**
* Returns the absolute URL of the page's largest icon, among its `icon` and
* `apple-touch-icon` links, by declared `sizes`; the first one when none
* declares a size. The generic provider stores it as the preview's
* `faviconUrl`, which the thumbnail placeholder draws.
* declares a size. A `mask-icon` is no candidate: it is a one-color shape for
* Safari to tint, black when drawn as is.
*
* An SVG icon wins over any size: it declares none, yet scales to any.
*/
export function extractBestIcon(
html: string,
baseUrl: string,
): string | undefined {
const linkRe = /<link[^>]+>/gi;
const relRe = /\brel=["']([^"']+)["']/i;
const hrefRe = /\bhref=["']([^"']+)["']/i;
const sizesRe = /\bsizes=["']([^"']+)["']/i;
const candidates: { href: string; area: number }[] = [];
let m: RegExpExecArray | null;
while ((m = linkRe.exec(html)) !== null) {
const tag = m[0];
const rel = relRe.exec(tag)?.[1] ?? "";
if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
const href = hrefRe.exec(tag)?.[1];
function bestIconOf(links: Attrs[], baseUrl: string): string | undefined {
let best: { href: string; area: number } | undefined;
for (const link of links) {
const rel = relTokens(link);
if (
!rel.includes("icon") && !rel.includes("apple-touch-icon") &&
!rel.includes("apple-touch-icon-precomposed")
) continue;
const href = absoluteHttpUrl(link.get("href"), baseUrl);
if (!href) continue;
const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
const sm = sizesStr.match(/(\d+)x(\d+)/i);
const sizes = (link.get("sizes") ?? "").toLowerCase();
const isSvg = /\.svg(\?|$)/i.test(href) ||
/\btype=["']image\/svg\+xml["']/i.test(tag);
const area = isSvg
? Number.MAX_SAFE_INTEGER
: sm
? parseInt(sm[1]) * parseInt(sm[2])
: 0;
link.get("type")?.toLowerCase() === "image/svg+xml" ||
sizes === "any";
const area = isSvg ? Number.MAX_SAFE_INTEGER : Math.max(
0,
...[...sizes.matchAll(/(\d+)x(\d+)/g)].map(([, w, h]) => +w * +h),
);
if (!best || area > best.area) best = { href, area };
}
return best?.href;
}
/**
* Returned by the JSON-LD reading of {@link parsePage}, which the generic
* preview ranks below the page's OG and Twitter tags.
*/
export interface JsonLdResult {
/** From `name`, or else `headline`. */
title?: string;
description?: string;
/** From `image`, or else `thumbnailUrl` or `thumbnail`. */
thumbnailUrl?: string;
}
/**
* The JSON-LD types that describe the site around a page rather than the
* page itself. Their name is the site's or the company's, so a node of one of
* them is read only when no other node has a title or an image.
*/
const SITE_LD_TYPES = new Set([
"WebSite",
"Organization",
"Corporation",
"NewsMediaOrganization",
"Person",
"BreadcrumbList",
"SearchAction",
"ImageObject",
"SiteNavigationElement",
"WPHeader",
"WPFooter",
]);
function ldString(v: unknown): string | undefined {
if (typeof v === "string") return nonBlank(v);
if (Array.isArray(v) && typeof v[0] === "string") return nonBlank(v[0]);
return undefined;
}
function ldImage(v: unknown, baseUrl: string): string | undefined {
if (typeof v === "string") return absoluteHttpUrl(v, baseUrl);
if (Array.isArray(v)) return ldImage(v[0], baseUrl);
if (v && typeof v === "object") {
const o = v as Record<string, unknown>;
return ldImage(o.url ?? o.contentUrl, baseUrl);
}
return undefined;
}
/** Lists every node of `data`, through arrays and `@graph`, in order. */
function ldNodes(data: unknown): Record<string, unknown>[] {
if (Array.isArray(data)) return data.flatMap(ldNodes);
if (!data || typeof data !== "object") return [];
const o = data as Record<string, unknown>;
return o["@graph"] ? ldNodes(o["@graph"]) : [o];
}
/**
* Returns the first JSON-LD node about the page that has a title or an image,
* else the first about its site ({@link SITE_LD_TYPES}); empty when there is
* none. An image may be a URL, a list of them or an `ImageObject`. Blocks that
* aren't valid JSON are skipped.
*/
function jsonLdOf(blocks: string[], baseUrl: string): JsonLdResult {
let fallback: JsonLdResult | undefined;
for (const block of blocks) {
let data: unknown;
try {
candidates.push({ href: new URL(href, baseUrl).toString(), area });
data = JSON.parse(block);
} catch {
continue;
}
for (const node of ldNodes(data)) {
const result: JsonLdResult = {
title: ldString(node.name ?? node.headline),
description: ldString(node.description),
thumbnailUrl: ldImage(
node.image ?? node.thumbnailUrl ?? node.thumbnail,
baseUrl,
),
};
if (!result.title && !result.thumbnailUrl) continue;
const types = [node["@type"]].flat();
if (!types.some((t) => typeof t === "string" && SITE_LD_TYPES.has(t))) {
return result;
}
fallback ??= result;
}
}
return fallback ?? {};
}
if (candidates.length === 0) return undefined;
candidates.sort((a, b) => b.area - a.area);
return candidates[0].href;
/**
* Returns the source of an `<img>`: its `src`, or for a lazy-loaded image
* whose `src` is a `data:` placeholder, its `data-src` or `data-lazy-src`.
*/
function imageSource(img: Attrs, baseUrl: string): string | undefined {
const src = img.get("src")?.trim();
return src && !src.startsWith("data:")
? absoluteHttpUrl(src, baseUrl)
: absoluteHttpUrl(img.get("data-src") ?? img.get("data-lazy-src"), baseUrl);
}
/** Returns an `<img>`'s declared `name` dimension in pixels; 0 when none. */
function declaredSize(img: Attrs, name: "width" | "height"): number {
return parseInt(img.get(name) ?? "") || 0;
}
/**
* Returns the absolute URL of the first `<img>` whose declared width and height
* are both at least `minSize` pixels; an image without them doesn't count.
*/
function largeImageOf(
imgs: Attrs[],
baseUrl: string,
minSize = 200,
): string | undefined {
for (const img of imgs) {
if (
declaredSize(img, "width") < minSize ||
declaredSize(img, "height") < minSize
) continue;
const src = imageSource(img, baseUrl);
if (src) return src;
}
return undefined;
}
/** Filename words that mark an image as page furniture rather than content. */
@@ -297,6 +440,11 @@ const UI_IMAGE_KEYWORDS = new Set([
"pixel",
"bg",
"background",
"placeholder",
"spacer",
"blank",
"lazy",
"loading",
]);
/**
@@ -314,25 +462,86 @@ function isUiImage(src: string): boolean {
/**
* Returns the absolute URL of the first `<img>` that looks like content rather
* than a logo, icon or other page furniture. Data URIs are skipped. The generic
* provider's last resort for a thumbnail.
* than a logo, icon, tracking pixel or other page furniture: one declared
* smaller than 50 pixels either way doesn't count. The generic preview's last
* resort for a thumbnail.
*/
export function extractFirstContentImage(
html: string,
function firstContentImageOf(
imgs: Attrs[],
baseUrl: string,
): string | undefined {
const imgPattern = /<img[^>]+>/gi;
let match: RegExpExecArray | null;
while ((match = imgPattern.exec(html)) !== null) {
const tag = match[0];
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
if (!src || src.startsWith("data:")) continue;
if (isUiImage(src)) continue;
try {
return new URL(src, baseUrl).toString();
} catch {
continue;
}
for (const img of imgs) {
const w = declaredSize(img, "width");
const h = declaredSize(img, "height");
if ((w && w < 50) || (h && h < 50)) continue;
const src = imageSource(img, baseUrl);
if (src && !isUiImage(src)) return src;
}
return undefined;
}
/**
* Returned by {@link parsePage}: what a page says about itself, entities
* decoded, values trimmed and URLs absolute.
*/
export interface Page {
/**
* Returns the `content` of the first `<meta>` whose `property` or `name` is
* `key`, in any case: sites write `og:` tags under either attribute, and
* Twitter's too.
*/
meta(key: string): string | undefined;
/** The text of the `<title>` element. */
title?: string;
jsonLd: JsonLdResult;
/** The `theme-color`, as written: pass it to {@link normalizeCssColor}. */
themeColor?: string;
/** The `color` of the Safari pinned-tab icon, as written. */
maskIconColor?: string;
/** The largest icon the page links, which the thumbnail placeholder draws. */
bestIcon?: string;
/** The first image declared at least 200 pixels both ways. */
largeImage?: string;
/** The first image that doesn't look like page furniture. */
firstContentImage?: string;
}
/**
* Reads `html`, served from `pageUrl`, in one pass. The generic preview and
* the providers built on it read a page through it.
*/
export function parsePage(html: string, pageUrl: string): Page {
const scan = scanTags(html);
const metas = new Map<string, string>();
for (const meta of scan.metas) {
const content = nonBlank(meta.get("content"));
if (!content) continue;
for (const attr of ["property", "name"]) {
const key = meta.get(attr)?.trim().toLowerCase();
if (key && !metas.has(key)) metas.set(key, content);
}
}
return {
meta: (key) => metas.get(key.toLowerCase()),
title: nonBlank(scan.title && decodeHtmlEntities(scan.title)),
jsonLd: jsonLdOf(scan.jsonLd, pageUrl),
themeColor: themeColorOf(scan.metas),
maskIconColor: maskIconColorOf(scan.links),
bestIcon: bestIconOf(scan.links, pageUrl),
largeImage: largeImageOf(scan.imgs, pageUrl),
firstContentImage: firstContentImageOf(scan.imgs, pageUrl),
};
}
/**
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
* common named color; undefined for any other form. The generic preview
* stores it as the preview's `accentColor`, a tint over the app's own surface,
* so alpha is dropped, and a fully transparent one is no color at all.
*/
export function normalizeCssColor(raw: string | undefined): string | undefined {
const color = raw ? parseCssColor(raw) : undefined;
return color && color.a > 0 ? hexOf(color) : undefined;
}

46
api/remote/providers/bandcamp.ts
View File

@@ -1,15 +1,35 @@
/**
* Previews Bandcamp links from their page's OG tags, for
* `rich-content-service.ts`.
* Previews Bandcamp links from their page, for `rich-content-service.ts`.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage } from "../fetch.ts";
import { BANDCAMP_HOST_RE } from "../bandcamp-tralbum.ts";
import { genericPreview } from "./generic.ts";
/**
* Returns the embed player's URL a page's `og:video` names, over https; only
* a player on bandcamp.com counts, since the frontend frames it.
*/
function embeddedPlayerUrl(raw: string | undefined): string | undefined {
if (!raw) return undefined;
try {
const url = new URL(raw);
if (
(url.protocol !== "https:" && url.protocol !== "http:") ||
url.hostname !== "bandcamp.com" ||
!url.pathname.startsWith("/EmbeddedPlayer/")
) return undefined;
url.protocol = "https:";
return url.href;
} catch {
return undefined;
}
}
/**
* Claims bandcamp.com and its subdomains, where artists have their pages, and
@@ -29,24 +49,16 @@ export const bandcampProvider: RichContentProvider = {
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
const fetched = await fetchPage(url);
if (fetched.html === undefined) {
return { type: "bandcamp", siteName: "Bandcamp", url };
}
const html = await readTextCapped(res);
const page = parsePage(fetched.html, fetched.url);
return {
...genericPreview(page, url, fetched.url),
type: "bandcamp",
siteName: "Bandcamp",
url,
title: extractOgTag(html, "title"),
description: extractOgTag(html, "description"),
thumbnailUrl: extractOgTag(html, "image"),
embedUrl: extractOgTag(html, "video") ?? undefined,
embedUrl: embeddedPlayerUrl(page.meta("og:video")),
};
},
};

176
api/remote/providers/generic.ts
View File

@@ -1,6 +1,8 @@
/**
* Previews any page no other provider claims, for `rich-content-service.ts`,
* from whatever metadata its HTML carries.
* from whatever metadata its HTML carries. Its reading of a page,
* {@link genericPreview}, is also the base of the Bandcamp, SoundCloud and
* YouTube channel previews.
*
* @module
*/
@@ -9,25 +11,97 @@ import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { siteHost } from "../../../shared/url.ts";
import {
extractBestIcon,
extractFirstContentImage,
extractJsonLd,
extractLargeImage,
extractMaskIconColor,
extractMetaName,
extractOgTag,
extractPageTitle,
extractThemeColor,
absoluteHttpUrl,
normalizeCssColor,
type Page,
parsePage,
} from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { fetchPage } from "../fetch.ts";
/** Returns the path of `url` without its trailing slashes, `/` for the root. */
function pathOf(url: string): string {
return new URL(url).pathname.replace(/\/+$/, "") || "/";
}
/**
* Matches every URL, so it comes last in the service's list. Each field falls
* back from OG tags to Twitter tags, JSON-LD and the markup itself. The icon
* and brand color feed the placeholder the frontend draws when there is no
* thumbnail.
* Strips a trailing ` | Site`, ` – Site` or the like from a `<title>`, which
* names the page and then its site; the site is in the preview's badge.
*/
function withoutSiteSuffix(title: string, siteName: string): string {
const escaped = siteName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const suffix = new RegExp(`\\s+[|–—\\-·•]\\s+${escaped}\\s*$`, "i");
return title.replace(suffix, "") || title;
}
/**
* Returns the preview of the page `url` that `page` read, served from
* `finalUrl` after redirects. Each field falls back from OG tags to Twitter
* tags, JSON-LD and the markup itself. The icon and brand color feed the
* placeholder the frontend draws when there is no thumbnail.
*/
export function genericPreview(
page: Page,
url: string,
finalUrl: string,
): RichContent {
// An og:url naming the home page from another page means the OG tags are
// the site's defaults, not this page's: their title and image are skipped.
// Any other og:url is the page's canonical address, often with a slug the
// link left out, and its tags are the page's.
const ogUrl = page.meta("og:url");
let useOg = true;
try {
useOg = !ogUrl || pathOf(new URL(ogUrl, finalUrl).href) !== "/" ||
pathOf(finalUrl) === "/";
} catch { /* an og:url that doesn't parse names no other page */ }
const ogSiteName = page.meta("og:site_name");
const pageTitle = page.title && ogSiteName
? withoutSiteSuffix(page.title, ogSiteName)
: page.title;
const title = (useOg ? page.meta("og:title") : undefined) ??
page.meta("twitter:title") ??
page.jsonLd.title ??
pageTitle;
const description = page.meta("og:description") ??
page.meta("twitter:description") ??
page.jsonLd.description ??
page.meta("description");
// Never the favicon: a 16×16 icon cropped into a thumbnail looks broken,
// and the placeholder drawn instead shows the icon properly. A tag naming
// no usable URL falls through to the next.
const thumbnailUrl =
(useOg ? absoluteHttpUrl(page.meta("og:image"), finalUrl) : undefined) ??
absoluteHttpUrl(page.meta("twitter:image"), finalUrl) ??
page.jsonLd.thumbnailUrl ??
page.largeImage ??
page.firstContentImage;
// `/favicon.ico` is a guess. When it doesn't load, the placeholder shows
// the site's initial.
const faviconUrl = page.bestIcon ?? `${new URL(finalUrl).origin}/favicon.ico`;
const accentColor = normalizeCssColor(
page.themeColor ??
page.meta("msapplication-TileColor") ??
page.maskIconColor,
);
return {
type: "generic",
url,
title,
description,
thumbnailUrl,
faviconUrl,
accentColor,
siteName: ogSiteName ?? siteHost(url),
};
}
/** Matches every URL, so it comes last in the service's list. */
export const genericProvider: RichContentProvider = {
name: "generic",
@@ -36,72 +110,12 @@ export const genericProvider: RichContentProvider = {
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "generic", url };
}
const html = await readTextCapped(res);
const ld = extractJsonLd(html);
// An og:url naming another page (often the home page) means the OG tags
// are the site's defaults, not this page's: its title and image are
// skipped.
const ogUrl = extractOgTag(html, "url");
const useOg = !ogUrl || (() => {
try {
const ogPath = new URL(ogUrl).pathname.replace(/\/+$/, "") || "/";
const pagePath = new URL(url).pathname.replace(/\/+$/, "") || "/";
return ogPath === pagePath;
} catch {
return true;
}
})();
const title = (useOg ? extractOgTag(html, "title") : undefined) ??
extractMetaName(html, "twitter:title") ??
ld.title ??
extractPageTitle(html);
const siteName = extractOgTag(html, "site_name") ??
siteHost(url);
const description = extractOgTag(html, "description") ??
extractMetaName(html, "twitter:description") ??
ld.description ??
extractMetaName(html, "description");
// Never the favicon: a 16×16 icon cropped into a thumbnail looks broken,
// and the placeholder drawn instead shows the icon properly.
const thumbnailUrl = (useOg ? extractOgTag(html, "image") : undefined) ??
extractMetaName(html, "twitter:image") ??
ld.thumbnailUrl ??
extractLargeImage(html, url) ??
extractFirstContentImage(html, url);
// `/favicon.ico` is a guess. When it doesn't load, the placeholder shows
// the site's initial.
const faviconUrl = extractBestIcon(html, url) ??
`${new URL(url).origin}/favicon.ico`;
const accentColor = normalizeCssColor(
extractThemeColor(html) ??
extractMetaName(html, "msapplication-TileColor") ??
extractMaskIconColor(html),
);
return {
type: "generic",
const fetched = await fetchPage(url);
if (fetched.html === undefined) return { type: "generic", url };
return genericPreview(
parsePage(fetched.html, fetched.url),
url,
title,
description,
thumbnailUrl,
faviconUrl,
accentColor,
siteName,
};
fetched.url,
);
},
};

7
api/remote/providers/provider.ts
View File

@@ -18,9 +18,10 @@ export interface RichContentProvider {
matches(url: string): boolean;
/**
* Returns the preview of `url`, which ends up in the dump's `rich_content` or
* the create form's preview. It rejects when the page can't be fetched, and
* the service then falls back to a stub. A page that isn't HTML yields a
* preview without metadata.
* the create form's preview. It rejects when the page can't be fetched or
* answers with an error status, and the service then falls back to a stub.
* A page that isn't HTML yields a preview without metadata. The service
* tidies what it returns, so a provider leaves text as the page wrote it.
*/
fetch(url: string): Promise<RichContent>;
}

5
api/remote/providers/self.ts
View File

@@ -1,7 +1,8 @@
/**
* Previews links to this site's own dump, user and playlist pages, as
* `parsePagePath` reads them, from the database, for `rich-content-service.ts`. No request goes out, so this works
* even when the site runs on a private host that `fetchWithTimeout` refuses.
* `parsePagePath` reads them, from the database, for
* `rich-content-service.ts`. No request goes out, so this works even when the
* site runs on a private host that `fetchWithTimeout` refuses.
*
* @module
*/

66
api/remote/providers/soundcloud.ts
View File

@@ -1,50 +1,72 @@
/**
* Previews SoundCloud links from their page's OG tags, for
* `rich-content-service.ts`.
* Previews SoundCloud links from their page, for `rich-content-service.ts`.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage } from "../fetch.ts";
import { genericPreview } from "./generic.ts";
/** SoundCloud's own hosts, the short links' `on.` included. */
const SOUNDCLOUD_HOSTS = new Set([
"soundcloud.com",
"www.soundcloud.com",
"m.soundcloud.com",
"on.soundcloud.com",
]);
/**
* Claims soundcloud.com only, not www. or m.soundcloud.com, and builds the
* embed player's URL from the link itself.
* Returns the soundcloud.com address of the page, which the embed player
* takes: its `og:url`, or else the URL it was served from. A short link only
* redirects, so it is never the answer; tracking parameters are dropped.
*/
function pageAddress(ogUrl: string | undefined, finalUrl: string): string {
for (const raw of [ogUrl, finalUrl]) {
if (!raw) continue;
try {
const url = new URL(raw);
if (
!SOUNDCLOUD_HOSTS.has(url.hostname) ||
url.hostname === "on.soundcloud.com"
) continue;
return `https://soundcloud.com${url.pathname}`;
} catch {
continue;
}
}
return finalUrl;
}
/**
* Claims SoundCloud's hosts, short links included, and builds the embed
* player's URL from the page's address.
*/
export const soundcloudProvider: RichContentProvider = {
name: "soundcloud",
matches(url: string): boolean {
try {
return new URL(url).hostname === "soundcloud.com";
return SOUNDCLOUD_HOSTS.has(new URL(url).hostname);
} catch {
return false;
}
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "soundcloud", siteName: "SoundCloud", url };
}
const html = await readTextCapped(res);
const fetched = await fetchPage(url);
const page = fetched.html === undefined
? undefined
: parsePage(fetched.html, fetched.url);
const address = pageAddress(page?.meta("og:url"), fetched.url);
return {
...(page ? genericPreview(page, url, fetched.url) : { url }),
type: "soundcloud",
siteName: "SoundCloud",
url,
title: extractOgTag(html, "title"),
description: extractOgTag(html, "description"),
thumbnailUrl: extractOgTag(html, "image"),
embedUrl: `https://w.soundcloud.com/player/?url=${
encodeURIComponent(url)
encodeURIComponent(address)
}&visual=true&auto_play=false`,
};
},

30
api/remote/providers/youtube.ts
View File

@@ -1,17 +1,18 @@
/**
* Previews YouTube videos, playlists and channels, for
* `rich-content-service.ts`. Videos and playlists are read from YouTube's
* oEmbed endpoint, channels from their page's OG tags.
* oEmbed endpoint, channels from their page.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage, fetchWithTimeout } from "../fetch.ts";
import { youtubeHost, youtubeVideoId } from "../../lib/canonical-url.ts";
import { siteHostOf } from "../../../shared/url.ts";
import { genericPreview } from "./generic.ts";
/** Returns the id of the video `url` points at, in any shape YouTube uses. */
function extractVideoId(url: string): string | null {
@@ -123,27 +124,17 @@ export const youtubeProvider: RichContentProvider = {
if (channelPath && !videoId) {
// oEmbed doesn't take channel URLs.
let title: string | undefined;
let thumbnailUrl: string | undefined;
let channel: RichContent | undefined;
try {
const res = await fetchWithTimeout(url);
if (res.ok) {
const html = await readTextCapped(res);
title = extractOgTag(html, "title");
thumbnailUrl = extractOgTag(html, "image");
} else {
await res.body?.cancel();
const fetched = await fetchPage(url);
if (fetched.html !== undefined) {
const page = parsePage(fetched.html, fetched.url);
channel = genericPreview(page, url, fetched.url);
}
} catch {
// YouTube unreachable: the preview goes without.
}
return {
type: "youtube",
siteName: "YouTube",
url,
title,
thumbnailUrl,
};
return { ...channel, type: "youtube", siteName: "YouTube", url };
}
if (listId && !videoId) {
@@ -172,7 +163,6 @@ export const youtubeProvider: RichContentProvider = {
type: "youtube",
siteName: "YouTube",
url,
videoId: videoId!,
title,
thumbnailUrl,
embedUrl: `https://www.youtube.com/embed/${videoId}?${embedParams}`,

61
api/services/rich-content-service.ts
View File

@@ -84,6 +84,65 @@ export function tryFetchRichContent(
return result;
}
/** The longest title a preview keeps, in characters. */
const MAX_TITLE_LENGTH = 300;
/**
* The longest description a preview keeps, in characters: it is stored with
* the dump, searched and sent with every feed, and some pages put a whole
* article in theirs.
*/
const MAX_DESCRIPTION_LENGTH = 1000;
/**
* Returns `text` with its whitespace collapsed, cut to `max` characters with
* an ellipsis; undefined when nothing is left.
*/
function tidyText(
text: string | undefined,
max: number,
): string | undefined {
const collapsed = text?.replace(/\s+/g, " ").trim();
if (!collapsed) return undefined;
const chars = [...collapsed];
return chars.length > max
? chars.slice(0, max - 1).join("").trimEnd() + "…"
: collapsed;
}
/** Returns `url` if it is an absolute http(s) URL, and undefined otherwise. */
function httpUrl(url: string | undefined): string | undefined {
if (!url) return undefined;
try {
const { protocol } = new URL(url);
return protocol === "http:" || protocol === "https:" ? url : undefined;
} catch {
return undefined;
}
}
/**
* Applies the rules every preview follows, whichever provider made it: text
* on one line and of bounded length, and only http(s) URLs for the images and
* the player, which the frontend loads.
*/
function tidy(content: RichContent): RichContent {
const tidied: RichContent = {
...content,
title: tidyText(content.title, MAX_TITLE_LENGTH),
description: tidyText(content.description, MAX_DESCRIPTION_LENGTH),
siteName: tidyText(content.siteName, MAX_TITLE_LENGTH),
thumbnailUrl: httpUrl(content.thumbnailUrl),
faviconUrl: httpUrl(content.faviconUrl),
embedUrl: httpUrl(content.embedUrl),
};
// Stored as JSON: a field left undefined would only be dropped there.
for (const key of Object.keys(tidied) as (keyof RichContent)[]) {
if (tidied[key] === undefined) delete tidied[key];
}
return tidied;
}
/**
* Fetches the preview of `url` through the first provider that matches, or
* makes the failure stub when the provider rejects.
@@ -93,7 +152,7 @@ async function fetchRichContentUncached(
): Promise<FetchRichContentResult> {
try {
const provider = providers.find((p) => p.matches(url))!;
return { ok: true, content: await provider.fetch(url) };
return { ok: true, content: tidy(await provider.fetch(url)) };
} catch (err) {
console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err);
const siteName = siteHost(url);

33
api/test/remote/fetch.test.ts
View File

@@ -1,6 +1,7 @@
import { stubNetwork } from "../helpers.ts";
import { html, stubNetwork } from "../helpers.ts";
import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert";
import {
fetchPage,
fetchWithTimeout,
isValidHttpUrl,
readTextCapped,
@@ -110,3 +111,33 @@ Deno.test("readTextCapped: decodes the charset the header or a <meta> declares",
"caf<61>",
);
});
Deno.test("fetchPage: the page and where it was served from", async () => {
using _net = stubNetwork((url) => {
if (url.pathname === "/moved") {
return new Response(null, {
status: 301,
headers: { location: "https://www.example.com/here" },
});
}
if (url.pathname === "/gone") return html("<title>Gone</title>", 410);
if (url.pathname === "/data") {
return new Response("{}", {
headers: { "content-type": "application/json" },
});
}
return html("<title>Here</title>");
});
assertEquals(await fetchPage("https://example.com/moved"), {
url: "https://www.example.com/here",
html: "<title>Here</title>",
});
assertEquals(await fetchPage("https://example.com/data"), {
url: "https://example.com/data",
});
await assertRejects(
() => fetchPage("https://example.com/gone"),
Error,
"410",
);
});

228
api/test/remote/html.test.ts
View File

@@ -1,68 +1,122 @@
import "../setup.ts";
import { assertEquals } from "@std/assert";
import {
extractBestIcon,
extractFirstContentImage,
extractJsonLd,
extractLargeImage,
extractMaskIconColor,
extractMetaName,
extractOgTag,
extractPageTitle,
extractThemeColor,
absoluteHttpUrl,
decodeHtmlEntities,
normalizeCssColor,
parsePage,
} from "../../remote/html.ts";
const BASE = "https://example.com/articles/1";
Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => {
const page = `
const read = (html: string) => parsePage(html, BASE);
Deno.test("parsePage meta: either attribute, either order, entities decoded", () => {
const page = read(`
<meta property="og:title" content="Tom &amp; Jerry &#8212; &#x2764;">
<meta content='Desc' property='og:description'>
<meta name="twitter:title" content="TW">
<meta name="og:title" content="wrong attribute">`;
assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤");
assertEquals(extractOgTag(page, "description"), "Desc");
assertEquals(extractOgTag(page, "image"), undefined);
assertEquals(extractMetaName(page, "twitter:title"), "TW");
<meta name="og:image" content="https://example.com/by-name.jpg">
<meta property="og:title" content="second, ignored">`);
assertEquals(page.meta("og:title"), "Tom & Jerry — ❤");
assertEquals(page.meta("og:description"), "Desc");
assertEquals(page.meta("twitter:title"), "TW");
// Sites write og: tags under `name` too.
assertEquals(page.meta("og:image"), "https://example.com/by-name.jpg");
assertEquals(page.meta("OG:Title"), "Tom & Jerry — ❤");
assertEquals(page.meta("og:url"), undefined);
});
Deno.test("extractOgTag: content never spans into the next tag", () => {
const page = `<meta property="og:title"><meta name="x" content="leak">`;
assertEquals(extractOgTag(page, "title"), undefined);
Deno.test("parsePage meta: attributes read as a browser reads them", () => {
const page = read(`
<meta property=og:title content=Unquoted>
<meta property="og:description" content="a > b, still the tag">
<meta data-property="og:image" content="https://example.com/no.jpg">
<meta property="og:site_name"><meta name="x" content="leak">
<meta property="og:type" content=" ">`);
assertEquals(page.meta("og:title"), "Unquoted");
assertEquals(page.meta("og:description"), "a > b, still the tag");
assertEquals(page.meta("og:image"), undefined);
assertEquals(page.meta("og:site_name"), undefined);
// A blank value is no value, so a fallback can take over.
assertEquals(page.meta("og:type"), undefined);
});
Deno.test("extractPageTitle", () => {
Deno.test("parsePage: tags in comments and scripts aren't read", () => {
const page = read(`
<!-- <meta property="og:title" content="commented"> -->
<script>document.write('<meta property="og:title" content="scripted">')</script>
<meta property="og:title" content="Real">`);
assertEquals(page.meta("og:title"), "Real");
// An unclosed comment runs to the end of the page.
assertEquals(
extractPageTitle("<title lang=en> A &lt;b&gt; </title>"),
"A <b>",
read(`<!-- <meta property="og:title" content="x">`).meta("og:title"),
undefined,
);
assertEquals(extractPageTitle("<p>none</p>"), undefined);
});
Deno.test("extractThemeColor: unscoped, then light, then any", () => {
Deno.test("parsePage title", () => {
assertEquals(read("<title lang=en> A &lt;b&gt; </title>").title, "A <b>");
assertEquals(read("<title> </title>").title, undefined);
assertEquals(read("<p>none</p>").title, undefined);
});
Deno.test("decodeHtmlEntities: one pass, the names pages use, nothing thrown", () => {
assertEquals(
extractThemeColor(`
decodeHtmlEntities("Caf&eacute; &Agrave; l&rsquo;&eacute;t&eacute;"),
"Café À l’été",
);
assertEquals(
decodeHtmlEntities("&laquo;&nbsp;Oui&hellip;&raquo;"),
"« Oui…»",
);
assertEquals(decodeHtmlEntities("&ccedil;a &AMP; &Ntilde;"), "ça & Ñ");
// `&amp;quot;` is a literal `&quot;` on the page.
assertEquals(decodeHtmlEntities("&amp;quot;"), "&quot;");
// No such code point or name: left as written.
assertEquals(decodeHtmlEntities("&#99999999;"), "&#99999999;");
assertEquals(decodeHtmlEntities("&#xD800;"), "&#xD800;");
assertEquals(decodeHtmlEntities("&bogus;"), "&bogus;");
});
Deno.test("absoluteHttpUrl", () => {
assertEquals(absoluteHttpUrl("/og.png", BASE), "https://example.com/og.png");
assertEquals(
absoluteHttpUrl("//cdn.test/a.jpg", BASE),
"https://cdn.test/a.jpg",
);
assertEquals(
absoluteHttpUrl(" og.png ", BASE),
"https://example.com/articles/og.png",
);
assertEquals(absoluteHttpUrl("javascript:alert(1)", BASE), undefined);
assertEquals(absoluteHttpUrl("data:image/png;base64,xx", BASE), undefined);
assertEquals(absoluteHttpUrl("http://[bad", BASE), undefined);
assertEquals(absoluteHttpUrl("", BASE), undefined);
});
Deno.test("parsePage themeColor: unscoped, then light, then any", () => {
assertEquals(
read(`
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
<meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`),
<meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`)
.themeColor,
"#fff",
);
assertEquals(
extractThemeColor(`
read(`
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
<meta name="theme-color" content="#123">`),
<meta name="theme-color" content="#123">`).themeColor,
"#123",
);
assertEquals(
extractThemeColor(
read(
`<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
),
).themeColor,
"#000",
);
assertEquals(
extractMaskIconColor(
`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`,
),
read(`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`).maskIconColor,
"#5bbad5",
);
});
@@ -82,78 +136,106 @@ Deno.test("normalizeCssColor", () => {
assertEquals(normalizeCssColor(undefined), undefined);
});
Deno.test("extractJsonLd: @graph, arrays and image shapes", () => {
Deno.test("parsePage jsonLd: @graph, arrays and image shapes", () => {
const ld = (data: unknown) =>
`<script type="application/ld+json">${JSON.stringify(data)}</script>`;
assertEquals(
extractJsonLd(
ld({
"@graph": [{ "@type": "WebSite" }, {
headline: "Head",
image: { url: "https://i/1.jpg" },
}],
}),
),
read(ld({
"@graph": [{ "@type": "WebSite" }, {
headline: "Head",
image: { url: "https://i/1.jpg" },
}],
})).jsonLd,
{ title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
);
assertEquals(
extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl,
read(ld({ name: ["N"], image: ["https://i/2.jpg"] })).jsonLd.thumbnailUrl,
"https://i/2.jpg",
);
assertEquals(
extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl,
undefined,
read(ld({ name: "N", image: "/relative.jpg" })).jsonLd.thumbnailUrl,
"https://example.com/relative.jpg",
);
// Invalid JSON is skipped for the next block.
assertEquals(
extractJsonLd(
read(
`<script type="application/ld+json">{oops</script>${
ld({ name: "Second" })
}`,
).title,
).jsonLd.title,
"Second",
);
assertEquals(extractJsonLd("<p></p>"), {});
assertEquals(read("<p></p>").jsonLd, {});
});
Deno.test("image extraction: large images, content images, relative URLs", () => {
const page = `
Deno.test("parsePage jsonLd: the page's node beats the site's", () => {
const ld = (data: unknown) =>
`<script type="application/ld+json">${JSON.stringify(data)}</script>`;
const page = read(
ld({ "@type": "Organization", name: "Acme", logo: "https://a/l.png" }) +
ld({
"@graph": [
{ "@type": ["WebSite"], name: "Acme site" },
{ "@type": "NewsArticle", headline: "The news" },
],
}),
);
assertEquals(page.jsonLd.title, "The news");
// With nothing else, the site's node still names the page.
assertEquals(
read(ld({ "@type": "Organization", name: "Acme" })).jsonLd.title,
"Acme",
);
});
Deno.test("parsePage images: large images, content images, relative URLs", () => {
const page = read(`
<img src="data:image/png;base64,xx" width="500" height="500">
<img src="/small.jpg" width="50" height="50">
<img src="/pixel.gif" width="1" height="1">
<img src="/small.jpg" width="60" height="60">
<img src="/img/site-logo.png">
<img src="/icons/sprite.svg">
<img src="/photos/cat.jpg" width="640" height="480">`;
<img src="/photos/cat.jpg" width="640" height="480">`);
assertEquals(page.largeImage, "https://example.com/photos/cat.jpg");
assertEquals(page.firstContentImage, "https://example.com/small.jpg");
assertEquals(
extractLargeImage(page, BASE),
"https://example.com/photos/cat.jpg",
);
assertEquals(
extractFirstContentImage(page, BASE),
"https://example.com/small.jpg",
);
assertEquals(
extractFirstContentImage(`<img src="//cdn.test/a.jpg">`, BASE),
read(`<img src="//cdn.test/a.jpg">`).firstContentImage,
"https://cdn.test/a.jpg",
);
});
Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
Deno.test("parsePage images: lazy images by their data-src", () => {
const page = read(`
<img data-src="/wrong.jpg" src="/img/lazy-placeholder.gif">
<img src="data:image/gif;base64,R0lGOD" data-src="/real.jpg" width="800" height="600">`);
assertEquals(page.largeImage, "https://example.com/real.jpg");
assertEquals(page.firstContentImage, "https://example.com/real.jpg");
});
Deno.test("parsePage bestIcon: SVG beats everything, else the largest", () => {
assertEquals(
extractBestIcon(
`<link rel="icon" href="/16.png" sizes="16x16">
read(`<link rel="icon" href="/16.png" sizes="16x16">
<link rel="apple-touch-icon" href="/180.png" sizes="180x180">
<link rel="stylesheet" href="/x.css">`,
BASE,
),
<link rel="stylesheet" href="/x.css">`).bestIcon,
"https://example.com/180.png",
);
assertEquals(
extractBestIcon(
`<link rel="icon" href="/180.png" sizes="180x180">
<link rel="icon" type="image/svg+xml" href="/icon">`,
BASE,
),
read(`<link rel="icon" href="/180.png" sizes="180x180">
<link rel="icon" type="image/svg+xml" href="/icon">`).bestIcon,
"https://example.com/icon",
);
assertEquals(extractBestIcon("<p></p>", BASE), undefined);
assertEquals(
read(`<link rel="shortcut icon" href="/f.ico" sizes="16x16 48x48">
<link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
"https://example.com/f.ico",
);
assertEquals(read("<p></p>").bestIcon, undefined);
});
Deno.test("parsePage bestIcon: a mask icon is never the icon", () => {
assertEquals(
read(`<link rel="mask-icon" href="/pinned.svg" color="#000">
<link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
"https://example.com/32.png",
);
});

12
api/test/remote/providers/bandcamp.test.ts
View File

@@ -20,3 +20,15 @@ Deno.test("bandcamp provider: bandcamp.com and its subdomains only", () => {
bandcampProvider.matches("https://bandcamp.com.evil.example/album/x"),
);
});
Deno.test("bandcamp provider: falls back like any page, embeds only Bandcamp's player", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:title" content="Album">
<meta name="description" content="An album by Artist">
<meta property="og:video" content="https://evil.example/EmbeddedPlayer/x">`)
);
const bc = await fetchRichContent("https://artist.bandcamp.com/album/b");
assertEquals(bc?.description, "An album by Artist");
assertEquals(bc?.siteName, "Bandcamp");
assertEquals(bc?.embedUrl, undefined);
});

66
api/test/remote/providers/generic.test.ts
View File

@@ -1,5 +1,5 @@
import { html, stubNetwork } from "../../helpers.ts";
import { assertEquals } from "@std/assert";
import { assert, assertEquals } from "@std/assert";
import { fetchRichContent } from "../../../services/rich-content-service.ts";
const BASE = "https://example.com/articles/1";
@@ -51,3 +51,67 @@ Deno.test("generic provider: non-HTML responses carry just the URL", async () =>
);
assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
});
Deno.test("generic provider: a canonical og:url with a slug keeps the OG tags", async () => {
using _ = stubNetwork(() =>
html(`
<meta property="og:url" content="https://example.com/q/123/how-to-x">
<meta property="og:title" content="How to X">
<meta property="og:image" content="https://example.com/x.jpg">
<title>How to X - Example</title>`)
);
const content = await fetchRichContent("https://example.com/q/123");
assertEquals(content?.title, "How to X");
assertEquals(content?.thumbnailUrl, "https://example.com/x.jpg");
});
Deno.test("generic provider: relative URLs resolve against the page after redirects", async () => {
using _ = stubNetwork((url) =>
url.hostname === "old.example"
? new Response(null, {
status: 301,
headers: { location: "https://new.example/b/page" },
})
: html(`<meta property="og:image" content="/og.png">
<meta name="twitter:image" content="https://new.example/tw.png">`)
);
const content = await fetchRichContent("https://old.example/a");
assertEquals(content?.url, "https://old.example/a");
assertEquals(content?.thumbnailUrl, "https://new.example/og.png");
assertEquals(content?.faviconUrl, "https://new.example/favicon.ico");
});
Deno.test("generic provider: an og:image that isn't http(s) falls through", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:image" content="javascript:alert(1)">
<meta name="twitter:image" content="//cdn.example/tw.png">`)
);
assertEquals(
(await fetchRichContent(BASE))?.thumbnailUrl,
"https://cdn.example/tw.png",
);
});
Deno.test("generic provider: a <title> loses its site suffix", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:site_name" content="Le Monde">
<title>Un article &eacute;crit — Le Monde</title>`)
);
const content = await fetchRichContent(BASE);
assertEquals(content?.title, "Un article écrit");
assertEquals(content?.siteName, "Le Monde");
});
Deno.test("generic provider: text on one line, and of bounded length", async () => {
using _ = stubNetwork(() =>
html(`<title>
A title
on two lines
</title>
<meta name="description" content="${"word ".repeat(400)}">`)
);
const content = await fetchRichContent(BASE);
assertEquals(content?.title, "A title on two lines");
assertEquals([...content!.description!].length, 1000);
assert(content!.description!.endsWith("word…"));
});

65
api/test/remote/providers/soundcloud.test.ts
View File

@@ -1,16 +1,69 @@
import { html, stubNetwork } from "../../helpers.ts";
import { assertEquals, assertStringIncludes } from "@std/assert";
import { assert, assertEquals, assertFalse } from "@std/assert";
import { fetchRichContent } from "../../../services/rich-content-service.ts";
import { soundcloudProvider } from "../../../remote/providers/soundcloud.ts";
const embedOf = (url: string) =>
`https://w.soundcloud.com/player/?url=${
encodeURIComponent(url)
}&visual=true&auto_play=false`;
Deno.test("soundcloud provider", async () => {
using _ = stubNetwork(() =>
html(`<meta property="og:title" content="Album">
<meta property="og:video" content="https://bandcamp.com/EmbeddedPlayer/album=1">`)
html(`<meta property="og:title" content="Track">
<meta property="og:image" content="https://i1.sndcdn.com/a.jpg">
<meta name="description" content="Listen to Track">`)
);
const sc = await fetchRichContent("https://soundcloud.com/artist/track");
assertEquals(sc?.type, "soundcloud");
assertStringIncludes(
sc?.embedUrl ?? "",
encodeURIComponent("https://soundcloud.com/artist/track"),
assertEquals(sc?.siteName, "SoundCloud");
assertEquals(sc?.title, "Track");
assertEquals(sc?.description, "Listen to Track");
assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/track"));
});
Deno.test("soundcloud provider: a short link embeds the page it leads to", async () => {
using _ = stubNetwork((url) =>
url.hostname === "on.soundcloud.com"
? new Response(null, {
status: 302,
headers: {
location: "https://m.soundcloud.com/artist/track?si=abc&utm_source=x",
},
})
: html(`<meta property="og:title" content="Track">`)
);
const sc = await fetchRichContent("https://on.soundcloud.com/AbCd");
assertEquals(sc?.type, "soundcloud");
assertEquals(sc?.url, "https://on.soundcloud.com/AbCd");
assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/track"));
});
Deno.test("soundcloud provider: the page's og:url names what is embedded", async () => {
using _ = stubNetwork(() =>
html(
`<meta property="og:url" content="https://soundcloud.com/artist/real">`,
)
);
const sc = await fetchRichContent("https://soundcloud.com/artist/real?in=x");
assertEquals(sc?.embedUrl, embedOf("https://soundcloud.com/artist/real"));
});
Deno.test("soundcloud provider: SoundCloud's hosts only", () => {
for (
const url of [
"https://soundcloud.com/a/t",
"https://www.soundcloud.com/a/t",
"https://m.soundcloud.com/a/t",
"https://on.soundcloud.com/AbCd",
]
) {
assert(soundcloudProvider.matches(url), url);
}
assertFalse(
soundcloudProvider.matches("https://soundcloud.com.evil.example/"),
);
assertFalse(
soundcloudProvider.matches("https://api.soundcloud.com/tracks/1"),
);
});

5
api/test/remote/providers/youtube.test.ts
View File

@@ -13,7 +13,6 @@ Deno.test("youtube provider: video with start time and playlist", async () => {
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL123&t=1m30s",
);
assertEquals(content?.type, "youtube");
assertEquals(content?.videoId, "dQw4w9WgXcQ");
assertEquals(content?.title, "Never Gonna");
assertEquals(
content?.thumbnailUrl,
@@ -41,8 +40,8 @@ Deno.test("youtube provider: youtu.be, shorts, playlists and channels", async ()
"https://www.youtube.com/embed/abc123?rel=0&start=42",
);
assertEquals(
(await fetchRichContent("https://youtube.com/shorts/xyz"))?.videoId,
"xyz",
(await fetchRichContent("https://youtube.com/shorts/xyz"))?.embedUrl,
"https://www.youtube.com/embed/xyz?rel=0",
);
const list = await fetchRichContent(
"https://www.youtube.com/playlist?list=PLabc",

12
api/test/services/rich-content-service.test.ts
View File

@@ -41,3 +41,15 @@ Deno.test("tryFetchRichContent: an unreachable page is ok:false with a stub", as
},
});
});
Deno.test("tryFetchRichContent: an error page is unreachable, not a preview", async () => {
using net = stubNetwork(() => html(`<title>404 Not Found</title>`, 404));
const url = "https://example.com/gone";
assertEquals(await tryFetchRichContent(url), {
ok: false,
content: { type: "generic", url, siteName: "example.com" },
});
// Not kept: the page may be back next time.
await tryFetchRichContent(url);
assertEquals(net.calls.length, 2);
});

19
shared/dump.ts
View File

@@ -7,17 +7,20 @@
* @module
*/
/**
* Names the kind of link a {@link RichContent} previews: a site the player
* plays its own way, or `generic` for any other page, this site's included.
*/
export type RichContentType = "youtube" | "bandcamp" | "soundcloud" | "generic";
/**
* Built by a provider of `rich-content-service.ts` when a link is posted or
* previewed, and stored with the dump as JSON. The frontend's
* `RichContentCard` and player read it.
*/
export interface RichContent {
/**
* The provider that read the page: `youtube`, `bandcamp`, `soundcloud` or
* `generic`. The player picks how to play the link by it.
*/
type: string;
/** The player picks how to play the link by it. */
type: RichContentType;
url: string;
siteName?: string;
title?: string;
@@ -32,8 +35,10 @@ export interface RichContent {
faviconUrl?: string;
/** The page's declared brand color, normalized to `#rrggbb`. */
accentColor?: string;
/** Set for a YouTube link only. */
videoId?: string;
/**
* The player a YouTube, Bandcamp or SoundCloud link plays in, framed by the
* frontend.
*/
embedUrl?: string;
}

2
src/test/components/dump/JournalCard.test.tsx
View File

@@ -21,7 +21,7 @@ describe("JournalCard", () => {
<JournalCard
dump={makeDump({
richContent: {
type: "link",
type: "generic",
url: "u",
thumbnailUrl: "https://i.test/a.png",
},

7
src/test/components/dump/RichContentCard.test.tsx
View File

@@ -3,6 +3,7 @@ import { screen } from "@testing-library/react";
import { userEvent } from "@testing-library/user-event";
import { RichContentCard } from "../../../components/dump/RichContentCard.tsx";
import { renderWithContexts } from "../../render.tsx";
import type { RichContent } from "../../../../shared/dump.ts";
describe("RichContentCard", () => {
const yt = {
@@ -13,7 +14,7 @@ describe("RichContentCard", () => {
description: "About it",
thumbnailUrl: "https://i.ytimg.com/x.jpg",
embedUrl: "https://www.youtube.com/embed/x",
};
} satisfies RichContent;
test("full card: play button, source link and details", async () => {
const play = vi.fn();
@@ -52,7 +53,7 @@ describe("RichContentCard", () => {
renderWithContexts(
<RichContentCard
richContent={{
type: "link",
type: "generic",
url: "https://blog.test/p",
siteName: "Blog",
}}
@@ -85,7 +86,7 @@ describe("RichContentCard", () => {
type: "bandcamp",
url: "https://a.bandcamp.com/album/x",
embedUrl: "https://bandcamp.com/EmbeddedPlayer/1",
};
} satisfies RichContent;
renderWithContexts(<RichContentCard richContent={bc} compact />, {
player: {
togglePlay,

4
src/test/pages/Dump.test.tsx
View File

@@ -337,7 +337,7 @@ describe("DumpEdit page", () => {
comment: "old why",
categoryIds: [],
richContent: {
type: "link",
type: "generic",
url: "https://a.test/",
title: "Page's own title",
},
@@ -452,7 +452,7 @@ describe("DumpEdit page", () => {
wire({
...urlDump(),
richContent: {
type: "link",
type: "generic",
url: "https://a.test/",
title: "Fresh title",
description: "New blurb",

4
src/test/utils/feed/journalLayout.test.ts
View File

@@ -45,7 +45,7 @@ function feed(n: number, withArt: (i: number) => boolean): Dump[] {
// Strictly decreasing heat, so the ranking is the index order.
createdAt: new Date(Date.now() - (i + 1) * HOUR),
richContent: withArt(i)
? { type: "link", url: "https://x", thumbnailUrl: "https://x/t.png" }
? { type: "generic", url: "https://x", thumbnailUrl: "https://x/t.png" }
: undefined,
}));
}
@@ -66,7 +66,7 @@ describe("hasThumbnail / hasQuote", () => {
expect(hasThumbnail(makeDump({ thumbnailMime: "image/webp" }))).toBe(true);
expect(
hasThumbnail(
makeDump({ richContent: { type: "x", url: "u", thumbnailUrl: "t" } }),
makeDump({ richContent: { type: "generic", url: "u", thumbnailUrl: "t" } }),
),
).toBe(true);
expect(hasThumbnail(makeDump())).toBe(false);