From 2c3ba6ad1188d1de66ed6ba90845aa466c36e885 Mon Sep 17 00:00:00 2001 From: khannurien Date: Fri, 9 Oct 2026 20:46:32 +0000 Subject: [PATCH] =?UTF-8?q?v4:=20link=20previews=20are=20read=20once,=20in?= =?UTF-8?q?=20one=20place,=20and=20no=20longer=20take=20an=20error=20page,?= =?UTF-8?q?=20a=20relative=20image=20or=20a=20mask=20icon=20for=20the=20pa?= =?UTF-8?q?ge's=20=E2=80=94=20each=20provider=20fetched=20its=20page=20and?= =?UTF-8?q?=20scanned=20it=20with=20regular=20expressions=20of=20its=20own?= =?UTF-8?q?,=20the=20generic=20one=20some=20fifteen=20times,=20and=20Bandc?= =?UTF-8?q?amp=20and=20SoundCloud=20read=20only=20OG=20tags.=20fetchPage?= =?UTF-8?q?=20(remote/fetch.ts)=20now=20fetches=20every=20page,=20follows?= =?UTF-8?q?=20redirects=20to=20the=20address=20it=20was=20served=20from,?= =?UTF-8?q?=20and=20refuses=20an=20error=20status,=20so=20a=20404=20or=20a?= =?UTF-8?q?=20bot=20challenge=20is=20unreachable=20rather=20than=20a=20pre?= =?UTF-8?q?view=20titled=20"Just=20a=20moment...",=20and=20a=20refresh=20n?= =?UTF-8?q?o=20longer=20overwrites=20good=20metadata=20with=20it.=20parseP?= =?UTF-8?q?age=20(remote/html.ts)=20reads=20it=20in=20one=20linear=20pass?= =?UTF-8?q?=20with=20a=20real=20attribute=20reader:=20unquoted=20values,?= =?UTF-8?q?=20a=20>=20inside=20quotes,=20og:=20tags=20under=20name=20as=20?= =?UTF-8?q?well=20as=20property,=20and=20nothing=20from=20comments=20or=20?= =?UTF-8?q?scripts.=20Every=20URL=20a=20page=20names=20is=20resolved=20aga?= =?UTF-8?q?inst=20where=20it=20was=20served=20from,=20so=20a=20relative=20?= =?UTF-8?q?og:image=20works;=20entities=20are=20decoded=20in=20one=20pass,?= =?UTF-8?q?=20é=20and=20’=20included,=20and=20one=20that=20na?= =?UTF-8?q?mes=20no=20character=20no=20longer=20fails=20the=20preview=20(b?= =?UTF-8?q?andcamp-tralbum.ts=20shares=20the=20decoder).=20The=20mask=20ic?= =?UTF-8?q?on=20is=20no=20longer=20the=20favicon,=20JSON-LD=20prefers=20th?= =?UTF-8?q?e=20page's=20node=20over=20the=20site's,=20tracking=20pixels=20?= =?UTF-8?q?and=20lazy=20placeholders=20aren't=20thumbnails,=20an=20og:url?= =?UTF-8?q?=20with=20a=20slug=20keeps=20the=20OG=20tags,=20and=20a=20=20loses=20its=20site=20suffix.=20genericPreview=20is=20the?= =?UTF-8?q?=20base=20of=20the=20Bandcamp,=20SoundCloud=20and=20YouTube=20c?= =?UTF-8?q?hannel=20previews;=20SoundCloud=20takes=20www.,=20m.=20and=20on?= =?UTF-8?q?.soundcloud.com=20short=20links,=20embedding=20the=20page=20the?= =?UTF-8?q?y=20lead=20to,=20and=20Bandcamp=20keeps=20only=20its=20own=20pl?= =?UTF-8?q?ayer.=20The=20service=20tidies=20every=20preview:=20text=20on?= =?UTF-8?q?=20one=20line,=20titles=20and=20descriptions=20capped,=20only?= =?UTF-8?q?=20http(s)=20images=20and=20players.=20RichContent.type=20is=20?= =?UTF-8?q?now=20RichContentType,=20and=20the=20unused=20videoId=20is=20go?= =?UTF-8?q?ne.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- api/remote/bandcamp-tralbum.ts | 27 +- api/remote/fetch.ts | 72 +- api/remote/html.ts | 683 ++++++++++++------ api/remote/providers/bandcamp.ts | 46 +- api/remote/providers/generic.ts | 176 ++--- api/remote/providers/provider.ts | 7 +- api/remote/providers/self.ts | 5 +- api/remote/providers/soundcloud.ts | 66 +- api/remote/providers/youtube.ts | 30 +- api/services/rich-content-service.ts | 61 +- api/test/remote/fetch.test.ts | 33 +- api/test/remote/html.test.ts | 228 ++++-- api/test/remote/providers/bandcamp.test.ts | 12 + api/test/remote/providers/generic.test.ts | 66 +- api/test/remote/providers/soundcloud.test.ts | 65 +- api/test/remote/providers/youtube.test.ts | 5 +- .../services/rich-content-service.test.ts | 12 + shared/dump.ts | 19 +- src/test/components/dump/JournalCard.test.tsx | 2 +- .../components/dump/RichContentCard.test.tsx | 7 +- src/test/pages/Dump.test.tsx | 4 +- src/test/utils/feed/journalLayout.test.ts | 4 +- 22 files changed, 1111 insertions(+), 519 deletions(-) diff --git a/api/remote/bandcamp-tralbum.ts b/api/remote/bandcamp-tralbum.ts index 2da7c009..a8ff3991 100644 --- a/api/remote/bandcamp-tralbum.ts +++ b/api/remote/bandcamp-tralbum.ts @@ -8,6 +8,7 @@ */ import type { Tralbum, TralbumTrack } from "../../shared/player.ts"; +import { decodeHtmlEntities } from "./html.ts"; /** * Matches the hostnames `bandcamp-streams.ts` accepts: bandcamp.com and its @@ -16,30 +17,6 @@ import type { Tralbum, TralbumTrack } from "../../shared/player.ts"; */ export const BANDCAMP_HOST_RE = /(?:^|\.)bandcamp\.com$/; -/** - * Decodes HTML entities in a single pass. `html.ts` decodes `&` before - * `"`, which turns a title's literal `"` (escaped as `&quot;`) - * into a bare `"` that breaks the JSON. - */ -function decodeEntitiesOnce(input: string): string { - const named: Record = { - quot: '"', - apos: "'", - amp: "&", - lt: "<", - gt: ">", - nbsp: " ", - }; - return input.replace( - /&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|([a-zA-Z]+));/g, - (match, dec: string, hex: string, name: string) => { - if (dec) return String.fromCodePoint(Number(dec)); - if (hex) return String.fromCodePoint(parseInt(hex, 16)); - return named[name.toLowerCase()] ?? match; - }, - ); -} - /** Returns the URL of the artwork `artId`; undefined when it isn't numeric. */ function artworkUrl(artId: unknown): string | undefined { if (typeof artId !== "number" && typeof artId !== "string") return undefined; @@ -60,7 +37,7 @@ export function parseTralbum(html: string, sourceUrl: string): Tralbum | null { let raw: Record; try { - raw = JSON.parse(decodeEntitiesOnce(match[2])); + raw = JSON.parse(decodeHtmlEntities(match[2])); } catch { return null; } diff --git a/api/remote/fetch.ts b/api/remote/fetch.ts index 6109f778..f2e05ff4 100644 --- a/api/remote/fetch.ts +++ b/api/remote/fetch.ts @@ -175,7 +175,7 @@ function declaredCharset(contentType: string, bytes: Uint8Array): string { /** * Reads the body of `res` as text, up to `maxBytes`, in the charset the page - * declares; the rest is cancelled, never downloaded. The providers and the + * declares; the rest is cancelled, never downloaded. {@link fetchPage} and the * Bandcamp and YouTube resolvers read pages through it. */ export async function readTextCapped( @@ -235,6 +235,33 @@ async function fetchOnce( }); } +/** + * Fetches `url` as {@link fetchWithTimeout} does, and also returns the URL the + * last hop answered from, which {@link fetchPage} resolves the page's links + * against. + */ +async function fetchFollowing( + url: string, + timeoutMs: number, +): Promise<{ res: Response; url: URL }> { + let current = new URL(url); + for (let hop = 0;; hop++) { + if (current.protocol !== "http:" && current.protocol !== "https:") { + throw new TypeError(`Unsupported protocol ${current.protocol}`); + } + const address = await resolvePublicAddress(current); + const res = await fetchOnce(current, address, timeoutMs); + const location = res.headers.get("location"); + if (res.status < 300 || res.status >= 400 || !location) { + return { res, url: current }; + } + + await res.body?.cancel(); + if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects"); + current = new URL(location, current); + } +} + /** * Fetches a user-supplied URL, following up to five redirects, each of which * must resolve to a public address too. `timeoutMs` bounds each hop, body @@ -249,20 +276,41 @@ export async function fetchWithTimeout( url: string, timeoutMs = 5000, ): Promise { - let current = new URL(url); - for (let hop = 0;; hop++) { - if (current.protocol !== "http:" && current.protocol !== "https:") { - throw new TypeError(`Unsupported protocol ${current.protocol}`); - } - const address = await resolvePublicAddress(current); - const res = await fetchOnce(current, address, timeoutMs); - const location = res.headers.get("location"); - if (res.status < 300 || res.status >= 400 || !location) return res; + return (await fetchFollowing(url, timeoutMs)).res; +} +/** + * Returned by {@link fetchPage} to the link preview providers. + */ +export interface FetchedPage { + /** + * The URL the page was served from, after redirects: the base its relative + * links resolve against. + */ + url: string; + /** The page's text, up to {@link MAX_PAGE_BYTES}; missing when it isn't HTML. */ + html?: string; +} + +/** + * Fetches a page for its link preview, through {@link fetchWithTimeout}. A + * response that isn't HTML is cancelled unread. + * + * @throws {Error} When the page answers with an error status, so that an error + * page never becomes a preview; and as {@link fetchWithTimeout} does. + */ +export async function fetchPage(url: string): Promise { + const { res, url: finalUrl } = await fetchFollowing(url, 5000); + if (!res.ok) { await res.body?.cancel(); - if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects"); - current = new URL(location, current); + throw new Error(`${finalUrl} answered ${res.status}`); } + const contentType = res.headers.get("content-type") ?? ""; + if (!/^(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) { + await res.body?.cancel(); + return { url: finalUrl.href }; + } + return { url: finalUrl.href, html: await readTextCapped(res) }; } /** diff --git a/api/remote/html.ts b/api/remote/html.ts index 57e2f9d6..5c0a8c6a 100644 --- a/api/remote/html.ts +++ b/api/remote/html.ts @@ -1,8 +1,9 @@ /** - * Reads a link preview's metadata out of a page's HTML with regular - * expressions, without parsing the document. The generic provider uses most of - * it; the Bandcamp, SoundCloud and YouTube providers read only OG tags. Each - * function returns undefined when the page doesn't say. + * Reads a link preview's metadata out of a page's HTML. {@link parsePage} + * scans the page once for the tags that carry it (``, ``, ``, + * `` and JSON-LD scripts) without building a document, and returns a + * {@link Page} that the providers' `genericPreview` picks from. Every field is + * undefined when the page doesn't say. * * @module */ @@ -10,71 +11,220 @@ import { hexOf, parseCssColor } from "../lib/css-color.ts"; /** - * Decodes the five XML entities and numeric ones in `str`. `&` goes first, - * so `&quot;` decodes twice; `bandcamp-tralbum.ts`, which decodes JSON, - * has its own single-pass decoder. + * The named entities decoded besides numeric ones: the XML five, and those + * page titles and descriptions actually use, Latin-1 letters and typography. */ -function decodeHtmlEntities(str: string): string { - return str - .replace(/&/gi, "&") - .replace(/</gi, "<") - .replace(/>/gi, ">") - .replace(/"/gi, '"') - .replace(/'/gi, "'") - .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec))) - .replace( - /&#x([0-9a-f]+);/gi, - (_, hex) => String.fromCodePoint(parseInt(hex, 16)), - ); -} +const NAMED_ENTITIES: Record<string, string> = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", + aelig: "æ", + AElig: "Æ", + oelig: "œ", + OElig: "Œ", + szlig: "ß", + oslash: "ø", + Oslash: "Ø", + eth: "ð", + ETH: "Ð", + thorn: "þ", + THORN: "Þ", + hellip: "…", + lsquo: "‘", + rsquo: "’", + sbquo: "‚", + ldquo: "“", + rdquo: "”", + bdquo: "„", + laquo: "«", + raquo: "»", + lsaquo: "‹", + rsaquo: "›", + ndash: "–", + mdash: "—", + bull: "•", + middot: "·", + dagger: "†", + Dagger: "‡", + prime: "′", + Prime: "″", + euro: "€", + pound: "£", + yen: "¥", + cent: "¢", + copy: "©", + reg: "®", + trade: "™", + deg: "°", + plusmn: "±", + times: "×", + divide: "÷", + micro: "µ", + para: "¶", + sect: "§", + iexcl: "¡", + iquest: "¿", + ordf: "ª", + ordm: "º", + sup2: "²", + sup3: "³", + frac14: "¼", + frac12: "½", + frac34: "¾", + shy: "­", + ensp: " ", + emsp: " ", + thinsp: " ", + zwnj: "‌", + zwj: "‍", +}; -/** - * Returns the `content` of the first `<meta>` tag whose `attr` equals `value`, - * in whichever order the attributes come. Each tag is matched on its own, so - * the capture can't run into the next tag. - */ -function extractMetaContent( - html: string, - attr: string, - value: string, -): string | undefined { - const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i"); - const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i; - const tagRe = /<meta\b[^>]*>/gi; - let m: RegExpExecArray | null; - while ((m = tagRe.exec(html)) !== null) { - const tag = m[0]; - if (!keyRe.test(tag)) continue; - const content = contentRe.exec(tag); - if (content) return decodeHtmlEntities(content[2]); +// The accented letters, `é` and the like, composed from their letter +// and accent rather than listed one by one. +for ( + const [accent, mark, letters] of [ + ["grave", "̀", "aeiouAEIOU"], + ["acute", "́", "aeiouyAEIOUY"], + ["circ", "̂", "aeiouAEIOU"], + ["tilde", "̃", "anoANO"], + ["uml", "̈", "aeiouyAEIOUY"], + ["ring", "̊", "aA"], + ["cedil", "̧", "cC"], + ] +) { + for (const letter of letters) { + NAMED_ENTITIES[letter + accent] = (letter + mark).normalize("NFC"); } - return undefined; } -/** Returns the content of `<meta property="og:<tag>">`, entities decoded. */ -export function extractOgTag( - html: string, - tag: string, -): string | undefined { - return extractMetaContent(html, "property", `og:${tag}`); +/** Returns the character of code point `n`, or undefined when there is none. */ +function fromCodePoint(n: number): string | undefined { + if (n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) { + return undefined; + } + return String.fromCodePoint(n); } /** - * Returns the content of `<meta name="<name>">`, entities decoded. Twitter - * card tags use `name`, not `property`. + * Decodes the entities in `str` in a single pass, so `&quot;` reads + * `"`. An entity it doesn't know, or a code point that doesn't exist, is + * left as written. Also used by `bandcamp-tralbum.ts` on the JSON a page + * carries in an attribute. */ -export function extractMetaName( - html: string, - name: string, -): string | undefined { - return extractMetaContent(html, "name", name); +export function decodeHtmlEntities(str: string): string { + return str.replace( + /&(?:#(\d{1,8})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z][a-zA-Z0-9]{1,31}));/g, + (match, dec?: string, hex?: string, name?: string) => { + if (dec) return fromCodePoint(Number(dec)) ?? match; + if (hex) return fromCodePoint(parseInt(hex, 16)) ?? match; + return NAMED_ENTITIES[name!] ?? NAMED_ENTITIES[name!.toLowerCase()] ?? + match; + }, + ); } -/** Returns the text of the `<title>` element, trimmed and entities decoded. */ -export function extractPageTitle(html: string): string | undefined { - const match = html.match(/<title[^>]*>([^<]+)<\/title>/i); - return match ? decodeHtmlEntities(match[1].trim()) : undefined; +/** + * Returns `raw` resolved against `base` as an absolute http(s) URL; undefined + * for a `data:` URI, another scheme or a URL that doesn't parse. Every URL a + * page names goes through it, so `/og.png` and `//cdn/og.png` work. + */ +export function absoluteHttpUrl( + raw: string | undefined, + base: string, +): string | undefined { + if (!raw?.trim()) return undefined; + try { + const url = new URL(raw.trim(), base); + return url.protocol === "http:" || url.protocol === "https:" + ? url.href + : undefined; + } catch { + return undefined; + } +} + +/** A tag's attributes by lowercase name, values decoded; the first one wins. */ +type Attrs = Map<string, string>; + +/** + * Matches the start of a tag the scan reads, or of a comment. Quoted values + * may hold `>`, but no tag spans a `<`, so each attempt stops at the next one + * and the scan stays linear in the page's size. + */ +const TAG_RE = + /<(meta|link|img|title|script)\b((?:[^<>"']|"[^"<]*"|'[^'<]*')*)>|<!--/gi; + +/** Matches one attribute: unquoted, single- or double-quoted, or bare. */ +const ATTR_RE = + /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g; + +function parseAttrs(source: string): Attrs { + const attrs: Attrs = new Map(); + for (const m of source.matchAll(ATTR_RE)) { + const name = m[1].toLowerCase(); + if (attrs.has(name)) continue; + attrs.set(name, decodeHtmlEntities(m[2] ?? m[3] ?? m[4] ?? "")); + } + return attrs; +} + +/** What {@link scanTags} found, in page order. */ +interface Scan { + metas: Attrs[]; + links: Attrs[]; + imgs: Attrs[]; + /** The raw text of the first `<title>`. */ + title?: string; + /** The bodies of the `application/ld+json` scripts. */ + jsonLd: string[]; +} + +/** + * Collects the tags {@link parsePage} reads in one pass. Comments and the + * bodies of scripts and the title are skipped over, so a tag written inside + * them isn't read; one left open runs to the end of the page, as in a browser. + */ +function scanTags(html: string): Scan { + const scan: Scan = { metas: [], links: [], imgs: [], jsonLd: [] }; + const re = new RegExp(TAG_RE); + let m: RegExpExecArray | null; + while ((m = re.exec(html)) !== null) { + const tag = m[1]?.toLowerCase(); + if (!tag) { + const end = html.indexOf("-->", re.lastIndex); + if (end < 0) break; + re.lastIndex = end + 3; + continue; + } + const attrs = parseAttrs(m[2]); + if (tag === "meta") scan.metas.push(attrs); + else if (tag === "link") scan.links.push(attrs); + else if (tag === "img") scan.imgs.push(attrs); + else { + const close = new RegExp(`</${tag}\\s*>`, "gi"); + close.lastIndex = re.lastIndex; + const end = close.exec(html); + if (!end) break; + const body = html.slice(re.lastIndex, end.index); + if (tag === "title") scan.title ??= body; + else if ( + attrs.get("type")?.trim().toLowerCase() === "application/ld+json" + ) { + scan.jsonLd.push(body); + } + re.lastIndex = close.lastIndex; + } + } + return scan; +} + +/** Returns `value` trimmed, or undefined when nothing is left. */ +function nonBlank(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; } /** @@ -82,21 +232,15 @@ export function extractPageTitle(html: string): string | undefined { * {@link normalizeCssColor}. Sites often ship one per color scheme with the * dark one first, so the unscoped tag wins, then the light one, then any. */ -export function extractThemeColor(html: string): string | undefined { - const tagRe = /<meta\b[^>]*>/gi; - const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i; - const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i; +function themeColorOf(metas: Attrs[]): string | undefined { let unscoped: string | undefined; let light: string | undefined; let any: string | undefined; - - let m: RegExpExecArray | null; - while ((m = tagRe.exec(html)) !== null) { - const tag = m[0]; - if (!/\bname=["']theme-color["']/i.test(tag)) continue; - const content = contentRe.exec(tag)?.[2]; + for (const meta of metas) { + if (meta.get("name")?.toLowerCase() !== "theme-color") continue; + const content = nonBlank(meta.get("content")); if (!content) continue; - const media = mediaRe.exec(tag)?.[2]; + const media = meta.get("media"); if (!media) unscoped ??= content; else if (/light/i.test(media)) light ??= content; any ??= content; @@ -104,180 +248,179 @@ export function extractThemeColor(html: string): string | undefined { return unscoped ?? light ?? any; } +/** Returns the lowercase tokens of a `<link>`'s `rel`. */ +function relTokens(link: Attrs): string[] { + return (link.get("rel") ?? "").toLowerCase().split(/\s+/); +} + /** * Returns the `color` of `<link rel="mask-icon">`, Safari's pinned-tab icon, - * as written. The generic provider's last resort for a brand color. + * as written. The generic preview's last resort for a brand color. */ -export function extractMaskIconColor(html: string): string | undefined { - const linkRe = /<link[^>]+>/gi; - let m: RegExpExecArray | null; - while ((m = linkRe.exec(html)) !== null) { - const tag = m[0]; - if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue; - const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2]; +function maskIconColorOf(links: Attrs[]): string | undefined { + for (const link of links) { + if (!relTokens(link).includes("mask-icon")) continue; + const color = nonBlank(link.get("color")); if (color) return color; } return undefined; } -/** - * Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or - * common named color; undefined for any other form. The generic provider - * stores it as the preview's `accentColor`, a tint over the app's own surface, - * so alpha is dropped, and a fully transparent one is no color at all. - */ -export function normalizeCssColor(raw: string | undefined): string | undefined { - const color = raw ? parseCssColor(raw) : undefined; - return color && color.a > 0 ? hexOf(color) : undefined; -} - -/** - * Returned to the generic provider by {@link extractJsonLd}, which ranks each - * field below the page's OG and Twitter tags. - */ -export interface JsonLdResult { - /** From `name`, or else `headline`. */ - title?: string; - description?: string; - /** - * From `image`, or else `thumbnailUrl` or `thumbnail`, and only when it is - * an absolute URL. - */ - thumbnailUrl?: string; -} - -function ldString(v: unknown): string | undefined { - if (typeof v === "string" && v.trim()) return v.trim(); - if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) { - return v[0].trim(); - } - return undefined; -} - -function ldImage(v: unknown): string | undefined { - if ( - typeof v === "string" && - (v.startsWith("http://") || v.startsWith("https://")) - ) return v; - if (Array.isArray(v)) return ldImage(v[0]); - if (v && typeof v === "object") { - const o = v as Record<string, unknown>; - return ldImage(o.url ?? o.contentUrl); - } - return undefined; -} - -function ldExtractNode(data: unknown): JsonLdResult { - if (Array.isArray(data)) { - for (const item of data) { - const r = ldExtractNode(item); - if (r.title || r.thumbnailUrl) return r; - } - return {}; - } - if (!data || typeof data !== "object") return {}; - const o = data as Record<string, unknown>; - if (o["@graph"]) return ldExtractNode(o["@graph"]); - return { - title: ldString(o.name ?? o.headline), - description: ldString(o.description), - thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail), - }; -} - -/** - * Returns the first JSON-LD node in `html` that has a title or an image, - * looking through `@graph` and arrays; empty when there is none. An image may - * be a URL, a list of them or an `ImageObject`. Blocks that aren't valid JSON - * are skipped. - */ -export function extractJsonLd(html: string): JsonLdResult { - const pattern = - /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi; - let match: RegExpExecArray | null; - while ((match = pattern.exec(html)) !== null) { - try { - const result = ldExtractNode(JSON.parse(match[1])); - if (result.title || result.thumbnailUrl) return result; - } catch { /* invalid JSON: try the next block */ } - } - return {}; -} - -/** - * Returns the absolute URL of the first `<img>` whose declared width and height - * are both at least `minSize` pixels; an image without them doesn't count. - * Data URIs are skipped. - */ -export function extractLargeImage( - html: string, - baseUrl: string, - minSize = 200, -): string | undefined { - const imgPattern = /<img[^>]+>/gi; - let match: RegExpExecArray | null; - while ((match = imgPattern.exec(html)) !== null) { - const tag = match[0]; - const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1]; - if (!src || src.startsWith("data:")) continue; - const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0"); - const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0"); - if (w >= minSize && h >= minSize) { - try { - return new URL(src, baseUrl).toString(); - } catch { - continue; - } - } - } - return undefined; -} - /** * Returns the absolute URL of the page's largest icon, among its `icon` and * `apple-touch-icon` links, by declared `sizes`; the first one when none - * declares a size. The generic provider stores it as the preview's - * `faviconUrl`, which the thumbnail placeholder draws. + * declares a size. A `mask-icon` is no candidate: it is a one-color shape for + * Safari to tint, black when drawn as is. * * An SVG icon wins over any size: it declares none, yet scales to any. */ -export function extractBestIcon( - html: string, - baseUrl: string, -): string | undefined { - const linkRe = /<link[^>]+>/gi; - const relRe = /\brel=["']([^"']+)["']/i; - const hrefRe = /\bhref=["']([^"']+)["']/i; - const sizesRe = /\bsizes=["']([^"']+)["']/i; - - const candidates: { href: string; area: number }[] = []; - - let m: RegExpExecArray | null; - while ((m = linkRe.exec(html)) !== null) { - const tag = m[0]; - const rel = relRe.exec(tag)?.[1] ?? ""; - if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue; - const href = hrefRe.exec(tag)?.[1]; +function bestIconOf(links: Attrs[], baseUrl: string): string | undefined { + let best: { href: string; area: number } | undefined; + for (const link of links) { + const rel = relTokens(link); + if ( + !rel.includes("icon") && !rel.includes("apple-touch-icon") && + !rel.includes("apple-touch-icon-precomposed") + ) continue; + const href = absoluteHttpUrl(link.get("href"), baseUrl); if (!href) continue; - const sizesStr = sizesRe.exec(tag)?.[1] ?? ""; - const sm = sizesStr.match(/(\d+)x(\d+)/i); + const sizes = (link.get("sizes") ?? "").toLowerCase(); const isSvg = /\.svg(\?|$)/i.test(href) || - /\btype=["']image\/svg\+xml["']/i.test(tag); - const area = isSvg - ? Number.MAX_SAFE_INTEGER - : sm - ? parseInt(sm[1]) * parseInt(sm[2]) - : 0; + link.get("type")?.toLowerCase() === "image/svg+xml" || + sizes === "any"; + const area = isSvg ? Number.MAX_SAFE_INTEGER : Math.max( + 0, + ...[...sizes.matchAll(/(\d+)x(\d+)/g)].map(([, w, h]) => +w * +h), + ); + if (!best || area > best.area) best = { href, area }; + } + return best?.href; +} + +/** + * Returned by the JSON-LD reading of {@link parsePage}, which the generic + * preview ranks below the page's OG and Twitter tags. + */ +export interface JsonLdResult { + /** From `name`, or else `headline`. */ + title?: string; + description?: string; + /** From `image`, or else `thumbnailUrl` or `thumbnail`. */ + thumbnailUrl?: string; +} + +/** + * The JSON-LD types that describe the site around a page rather than the + * page itself. Their name is the site's or the company's, so a node of one of + * them is read only when no other node has a title or an image. + */ +const SITE_LD_TYPES = new Set([ + "WebSite", + "Organization", + "Corporation", + "NewsMediaOrganization", + "Person", + "BreadcrumbList", + "SearchAction", + "ImageObject", + "SiteNavigationElement", + "WPHeader", + "WPFooter", +]); + +function ldString(v: unknown): string | undefined { + if (typeof v === "string") return nonBlank(v); + if (Array.isArray(v) && typeof v[0] === "string") return nonBlank(v[0]); + return undefined; +} + +function ldImage(v: unknown, baseUrl: string): string | undefined { + if (typeof v === "string") return absoluteHttpUrl(v, baseUrl); + if (Array.isArray(v)) return ldImage(v[0], baseUrl); + if (v && typeof v === "object") { + const o = v as Record<string, unknown>; + return ldImage(o.url ?? o.contentUrl, baseUrl); + } + return undefined; +} + +/** Lists every node of `data`, through arrays and `@graph`, in order. */ +function ldNodes(data: unknown): Record<string, unknown>[] { + if (Array.isArray(data)) return data.flatMap(ldNodes); + if (!data || typeof data !== "object") return []; + const o = data as Record<string, unknown>; + return o["@graph"] ? ldNodes(o["@graph"]) : [o]; +} + +/** + * Returns the first JSON-LD node about the page that has a title or an image, + * else the first about its site ({@link SITE_LD_TYPES}); empty when there is + * none. An image may be a URL, a list of them or an `ImageObject`. Blocks that + * aren't valid JSON are skipped. + */ +function jsonLdOf(blocks: string[], baseUrl: string): JsonLdResult { + let fallback: JsonLdResult | undefined; + for (const block of blocks) { + let data: unknown; try { - candidates.push({ href: new URL(href, baseUrl).toString(), area }); + data = JSON.parse(block); } catch { continue; } + for (const node of ldNodes(data)) { + const result: JsonLdResult = { + title: ldString(node.name ?? node.headline), + description: ldString(node.description), + thumbnailUrl: ldImage( + node.image ?? node.thumbnailUrl ?? node.thumbnail, + baseUrl, + ), + }; + if (!result.title && !result.thumbnailUrl) continue; + const types = [node["@type"]].flat(); + if (!types.some((t) => typeof t === "string" && SITE_LD_TYPES.has(t))) { + return result; + } + fallback ??= result; + } } + return fallback ?? {}; +} - if (candidates.length === 0) return undefined; - candidates.sort((a, b) => b.area - a.area); - return candidates[0].href; +/** + * Returns the source of an `<img>`: its `src`, or for a lazy-loaded image + * whose `src` is a `data:` placeholder, its `data-src` or `data-lazy-src`. + */ +function imageSource(img: Attrs, baseUrl: string): string | undefined { + const src = img.get("src")?.trim(); + return src && !src.startsWith("data:") + ? absoluteHttpUrl(src, baseUrl) + : absoluteHttpUrl(img.get("data-src") ?? img.get("data-lazy-src"), baseUrl); +} + +/** Returns an `<img>`'s declared `name` dimension in pixels; 0 when none. */ +function declaredSize(img: Attrs, name: "width" | "height"): number { + return parseInt(img.get(name) ?? "") || 0; +} + +/** + * Returns the absolute URL of the first `<img>` whose declared width and height + * are both at least `minSize` pixels; an image without them doesn't count. + */ +function largeImageOf( + imgs: Attrs[], + baseUrl: string, + minSize = 200, +): string | undefined { + for (const img of imgs) { + if ( + declaredSize(img, "width") < minSize || + declaredSize(img, "height") < minSize + ) continue; + const src = imageSource(img, baseUrl); + if (src) return src; + } + return undefined; } /** Filename words that mark an image as page furniture rather than content. */ @@ -297,6 +440,11 @@ const UI_IMAGE_KEYWORDS = new Set([ "pixel", "bg", "background", + "placeholder", + "spacer", + "blank", + "lazy", + "loading", ]); /** @@ -314,25 +462,86 @@ function isUiImage(src: string): boolean { /** * Returns the absolute URL of the first `<img>` that looks like content rather - * than a logo, icon or other page furniture. Data URIs are skipped. The generic - * provider's last resort for a thumbnail. + * than a logo, icon, tracking pixel or other page furniture: one declared + * smaller than 50 pixels either way doesn't count. The generic preview's last + * resort for a thumbnail. */ -export function extractFirstContentImage( - html: string, +function firstContentImageOf( + imgs: Attrs[], baseUrl: string, ): string | undefined { - const imgPattern = /<img[^>]+>/gi; - let match: RegExpExecArray | null; - while ((match = imgPattern.exec(html)) !== null) { - const tag = match[0]; - const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1]; - if (!src || src.startsWith("data:")) continue; - if (isUiImage(src)) continue; - try { - return new URL(src, baseUrl).toString(); - } catch { - continue; - } + for (const img of imgs) { + const w = declaredSize(img, "width"); + const h = declaredSize(img, "height"); + if ((w && w < 50) || (h && h < 50)) continue; + const src = imageSource(img, baseUrl); + if (src && !isUiImage(src)) return src; } return undefined; } + +/** + * Returned by {@link parsePage}: what a page says about itself, entities + * decoded, values trimmed and URLs absolute. + */ +export interface Page { + /** + * Returns the `content` of the first `<meta>` whose `property` or `name` is + * `key`, in any case: sites write `og:` tags under either attribute, and + * Twitter's too. + */ + meta(key: string): string | undefined; + /** The text of the `<title>` element. */ + title?: string; + jsonLd: JsonLdResult; + /** The `theme-color`, as written: pass it to {@link normalizeCssColor}. */ + themeColor?: string; + /** The `color` of the Safari pinned-tab icon, as written. */ + maskIconColor?: string; + /** The largest icon the page links, which the thumbnail placeholder draws. */ + bestIcon?: string; + /** The first image declared at least 200 pixels both ways. */ + largeImage?: string; + /** The first image that doesn't look like page furniture. */ + firstContentImage?: string; +} + +/** + * Reads `html`, served from `pageUrl`, in one pass. The generic preview and + * the providers built on it read a page through it. + */ +export function parsePage(html: string, pageUrl: string): Page { + const scan = scanTags(html); + + const metas = new Map<string, string>(); + for (const meta of scan.metas) { + const content = nonBlank(meta.get("content")); + if (!content) continue; + for (const attr of ["property", "name"]) { + const key = meta.get(attr)?.trim().toLowerCase(); + if (key && !metas.has(key)) metas.set(key, content); + } + } + + return { + meta: (key) => metas.get(key.toLowerCase()), + title: nonBlank(scan.title && decodeHtmlEntities(scan.title)), + jsonLd: jsonLdOf(scan.jsonLd, pageUrl), + themeColor: themeColorOf(scan.metas), + maskIconColor: maskIconColorOf(scan.links), + bestIcon: bestIconOf(scan.links, pageUrl), + largeImage: largeImageOf(scan.imgs, pageUrl), + firstContentImage: firstContentImageOf(scan.imgs, pageUrl), + }; +} + +/** + * Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or + * common named color; undefined for any other form. The generic preview + * stores it as the preview's `accentColor`, a tint over the app's own surface, + * so alpha is dropped, and a fully transparent one is no color at all. + */ +export function normalizeCssColor(raw: string | undefined): string | undefined { + const color = raw ? parseCssColor(raw) : undefined; + return color && color.a > 0 ? hexOf(color) : undefined; +} diff --git a/api/remote/providers/bandcamp.ts b/api/remote/providers/bandcamp.ts index 65c5232a..14d936c1 100644 --- a/api/remote/providers/bandcamp.ts +++ b/api/remote/providers/bandcamp.ts @@ -1,15 +1,35 @@ /** - * Previews Bandcamp links from their page's OG tags, for - * `rich-content-service.ts`. + * Previews Bandcamp links from their page, for `rich-content-service.ts`. * * @module */ import type { RichContent } from "../../../shared/dump.ts"; import type { RichContentProvider } from "./provider.ts"; -import { extractOgTag } from "../html.ts"; -import { fetchWithTimeout, readTextCapped } from "../fetch.ts"; +import { parsePage } from "../html.ts"; +import { fetchPage } from "../fetch.ts"; import { BANDCAMP_HOST_RE } from "../bandcamp-tralbum.ts"; +import { genericPreview } from "./generic.ts"; + +/** + * Returns the embed player's URL a page's `og:video` names, over https; only + * a player on bandcamp.com counts, since the frontend frames it. + */ +function embeddedPlayerUrl(raw: string | undefined): string | undefined { + if (!raw) return undefined; + try { + const url = new URL(raw); + if ( + (url.protocol !== "https:" && url.protocol !== "http:") || + url.hostname !== "bandcamp.com" || + !url.pathname.startsWith("/EmbeddedPlayer/") + ) return undefined; + url.protocol = "https:"; + return url.href; + } catch { + return undefined; + } +} /** * Claims bandcamp.com and its subdomains, where artists have their pages, and @@ -29,24 +49,16 @@ export const bandcampProvider: RichContentProvider = { }, async fetch(url: string): Promise<RichContent> { - const res = await fetchWithTimeout(url); - const contentType = res.headers.get("content-type") ?? ""; - - if (!contentType.startsWith("text/html")) { - await res.body?.cancel(); + const fetched = await fetchPage(url); + if (fetched.html === undefined) { return { type: "bandcamp", siteName: "Bandcamp", url }; } - - const html = await readTextCapped(res); - + const page = parsePage(fetched.html, fetched.url); return { + ...genericPreview(page, url, fetched.url), type: "bandcamp", siteName: "Bandcamp", - url, - title: extractOgTag(html, "title"), - description: extractOgTag(html, "description"), - thumbnailUrl: extractOgTag(html, "image"), - embedUrl: extractOgTag(html, "video") ?? undefined, + embedUrl: embeddedPlayerUrl(page.meta("og:video")), }; }, }; diff --git a/api/remote/providers/generic.ts b/api/remote/providers/generic.ts index c3b58c38..1a2067f7 100644 --- a/api/remote/providers/generic.ts +++ b/api/remote/providers/generic.ts @@ -1,6 +1,8 @@ /** * Previews any page no other provider claims, for `rich-content-service.ts`, - * from whatever metadata its HTML carries. + * from whatever metadata its HTML carries. Its reading of a page, + * {@link genericPreview}, is also the base of the Bandcamp, SoundCloud and + * YouTube channel previews. * * @module */ @@ -9,25 +11,97 @@ import type { RichContent } from "../../../shared/dump.ts"; import type { RichContentProvider } from "./provider.ts"; import { siteHost } from "../../../shared/url.ts"; import { - extractBestIcon, - extractFirstContentImage, - extractJsonLd, - extractLargeImage, - extractMaskIconColor, - extractMetaName, - extractOgTag, - extractPageTitle, - extractThemeColor, + absoluteHttpUrl, normalizeCssColor, + type Page, + parsePage, } from "../html.ts"; -import { fetchWithTimeout, readTextCapped } from "../fetch.ts"; +import { fetchPage } from "../fetch.ts"; + +/** Returns the path of `url` without its trailing slashes, `/` for the root. */ +function pathOf(url: string): string { + return new URL(url).pathname.replace(/\/+$/, "") || "/"; +} /** - * Matches every URL, so it comes last in the service's list. Each field falls - * back from OG tags to Twitter tags, JSON-LD and the markup itself. The icon - * and brand color feed the placeholder the frontend draws when there is no - * thumbnail. + * Strips a trailing ` | Site`, ` – Site` or the like from a `<title>`, which + * names the page and then its site; the site is in the preview's badge. */ +function withoutSiteSuffix(title: string, siteName: string): string { + const escaped = siteName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const suffix = new RegExp(`\\s+[|–—\\-·•]\\s+${escaped}\\s*$`, "i"); + return title.replace(suffix, "") || title; +} + +/** + * Returns the preview of the page `url` that `page` read, served from + * `finalUrl` after redirects. Each field falls back from OG tags to Twitter + * tags, JSON-LD and the markup itself. The icon and brand color feed the + * placeholder the frontend draws when there is no thumbnail. + */ +export function genericPreview( + page: Page, + url: string, + finalUrl: string, +): RichContent { + // An og:url naming the home page from another page means the OG tags are + // the site's defaults, not this page's: their title and image are skipped. + // Any other og:url is the page's canonical address, often with a slug the + // link left out, and its tags are the page's. + const ogUrl = page.meta("og:url"); + let useOg = true; + try { + useOg = !ogUrl || pathOf(new URL(ogUrl, finalUrl).href) !== "/" || + pathOf(finalUrl) === "/"; + } catch { /* an og:url that doesn't parse names no other page */ } + + const ogSiteName = page.meta("og:site_name"); + const pageTitle = page.title && ogSiteName + ? withoutSiteSuffix(page.title, ogSiteName) + : page.title; + const title = (useOg ? page.meta("og:title") : undefined) ?? + page.meta("twitter:title") ?? + page.jsonLd.title ?? + pageTitle; + + const description = page.meta("og:description") ?? + page.meta("twitter:description") ?? + page.jsonLd.description ?? + page.meta("description"); + + // Never the favicon: a 16×16 icon cropped into a thumbnail looks broken, + // and the placeholder drawn instead shows the icon properly. A tag naming + // no usable URL falls through to the next. + const thumbnailUrl = + (useOg ? absoluteHttpUrl(page.meta("og:image"), finalUrl) : undefined) ?? + absoluteHttpUrl(page.meta("twitter:image"), finalUrl) ?? + page.jsonLd.thumbnailUrl ?? + page.largeImage ?? + page.firstContentImage; + + // `/favicon.ico` is a guess. When it doesn't load, the placeholder shows + // the site's initial. + const faviconUrl = page.bestIcon ?? `${new URL(finalUrl).origin}/favicon.ico`; + + const accentColor = normalizeCssColor( + page.themeColor ?? + page.meta("msapplication-TileColor") ?? + page.maskIconColor, + ); + + return { + type: "generic", + url, + title, + description, + thumbnailUrl, + faviconUrl, + accentColor, + siteName: ogSiteName ?? siteHost(url), + }; +} + +/** Matches every URL, so it comes last in the service's list. */ export const genericProvider: RichContentProvider = { name: "generic", @@ -36,72 +110,12 @@ export const genericProvider: RichContentProvider = { }, async fetch(url: string): Promise<RichContent> { - const res = await fetchWithTimeout(url); - const contentType = res.headers.get("content-type") ?? ""; - - if (!contentType.startsWith("text/html")) { - await res.body?.cancel(); - return { type: "generic", url }; - } - - const html = await readTextCapped(res); - const ld = extractJsonLd(html); - - // An og:url naming another page (often the home page) means the OG tags - // are the site's defaults, not this page's: its title and image are - // skipped. - const ogUrl = extractOgTag(html, "url"); - const useOg = !ogUrl || (() => { - try { - const ogPath = new URL(ogUrl).pathname.replace(/\/+$/, "") || "/"; - const pagePath = new URL(url).pathname.replace(/\/+$/, "") || "/"; - return ogPath === pagePath; - } catch { - return true; - } - })(); - - const title = (useOg ? extractOgTag(html, "title") : undefined) ?? - extractMetaName(html, "twitter:title") ?? - ld.title ?? - extractPageTitle(html); - - const siteName = extractOgTag(html, "site_name") ?? - siteHost(url); - - const description = extractOgTag(html, "description") ?? - extractMetaName(html, "twitter:description") ?? - ld.description ?? - extractMetaName(html, "description"); - - // Never the favicon: a 16×16 icon cropped into a thumbnail looks broken, - // and the placeholder drawn instead shows the icon properly. - const thumbnailUrl = (useOg ? extractOgTag(html, "image") : undefined) ?? - extractMetaName(html, "twitter:image") ?? - ld.thumbnailUrl ?? - extractLargeImage(html, url) ?? - extractFirstContentImage(html, url); - - // `/favicon.ico` is a guess. When it doesn't load, the placeholder shows - // the site's initial. - const faviconUrl = extractBestIcon(html, url) ?? - `${new URL(url).origin}/favicon.ico`; - - const accentColor = normalizeCssColor( - extractThemeColor(html) ?? - extractMetaName(html, "msapplication-TileColor") ?? - extractMaskIconColor(html), - ); - - return { - type: "generic", + const fetched = await fetchPage(url); + if (fetched.html === undefined) return { type: "generic", url }; + return genericPreview( + parsePage(fetched.html, fetched.url), url, - title, - description, - thumbnailUrl, - faviconUrl, - accentColor, - siteName, - }; + fetched.url, + ); }, }; diff --git a/api/remote/providers/provider.ts b/api/remote/providers/provider.ts index 525c534a..25a60f93 100644 --- a/api/remote/providers/provider.ts +++ b/api/remote/providers/provider.ts @@ -18,9 +18,10 @@ export interface RichContentProvider { matches(url: string): boolean; /** * Returns the preview of `url`, which ends up in the dump's `rich_content` or - * the create form's preview. It rejects when the page can't be fetched, and - * the service then falls back to a stub. A page that isn't HTML yields a - * preview without metadata. + * the create form's preview. It rejects when the page can't be fetched or + * answers with an error status, and the service then falls back to a stub. + * A page that isn't HTML yields a preview without metadata. The service + * tidies what it returns, so a provider leaves text as the page wrote it. */ fetch(url: string): Promise<RichContent>; } diff --git a/api/remote/providers/self.ts b/api/remote/providers/self.ts index 9694c95f..fdce7c1c 100644 --- a/api/remote/providers/self.ts +++ b/api/remote/providers/self.ts @@ -1,7 +1,8 @@ /** * Previews links to this site's own dump, user and playlist pages, as - * `parsePagePath` reads them, from the database, for `rich-content-service.ts`. No request goes out, so this works - * even when the site runs on a private host that `fetchWithTimeout` refuses. + * `parsePagePath` reads them, from the database, for + * `rich-content-service.ts`. No request goes out, so this works even when the + * site runs on a private host that `fetchWithTimeout` refuses. * * @module */ diff --git a/api/remote/providers/soundcloud.ts b/api/remote/providers/soundcloud.ts index f152838e..898cca1b 100644 --- a/api/remote/providers/soundcloud.ts +++ b/api/remote/providers/soundcloud.ts @@ -1,50 +1,72 @@ /** - * Previews SoundCloud links from their page's OG tags, for - * `rich-content-service.ts`. + * Previews SoundCloud links from their page, for `rich-content-service.ts`. * * @module */ import type { RichContent } from "../../../shared/dump.ts"; import type { RichContentProvider } from "./provider.ts"; -import { extractOgTag } from "../html.ts"; -import { fetchWithTimeout, readTextCapped } from "../fetch.ts"; +import { parsePage } from "../html.ts"; +import { fetchPage } from "../fetch.ts"; +import { genericPreview } from "./generic.ts"; + +/** SoundCloud's own hosts, the short links' `on.` included. */ +const SOUNDCLOUD_HOSTS = new Set([ + "soundcloud.com", + "www.soundcloud.com", + "m.soundcloud.com", + "on.soundcloud.com", +]); /** - * Claims soundcloud.com only, not www. or m.soundcloud.com, and builds the - * embed player's URL from the link itself. + * Returns the soundcloud.com address of the page, which the embed player + * takes: its `og:url`, or else the URL it was served from. A short link only + * redirects, so it is never the answer; tracking parameters are dropped. + */ +function pageAddress(ogUrl: string | undefined, finalUrl: string): string { + for (const raw of [ogUrl, finalUrl]) { + if (!raw) continue; + try { + const url = new URL(raw); + if ( + !SOUNDCLOUD_HOSTS.has(url.hostname) || + url.hostname === "on.soundcloud.com" + ) continue; + return `https://soundcloud.com${url.pathname}`; + } catch { + continue; + } + } + return finalUrl; +} + +/** + * Claims SoundCloud's hosts, short links included, and builds the embed + * player's URL from the page's address. */ export const soundcloudProvider: RichContentProvider = { name: "soundcloud", matches(url: string): boolean { try { - return new URL(url).hostname === "soundcloud.com"; + return SOUNDCLOUD_HOSTS.has(new URL(url).hostname); } catch { return false; } }, async fetch(url: string): Promise<RichContent> { - const res = await fetchWithTimeout(url); - const contentType = res.headers.get("content-type") ?? ""; - - if (!contentType.startsWith("text/html")) { - await res.body?.cancel(); - return { type: "soundcloud", siteName: "SoundCloud", url }; - } - - const html = await readTextCapped(res); - + const fetched = await fetchPage(url); + const page = fetched.html === undefined + ? undefined + : parsePage(fetched.html, fetched.url); + const address = pageAddress(page?.meta("og:url"), fetched.url); return { + ...(page ? genericPreview(page, url, fetched.url) : { url }), type: "soundcloud", siteName: "SoundCloud", - url, - title: extractOgTag(html, "title"), - description: extractOgTag(html, "description"), - thumbnailUrl: extractOgTag(html, "image"), embedUrl: `https://w.soundcloud.com/player/?url=${ - encodeURIComponent(url) + encodeURIComponent(address) }&visual=true&auto_play=false`, }; }, diff --git a/api/remote/providers/youtube.ts b/api/remote/providers/youtube.ts index 45d16945..b5e3cd63 100644 --- a/api/remote/providers/youtube.ts +++ b/api/remote/providers/youtube.ts @@ -1,17 +1,18 @@ /** * Previews YouTube videos, playlists and channels, for * `rich-content-service.ts`. Videos and playlists are read from YouTube's - * oEmbed endpoint, channels from their page's OG tags. + * oEmbed endpoint, channels from their page. * * @module */ import type { RichContent } from "../../../shared/dump.ts"; import type { RichContentProvider } from "./provider.ts"; -import { extractOgTag } from "../html.ts"; -import { fetchWithTimeout, readTextCapped } from "../fetch.ts"; +import { parsePage } from "../html.ts"; +import { fetchPage, fetchWithTimeout } from "../fetch.ts"; import { youtubeHost, youtubeVideoId } from "../../lib/canonical-url.ts"; import { siteHostOf } from "../../../shared/url.ts"; +import { genericPreview } from "./generic.ts"; /** Returns the id of the video `url` points at, in any shape YouTube uses. */ function extractVideoId(url: string): string | null { @@ -123,27 +124,17 @@ export const youtubeProvider: RichContentProvider = { if (channelPath && !videoId) { // oEmbed doesn't take channel URLs. - let title: string | undefined; - let thumbnailUrl: string | undefined; + let channel: RichContent | undefined; try { - const res = await fetchWithTimeout(url); - if (res.ok) { - const html = await readTextCapped(res); - title = extractOgTag(html, "title"); - thumbnailUrl = extractOgTag(html, "image"); - } else { - await res.body?.cancel(); + const fetched = await fetchPage(url); + if (fetched.html !== undefined) { + const page = parsePage(fetched.html, fetched.url); + channel = genericPreview(page, url, fetched.url); } } catch { // YouTube unreachable: the preview goes without. } - return { - type: "youtube", - siteName: "YouTube", - url, - title, - thumbnailUrl, - }; + return { ...channel, type: "youtube", siteName: "YouTube", url }; } if (listId && !videoId) { @@ -172,7 +163,6 @@ export const youtubeProvider: RichContentProvider = { type: "youtube", siteName: "YouTube", url, - videoId: videoId!, title, thumbnailUrl, embedUrl: `https://www.youtube.com/embed/${videoId}?${embedParams}`, diff --git a/api/services/rich-content-service.ts b/api/services/rich-content-service.ts index 3ea1629d..e7cc0e70 100644 --- a/api/services/rich-content-service.ts +++ b/api/services/rich-content-service.ts @@ -84,6 +84,65 @@ export function tryFetchRichContent( return result; } +/** The longest title a preview keeps, in characters. */ +const MAX_TITLE_LENGTH = 300; + +/** + * The longest description a preview keeps, in characters: it is stored with + * the dump, searched and sent with every feed, and some pages put a whole + * article in theirs. + */ +const MAX_DESCRIPTION_LENGTH = 1000; + +/** + * Returns `text` with its whitespace collapsed, cut to `max` characters with + * an ellipsis; undefined when nothing is left. + */ +function tidyText( + text: string | undefined, + max: number, +): string | undefined { + const collapsed = text?.replace(/\s+/g, " ").trim(); + if (!collapsed) return undefined; + const chars = [...collapsed]; + return chars.length > max + ? chars.slice(0, max - 1).join("").trimEnd() + "…" + : collapsed; +} + +/** Returns `url` if it is an absolute http(s) URL, and undefined otherwise. */ +function httpUrl(url: string | undefined): string | undefined { + if (!url) return undefined; + try { + const { protocol } = new URL(url); + return protocol === "http:" || protocol === "https:" ? url : undefined; + } catch { + return undefined; + } +} + +/** + * Applies the rules every preview follows, whichever provider made it: text + * on one line and of bounded length, and only http(s) URLs for the images and + * the player, which the frontend loads. + */ +function tidy(content: RichContent): RichContent { + const tidied: RichContent = { + ...content, + title: tidyText(content.title, MAX_TITLE_LENGTH), + description: tidyText(content.description, MAX_DESCRIPTION_LENGTH), + siteName: tidyText(content.siteName, MAX_TITLE_LENGTH), + thumbnailUrl: httpUrl(content.thumbnailUrl), + faviconUrl: httpUrl(content.faviconUrl), + embedUrl: httpUrl(content.embedUrl), + }; + // Stored as JSON: a field left undefined would only be dropped there. + for (const key of Object.keys(tidied) as (keyof RichContent)[]) { + if (tidied[key] === undefined) delete tidied[key]; + } + return tidied; +} + /** * Fetches the preview of `url` through the first provider that matches, or * makes the failure stub when the provider rejects. @@ -93,7 +152,7 @@ async function fetchRichContentUncached( ): Promise<FetchRichContentResult> { try { const provider = providers.find((p) => p.matches(url))!; - return { ok: true, content: await provider.fetch(url) }; + return { ok: true, content: tidy(await provider.fetch(url)) }; } catch (err) { console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err); const siteName = siteHost(url); diff --git a/api/test/remote/fetch.test.ts b/api/test/remote/fetch.test.ts index b8281629..6ef1b0f5 100644 --- a/api/test/remote/fetch.test.ts +++ b/api/test/remote/fetch.test.ts @@ -1,6 +1,7 @@ -import { stubNetwork } from "../helpers.ts"; +import { html, stubNetwork } from "../helpers.ts"; import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert"; import { + fetchPage, fetchWithTimeout, isValidHttpUrl, readTextCapped, @@ -110,3 +111,33 @@ Deno.test("readTextCapped: decodes the charset the header or a <meta> declares", "caf�", ); }); + +Deno.test("fetchPage: the page and where it was served from", async () => { + using _net = stubNetwork((url) => { + if (url.pathname === "/moved") { + return new Response(null, { + status: 301, + headers: { location: "https://www.example.com/here" }, + }); + } + if (url.pathname === "/gone") return html("<title>Gone", 410); + if (url.pathname === "/data") { + return new Response("{}", { + headers: { "content-type": "application/json" }, + }); + } + return html("Here"); + }); + assertEquals(await fetchPage("https://example.com/moved"), { + url: "https://www.example.com/here", + html: "Here", + }); + assertEquals(await fetchPage("https://example.com/data"), { + url: "https://example.com/data", + }); + await assertRejects( + () => fetchPage("https://example.com/gone"), + Error, + "410", + ); +}); diff --git a/api/test/remote/html.test.ts b/api/test/remote/html.test.ts index e34dddcc..e535b3e8 100644 --- a/api/test/remote/html.test.ts +++ b/api/test/remote/html.test.ts @@ -1,68 +1,122 @@ import "../setup.ts"; import { assertEquals } from "@std/assert"; import { - extractBestIcon, - extractFirstContentImage, - extractJsonLd, - extractLargeImage, - extractMaskIconColor, - extractMetaName, - extractOgTag, - extractPageTitle, - extractThemeColor, + absoluteHttpUrl, + decodeHtmlEntities, normalizeCssColor, + parsePage, } from "../../remote/html.ts"; const BASE = "https://example.com/articles/1"; -Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => { - const page = ` +const read = (html: string) => parsePage(html, BASE); + +Deno.test("parsePage meta: either attribute, either order, entities decoded", () => { + const page = read(` - `; - assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤"); - assertEquals(extractOgTag(page, "description"), "Desc"); - assertEquals(extractOgTag(page, "image"), undefined); - assertEquals(extractMetaName(page, "twitter:title"), "TW"); + + `); + assertEquals(page.meta("og:title"), "Tom & Jerry — ❤"); + assertEquals(page.meta("og:description"), "Desc"); + assertEquals(page.meta("twitter:title"), "TW"); + // Sites write og: tags under `name` too. + assertEquals(page.meta("og:image"), "https://example.com/by-name.jpg"); + assertEquals(page.meta("OG:Title"), "Tom & Jerry — ❤"); + assertEquals(page.meta("og:url"), undefined); }); -Deno.test("extractOgTag: content never spans into the next tag", () => { - const page = ``; - assertEquals(extractOgTag(page, "title"), undefined); +Deno.test("parsePage meta: attributes read as a browser reads them", () => { + const page = read(` + + + + + `); + assertEquals(page.meta("og:title"), "Unquoted"); + assertEquals(page.meta("og:description"), "a > b, still the tag"); + assertEquals(page.meta("og:image"), undefined); + assertEquals(page.meta("og:site_name"), undefined); + // A blank value is no value, so a fallback can take over. + assertEquals(page.meta("og:type"), undefined); }); -Deno.test("extractPageTitle", () => { +Deno.test("parsePage: tags in comments and scripts aren't read", () => { + const page = read(` + + + `); + assertEquals(page.meta("og:title"), "Real"); + // An unclosed comment runs to the end of the page. assertEquals( - extractPageTitle(" A <b> "), - "A ", + read(`