vincent/gerbeur
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
api/remote/bandcamp-tralbum.ts
View File

@@ -8,6 +8,7 @@
*/
import type { Tralbum, TralbumTrack } from "../../shared/player.ts";
import { decodeHtmlEntities } from "./html.ts";
/**
* Matches the hostnames `bandcamp-streams.ts` accepts: bandcamp.com and its
@@ -16,30 +17,6 @@ import type { Tralbum, TralbumTrack } from "../../shared/player.ts";
*/
export const BANDCAMP_HOST_RE = /(?:^|\.)bandcamp\.com$/;
/**
* Decodes HTML entities in a single pass. `html.ts` decodes `&amp;` before
* `&quot;`, which turns a title's literal `&quot;` (escaped as `&amp;quot;`)
* into a bare `"` that breaks the JSON.
*/
function decodeEntitiesOnce(input: string): string {
const named: Record<string, string> = {
quot: '"',
apos: "'",
amp: "&",
lt: "<",
gt: ">",
nbsp: " ",
};
return input.replace(
/&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|([a-zA-Z]+));/g,
(match, dec: string, hex: string, name: string) => {
if (dec) return String.fromCodePoint(Number(dec));
if (hex) return String.fromCodePoint(parseInt(hex, 16));
return named[name.toLowerCase()] ?? match;
},
);
}
/** Returns the URL of the artwork `artId`; undefined when it isn't numeric. */
function artworkUrl(artId: unknown): string | undefined {
if (typeof artId !== "number" && typeof artId !== "string") return undefined;
@@ -60,7 +37,7 @@ export function parseTralbum(html: string, sourceUrl: string): Tralbum | null {
let raw: Record<string, unknown>;
try {
raw = JSON.parse(decodeEntitiesOnce(match[2]));
raw = JSON.parse(decodeHtmlEntities(match[2]));
} catch {
return null;
}

72
api/remote/fetch.ts
View File

@@ -175,7 +175,7 @@ function declaredCharset(contentType: string, bytes: Uint8Array): string {
/**
* Reads the body of `res` as text, up to `maxBytes`, in the charset the page
* declares; the rest is cancelled, never downloaded. The providers and the
* declares; the rest is cancelled, never downloaded. {@link fetchPage} and the
* Bandcamp and YouTube resolvers read pages through it.
*/
export async function readTextCapped(
@@ -235,6 +235,33 @@ async function fetchOnce(
});
}
/**
* Fetches `url` as {@link fetchWithTimeout} does, and also returns the URL the
* last hop answered from, which {@link fetchPage} resolves the page's links
* against.
*/
async function fetchFollowing(
url: string,
timeoutMs: number,
): Promise<{ res: Response; url: URL }> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) {
return { res, url: current };
}
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
}
}
/**
* Fetches a user-supplied URL, following up to five redirects, each of which
* must resolve to a public address too. `timeoutMs` bounds each hop, body
@@ -249,20 +276,41 @@ export async function fetchWithTimeout(
url: string,
timeoutMs = 5000,
): Promise<Response> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) return res;
return (await fetchFollowing(url, timeoutMs)).res;
}
/**
* Returned by {@link fetchPage} to the link preview providers.
*/
export interface FetchedPage {
/**
* The URL the page was served from, after redirects: the base its relative
* links resolve against.
*/
url: string;
/** The page's text, up to {@link MAX_PAGE_BYTES}; missing when it isn't HTML. */
html?: string;
}
/**
* Fetches a page for its link preview, through {@link fetchWithTimeout}. A
* response that isn't HTML is cancelled unread.
*
* @throws {Error} When the page answers with an error status, so that an error
* page never becomes a preview; and as {@link fetchWithTimeout} does.
*/
export async function fetchPage(url: string): Promise<FetchedPage> {
const { res, url: finalUrl } = await fetchFollowing(url, 5000);
if (!res.ok) {
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
throw new Error(`${finalUrl} answered ${res.status}`);
}
const contentType = res.headers.get("content-type") ?? "";
if (!/^(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
await res.body?.cancel();
return { url: finalUrl.href };
}
return { url: finalUrl.href, html: await readTextCapped(res) };
}
/**

683
api/remote/html.ts
View File

@@ -1,8 +1,9 @@
/**
* Reads a link preview's metadata out of a page's HTML with regular
* expressions, without parsing the document. The generic provider uses most of
* it; the Bandcamp, SoundCloud and YouTube providers read only OG tags. Each
* function returns undefined when the page doesn't say.
* Reads a link preview's metadata out of a page's HTML. {@link parsePage}
* scans the page once for the tags that carry it (`<meta>`, `<link>`, `<img>`,
* `<title>` and JSON-LD scripts) without building a document, and returns a
* {@link Page} that the providers' `genericPreview` picks from. Every field is
* undefined when the page doesn't say.
*
* @module
*/
@@ -10,71 +11,220 @@
import { hexOf, parseCssColor } from "../lib/css-color.ts";
/**
* Decodes the five XML entities and numeric ones in `str`. `&amp;` goes first,
* so `&amp;quot;` decodes twice; `bandcamp-tralbum.ts`, which decodes JSON,
* has its own single-pass decoder.
* The named entities decoded besides numeric ones: the XML five, and those
* page titles and descriptions actually use, Latin-1 letters and typography.
*/
function decodeHtmlEntities(str: string): string {
return str
.replace(/&amp;/gi, "&")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">")
.replace(/&quot;/gi, '"')
.replace(/&apos;/gi, "'")
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
.replace(
/&#x([0-9a-f]+);/gi,
(_, hex) => String.fromCodePoint(parseInt(hex, 16)),
);
}
const NAMED_ENTITIES: Record<string, string> = {
amp: "&",
lt: "<",
gt: ">",
quot: '"',
apos: "'",
nbsp: " ",
aelig: "æ",
AElig: "Æ",
oelig: "œ",
OElig: "Œ",
szlig: "ß",
oslash: "ø",
Oslash: "Ø",
eth: "ð",
ETH: "Ð",
thorn: "þ",
THORN: "Þ",
hellip: "…",
lsquo: "‘",
rsquo: "’",
sbquo: "‚",
ldquo: "“",
rdquo: "”",
bdquo: "„",
laquo: "«",
raquo: "»",
lsaquo: "‹",
rsaquo: "›",
ndash: "–",
mdash: "—",
bull: "•",
middot: "·",
dagger: "†",
Dagger: "‡",
prime: "′",
Prime: "″",
euro: "€",
pound: "£",
yen: "¥",
cent: "¢",
copy: "©",
reg: "®",
trade: "™",
deg: "°",
plusmn: "±",
times: "×",
divide: "÷",
micro: "µ",
para: "¶",
sect: "§",
iexcl: "¡",
iquest: "¿",
ordf: "ª",
ordm: "º",
sup2: "²",
sup3: "³",
frac14: "¼",
frac12: "½",
frac34: "¾",
shy: "­",
ensp: " ",
emsp: " ",
thinsp: " ",
zwnj: "‌",
zwj: "‍",
};
/**
* Returns the `content` of the first `<meta>` tag whose `attr` equals `value`,
* in whichever order the attributes come. Each tag is matched on its own, so
* the capture can't run into the next tag.
*/
function extractMetaContent(
html: string,
attr: string,
value: string,
): string | undefined {
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
const tagRe = /<meta\b[^>]*>/gi;
let m: RegExpExecArray | null;
while ((m = tagRe.exec(html)) !== null) {
const tag = m[0];
if (!keyRe.test(tag)) continue;
const content = contentRe.exec(tag);
if (content) return decodeHtmlEntities(content[2]);
// The accented letters, `&eacute;` and the like, composed from their letter
// and accent rather than listed one by one.
for (
const [accent, mark, letters] of [
["grave", "̀", "aeiouAEIOU"],
["acute", "́", "aeiouyAEIOUY"],
["circ", "̂", "aeiouAEIOU"],
["tilde", "̃", "anoANO"],
["uml", "̈", "aeiouyAEIOUY"],
["ring", "̊", "aA"],
["cedil", "̧", "cC"],
]
) {
for (const letter of letters) {
NAMED_ENTITIES[letter + accent] = (letter + mark).normalize("NFC");
}
return undefined;
}
/** Returns the content of `<meta property="og:<tag>">`, entities decoded. */
export function extractOgTag(
html: string,
tag: string,
): string | undefined {
return extractMetaContent(html, "property", `og:${tag}`);
/** Returns the character of code point `n`, or undefined when there is none. */
function fromCodePoint(n: number): string | undefined {
if (n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) {
return undefined;
}
return String.fromCodePoint(n);
}
/**
* Returns the content of `<meta name="<name>">`, entities decoded. Twitter
* card tags use `name`, not `property`.
* Decodes the entities in `str` in a single pass, so `&amp;quot;` reads
* `&quot;`. An entity it doesn't know, or a code point that doesn't exist, is
* left as written. Also used by `bandcamp-tralbum.ts` on the JSON a page
* carries in an attribute.
*/
export function extractMetaName(
html: string,
name: string,
): string | undefined {
return extractMetaContent(html, "name", name);
export function decodeHtmlEntities(str: string): string {
return str.replace(
/&(?:#(\d{1,8})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z][a-zA-Z0-9]{1,31}));/g,
(match, dec?: string, hex?: string, name?: string) => {
if (dec) return fromCodePoint(Number(dec)) ?? match;
if (hex) return fromCodePoint(parseInt(hex, 16)) ?? match;
return NAMED_ENTITIES[name!] ?? NAMED_ENTITIES[name!.toLowerCase()] ??
match;
},
);
}
/** Returns the text of the `<title>` element, trimmed and entities decoded. */
export function extractPageTitle(html: string): string | undefined {
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
return match ? decodeHtmlEntities(match[1].trim()) : undefined;
/**
* Returns `raw` resolved against `base` as an absolute http(s) URL; undefined
* for a `data:` URI, another scheme or a URL that doesn't parse. Every URL a
* page names goes through it, so `/og.png` and `//cdn/og.png` work.
*/
export function absoluteHttpUrl(
raw: string | undefined,
base: string,
): string | undefined {
if (!raw?.trim()) return undefined;
try {
const url = new URL(raw.trim(), base);
return url.protocol === "http:" || url.protocol === "https:"
? url.href
: undefined;
} catch {
return undefined;
}
}
/** A tag's attributes by lowercase name, values decoded; the first one wins. */
type Attrs = Map<string, string>;
/**
* Matches the start of a tag the scan reads, or of a comment. Quoted values
* may hold `>`, but no tag spans a `<`, so each attempt stops at the next one
* and the scan stays linear in the page's size.
*/
const TAG_RE =
/<(meta|link|img|title|script)\b((?:[^<>"']|"[^"<]*"|'[^'<]*')*)>|<!--/gi;
/** Matches one attribute: unquoted, single- or double-quoted, or bare. */
const ATTR_RE =
/([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
function parseAttrs(source: string): Attrs {
const attrs: Attrs = new Map();
for (const m of source.matchAll(ATTR_RE)) {
const name = m[1].toLowerCase();
if (attrs.has(name)) continue;
attrs.set(name, decodeHtmlEntities(m[2] ?? m[3] ?? m[4] ?? ""));
}
return attrs;
}
/** What {@link scanTags} found, in page order. */
interface Scan {
metas: Attrs[];
links: Attrs[];
imgs: Attrs[];
/** The raw text of the first `<title>`. */
title?: string;
/** The bodies of the `application/ld+json` scripts. */
jsonLd: string[];
}
/**
* Collects the tags {@link parsePage} reads in one pass. Comments and the
* bodies of scripts and the title are skipped over, so a tag written inside
* them isn't read; one left open runs to the end of the page, as in a browser.
*/
function scanTags(html: string): Scan {
const scan: Scan = { metas: [], links: [], imgs: [], jsonLd: [] };
const re = new RegExp(TAG_RE);
let m: RegExpExecArray | null;
while ((m = re.exec(html)) !== null) {
const tag = m[1]?.toLowerCase();
if (!tag) {
const end = html.indexOf("-->", re.lastIndex);
if (end < 0) break;
re.lastIndex = end + 3;
continue;
}
const attrs = parseAttrs(m[2]);
if (tag === "meta") scan.metas.push(attrs);
else if (tag === "link") scan.links.push(attrs);
else if (tag === "img") scan.imgs.push(attrs);
else {
const close = new RegExp(`</${tag}\\s*>`, "gi");
close.lastIndex = re.lastIndex;
const end = close.exec(html);
if (!end) break;
const body = html.slice(re.lastIndex, end.index);
if (tag === "title") scan.title ??= body;
else if (
attrs.get("type")?.trim().toLowerCase() === "application/ld+json"
) {
scan.jsonLd.push(body);
}
re.lastIndex = close.lastIndex;
}
}
return scan;
}
/** Returns `value` trimmed, or undefined when nothing is left. */
function nonBlank(value: string | undefined): string | undefined {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
}
/**
@@ -82,21 +232,15 @@ export function extractPageTitle(html: string): string | undefined {
* {@link normalizeCssColor}. Sites often ship one per color scheme with the
* dark one first, so the unscoped tag wins, then the light one, then any.
*/
export function extractThemeColor(html: string): string | undefined {
const tagRe = /<meta\b[^>]*>/gi;
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
function themeColorOf(metas: Attrs[]): string | undefined {
let unscoped: string | undefined;
let light: string | undefined;
let any: string | undefined;
let m: RegExpExecArray | null;
while ((m = tagRe.exec(html)) !== null) {
const tag = m[0];
if (!/\bname=["']theme-color["']/i.test(tag)) continue;
const content = contentRe.exec(tag)?.[2];
for (const meta of metas) {
if (meta.get("name")?.toLowerCase() !== "theme-color") continue;
const content = nonBlank(meta.get("content"));
if (!content) continue;
const media = mediaRe.exec(tag)?.[2];
const media = meta.get("media");
if (!media) unscoped ??= content;
else if (/light/i.test(media)) light ??= content;
any ??= content;
@@ -104,180 +248,179 @@ export function extractThemeColor(html: string): string | undefined {
return unscoped ?? light ?? any;
}
/** Returns the lowercase tokens of a `<link>`'s `rel`. */
function relTokens(link: Attrs): string[] {
return (link.get("rel") ?? "").toLowerCase().split(/\s+/);
}
/**
* Returns the `color` of `<link rel="mask-icon">`, Safari's pinned-tab icon,
* as written. The generic provider's last resort for a brand color.
* as written. The generic preview's last resort for a brand color.
*/
export function extractMaskIconColor(html: string): string | undefined {
const linkRe = /<link[^>]+>/gi;
let m: RegExpExecArray | null;
while ((m = linkRe.exec(html)) !== null) {
const tag = m[0];
if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
function maskIconColorOf(links: Attrs[]): string | undefined {
for (const link of links) {
if (!relTokens(link).includes("mask-icon")) continue;
const color = nonBlank(link.get("color"));
if (color) return color;
}
return undefined;
}
/**
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
* common named color; undefined for any other form. The generic provider
* stores it as the preview's `accentColor`, a tint over the app's own surface,
* so alpha is dropped, and a fully transparent one is no color at all.
*/
export function normalizeCssColor(raw: string | undefined): string | undefined {
const color = raw ? parseCssColor(raw) : undefined;
return color && color.a > 0 ? hexOf(color) : undefined;
}
/**
* Returned to the generic provider by {@link extractJsonLd}, which ranks each
* field below the page's OG and Twitter tags.
*/
export interface JsonLdResult {
/** From `name`, or else `headline`. */
title?: string;
description?: string;
/**
* From `image`, or else `thumbnailUrl` or `thumbnail`, and only when it is
* an absolute URL.
*/
thumbnailUrl?: string;
}
function ldString(v: unknown): string | undefined {
if (typeof v === "string" && v.trim()) return v.trim();
if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
return v[0].trim();
}
return undefined;
}
function ldImage(v: unknown): string | undefined {
if (
typeof v === "string" &&
(v.startsWith("http://") || v.startsWith("https://"))
) return v;
if (Array.isArray(v)) return ldImage(v[0]);
if (v && typeof v === "object") {
const o = v as Record<string, unknown>;
return ldImage(o.url ?? o.contentUrl);
}
return undefined;
}
function ldExtractNode(data: unknown): JsonLdResult {
if (Array.isArray(data)) {
for (const item of data) {
const r = ldExtractNode(item);
if (r.title || r.thumbnailUrl) return r;
}
return {};
}
if (!data || typeof data !== "object") return {};
const o = data as Record<string, unknown>;
if (o["@graph"]) return ldExtractNode(o["@graph"]);
return {
title: ldString(o.name ?? o.headline),
description: ldString(o.description),
thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
};
}
/**
* Returns the first JSON-LD node in `html` that has a title or an image,
* looking through `@graph` and arrays; empty when there is none. An image may
* be a URL, a list of them or an `ImageObject`. Blocks that aren't valid JSON
* are skipped.
*/
export function extractJsonLd(html: string): JsonLdResult {
const pattern =
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
let match: RegExpExecArray | null;
while ((match = pattern.exec(html)) !== null) {
try {
const result = ldExtractNode(JSON.parse(match[1]));
if (result.title || result.thumbnailUrl) return result;
} catch { /* invalid JSON: try the next block */ }
}
return {};
}
/**
* Returns the absolute URL of the first `<img>` whose declared width and height
* are both at least `minSize` pixels; an image without them doesn't count.
* Data URIs are skipped.
*/
export function extractLargeImage(
html: string,
baseUrl: string,
minSize = 200,
): string | undefined {
const imgPattern = /<img[^>]+>/gi;
let match: RegExpExecArray | null;
while ((match = imgPattern.exec(html)) !== null) {
const tag = match[0];
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
if (!src || src.startsWith("data:")) continue;
const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
if (w >= minSize && h >= minSize) {
try {
return new URL(src, baseUrl).toString();
} catch {
continue;
}
}
}
return undefined;
}
/**
* Returns the absolute URL of the page's largest icon, among its `icon` and
* `apple-touch-icon` links, by declared `sizes`; the first one when none
* declares a size. The generic provider stores it as the preview's
* `faviconUrl`, which the thumbnail placeholder draws.
* declares a size. A `mask-icon` is no candidate: it is a one-color shape for
* Safari to tint, black when drawn as is.
*
* An SVG icon wins over any size: it declares none, yet scales to any.
*/
export function extractBestIcon(
html: string,
baseUrl: string,
): string | undefined {
const linkRe = /<link[^>]+>/gi;
const relRe = /\brel=["']([^"']+)["']/i;
const hrefRe = /\bhref=["']([^"']+)["']/i;
const sizesRe = /\bsizes=["']([^"']+)["']/i;
const candidates: { href: string; area: number }[] = [];
let m: RegExpExecArray | null;
while ((m = linkRe.exec(html)) !== null) {
const tag = m[0];
const rel = relRe.exec(tag)?.[1] ?? "";
if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
const href = hrefRe.exec(tag)?.[1];
function bestIconOf(links: Attrs[], baseUrl: string): string | undefined {
let best: { href: string; area: number } | undefined;
for (const link of links) {
const rel = relTokens(link);
if (
!rel.includes("icon") && !rel.includes("apple-touch-icon") &&
!rel.includes("apple-touch-icon-precomposed")
) continue;
const href = absoluteHttpUrl(link.get("href"), baseUrl);
if (!href) continue;
const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
const sm = sizesStr.match(/(\d+)x(\d+)/i);
const sizes = (link.get("sizes") ?? "").toLowerCase();
const isSvg = /\.svg(\?|$)/i.test(href) ||
/\btype=["']image\/svg\+xml["']/i.test(tag);
const area = isSvg
? Number.MAX_SAFE_INTEGER
: sm
? parseInt(sm[1]) * parseInt(sm[2])
: 0;
link.get("type")?.toLowerCase() === "image/svg+xml" ||
sizes === "any";
const area = isSvg ? Number.MAX_SAFE_INTEGER : Math.max(
0,
...[...sizes.matchAll(/(\d+)x(\d+)/g)].map(([, w, h]) => +w * +h),
);
if (!best || area > best.area) best = { href, area };
}
return best?.href;
}
/**
* Returned by the JSON-LD reading of {@link parsePage}, which the generic
* preview ranks below the page's OG and Twitter tags.
*/
export interface JsonLdResult {
/** From `name`, or else `headline`. */
title?: string;
description?: string;
/** From `image`, or else `thumbnailUrl` or `thumbnail`. */
thumbnailUrl?: string;
}
/**
* The JSON-LD types that describe the site around a page rather than the
* page itself. Their name is the site's or the company's, so a node of one of
* them is read only when no other node has a title or an image.
*/
const SITE_LD_TYPES = new Set([
"WebSite",
"Organization",
"Corporation",
"NewsMediaOrganization",
"Person",
"BreadcrumbList",
"SearchAction",
"ImageObject",
"SiteNavigationElement",
"WPHeader",
"WPFooter",
]);
function ldString(v: unknown): string | undefined {
if (typeof v === "string") return nonBlank(v);
if (Array.isArray(v) && typeof v[0] === "string") return nonBlank(v[0]);
return undefined;
}
function ldImage(v: unknown, baseUrl: string): string | undefined {
if (typeof v === "string") return absoluteHttpUrl(v, baseUrl);
if (Array.isArray(v)) return ldImage(v[0], baseUrl);
if (v && typeof v === "object") {
const o = v as Record<string, unknown>;
return ldImage(o.url ?? o.contentUrl, baseUrl);
}
return undefined;
}
/** Lists every node of `data`, through arrays and `@graph`, in order. */
function ldNodes(data: unknown): Record<string, unknown>[] {
if (Array.isArray(data)) return data.flatMap(ldNodes);
if (!data || typeof data !== "object") return [];
const o = data as Record<string, unknown>;
return o["@graph"] ? ldNodes(o["@graph"]) : [o];
}
/**
* Returns the first JSON-LD node about the page that has a title or an image,
* else the first about its site ({@link SITE_LD_TYPES}); empty when there is
* none. An image may be a URL, a list of them or an `ImageObject`. Blocks that
* aren't valid JSON are skipped.
*/
function jsonLdOf(blocks: string[], baseUrl: string): JsonLdResult {
let fallback: JsonLdResult | undefined;
for (const block of blocks) {
let data: unknown;
try {
candidates.push({ href: new URL(href, baseUrl).toString(), area });
data = JSON.parse(block);
} catch {
continue;
}
for (const node of ldNodes(data)) {
const result: JsonLdResult = {
title: ldString(node.name ?? node.headline),
description: ldString(node.description),
thumbnailUrl: ldImage(
node.image ?? node.thumbnailUrl ?? node.thumbnail,
baseUrl,
),
};
if (!result.title && !result.thumbnailUrl) continue;
const types = [node["@type"]].flat();
if (!types.some((t) => typeof t === "string" && SITE_LD_TYPES.has(t))) {
return result;
}
fallback ??= result;
}
}
return fallback ?? {};
}
if (candidates.length === 0) return undefined;
candidates.sort((a, b) => b.area - a.area);
return candidates[0].href;
/**
* Returns the source of an `<img>`: its `src`, or for a lazy-loaded image
* whose `src` is a `data:` placeholder, its `data-src` or `data-lazy-src`.
*/
function imageSource(img: Attrs, baseUrl: string): string | undefined {
const src = img.get("src")?.trim();
return src && !src.startsWith("data:")
? absoluteHttpUrl(src, baseUrl)
: absoluteHttpUrl(img.get("data-src") ?? img.get("data-lazy-src"), baseUrl);
}
/** Returns an `<img>`'s declared `name` dimension in pixels; 0 when none. */
function declaredSize(img: Attrs, name: "width" | "height"): number {
return parseInt(img.get(name) ?? "") || 0;
}
/**
* Returns the absolute URL of the first `<img>` whose declared width and height
* are both at least `minSize` pixels; an image without them doesn't count.
*/
function largeImageOf(
imgs: Attrs[],
baseUrl: string,
minSize = 200,
): string | undefined {
for (const img of imgs) {
if (
declaredSize(img, "width") < minSize ||
declaredSize(img, "height") < minSize
) continue;
const src = imageSource(img, baseUrl);
if (src) return src;
}
return undefined;
}
/** Filename words that mark an image as page furniture rather than content. */
@@ -297,6 +440,11 @@ const UI_IMAGE_KEYWORDS = new Set([
"pixel",
"bg",
"background",
"placeholder",
"spacer",
"blank",
"lazy",
"loading",
]);
/**
@@ -314,25 +462,86 @@ function isUiImage(src: string): boolean {
/**
* Returns the absolute URL of the first `<img>` that looks like content rather
* than a logo, icon or other page furniture. Data URIs are skipped. The generic
* provider's last resort for a thumbnail.
* than a logo, icon, tracking pixel or other page furniture: one declared
* smaller than 50 pixels either way doesn't count. The generic preview's last
* resort for a thumbnail.
*/
export function extractFirstContentImage(
html: string,
function firstContentImageOf(
imgs: Attrs[],
baseUrl: string,
): string | undefined {
const imgPattern = /<img[^>]+>/gi;
let match: RegExpExecArray | null;
while ((match = imgPattern.exec(html)) !== null) {
const tag = match[0];
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
if (!src || src.startsWith("data:")) continue;
if (isUiImage(src)) continue;
try {
return new URL(src, baseUrl).toString();
} catch {
continue;
}
for (const img of imgs) {
const w = declaredSize(img, "width");
const h = declaredSize(img, "height");
if ((w && w < 50) || (h && h < 50)) continue;
const src = imageSource(img, baseUrl);
if (src && !isUiImage(src)) return src;
}
return undefined;
}
/**
* Returned by {@link parsePage}: what a page says about itself, entities
* decoded, values trimmed and URLs absolute.
*/
export interface Page {
/**
* Returns the `content` of the first `<meta>` whose `property` or `name` is
* `key`, in any case: sites write `og:` tags under either attribute, and
* Twitter's too.
*/
meta(key: string): string | undefined;
/** The text of the `<title>` element. */
title?: string;
jsonLd: JsonLdResult;
/** The `theme-color`, as written: pass it to {@link normalizeCssColor}. */
themeColor?: string;
/** The `color` of the Safari pinned-tab icon, as written. */
maskIconColor?: string;
/** The largest icon the page links, which the thumbnail placeholder draws. */
bestIcon?: string;
/** The first image declared at least 200 pixels both ways. */
largeImage?: string;
/** The first image that doesn't look like page furniture. */
firstContentImage?: string;
}
/**
* Reads `html`, served from `pageUrl`, in one pass. The generic preview and
* the providers built on it read a page through it.
*/
export function parsePage(html: string, pageUrl: string): Page {
const scan = scanTags(html);
const metas = new Map<string, string>();
for (const meta of scan.metas) {
const content = nonBlank(meta.get("content"));
if (!content) continue;
for (const attr of ["property", "name"]) {
const key = meta.get(attr)?.trim().toLowerCase();
if (key && !metas.has(key)) metas.set(key, content);
}
}
return {
meta: (key) => metas.get(key.toLowerCase()),
title: nonBlank(scan.title && decodeHtmlEntities(scan.title)),
jsonLd: jsonLdOf(scan.jsonLd, pageUrl),
themeColor: themeColorOf(scan.metas),
maskIconColor: maskIconColorOf(scan.links),
bestIcon: bestIconOf(scan.links, pageUrl),
largeImage: largeImageOf(scan.imgs, pageUrl),
firstContentImage: firstContentImageOf(scan.imgs, pageUrl),
};
}
/**
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
* common named color; undefined for any other form. The generic preview
* stores it as the preview's `accentColor`, a tint over the app's own surface,
* so alpha is dropped, and a fully transparent one is no color at all.
*/
export function normalizeCssColor(raw: string | undefined): string | undefined {
const color = raw ? parseCssColor(raw) : undefined;
return color && color.a > 0 ? hexOf(color) : undefined;
}

46
api/remote/providers/bandcamp.ts
View File

@@ -1,15 +1,35 @@
/**
* Previews Bandcamp links from their page's OG tags, for
* `rich-content-service.ts`.
* Previews Bandcamp links from their page, for `rich-content-service.ts`.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage } from "../fetch.ts";
import { BANDCAMP_HOST_RE } from "../bandcamp-tralbum.ts";
import { genericPreview } from "./generic.ts";
/**
* Returns the embed player's URL a page's `og:video` names, over https; only
* a player on bandcamp.com counts, since the frontend frames it.
*/
function embeddedPlayerUrl(raw: string | undefined): string | undefined {
if (!raw) return undefined;
try {
const url = new URL(raw);
if (
(url.protocol !== "https:" && url.protocol !== "http:") ||
url.hostname !== "bandcamp.com" ||
!url.pathname.startsWith("/EmbeddedPlayer/")
) return undefined;
url.protocol = "https:";
return url.href;
} catch {
return undefined;
}
}
/**
* Claims bandcamp.com and its subdomains, where artists have their pages, and
@@ -29,24 +49,16 @@ export const bandcampProvider: RichContentProvider = {
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
const fetched = await fetchPage(url);
if (fetched.html === undefined) {
return { type: "bandcamp", siteName: "Bandcamp", url };
}
const html = await readTextCapped(res);
const page = parsePage(fetched.html, fetched.url);
return {
...genericPreview(page, url, fetched.url),
type: "bandcamp",
siteName: "Bandcamp",
url,
title: extractOgTag(html, "title"),
description: extractOgTag(html, "description"),
thumbnailUrl: extractOgTag(html, "image"),
embedUrl: extractOgTag(html, "video") ?? undefined,
embedUrl: embeddedPlayerUrl(page.meta("og:video")),
};
},
};

176
api/remote/providers/generic.ts
View File

@@ -1,6 +1,8 @@
/**
* Previews any page no other provider claims, for `rich-content-service.ts`,
* from whatever metadata its HTML carries.
* from whatever metadata its HTML carries. Its reading of a page,
* {@link genericPreview}, is also the base of the Bandcamp, SoundCloud and
* YouTube channel previews.
*
* @module
*/
@@ -9,25 +11,97 @@ import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { siteHost } from "../../../shared/url.ts";
import {
extractBestIcon,
extractFirstContentImage,
extractJsonLd,
extractLargeImage,
extractMaskIconColor,
extractMetaName,
extractOgTag,
extractPageTitle,
extractThemeColor,
absoluteHttpUrl,
normalizeCssColor,
type Page,
parsePage,
} from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { fetchPage } from "../fetch.ts";
/** Returns the path of `url` without its trailing slashes, `/` for the root. */
function pathOf(url: string): string {
return new URL(url).pathname.replace(/\/+$/, "") || "/";
}
/**
* Matches every URL, so it comes last in the service's list. Each field falls
* back from OG tags to Twitter tags, JSON-LD and the markup itself. The icon
* and brand color feed the placeholder the frontend draws when there is no
* thumbnail.
* Strips a trailing ` | Site`, ` – Site` or the like from a `<title>`, which
* names the page and then its site; the site is in the preview's badge.
*/
function withoutSiteSuffix(title: string, siteName: string): string {
const escaped = siteName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const suffix = new RegExp(`\\s+[|–—\\-·•]\\s+${escaped}\\s*$`, "i");
return title.replace(suffix, "") || title;
}
/**
* Returns the preview of the page `url` that `page` read, served from
* `finalUrl` after redirects. Each field falls back from OG tags to Twitter
* tags, JSON-LD and the markup itself. The icon and brand color feed the
* placeholder the frontend draws when there is no thumbnail.
*/
export function genericPreview(
page: Page,
url: string,
finalUrl: string,
): RichContent {
// An og:url naming the home page from another page means the OG tags are
// the site's defaults, not this page's: their title and image are skipped.
// Any other og:url is the page's canonical address, often with a slug the
// link left out, and its tags are the page's.
const ogUrl = page.meta("og:url");
let useOg = true;
try {
useOg = !ogUrl || pathOf(new URL(ogUrl, finalUrl).href) !== "/" ||
pathOf(finalUrl) === "/";
} catch { /* an og:url that doesn't parse names no other page */ }
const ogSiteName = page.meta("og:site_name");
const pageTitle = page.title && ogSiteName
? withoutSiteSuffix(page.title, ogSiteName)
: page.title;
const title = (useOg ? page.meta("og:title") : undefined) ??
page.meta("twitter:title") ??
page.jsonLd.title ??
pageTitle;
const description = page.meta("og:description") ??
page.meta("twitter:description") ??
page.jsonLd.description ??
page.meta("description");
// Never the favicon: a 16×16 icon cropped into a thumbnail looks broken,
// and the placeholder drawn instead shows the icon properly. A tag naming
// no usable URL falls through to the next.
const thumbnailUrl =
(useOg ? absoluteHttpUrl(page.meta("og:image"), finalUrl) : undefined) ??
absoluteHttpUrl(page.meta("twitter:image"), finalUrl) ??
page.jsonLd.thumbnailUrl ??
page.largeImage ??
page.firstContentImage;
// `/favicon.ico` is a guess. When it doesn't load, the placeholder shows
// the site's initial.
const faviconUrl = page.bestIcon ?? `${new URL(finalUrl).origin}/favicon.ico`;
const accentColor = normalizeCssColor(
page.themeColor ??
page.meta("msapplication-TileColor") ??
page.maskIconColor,
);
return {
type: "generic",
url,
title,
description,
thumbnailUrl,
faviconUrl,
accentColor,
siteName: ogSiteName ?? siteHost(url),
};
}
/** Matches every URL, so it comes last in the service's list. */
export const genericProvider: RichContentProvider = {
name: "generic",
@@ -36,72 +110,12 @@ export const genericProvider: RichContentProvider = {
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "generic", url };
}
const html = await readTextCapped(res);
const ld = extractJsonLd(html);
// An og:url naming another page (often the home page) means the OG tags
// are the site's defaults, not this page's: its title and image are
// skipped.
const ogUrl = extractOgTag(html, "url");
const useOg = !ogUrl || (() => {
try {
const ogPath = new URL(ogUrl).pathname.replace(/\/+$/, "") || "/";
const pagePath = new URL(url).pathname.replace(/\/+$/, "") || "/";
return ogPath === pagePath;
} catch {
return true;
}
})();
const title = (useOg ? extractOgTag(html, "title") : undefined) ??
extractMetaName(html, "twitter:title") ??
ld.title ??
extractPageTitle(html);
const siteName = extractOgTag(html, "site_name") ??
siteHost(url);
const description = extractOgTag(html, "description") ??
extractMetaName(html, "twitter:description") ??
ld.description ??
extractMetaName(html, "description");
// Never the favicon: a 16×16 icon cropped into a thumbnail looks broken,
// and the placeholder drawn instead shows the icon properly.
const thumbnailUrl = (useOg ? extractOgTag(html, "image") : undefined) ??
extractMetaName(html, "twitter:image") ??
ld.thumbnailUrl ??
extractLargeImage(html, url) ??
extractFirstContentImage(html, url);
// `/favicon.ico` is a guess. When it doesn't load, the placeholder shows
// the site's initial.
const faviconUrl = extractBestIcon(html, url) ??
`${new URL(url).origin}/favicon.ico`;
const accentColor = normalizeCssColor(
extractThemeColor(html) ??
extractMetaName(html, "msapplication-TileColor") ??
extractMaskIconColor(html),
);
return {
type: "generic",
const fetched = await fetchPage(url);
if (fetched.html === undefined) return { type: "generic", url };
return genericPreview(
parsePage(fetched.html, fetched.url),
url,
title,
description,
thumbnailUrl,
faviconUrl,
accentColor,
siteName,
};
fetched.url,
);
},
};

7
api/remote/providers/provider.ts
View File

@@ -18,9 +18,10 @@ export interface RichContentProvider {
matches(url: string): boolean;
/**
* Returns the preview of `url`, which ends up in the dump's `rich_content` or
* the create form's preview. It rejects when the page can't be fetched, and
* the service then falls back to a stub. A page that isn't HTML yields a
* preview without metadata.
* the create form's preview. It rejects when the page can't be fetched or
* answers with an error status, and the service then falls back to a stub.
* A page that isn't HTML yields a preview without metadata. The service
* tidies what it returns, so a provider leaves text as the page wrote it.
*/
fetch(url: string): Promise<RichContent>;
}

5
api/remote/providers/self.ts
View File

@@ -1,7 +1,8 @@
/**
* Previews links to this site's own dump, user and playlist pages, as
* `parsePagePath` reads them, from the database, for `rich-content-service.ts`. No request goes out, so this works
* even when the site runs on a private host that `fetchWithTimeout` refuses.
* `parsePagePath` reads them, from the database, for
* `rich-content-service.ts`. No request goes out, so this works even when the
* site runs on a private host that `fetchWithTimeout` refuses.
*
* @module
*/

66
api/remote/providers/soundcloud.ts
View File

@@ -1,50 +1,72 @@
/**
* Previews SoundCloud links from their page's OG tags, for
* `rich-content-service.ts`.
* Previews SoundCloud links from their page, for `rich-content-service.ts`.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage } from "../fetch.ts";
import { genericPreview } from "./generic.ts";
/** SoundCloud's own hosts, the short links' `on.` included. */
const SOUNDCLOUD_HOSTS = new Set([
"soundcloud.com",
"www.soundcloud.com",
"m.soundcloud.com",
"on.soundcloud.com",
]);
/**
* Claims soundcloud.com only, not www. or m.soundcloud.com, and builds the
* embed player's URL from the link itself.
* Returns the soundcloud.com address of the page, which the embed player
* takes: its `og:url`, or else the URL it was served from. A short link only
* redirects, so it is never the answer; tracking parameters are dropped.
*/
function pageAddress(ogUrl: string | undefined, finalUrl: string): string {
for (const raw of [ogUrl, finalUrl]) {
if (!raw) continue;
try {
const url = new URL(raw);
if (
!SOUNDCLOUD_HOSTS.has(url.hostname) ||
url.hostname === "on.soundcloud.com"
) continue;
return `https://soundcloud.com${url.pathname}`;
} catch {
continue;
}
}
return finalUrl;
}
/**
* Claims SoundCloud's hosts, short links included, and builds the embed
* player's URL from the page's address.
*/
export const soundcloudProvider: RichContentProvider = {
name: "soundcloud",
matches(url: string): boolean {
try {
return new URL(url).hostname === "soundcloud.com";
return SOUNDCLOUD_HOSTS.has(new URL(url).hostname);
} catch {
return false;
}
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "soundcloud", siteName: "SoundCloud", url };
}
const html = await readTextCapped(res);
const fetched = await fetchPage(url);
const page = fetched.html === undefined
? undefined
: parsePage(fetched.html, fetched.url);
const address = pageAddress(page?.meta("og:url"), fetched.url);
return {
...(page ? genericPreview(page, url, fetched.url) : { url }),
type: "soundcloud",
siteName: "SoundCloud",
url,
title: extractOgTag(html, "title"),
description: extractOgTag(html, "description"),
thumbnailUrl: extractOgTag(html, "image"),
embedUrl: `https://w.soundcloud.com/player/?url=${
encodeURIComponent(url)
encodeURIComponent(address)
}&visual=true&auto_play=false`,
};
},

30
api/remote/providers/youtube.ts
View File

@@ -1,17 +1,18 @@
/**
* Previews YouTube videos, playlists and channels, for
* `rich-content-service.ts`. Videos and playlists are read from YouTube's
* oEmbed endpoint, channels from their page's OG tags.
* oEmbed endpoint, channels from their page.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage, fetchWithTimeout } from "../fetch.ts";
import { youtubeHost, youtubeVideoId } from "../../lib/canonical-url.ts";
import { siteHostOf } from "../../../shared/url.ts";
import { genericPreview } from "./generic.ts";
/** Returns the id of the video `url` points at, in any shape YouTube uses. */
function extractVideoId(url: string): string | null {
@@ -123,27 +124,17 @@ export const youtubeProvider: RichContentProvider = {
if (channelPath && !videoId) {
// oEmbed doesn't take channel URLs.
let title: string | undefined;
let thumbnailUrl: string | undefined;
let channel: RichContent | undefined;
try {
const res = await fetchWithTimeout(url);
if (res.ok) {
const html = await readTextCapped(res);
title = extractOgTag(html, "title");
thumbnailUrl = extractOgTag(html, "image");
} else {
await res.body?.cancel();
const fetched = await fetchPage(url);
if (fetched.html !== undefined) {
const page = parsePage(fetched.html, fetched.url);
channel = genericPreview(page, url, fetched.url);
}
} catch {
// YouTube unreachable: the preview goes without.
}
return {
type: "youtube",
siteName: "YouTube",
url,
title,
thumbnailUrl,
};
return { ...channel, type: "youtube", siteName: "YouTube", url };
}
if (listId && !videoId) {
@@ -172,7 +163,6 @@ export const youtubeProvider: RichContentProvider = {
type: "youtube",
siteName: "YouTube",
url,
videoId: videoId!,
title,
thumbnailUrl,
embedUrl: `https://www.youtube.com/embed/${videoId}?${embedParams}`,