548 lines
16 KiB
TypeScript
548 lines
16 KiB
TypeScript
/**
|
||
* Reads a link preview's metadata out of a page's HTML. {@link parsePage}
|
||
* scans the page once for the tags that carry it (`<meta>`, `<link>`, `<img>`,
|
||
* `<title>` and JSON-LD scripts) without building a document, and returns a
|
||
* {@link Page} that the providers' `genericPreview` picks from. Every field is
|
||
* undefined when the page doesn't say.
|
||
*
|
||
* @module
|
||
*/
|
||
|
||
import { hexOf, parseCssColor } from "../lib/css-color.ts";
|
||
|
||
/**
|
||
* The named entities decoded besides numeric ones: the XML five, and those
|
||
* page titles and descriptions actually use, Latin-1 letters and typography.
|
||
*/
|
||
const NAMED_ENTITIES: Record<string, string> = {
|
||
amp: "&",
|
||
lt: "<",
|
||
gt: ">",
|
||
quot: '"',
|
||
apos: "'",
|
||
nbsp: " ",
|
||
aelig: "æ",
|
||
AElig: "Æ",
|
||
oelig: "œ",
|
||
OElig: "Œ",
|
||
szlig: "ß",
|
||
oslash: "ø",
|
||
Oslash: "Ø",
|
||
eth: "ð",
|
||
ETH: "Ð",
|
||
thorn: "þ",
|
||
THORN: "Þ",
|
||
hellip: "…",
|
||
lsquo: "‘",
|
||
rsquo: "’",
|
||
sbquo: "‚",
|
||
ldquo: "“",
|
||
rdquo: "”",
|
||
bdquo: "„",
|
||
laquo: "«",
|
||
raquo: "»",
|
||
lsaquo: "‹",
|
||
rsaquo: "›",
|
||
ndash: "–",
|
||
mdash: "—",
|
||
bull: "•",
|
||
middot: "·",
|
||
dagger: "†",
|
||
Dagger: "‡",
|
||
prime: "′",
|
||
Prime: "″",
|
||
euro: "€",
|
||
pound: "£",
|
||
yen: "¥",
|
||
cent: "¢",
|
||
copy: "©",
|
||
reg: "®",
|
||
trade: "™",
|
||
deg: "°",
|
||
plusmn: "±",
|
||
times: "×",
|
||
divide: "÷",
|
||
micro: "µ",
|
||
para: "¶",
|
||
sect: "§",
|
||
iexcl: "¡",
|
||
iquest: "¿",
|
||
ordf: "ª",
|
||
ordm: "º",
|
||
sup2: "²",
|
||
sup3: "³",
|
||
frac14: "¼",
|
||
frac12: "½",
|
||
frac34: "¾",
|
||
shy: "",
|
||
ensp: " ",
|
||
emsp: " ",
|
||
thinsp: " ",
|
||
zwnj: "",
|
||
zwj: "",
|
||
};
|
||
|
||
// The accented letters, `é` and the like, composed from their letter
|
||
// and accent rather than listed one by one.
|
||
for (
|
||
const [accent, mark, letters] of [
|
||
["grave", "̀", "aeiouAEIOU"],
|
||
["acute", "́", "aeiouyAEIOUY"],
|
||
["circ", "̂", "aeiouAEIOU"],
|
||
["tilde", "̃", "anoANO"],
|
||
["uml", "̈", "aeiouyAEIOUY"],
|
||
["ring", "̊", "aA"],
|
||
["cedil", "̧", "cC"],
|
||
]
|
||
) {
|
||
for (const letter of letters) {
|
||
NAMED_ENTITIES[letter + accent] = (letter + mark).normalize("NFC");
|
||
}
|
||
}
|
||
|
||
/** Returns the character of code point `n`, or undefined when there is none. */
|
||
function fromCodePoint(n: number): string | undefined {
|
||
if (n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) {
|
||
return undefined;
|
||
}
|
||
return String.fromCodePoint(n);
|
||
}
|
||
|
||
/**
|
||
* Decodes the entities in `str` in a single pass, so `&quot;` reads
|
||
* `"`. An entity it doesn't know, or a code point that doesn't exist, is
|
||
* left as written. Also used by `bandcamp-tralbum.ts` on the JSON a page
|
||
* carries in an attribute.
|
||
*/
|
||
export function decodeHtmlEntities(str: string): string {
|
||
return str.replace(
|
||
/&(?:#(\d{1,8})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z][a-zA-Z0-9]{1,31}));/g,
|
||
(match, dec?: string, hex?: string, name?: string) => {
|
||
if (dec) return fromCodePoint(Number(dec)) ?? match;
|
||
if (hex) return fromCodePoint(parseInt(hex, 16)) ?? match;
|
||
return NAMED_ENTITIES[name!] ?? NAMED_ENTITIES[name!.toLowerCase()] ??
|
||
match;
|
||
},
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Returns `raw` resolved against `base` as an absolute http(s) URL; undefined
|
||
* for a `data:` URI, another scheme or a URL that doesn't parse. Every URL a
|
||
* page names goes through it, so `/og.png` and `//cdn/og.png` work.
|
||
*/
|
||
export function absoluteHttpUrl(
|
||
raw: string | undefined,
|
||
base: string,
|
||
): string | undefined {
|
||
if (!raw?.trim()) return undefined;
|
||
try {
|
||
const url = new URL(raw.trim(), base);
|
||
return url.protocol === "http:" || url.protocol === "https:"
|
||
? url.href
|
||
: undefined;
|
||
} catch {
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
/** A tag's attributes by lowercase name, values decoded; the first one wins. */
|
||
type Attrs = Map<string, string>;
|
||
|
||
/**
|
||
* Matches the start of a tag the scan reads, or of a comment. Quoted values
|
||
* may hold `>`, but no tag spans a `<`, so each attempt stops at the next one
|
||
* and the scan stays linear in the page's size.
|
||
*/
|
||
const TAG_RE =
|
||
/<(meta|link|img|title|script)\b((?:[^<>"']|"[^"<]*"|'[^'<]*')*)>|<!--/gi;
|
||
|
||
/** Matches one attribute: unquoted, single- or double-quoted, or bare. */
|
||
const ATTR_RE =
|
||
/([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
|
||
|
||
function parseAttrs(source: string): Attrs {
|
||
const attrs: Attrs = new Map();
|
||
for (const m of source.matchAll(ATTR_RE)) {
|
||
const name = m[1].toLowerCase();
|
||
if (attrs.has(name)) continue;
|
||
attrs.set(name, decodeHtmlEntities(m[2] ?? m[3] ?? m[4] ?? ""));
|
||
}
|
||
return attrs;
|
||
}
|
||
|
||
/** What {@link scanTags} found, in page order. */
|
||
interface Scan {
|
||
metas: Attrs[];
|
||
links: Attrs[];
|
||
imgs: Attrs[];
|
||
/** The raw text of the first `<title>`. */
|
||
title?: string;
|
||
/** The bodies of the `application/ld+json` scripts. */
|
||
jsonLd: string[];
|
||
}
|
||
|
||
/**
|
||
* Collects the tags {@link parsePage} reads in one pass. Comments and the
|
||
* bodies of scripts and the title are skipped over, so a tag written inside
|
||
* them isn't read; one left open runs to the end of the page, as in a browser.
|
||
*/
|
||
function scanTags(html: string): Scan {
|
||
const scan: Scan = { metas: [], links: [], imgs: [], jsonLd: [] };
|
||
const re = new RegExp(TAG_RE);
|
||
let m: RegExpExecArray | null;
|
||
while ((m = re.exec(html)) !== null) {
|
||
const tag = m[1]?.toLowerCase();
|
||
if (!tag) {
|
||
const end = html.indexOf("-->", re.lastIndex);
|
||
if (end < 0) break;
|
||
re.lastIndex = end + 3;
|
||
continue;
|
||
}
|
||
const attrs = parseAttrs(m[2]);
|
||
if (tag === "meta") scan.metas.push(attrs);
|
||
else if (tag === "link") scan.links.push(attrs);
|
||
else if (tag === "img") scan.imgs.push(attrs);
|
||
else {
|
||
const close = new RegExp(`</${tag}\\s*>`, "gi");
|
||
close.lastIndex = re.lastIndex;
|
||
const end = close.exec(html);
|
||
if (!end) break;
|
||
const body = html.slice(re.lastIndex, end.index);
|
||
if (tag === "title") scan.title ??= body;
|
||
else if (
|
||
attrs.get("type")?.trim().toLowerCase() === "application/ld+json"
|
||
) {
|
||
scan.jsonLd.push(body);
|
||
}
|
||
re.lastIndex = close.lastIndex;
|
||
}
|
||
}
|
||
return scan;
|
||
}
|
||
|
||
/** Returns `value` trimmed, or undefined when nothing is left. */
|
||
function nonBlank(value: string | undefined): string | undefined {
|
||
const trimmed = value?.trim();
|
||
return trimmed ? trimmed : undefined;
|
||
}
|
||
|
||
/**
|
||
* Returns the page's `theme-color`, as written: pass it to
|
||
* {@link normalizeCssColor}. Sites often ship one per color scheme with the
|
||
* dark one first, so the unscoped tag wins, then the light one, then any.
|
||
*/
|
||
function themeColorOf(metas: Attrs[]): string | undefined {
|
||
let unscoped: string | undefined;
|
||
let light: string | undefined;
|
||
let any: string | undefined;
|
||
for (const meta of metas) {
|
||
if (meta.get("name")?.toLowerCase() !== "theme-color") continue;
|
||
const content = nonBlank(meta.get("content"));
|
||
if (!content) continue;
|
||
const media = meta.get("media");
|
||
if (!media) unscoped ??= content;
|
||
else if (/light/i.test(media)) light ??= content;
|
||
any ??= content;
|
||
}
|
||
return unscoped ?? light ?? any;
|
||
}
|
||
|
||
/** Returns the lowercase tokens of a `<link>`'s `rel`. */
|
||
function relTokens(link: Attrs): string[] {
|
||
return (link.get("rel") ?? "").toLowerCase().split(/\s+/);
|
||
}
|
||
|
||
/**
|
||
* Returns the `color` of `<link rel="mask-icon">`, Safari's pinned-tab icon,
|
||
* as written. The generic preview's last resort for a brand color.
|
||
*/
|
||
function maskIconColorOf(links: Attrs[]): string | undefined {
|
||
for (const link of links) {
|
||
if (!relTokens(link).includes("mask-icon")) continue;
|
||
const color = nonBlank(link.get("color"));
|
||
if (color) return color;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/**
|
||
* Returns the absolute URL of the page's largest icon, among its `icon` and
|
||
* `apple-touch-icon` links, by declared `sizes`; the first one when none
|
||
* declares a size. A `mask-icon` is no candidate: it is a one-color shape for
|
||
* Safari to tint, black when drawn as is.
|
||
*
|
||
* An SVG icon wins over any size: it declares none, yet scales to any.
|
||
*/
|
||
function bestIconOf(links: Attrs[], baseUrl: string): string | undefined {
|
||
let best: { href: string; area: number } | undefined;
|
||
for (const link of links) {
|
||
const rel = relTokens(link);
|
||
if (
|
||
!rel.includes("icon") && !rel.includes("apple-touch-icon") &&
|
||
!rel.includes("apple-touch-icon-precomposed")
|
||
) continue;
|
||
const href = absoluteHttpUrl(link.get("href"), baseUrl);
|
||
if (!href) continue;
|
||
const sizes = (link.get("sizes") ?? "").toLowerCase();
|
||
const isSvg = /\.svg(\?|$)/i.test(href) ||
|
||
link.get("type")?.toLowerCase() === "image/svg+xml" ||
|
||
sizes === "any";
|
||
const area = isSvg ? Number.MAX_SAFE_INTEGER : Math.max(
|
||
0,
|
||
...[...sizes.matchAll(/(\d+)x(\d+)/g)].map(([, w, h]) => +w * +h),
|
||
);
|
||
if (!best || area > best.area) best = { href, area };
|
||
}
|
||
return best?.href;
|
||
}
|
||
|
||
/**
|
||
* Returned by the JSON-LD reading of {@link parsePage}, which the generic
|
||
* preview ranks below the page's OG and Twitter tags.
|
||
*/
|
||
export interface JsonLdResult {
|
||
/** From `name`, or else `headline`. */
|
||
title?: string;
|
||
description?: string;
|
||
/** From `image`, or else `thumbnailUrl` or `thumbnail`. */
|
||
thumbnailUrl?: string;
|
||
}
|
||
|
||
/**
|
||
* The JSON-LD types that describe the site around a page rather than the
|
||
* page itself. Their name is the site's or the company's, so a node of one of
|
||
* them is read only when no other node has a title or an image.
|
||
*/
|
||
const SITE_LD_TYPES = new Set([
|
||
"WebSite",
|
||
"Organization",
|
||
"Corporation",
|
||
"NewsMediaOrganization",
|
||
"Person",
|
||
"BreadcrumbList",
|
||
"SearchAction",
|
||
"ImageObject",
|
||
"SiteNavigationElement",
|
||
"WPHeader",
|
||
"WPFooter",
|
||
]);
|
||
|
||
function ldString(v: unknown): string | undefined {
|
||
if (typeof v === "string") return nonBlank(v);
|
||
if (Array.isArray(v) && typeof v[0] === "string") return nonBlank(v[0]);
|
||
return undefined;
|
||
}
|
||
|
||
function ldImage(v: unknown, baseUrl: string): string | undefined {
|
||
if (typeof v === "string") return absoluteHttpUrl(v, baseUrl);
|
||
if (Array.isArray(v)) return ldImage(v[0], baseUrl);
|
||
if (v && typeof v === "object") {
|
||
const o = v as Record<string, unknown>;
|
||
return ldImage(o.url ?? o.contentUrl, baseUrl);
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/** Lists every node of `data`, through arrays and `@graph`, in order. */
|
||
function ldNodes(data: unknown): Record<string, unknown>[] {
|
||
if (Array.isArray(data)) return data.flatMap(ldNodes);
|
||
if (!data || typeof data !== "object") return [];
|
||
const o = data as Record<string, unknown>;
|
||
return o["@graph"] ? ldNodes(o["@graph"]) : [o];
|
||
}
|
||
|
||
/**
|
||
* Returns the first JSON-LD node about the page that has a title or an image,
|
||
* else the first about its site ({@link SITE_LD_TYPES}); empty when there is
|
||
* none. An image may be a URL, a list of them or an `ImageObject`. Blocks that
|
||
* aren't valid JSON are skipped.
|
||
*/
|
||
function jsonLdOf(blocks: string[], baseUrl: string): JsonLdResult {
|
||
let fallback: JsonLdResult | undefined;
|
||
for (const block of blocks) {
|
||
let data: unknown;
|
||
try {
|
||
data = JSON.parse(block);
|
||
} catch {
|
||
continue;
|
||
}
|
||
for (const node of ldNodes(data)) {
|
||
const result: JsonLdResult = {
|
||
title: ldString(node.name ?? node.headline),
|
||
description: ldString(node.description),
|
||
thumbnailUrl: ldImage(
|
||
node.image ?? node.thumbnailUrl ?? node.thumbnail,
|
||
baseUrl,
|
||
),
|
||
};
|
||
if (!result.title && !result.thumbnailUrl) continue;
|
||
const types = [node["@type"]].flat();
|
||
if (!types.some((t) => typeof t === "string" && SITE_LD_TYPES.has(t))) {
|
||
return result;
|
||
}
|
||
fallback ??= result;
|
||
}
|
||
}
|
||
return fallback ?? {};
|
||
}
|
||
|
||
/**
|
||
* Returns the source of an `<img>`: its `src`, or for a lazy-loaded image
|
||
* whose `src` is a `data:` placeholder, its `data-src` or `data-lazy-src`.
|
||
*/
|
||
function imageSource(img: Attrs, baseUrl: string): string | undefined {
|
||
const src = img.get("src")?.trim();
|
||
return src && !src.startsWith("data:")
|
||
? absoluteHttpUrl(src, baseUrl)
|
||
: absoluteHttpUrl(img.get("data-src") ?? img.get("data-lazy-src"), baseUrl);
|
||
}
|
||
|
||
/** Returns an `<img>`'s declared `name` dimension in pixels; 0 when none. */
|
||
function declaredSize(img: Attrs, name: "width" | "height"): number {
|
||
return parseInt(img.get(name) ?? "") || 0;
|
||
}
|
||
|
||
/**
|
||
* Returns the absolute URL of the first `<img>` whose declared width and height
|
||
* are both at least `minSize` pixels; an image without them doesn't count.
|
||
*/
|
||
function largeImageOf(
|
||
imgs: Attrs[],
|
||
baseUrl: string,
|
||
minSize = 200,
|
||
): string | undefined {
|
||
for (const img of imgs) {
|
||
if (
|
||
declaredSize(img, "width") < minSize ||
|
||
declaredSize(img, "height") < minSize
|
||
) continue;
|
||
const src = imageSource(img, baseUrl);
|
||
if (src) return src;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/** Filename words that mark an image as page furniture rather than content. */
|
||
const UI_IMAGE_KEYWORDS = new Set([
|
||
"logo",
|
||
"icon",
|
||
"sprite",
|
||
"favicon",
|
||
"avatar",
|
||
"banner",
|
||
"header",
|
||
"nav",
|
||
"menu",
|
||
"cart",
|
||
"search",
|
||
"tracking",
|
||
"pixel",
|
||
"bg",
|
||
"background",
|
||
"placeholder",
|
||
"spacer",
|
||
"blank",
|
||
"lazy",
|
||
"loading",
|
||
]);
|
||
|
||
/**
|
||
* Checks whether `src` looks like page furniture: an SVG, or a filename with
|
||
* one of the {@link UI_IMAGE_KEYWORDS} between its `-`, `_` or `.`.
|
||
*/
|
||
function isUiImage(src: string): boolean {
|
||
if (/\.svg(\?|$)/i.test(src)) return true;
|
||
const filename = src.split("?")[0].split("/").pop() ?? "";
|
||
const baseName = filename.replace(/\.[^.]+$/, "");
|
||
return baseName.toLowerCase().split(/[-_.]/).some((t) =>
|
||
UI_IMAGE_KEYWORDS.has(t)
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Returns the absolute URL of the first `<img>` that looks like content rather
|
||
* than a logo, icon, tracking pixel or other page furniture: one declared
|
||
* smaller than 50 pixels either way doesn't count. The generic preview's last
|
||
* resort for a thumbnail.
|
||
*/
|
||
function firstContentImageOf(
|
||
imgs: Attrs[],
|
||
baseUrl: string,
|
||
): string | undefined {
|
||
for (const img of imgs) {
|
||
const w = declaredSize(img, "width");
|
||
const h = declaredSize(img, "height");
|
||
if ((w && w < 50) || (h && h < 50)) continue;
|
||
const src = imageSource(img, baseUrl);
|
||
if (src && !isUiImage(src)) return src;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/**
|
||
* Returned by {@link parsePage}: what a page says about itself, entities
|
||
* decoded, values trimmed and URLs absolute.
|
||
*/
|
||
export interface Page {
|
||
/**
|
||
* Returns the `content` of the first `<meta>` whose `property` or `name` is
|
||
* `key`, in any case: sites write `og:` tags under either attribute, and
|
||
* Twitter's too.
|
||
*/
|
||
meta(key: string): string | undefined;
|
||
/** The text of the `<title>` element. */
|
||
title?: string;
|
||
jsonLd: JsonLdResult;
|
||
/** The `theme-color`, as written: pass it to {@link normalizeCssColor}. */
|
||
themeColor?: string;
|
||
/** The `color` of the Safari pinned-tab icon, as written. */
|
||
maskIconColor?: string;
|
||
/** The largest icon the page links, which the thumbnail placeholder draws. */
|
||
bestIcon?: string;
|
||
/** The first image declared at least 200 pixels both ways. */
|
||
largeImage?: string;
|
||
/** The first image that doesn't look like page furniture. */
|
||
firstContentImage?: string;
|
||
}
|
||
|
||
/**
|
||
* Reads `html`, served from `pageUrl`, in one pass. The generic preview and
|
||
* the providers built on it read a page through it.
|
||
*/
|
||
export function parsePage(html: string, pageUrl: string): Page {
|
||
const scan = scanTags(html);
|
||
|
||
const metas = new Map<string, string>();
|
||
for (const meta of scan.metas) {
|
||
const content = nonBlank(meta.get("content"));
|
||
if (!content) continue;
|
||
for (const attr of ["property", "name"]) {
|
||
const key = meta.get(attr)?.trim().toLowerCase();
|
||
if (key && !metas.has(key)) metas.set(key, content);
|
||
}
|
||
}
|
||
|
||
return {
|
||
meta: (key) => metas.get(key.toLowerCase()),
|
||
title: nonBlank(scan.title && decodeHtmlEntities(scan.title)),
|
||
jsonLd: jsonLdOf(scan.jsonLd, pageUrl),
|
||
themeColor: themeColorOf(scan.metas),
|
||
maskIconColor: maskIconColorOf(scan.links),
|
||
bestIcon: bestIconOf(scan.links, pageUrl),
|
||
largeImage: largeImageOf(scan.imgs, pageUrl),
|
||
firstContentImage: firstContentImageOf(scan.imgs, pageUrl),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
|
||
* common named color; undefined for any other form. The generic preview
|
||
* stores it as the preview's `accentColor`, a tint over the app's own surface,
|
||
* so alpha is dropped, and a fully transparent one is no color at all.
|
||
*/
|
||
export function normalizeCssColor(raw: string | undefined): string | undefined {
|
||
const color = raw ? parseCssColor(raw) : undefined;
|
||
return color && color.a > 0 ? hexOf(color) : undefined;
|
||
}
|