v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, é and ’ included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* Reads a link preview's metadata out of a page's HTML with regular
|
||||
* expressions, without parsing the document. The generic provider uses most of
|
||||
* it; the Bandcamp, SoundCloud and YouTube providers read only OG tags. Each
|
||||
* function returns undefined when the page doesn't say.
|
||||
* Reads a link preview's metadata out of a page's HTML. {@link parsePage}
|
||||
* scans the page once for the tags that carry it (`<meta>`, `<link>`, `<img>`,
|
||||
* `<title>` and JSON-LD scripts) without building a document, and returns a
|
||||
* {@link Page} that the providers' `genericPreview` picks from. Every field is
|
||||
* undefined when the page doesn't say.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
@@ -10,71 +11,220 @@
|
||||
import { hexOf, parseCssColor } from "../lib/css-color.ts";
|
||||
|
||||
/**
|
||||
* Decodes the five XML entities and numeric ones in `str`. `&` goes first,
|
||||
* so `&quot;` decodes twice; `bandcamp-tralbum.ts`, which decodes JSON,
|
||||
* has its own single-pass decoder.
|
||||
* The named entities decoded besides numeric ones: the XML five, and those
|
||||
* page titles and descriptions actually use, Latin-1 letters and typography.
|
||||
*/
|
||||
function decodeHtmlEntities(str: string): string {
|
||||
return str
|
||||
.replace(/&/gi, "&")
|
||||
.replace(/</gi, "<")
|
||||
.replace(/>/gi, ">")
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/'/gi, "'")
|
||||
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
|
||||
.replace(
|
||||
/&#x([0-9a-f]+);/gi,
|
||||
(_, hex) => String.fromCodePoint(parseInt(hex, 16)),
|
||||
);
|
||||
}
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: "&",
|
||||
lt: "<",
|
||||
gt: ">",
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: " ",
|
||||
aelig: "æ",
|
||||
AElig: "Æ",
|
||||
oelig: "œ",
|
||||
OElig: "Œ",
|
||||
szlig: "ß",
|
||||
oslash: "ø",
|
||||
Oslash: "Ø",
|
||||
eth: "ð",
|
||||
ETH: "Ð",
|
||||
thorn: "þ",
|
||||
THORN: "Þ",
|
||||
hellip: "…",
|
||||
lsquo: "‘",
|
||||
rsquo: "’",
|
||||
sbquo: "‚",
|
||||
ldquo: "“",
|
||||
rdquo: "”",
|
||||
bdquo: "„",
|
||||
laquo: "«",
|
||||
raquo: "»",
|
||||
lsaquo: "‹",
|
||||
rsaquo: "›",
|
||||
ndash: "–",
|
||||
mdash: "—",
|
||||
bull: "•",
|
||||
middot: "·",
|
||||
dagger: "†",
|
||||
Dagger: "‡",
|
||||
prime: "′",
|
||||
Prime: "″",
|
||||
euro: "€",
|
||||
pound: "£",
|
||||
yen: "¥",
|
||||
cent: "¢",
|
||||
copy: "©",
|
||||
reg: "®",
|
||||
trade: "™",
|
||||
deg: "°",
|
||||
plusmn: "±",
|
||||
times: "×",
|
||||
divide: "÷",
|
||||
micro: "µ",
|
||||
para: "¶",
|
||||
sect: "§",
|
||||
iexcl: "¡",
|
||||
iquest: "¿",
|
||||
ordf: "ª",
|
||||
ordm: "º",
|
||||
sup2: "²",
|
||||
sup3: "³",
|
||||
frac14: "¼",
|
||||
frac12: "½",
|
||||
frac34: "¾",
|
||||
shy: "",
|
||||
ensp: " ",
|
||||
emsp: " ",
|
||||
thinsp: " ",
|
||||
zwnj: "",
|
||||
zwj: "",
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the `content` of the first `<meta>` tag whose `attr` equals `value`,
|
||||
* in whichever order the attributes come. Each tag is matched on its own, so
|
||||
* the capture can't run into the next tag.
|
||||
*/
|
||||
function extractMetaContent(
|
||||
html: string,
|
||||
attr: string,
|
||||
value: string,
|
||||
): string | undefined {
|
||||
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
|
||||
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
|
||||
const tagRe = /<meta\b[^>]*>/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = tagRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
if (!keyRe.test(tag)) continue;
|
||||
const content = contentRe.exec(tag);
|
||||
if (content) return decodeHtmlEntities(content[2]);
|
||||
// The accented letters, `é` and the like, composed from their letter
|
||||
// and accent rather than listed one by one.
|
||||
for (
|
||||
const [accent, mark, letters] of [
|
||||
["grave", "̀", "aeiouAEIOU"],
|
||||
["acute", "́", "aeiouyAEIOUY"],
|
||||
["circ", "̂", "aeiouAEIOU"],
|
||||
["tilde", "̃", "anoANO"],
|
||||
["uml", "̈", "aeiouyAEIOUY"],
|
||||
["ring", "̊", "aA"],
|
||||
["cedil", "̧", "cC"],
|
||||
]
|
||||
) {
|
||||
for (const letter of letters) {
|
||||
NAMED_ENTITIES[letter + accent] = (letter + mark).normalize("NFC");
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Returns the content of `<meta property="og:<tag>">`, entities decoded. */
|
||||
export function extractOgTag(
|
||||
html: string,
|
||||
tag: string,
|
||||
): string | undefined {
|
||||
return extractMetaContent(html, "property", `og:${tag}`);
|
||||
/** Returns the character of code point `n`, or undefined when there is none. */
|
||||
function fromCodePoint(n: number): string | undefined {
|
||||
if (n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) {
|
||||
return undefined;
|
||||
}
|
||||
return String.fromCodePoint(n);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the content of `<meta name="<name>">`, entities decoded. Twitter
|
||||
* card tags use `name`, not `property`.
|
||||
* Decodes the entities in `str` in a single pass, so `&quot;` reads
|
||||
* `"`. An entity it doesn't know, or a code point that doesn't exist, is
|
||||
* left as written. Also used by `bandcamp-tralbum.ts` on the JSON a page
|
||||
* carries in an attribute.
|
||||
*/
|
||||
export function extractMetaName(
|
||||
html: string,
|
||||
name: string,
|
||||
): string | undefined {
|
||||
return extractMetaContent(html, "name", name);
|
||||
export function decodeHtmlEntities(str: string): string {
|
||||
return str.replace(
|
||||
/&(?:#(\d{1,8})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z][a-zA-Z0-9]{1,31}));/g,
|
||||
(match, dec?: string, hex?: string, name?: string) => {
|
||||
if (dec) return fromCodePoint(Number(dec)) ?? match;
|
||||
if (hex) return fromCodePoint(parseInt(hex, 16)) ?? match;
|
||||
return NAMED_ENTITIES[name!] ?? NAMED_ENTITIES[name!.toLowerCase()] ??
|
||||
match;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns the text of the `<title>` element, trimmed and entities decoded. */
|
||||
export function extractPageTitle(html: string): string | undefined {
|
||||
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
return match ? decodeHtmlEntities(match[1].trim()) : undefined;
|
||||
/**
|
||||
* Returns `raw` resolved against `base` as an absolute http(s) URL; undefined
|
||||
* for a `data:` URI, another scheme or a URL that doesn't parse. Every URL a
|
||||
* page names goes through it, so `/og.png` and `//cdn/og.png` work.
|
||||
*/
|
||||
export function absoluteHttpUrl(
|
||||
raw: string | undefined,
|
||||
base: string,
|
||||
): string | undefined {
|
||||
if (!raw?.trim()) return undefined;
|
||||
try {
|
||||
const url = new URL(raw.trim(), base);
|
||||
return url.protocol === "http:" || url.protocol === "https:"
|
||||
? url.href
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** A tag's attributes by lowercase name, values decoded; the first one wins. */
|
||||
type Attrs = Map<string, string>;
|
||||
|
||||
/**
|
||||
* Matches the start of a tag the scan reads, or of a comment. Quoted values
|
||||
* may hold `>`, but no tag spans a `<`, so each attempt stops at the next one
|
||||
* and the scan stays linear in the page's size.
|
||||
*/
|
||||
const TAG_RE =
|
||||
/<(meta|link|img|title|script)\b((?:[^<>"']|"[^"<]*"|'[^'<]*')*)>|<!--/gi;
|
||||
|
||||
/** Matches one attribute: unquoted, single- or double-quoted, or bare. */
|
||||
const ATTR_RE =
|
||||
/([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
|
||||
|
||||
function parseAttrs(source: string): Attrs {
|
||||
const attrs: Attrs = new Map();
|
||||
for (const m of source.matchAll(ATTR_RE)) {
|
||||
const name = m[1].toLowerCase();
|
||||
if (attrs.has(name)) continue;
|
||||
attrs.set(name, decodeHtmlEntities(m[2] ?? m[3] ?? m[4] ?? ""));
|
||||
}
|
||||
return attrs;
|
||||
}
|
||||
|
||||
/** What {@link scanTags} found, in page order. */
|
||||
interface Scan {
|
||||
metas: Attrs[];
|
||||
links: Attrs[];
|
||||
imgs: Attrs[];
|
||||
/** The raw text of the first `<title>`. */
|
||||
title?: string;
|
||||
/** The bodies of the `application/ld+json` scripts. */
|
||||
jsonLd: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects the tags {@link parsePage} reads in one pass. Comments and the
|
||||
* bodies of scripts and the title are skipped over, so a tag written inside
|
||||
* them isn't read; one left open runs to the end of the page, as in a browser.
|
||||
*/
|
||||
function scanTags(html: string): Scan {
|
||||
const scan: Scan = { metas: [], links: [], imgs: [], jsonLd: [] };
|
||||
const re = new RegExp(TAG_RE);
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(html)) !== null) {
|
||||
const tag = m[1]?.toLowerCase();
|
||||
if (!tag) {
|
||||
const end = html.indexOf("-->", re.lastIndex);
|
||||
if (end < 0) break;
|
||||
re.lastIndex = end + 3;
|
||||
continue;
|
||||
}
|
||||
const attrs = parseAttrs(m[2]);
|
||||
if (tag === "meta") scan.metas.push(attrs);
|
||||
else if (tag === "link") scan.links.push(attrs);
|
||||
else if (tag === "img") scan.imgs.push(attrs);
|
||||
else {
|
||||
const close = new RegExp(`</${tag}\\s*>`, "gi");
|
||||
close.lastIndex = re.lastIndex;
|
||||
const end = close.exec(html);
|
||||
if (!end) break;
|
||||
const body = html.slice(re.lastIndex, end.index);
|
||||
if (tag === "title") scan.title ??= body;
|
||||
else if (
|
||||
attrs.get("type")?.trim().toLowerCase() === "application/ld+json"
|
||||
) {
|
||||
scan.jsonLd.push(body);
|
||||
}
|
||||
re.lastIndex = close.lastIndex;
|
||||
}
|
||||
}
|
||||
return scan;
|
||||
}
|
||||
|
||||
/** Returns `value` trimmed, or undefined when nothing is left. */
|
||||
function nonBlank(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -82,21 +232,15 @@ export function extractPageTitle(html: string): string | undefined {
|
||||
* {@link normalizeCssColor}. Sites often ship one per color scheme with the
|
||||
* dark one first, so the unscoped tag wins, then the light one, then any.
|
||||
*/
|
||||
export function extractThemeColor(html: string): string | undefined {
|
||||
const tagRe = /<meta\b[^>]*>/gi;
|
||||
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
|
||||
const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
|
||||
function themeColorOf(metas: Attrs[]): string | undefined {
|
||||
let unscoped: string | undefined;
|
||||
let light: string | undefined;
|
||||
let any: string | undefined;
|
||||
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = tagRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
if (!/\bname=["']theme-color["']/i.test(tag)) continue;
|
||||
const content = contentRe.exec(tag)?.[2];
|
||||
for (const meta of metas) {
|
||||
if (meta.get("name")?.toLowerCase() !== "theme-color") continue;
|
||||
const content = nonBlank(meta.get("content"));
|
||||
if (!content) continue;
|
||||
const media = mediaRe.exec(tag)?.[2];
|
||||
const media = meta.get("media");
|
||||
if (!media) unscoped ??= content;
|
||||
else if (/light/i.test(media)) light ??= content;
|
||||
any ??= content;
|
||||
@@ -104,180 +248,179 @@ export function extractThemeColor(html: string): string | undefined {
|
||||
return unscoped ?? light ?? any;
|
||||
}
|
||||
|
||||
/** Returns the lowercase tokens of a `<link>`'s `rel`. */
|
||||
function relTokens(link: Attrs): string[] {
|
||||
return (link.get("rel") ?? "").toLowerCase().split(/\s+/);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the `color` of `<link rel="mask-icon">`, Safari's pinned-tab icon,
|
||||
* as written. The generic provider's last resort for a brand color.
|
||||
* as written. The generic preview's last resort for a brand color.
|
||||
*/
|
||||
export function extractMaskIconColor(html: string): string | undefined {
|
||||
const linkRe = /<link[^>]+>/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = linkRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
|
||||
const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
|
||||
function maskIconColorOf(links: Attrs[]): string | undefined {
|
||||
for (const link of links) {
|
||||
if (!relTokens(link).includes("mask-icon")) continue;
|
||||
const color = nonBlank(link.get("color"));
|
||||
if (color) return color;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
|
||||
* common named color; undefined for any other form. The generic provider
|
||||
* stores it as the preview's `accentColor`, a tint over the app's own surface,
|
||||
* so alpha is dropped, and a fully transparent one is no color at all.
|
||||
*/
|
||||
export function normalizeCssColor(raw: string | undefined): string | undefined {
|
||||
const color = raw ? parseCssColor(raw) : undefined;
|
||||
return color && color.a > 0 ? hexOf(color) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returned to the generic provider by {@link extractJsonLd}, which ranks each
|
||||
* field below the page's OG and Twitter tags.
|
||||
*/
|
||||
export interface JsonLdResult {
|
||||
/** From `name`, or else `headline`. */
|
||||
title?: string;
|
||||
description?: string;
|
||||
/**
|
||||
* From `image`, or else `thumbnailUrl` or `thumbnail`, and only when it is
|
||||
* an absolute URL.
|
||||
*/
|
||||
thumbnailUrl?: string;
|
||||
}
|
||||
|
||||
function ldString(v: unknown): string | undefined {
|
||||
if (typeof v === "string" && v.trim()) return v.trim();
|
||||
if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
|
||||
return v[0].trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function ldImage(v: unknown): string | undefined {
|
||||
if (
|
||||
typeof v === "string" &&
|
||||
(v.startsWith("http://") || v.startsWith("https://"))
|
||||
) return v;
|
||||
if (Array.isArray(v)) return ldImage(v[0]);
|
||||
if (v && typeof v === "object") {
|
||||
const o = v as Record<string, unknown>;
|
||||
return ldImage(o.url ?? o.contentUrl);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function ldExtractNode(data: unknown): JsonLdResult {
|
||||
if (Array.isArray(data)) {
|
||||
for (const item of data) {
|
||||
const r = ldExtractNode(item);
|
||||
if (r.title || r.thumbnailUrl) return r;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (!data || typeof data !== "object") return {};
|
||||
const o = data as Record<string, unknown>;
|
||||
if (o["@graph"]) return ldExtractNode(o["@graph"]);
|
||||
return {
|
||||
title: ldString(o.name ?? o.headline),
|
||||
description: ldString(o.description),
|
||||
thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first JSON-LD node in `html` that has a title or an image,
|
||||
* looking through `@graph` and arrays; empty when there is none. An image may
|
||||
* be a URL, a list of them or an `ImageObject`. Blocks that aren't valid JSON
|
||||
* are skipped.
|
||||
*/
|
||||
export function extractJsonLd(html: string): JsonLdResult {
|
||||
const pattern =
|
||||
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = pattern.exec(html)) !== null) {
|
||||
try {
|
||||
const result = ldExtractNode(JSON.parse(match[1]));
|
||||
if (result.title || result.thumbnailUrl) return result;
|
||||
} catch { /* invalid JSON: try the next block */ }
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the absolute URL of the first `<img>` whose declared width and height
|
||||
* are both at least `minSize` pixels; an image without them doesn't count.
|
||||
* Data URIs are skipped.
|
||||
*/
|
||||
export function extractLargeImage(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
minSize = 200,
|
||||
): string | undefined {
|
||||
const imgPattern = /<img[^>]+>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = imgPattern.exec(html)) !== null) {
|
||||
const tag = match[0];
|
||||
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
|
||||
if (!src || src.startsWith("data:")) continue;
|
||||
const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
|
||||
const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
|
||||
if (w >= minSize && h >= minSize) {
|
||||
try {
|
||||
return new URL(src, baseUrl).toString();
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the absolute URL of the page's largest icon, among its `icon` and
|
||||
* `apple-touch-icon` links, by declared `sizes`; the first one when none
|
||||
* declares a size. The generic provider stores it as the preview's
|
||||
* `faviconUrl`, which the thumbnail placeholder draws.
|
||||
* declares a size. A `mask-icon` is no candidate: it is a one-color shape for
|
||||
* Safari to tint, black when drawn as is.
|
||||
*
|
||||
* An SVG icon wins over any size: it declares none, yet scales to any.
|
||||
*/
|
||||
export function extractBestIcon(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
): string | undefined {
|
||||
const linkRe = /<link[^>]+>/gi;
|
||||
const relRe = /\brel=["']([^"']+)["']/i;
|
||||
const hrefRe = /\bhref=["']([^"']+)["']/i;
|
||||
const sizesRe = /\bsizes=["']([^"']+)["']/i;
|
||||
|
||||
const candidates: { href: string; area: number }[] = [];
|
||||
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = linkRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
const rel = relRe.exec(tag)?.[1] ?? "";
|
||||
if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
|
||||
const href = hrefRe.exec(tag)?.[1];
|
||||
function bestIconOf(links: Attrs[], baseUrl: string): string | undefined {
|
||||
let best: { href: string; area: number } | undefined;
|
||||
for (const link of links) {
|
||||
const rel = relTokens(link);
|
||||
if (
|
||||
!rel.includes("icon") && !rel.includes("apple-touch-icon") &&
|
||||
!rel.includes("apple-touch-icon-precomposed")
|
||||
) continue;
|
||||
const href = absoluteHttpUrl(link.get("href"), baseUrl);
|
||||
if (!href) continue;
|
||||
const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
|
||||
const sm = sizesStr.match(/(\d+)x(\d+)/i);
|
||||
const sizes = (link.get("sizes") ?? "").toLowerCase();
|
||||
const isSvg = /\.svg(\?|$)/i.test(href) ||
|
||||
/\btype=["']image\/svg\+xml["']/i.test(tag);
|
||||
const area = isSvg
|
||||
? Number.MAX_SAFE_INTEGER
|
||||
: sm
|
||||
? parseInt(sm[1]) * parseInt(sm[2])
|
||||
: 0;
|
||||
link.get("type")?.toLowerCase() === "image/svg+xml" ||
|
||||
sizes === "any";
|
||||
const area = isSvg ? Number.MAX_SAFE_INTEGER : Math.max(
|
||||
0,
|
||||
...[...sizes.matchAll(/(\d+)x(\d+)/g)].map(([, w, h]) => +w * +h),
|
||||
);
|
||||
if (!best || area > best.area) best = { href, area };
|
||||
}
|
||||
return best?.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returned by the JSON-LD reading of {@link parsePage}, which the generic
|
||||
* preview ranks below the page's OG and Twitter tags.
|
||||
*/
|
||||
export interface JsonLdResult {
|
||||
/** From `name`, or else `headline`. */
|
||||
title?: string;
|
||||
description?: string;
|
||||
/** From `image`, or else `thumbnailUrl` or `thumbnail`. */
|
||||
thumbnailUrl?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The JSON-LD types that describe the site around a page rather than the
|
||||
* page itself. Their name is the site's or the company's, so a node of one of
|
||||
* them is read only when no other node has a title or an image.
|
||||
*/
|
||||
const SITE_LD_TYPES = new Set([
|
||||
"WebSite",
|
||||
"Organization",
|
||||
"Corporation",
|
||||
"NewsMediaOrganization",
|
||||
"Person",
|
||||
"BreadcrumbList",
|
||||
"SearchAction",
|
||||
"ImageObject",
|
||||
"SiteNavigationElement",
|
||||
"WPHeader",
|
||||
"WPFooter",
|
||||
]);
|
||||
|
||||
function ldString(v: unknown): string | undefined {
|
||||
if (typeof v === "string") return nonBlank(v);
|
||||
if (Array.isArray(v) && typeof v[0] === "string") return nonBlank(v[0]);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function ldImage(v: unknown, baseUrl: string): string | undefined {
|
||||
if (typeof v === "string") return absoluteHttpUrl(v, baseUrl);
|
||||
if (Array.isArray(v)) return ldImage(v[0], baseUrl);
|
||||
if (v && typeof v === "object") {
|
||||
const o = v as Record<string, unknown>;
|
||||
return ldImage(o.url ?? o.contentUrl, baseUrl);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Lists every node of `data`, through arrays and `@graph`, in order. */
|
||||
function ldNodes(data: unknown): Record<string, unknown>[] {
|
||||
if (Array.isArray(data)) return data.flatMap(ldNodes);
|
||||
if (!data || typeof data !== "object") return [];
|
||||
const o = data as Record<string, unknown>;
|
||||
return o["@graph"] ? ldNodes(o["@graph"]) : [o];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first JSON-LD node about the page that has a title or an image,
|
||||
* else the first about its site ({@link SITE_LD_TYPES}); empty when there is
|
||||
* none. An image may be a URL, a list of them or an `ImageObject`. Blocks that
|
||||
* aren't valid JSON are skipped.
|
||||
*/
|
||||
function jsonLdOf(blocks: string[], baseUrl: string): JsonLdResult {
|
||||
let fallback: JsonLdResult | undefined;
|
||||
for (const block of blocks) {
|
||||
let data: unknown;
|
||||
try {
|
||||
candidates.push({ href: new URL(href, baseUrl).toString(), area });
|
||||
data = JSON.parse(block);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const node of ldNodes(data)) {
|
||||
const result: JsonLdResult = {
|
||||
title: ldString(node.name ?? node.headline),
|
||||
description: ldString(node.description),
|
||||
thumbnailUrl: ldImage(
|
||||
node.image ?? node.thumbnailUrl ?? node.thumbnail,
|
||||
baseUrl,
|
||||
),
|
||||
};
|
||||
if (!result.title && !result.thumbnailUrl) continue;
|
||||
const types = [node["@type"]].flat();
|
||||
if (!types.some((t) => typeof t === "string" && SITE_LD_TYPES.has(t))) {
|
||||
return result;
|
||||
}
|
||||
fallback ??= result;
|
||||
}
|
||||
}
|
||||
return fallback ?? {};
|
||||
}
|
||||
|
||||
if (candidates.length === 0) return undefined;
|
||||
candidates.sort((a, b) => b.area - a.area);
|
||||
return candidates[0].href;
|
||||
/**
|
||||
* Returns the source of an `<img>`: its `src`, or for a lazy-loaded image
|
||||
* whose `src` is a `data:` placeholder, its `data-src` or `data-lazy-src`.
|
||||
*/
|
||||
function imageSource(img: Attrs, baseUrl: string): string | undefined {
|
||||
const src = img.get("src")?.trim();
|
||||
return src && !src.startsWith("data:")
|
||||
? absoluteHttpUrl(src, baseUrl)
|
||||
: absoluteHttpUrl(img.get("data-src") ?? img.get("data-lazy-src"), baseUrl);
|
||||
}
|
||||
|
||||
/** Returns an `<img>`'s declared `name` dimension in pixels; 0 when none. */
|
||||
function declaredSize(img: Attrs, name: "width" | "height"): number {
|
||||
return parseInt(img.get(name) ?? "") || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the absolute URL of the first `<img>` whose declared width and height
|
||||
* are both at least `minSize` pixels; an image without them doesn't count.
|
||||
*/
|
||||
function largeImageOf(
|
||||
imgs: Attrs[],
|
||||
baseUrl: string,
|
||||
minSize = 200,
|
||||
): string | undefined {
|
||||
for (const img of imgs) {
|
||||
if (
|
||||
declaredSize(img, "width") < minSize ||
|
||||
declaredSize(img, "height") < minSize
|
||||
) continue;
|
||||
const src = imageSource(img, baseUrl);
|
||||
if (src) return src;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Filename words that mark an image as page furniture rather than content. */
|
||||
@@ -297,6 +440,11 @@ const UI_IMAGE_KEYWORDS = new Set([
|
||||
"pixel",
|
||||
"bg",
|
||||
"background",
|
||||
"placeholder",
|
||||
"spacer",
|
||||
"blank",
|
||||
"lazy",
|
||||
"loading",
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -314,25 +462,86 @@ function isUiImage(src: string): boolean {
|
||||
|
||||
/**
|
||||
* Returns the absolute URL of the first `<img>` that looks like content rather
|
||||
* than a logo, icon or other page furniture. Data URIs are skipped. The generic
|
||||
* provider's last resort for a thumbnail.
|
||||
* than a logo, icon, tracking pixel or other page furniture: one declared
|
||||
* smaller than 50 pixels either way doesn't count. The generic preview's last
|
||||
* resort for a thumbnail.
|
||||
*/
|
||||
export function extractFirstContentImage(
|
||||
html: string,
|
||||
function firstContentImageOf(
|
||||
imgs: Attrs[],
|
||||
baseUrl: string,
|
||||
): string | undefined {
|
||||
const imgPattern = /<img[^>]+>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = imgPattern.exec(html)) !== null) {
|
||||
const tag = match[0];
|
||||
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
|
||||
if (!src || src.startsWith("data:")) continue;
|
||||
if (isUiImage(src)) continue;
|
||||
try {
|
||||
return new URL(src, baseUrl).toString();
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const img of imgs) {
|
||||
const w = declaredSize(img, "width");
|
||||
const h = declaredSize(img, "height");
|
||||
if ((w && w < 50) || (h && h < 50)) continue;
|
||||
const src = imageSource(img, baseUrl);
|
||||
if (src && !isUiImage(src)) return src;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returned by {@link parsePage}: what a page says about itself, entities
|
||||
* decoded, values trimmed and URLs absolute.
|
||||
*/
|
||||
export interface Page {
|
||||
/**
|
||||
* Returns the `content` of the first `<meta>` whose `property` or `name` is
|
||||
* `key`, in any case: sites write `og:` tags under either attribute, and
|
||||
* Twitter's too.
|
||||
*/
|
||||
meta(key: string): string | undefined;
|
||||
/** The text of the `<title>` element. */
|
||||
title?: string;
|
||||
jsonLd: JsonLdResult;
|
||||
/** The `theme-color`, as written: pass it to {@link normalizeCssColor}. */
|
||||
themeColor?: string;
|
||||
/** The `color` of the Safari pinned-tab icon, as written. */
|
||||
maskIconColor?: string;
|
||||
/** The largest icon the page links, which the thumbnail placeholder draws. */
|
||||
bestIcon?: string;
|
||||
/** The first image declared at least 200 pixels both ways. */
|
||||
largeImage?: string;
|
||||
/** The first image that doesn't look like page furniture. */
|
||||
firstContentImage?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads `html`, served from `pageUrl`, in one pass. The generic preview and
|
||||
* the providers built on it read a page through it.
|
||||
*/
|
||||
export function parsePage(html: string, pageUrl: string): Page {
|
||||
const scan = scanTags(html);
|
||||
|
||||
const metas = new Map<string, string>();
|
||||
for (const meta of scan.metas) {
|
||||
const content = nonBlank(meta.get("content"));
|
||||
if (!content) continue;
|
||||
for (const attr of ["property", "name"]) {
|
||||
const key = meta.get(attr)?.trim().toLowerCase();
|
||||
if (key && !metas.has(key)) metas.set(key, content);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
meta: (key) => metas.get(key.toLowerCase()),
|
||||
title: nonBlank(scan.title && decodeHtmlEntities(scan.title)),
|
||||
jsonLd: jsonLdOf(scan.jsonLd, pageUrl),
|
||||
themeColor: themeColorOf(scan.metas),
|
||||
maskIconColor: maskIconColorOf(scan.links),
|
||||
bestIcon: bestIconOf(scan.links, pageUrl),
|
||||
largeImage: largeImageOf(scan.imgs, pageUrl),
|
||||
firstContentImage: firstContentImageOf(scan.imgs, pageUrl),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
|
||||
* common named color; undefined for any other form. The generic preview
|
||||
* stores it as the preview's `accentColor`, a tint over the app's own surface,
|
||||
* so alpha is dropped, and a fully transparent one is no color at all.
|
||||
*/
|
||||
export function normalizeCssColor(raw: string | undefined): string | undefined {
|
||||
const color = raw ? parseCssColor(raw) : undefined;
|
||||
return color && color.a > 0 ? hexOf(color) : undefined;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user