gerbeur/api/remote/html.ts

/**
 * Reads a link preview's metadata out of a page's HTML with regular
 * expressions, without parsing the document. The generic provider uses most of
 * it; the Bandcamp, SoundCloud and YouTube providers read only OG tags. Each
 * function returns undefined when the page doesn't say.
 *
 * @module
 */

import { hexOf, parseCssColor } from "../lib/css-color.ts";

/**
 * Decodes the five XML entities and numeric ones in `str`. `&` goes first,
 * so `"` decodes twice; `bandcamp-tralbum.ts`, which decodes JSON,
 * has its own single-pass decoder.
 */
function decodeHtmlEntities(str: string): string {
  return str
    .replace(/&/gi, "&")
    .replace(/&lt;/gi, "<")
    .replace(/&gt;/gi, ">")
    .replace(/&quot;/gi, '"')
    .replace(/&apos;/gi, "'")
    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
    .replace(
      /&#x([0-9a-f]+);/gi,
      (_, hex) => String.fromCodePoint(parseInt(hex, 16)),
    );
}

/**
 * Returns the `content` of the first `<meta>` tag whose `attr` equals `value`,
 * in whichever order the attributes come. Each tag is matched on its own, so
 * the capture can't run into the next tag.
 */
function extractMetaContent(
  html: string,
  attr: string,
  value: string,
): string | undefined {
  const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
  const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
  const tagRe = /<meta\b[^>]*>/gi;
  let m: RegExpExecArray | null;
  while ((m = tagRe.exec(html)) !== null) {
    const tag = m[0];
    if (!keyRe.test(tag)) continue;
    const content = contentRe.exec(tag);
    if (content) return decodeHtmlEntities(content[2]);
  }
  return undefined;
}

/** Returns the content of `<meta property="og:<tag>">`, entities decoded. */
export function extractOgTag(
  html: string,
  tag: string,
): string | undefined {
  return extractMetaContent(html, "property", `og:${tag}`);
}

/**
 * Returns the content of `<meta name="<name>">`, entities decoded. Twitter
 * card tags use `name`, not `property`.
 */
export function extractMetaName(
  html: string,
  name: string,
): string | undefined {
  return extractMetaContent(html, "name", name);
}

/** Returns the text of the `<title>` element, trimmed and entities decoded. */
export function extractPageTitle(html: string): string | undefined {
  const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
  return match ? decodeHtmlEntities(match[1].trim()) : undefined;
}

/**
 * Returns the page's `theme-color`, as written: pass it to
 * {@link normalizeCssColor}. Sites often ship one per color scheme with the
 * dark one first, so the unscoped tag wins, then the light one, then any.
 */
export function extractThemeColor(html: string): string | undefined {
  const tagRe = /<meta\b[^>]*>/gi;
  const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
  const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
  let unscoped: string | undefined;
  let light: string | undefined;
  let any: string | undefined;

  let m: RegExpExecArray | null;
  while ((m = tagRe.exec(html)) !== null) {
    const tag = m[0];
    if (!/\bname=["']theme-color["']/i.test(tag)) continue;
    const content = contentRe.exec(tag)?.[2];
    if (!content) continue;
    const media = mediaRe.exec(tag)?.[2];
    if (!media) unscoped ??= content;
    else if (/light/i.test(media)) light ??= content;
    any ??= content;
  }
  return unscoped ?? light ?? any;
}

/**
 * Returns the `color` of `<link rel="mask-icon">`, Safari's pinned-tab icon,
 * as written. The generic provider's last resort for a brand color.
 */
export function extractMaskIconColor(html: string): string | undefined {
  const linkRe = /<link[^>]+>/gi;
  let m: RegExpExecArray | null;
  while ((m = linkRe.exec(html)) !== null) {
    const tag = m[0];
    if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
    const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
    if (color) return color;
  }
  return undefined;
}

/**
 * Returns `raw` as a lowercase `#rrggbb`, from a hex, `rgb()`, `rgba()` or
 * common named color; undefined for any other form. The generic provider
 * stores it as the preview's `accentColor`, a tint over the app's own surface,
 * so alpha is dropped, and a fully transparent one is no color at all.
 */
export function normalizeCssColor(raw: string | undefined): string | undefined {
  const color = raw ? parseCssColor(raw) : undefined;
  return color && color.a > 0 ? hexOf(color) : undefined;
}

/**
 * Returned to the generic provider by {@link extractJsonLd}, which ranks each
 * field below the page's OG and Twitter tags.
 */
export interface JsonLdResult {
  /** From `name`, or else `headline`. */
  title?: string;
  description?: string;
  /**
   * From `image`, or else `thumbnailUrl` or `thumbnail`, and only when it is
   * an absolute URL.
   */
  thumbnailUrl?: string;
}

function ldString(v: unknown): string | undefined {
  if (typeof v === "string" && v.trim()) return v.trim();
  if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
    return v[0].trim();
  }
  return undefined;
}

function ldImage(v: unknown): string | undefined {
  if (
    typeof v === "string" &&
    (v.startsWith("http://") || v.startsWith("https://"))
  ) return v;
  if (Array.isArray(v)) return ldImage(v[0]);
  if (v && typeof v === "object") {
    const o = v as Record<string, unknown>;
    return ldImage(o.url ?? o.contentUrl);
  }
  return undefined;
}

function ldExtractNode(data: unknown): JsonLdResult {
  if (Array.isArray(data)) {
    for (const item of data) {
      const r = ldExtractNode(item);
      if (r.title || r.thumbnailUrl) return r;
    }
    return {};
  }
  if (!data || typeof data !== "object") return {};
  const o = data as Record<string, unknown>;
  if (o["@graph"]) return ldExtractNode(o["@graph"]);
  return {
    title: ldString(o.name ?? o.headline),
    description: ldString(o.description),
    thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
  };
}

/**
 * Returns the first JSON-LD node in `html` that has a title or an image,
 * looking through `@graph` and arrays; empty when there is none. An image may
 * be a URL, a list of them or an `ImageObject`. Blocks that aren't valid JSON
 * are skipped.
 */
export function extractJsonLd(html: string): JsonLdResult {
  const pattern =
    /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
  let match: RegExpExecArray | null;
  while ((match = pattern.exec(html)) !== null) {
    try {
      const result = ldExtractNode(JSON.parse(match[1]));
      if (result.title || result.thumbnailUrl) return result;
    } catch { /* invalid JSON: try the next block */ }
  }
  return {};
}

/**
 * Returns the absolute URL of the first `<img>` whose declared width and height
 * are both at least `minSize` pixels; an image without them doesn't count.
 * Data URIs are skipped.
 */
export function extractLargeImage(
  html: string,
  baseUrl: string,
  minSize = 200,
): string | undefined {
  const imgPattern = /<img[^>]+>/gi;
  let match: RegExpExecArray | null;
  while ((match = imgPattern.exec(html)) !== null) {
    const tag = match[0];
    const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
    if (!src || src.startsWith("data:")) continue;
    const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
    const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
    if (w >= minSize && h >= minSize) {
      try {
        return new URL(src, baseUrl).toString();
      } catch {
        continue;
      }
    }
  }
  return undefined;
}

/**
 * Returns the absolute URL of the page's largest icon, among its `icon` and
 * `apple-touch-icon` links, by declared `sizes`; the first one when none
 * declares a size. The generic provider stores it as the preview's
 * `faviconUrl`, which the thumbnail placeholder draws.
 *
 * An SVG icon wins over any size: it declares none, yet scales to any.
 */
export function extractBestIcon(
  html: string,
  baseUrl: string,
): string | undefined {
  const linkRe = /<link[^>]+>/gi;
  const relRe = /\brel=["']([^"']+)["']/i;
  const hrefRe = /\bhref=["']([^"']+)["']/i;
  const sizesRe = /\bsizes=["']([^"']+)["']/i;

  const candidates: { href: string; area: number }[] = [];

  let m: RegExpExecArray | null;
  while ((m = linkRe.exec(html)) !== null) {
    const tag = m[0];
    const rel = relRe.exec(tag)?.[1] ?? "";
    if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
    const href = hrefRe.exec(tag)?.[1];
    if (!href) continue;
    const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
    const sm = sizesStr.match(/(\d+)x(\d+)/i);
    const isSvg = /\.svg(\?|$)/i.test(href) ||
      /\btype=["']image\/svg\+xml["']/i.test(tag);
    const area = isSvg
      ? Number.MAX_SAFE_INTEGER
      : sm
      ? parseInt(sm[1]) * parseInt(sm[2])
      : 0;
    try {
      candidates.push({ href: new URL(href, baseUrl).toString(), area });
    } catch {
      continue;
    }
  }

  if (candidates.length === 0) return undefined;
  candidates.sort((a, b) => b.area - a.area);
  return candidates[0].href;
}

/** Filename words that mark an image as page furniture rather than content. */
const UI_IMAGE_KEYWORDS = new Set([
  "logo",
  "icon",
  "sprite",
  "favicon",
  "avatar",
  "banner",
  "header",
  "nav",
  "menu",
  "cart",
  "search",
  "tracking",
  "pixel",
  "bg",
  "background",
]);

/**
 * Checks whether `src` looks like page furniture: an SVG, or a filename with
 * one of the {@link UI_IMAGE_KEYWORDS} between its `-`, `_` or `.`.
 */
function isUiImage(src: string): boolean {
  if (/\.svg(\?|$)/i.test(src)) return true;
  const filename = src.split("?")[0].split("/").pop() ?? "";
  const baseName = filename.replace(/\.[^.]+$/, "");
  return baseName.toLowerCase().split(/[-_.]/).some((t) =>
    UI_IMAGE_KEYWORDS.has(t)
  );
}

/**
 * Returns the absolute URL of the first `<img>` that looks like content rather
 * than a logo, icon or other page furniture. Data URIs are skipped. The generic
 * provider's last resort for a thumbnail.
 */
export function extractFirstContentImage(
  html: string,
  baseUrl: string,
): string | undefined {
  const imgPattern = /<img[^>]+>/gi;
  let match: RegExpExecArray | null;
  while ((match = imgPattern.exec(html)) !== null) {
    const tag = match[0];
    const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
    if (!src || src.startsWith("data:")) continue;
    if (isUiImage(src)) continue;
    try {
      return new URL(src, baseUrl).toString();
    } catch {
      continue;
    }
  }
  return undefined;
}