gerbeur/api/services/rich-content-service.ts

import type { RichContent } from "../model/interfaces.ts";
import { youtubeProvider } from "./providers/youtube.ts";
import { bandcampProvider } from "./providers/bandcamp.ts";
import { soundcloudProvider } from "./providers/soundcloud.ts";
import { selfProvider } from "./providers/self.ts";
import { genericProvider } from "./providers/generic.ts";
import {
  isPrivateHostname,
  resolvePublicAddress,
} from "../lib/public-address.ts";
import { isSelfUrl } from "../lib/self-url.ts";

export interface RichContentProvider {
  name: string;
  matches(url: string): boolean;
  fetch(url: string): Promise<RichContent>;
}

/**
 * Register providers in priority order. The first match wins.
 * `selfProvider` resolves gerbeur URLs directly from the DB (no HTTP round-trip).
 * `genericProvider` must stay last — it always matches.
 */
const providers: RichContentProvider[] = [
  youtubeProvider,
  bandcampProvider,
  soundcloudProvider,
  selfProvider,
  genericProvider,
];

// Shared utilities exported for use by providers

const FETCH_HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
  "Accept":
    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
};

const MAX_REDIRECTS = 5;

/**
 * Fetch `url` via a `curl --insecure` subprocess and return a minimal Response.
 * Used as a last resort when Deno's rustls rejects the server's TLS certificate
 * due to an unsupported algorithm (e.g. UnsupportedSignatureAlgorithm).
 * Returns null if curl is unavailable or exits non-zero.
 *
 * `address` pins the connection to the IP already checked by
 * resolvePublicAddress, and curl doesn't follow redirects itself: a 3xx comes
 * back with its Location so fetchWithTimeout can check the next hop too.
 */
async function fetchViaCurl(
  url: URL,
  address: string,
  timeoutMs: number,
): Promise<Response | null> {
  const tmpPath = await Deno.makeTempFile();
  const port = url.port || (url.protocol === "https:" ? "443" : "80");
  const pinned = address.includes(":") ? `[${address}]` : address;
  try {
    const { code, stdout } = await new Deno.Command("curl", {
      args: [
        "--silent",
        "--insecure",
        "--proto",
        "=http,https",
        "--resolve",
        `${url.hostname.replace(/^\[(.*)\]$/, "$1")}:${port}:${pinned}`,
        "--max-time",
        String(Math.ceil(timeoutMs / 1000)),
        "--user-agent",
        FETCH_HEADERS["User-Agent"],
        "--header",
        `Accept: ${FETCH_HEADERS["Accept"]}`,
        "--header",
        `Accept-Language: ${FETCH_HEADERS["Accept-Language"]}`,
        "--output",
        tmpPath,
        "--write-out",
        "%{http_code}\n%{redirect_url}\n%{content_type}",
        url.href,
      ],
      stdout: "piped",
      stderr: "null",
    }).output();

    if (code !== 0) return null;

    const [status, location, contentType = ""] = new TextDecoder()
      .decode(stdout).split("\n").map((l) => l.trim());
    const statusCode = Number(status);
    if (!(statusCode >= 200 && statusCode <= 599)) return null;
    const headers = new Headers();
    if (location) headers.set("location", location);
    if (contentType) headers.set("content-type", contentType);

    const bytes = await Deno.readFile(tmpPath);
    if (!contentType.startsWith("text/")) {
      return new Response(bytes, { status: statusCode, headers });
    }
    // Decode using the charset declared in the Content-Type header so that
    // pages served in ISO-8859-1, windows-1252, etc. are read correctly.
    const charset = /charset=([\w-]+)/i.exec(contentType)?.[1] ?? "utf-8";
    let bodyText: string;
    try {
      bodyText = new TextDecoder(charset, { fatal: false }).decode(bytes);
    } catch {
      bodyText = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
    }
    headers.set("content-type", contentType.replace(/;.*$/, "; charset=utf-8"));
    return new Response(bodyText, { status: statusCode, headers });
  } catch {
    return null;
  } finally {
    await Deno.remove(tmpPath).catch(() => {});
  }
}

/** One request, no redirects followed. */
async function fetchOnce(
  url: URL,
  address: string,
  timeoutMs: number,
): Promise<Response> {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    return await fetch(url, {
      signal: controller.signal,
      headers: FETCH_HEADERS,
      redirect: "manual",
    });
  } catch (err) {
    if (!(err instanceof TypeError && err.message.includes("certificate"))) {
      throw err;
    }

    // Deno/rustls rejected the server's TLS certificate (expired, self-signed,
    // or an unsupported signature algorithm) and offers no way to override
    // verification programmatically. Fall back to a `curl --insecure`
    // subprocess, which uses its own TLS stack.
    const curlRes = await fetchViaCurl(url, address, timeoutMs);
    if (curlRes) return curlRes;

    throw err;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Fetch a user-supplied URL. Every hop — the URL itself and each redirect —
 * must resolve to a public address (see resolvePublicAddress), so a link or a
 * redirect can't point the server at its own network.
 */
export async function fetchWithTimeout(
  url: string,
  timeoutMs = 5000,
): Promise<Response> {
  let current = new URL(url);
  for (let hop = 0;; hop++) {
    if (current.protocol !== "http:" && current.protocol !== "https:") {
      throw new TypeError(`Unsupported protocol ${current.protocol}`);
    }
    const address = await resolvePublicAddress(current);
    const res = await fetchOnce(current, address, timeoutMs);
    const location = res.headers.get("location");
    if (res.status < 300 || res.status >= 400 || !location) return res;

    await res.body?.cancel();
    if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
    current = new URL(location, current);
  }
}

function decodeHtmlEntities(str: string): string {
  return str
    .replace(/&amp;/gi, "&")
    .replace(/&lt;/gi, "<")
    .replace(/&gt;/gi, ">")
    .replace(/&quot;/gi, '"')
    .replace(/&apos;/gi, "'")
    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
    .replace(
      /&#x([0-9a-f]+);/gi,
      (_, hex) => String.fromCodePoint(parseInt(hex, 16)),
    );
}

/**
 * Find the first `<meta>` tag whose `attr` equals `value` and return its
 * `content` (either attribute ordering). Each tag is matched in isolation so
 * the `content` capture can never span across tag boundaries.
 */
function extractMetaContent(
  html: string,
  attr: string,
  value: string,
): string | undefined {
  const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
  const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
  const tagRe = /<meta\b[^>]*>/gi;
  let m: RegExpExecArray | null;
  while ((m = tagRe.exec(html)) !== null) {
    const tag = m[0];
    if (!keyRe.test(tag)) continue;
    const content = contentRe.exec(tag);
    if (content) return decodeHtmlEntities(content[2]);
  }
  return undefined;
}

export function extractOgTag(
  html: string,
  tag: string,
): string | undefined {
  return extractMetaContent(html, "property", `og:${tag}`);
}

/** Extract content from `<meta name="…" content="…">` (both attribute orderings). */
export function extractMetaName(
  html: string,
  name: string,
): string | undefined {
  return extractMetaContent(html, "name", name);
}

/** Extract the text content of the `<title>` element. */
export function extractPageTitle(html: string): string | undefined {
  const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
  return match ? decodeHtmlEntities(match[1].trim()) : undefined;
}

// ── Brand color helpers ───────────────────────────────────────────────────────

/**
 * Find the page's `theme-color`.
 *
 * Sites routinely ship the tag twice, scoped by `media`, and often list the
 * dark-scheme one first — so unlike `extractMetaName` this can't just take the
 * first match. Prefer the unscoped tag, then the light-scheme one, then any.
 */
export function extractThemeColor(html: string): string | undefined {
  const tagRe = /<meta\b[^>]*>/gi;
  const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
  const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
  let unscoped: string | undefined;
  let light: string | undefined;
  let any: string | undefined;

  let m: RegExpExecArray | null;
  while ((m = tagRe.exec(html)) !== null) {
    const tag = m[0];
    if (!/\bname=["']theme-color["']/i.test(tag)) continue;
    const content = contentRe.exec(tag)?.[2];
    if (!content) continue;
    const media = mediaRe.exec(tag)?.[2];
    if (!media) unscoped ??= content;
    else if (/light/i.test(media)) light ??= content;
    any ??= content;
  }
  return unscoped ?? light ?? any;
}

/** Extract the `color` of `<link rel="mask-icon">` (Safari pinned tabs). */
export function extractMaskIconColor(html: string): string | undefined {
  const linkRe = /<link[^>]+>/gi;
  let m: RegExpExecArray | null;
  while ((m = linkRe.exec(html)) !== null) {
    const tag = m[0];
    if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
    const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
    if (color) return color;
  }
  return undefined;
}

// The handful of named colors that actually show up in `theme-color`. The full
// 148-name table isn't worth carrying for the long tail.
const NAMED_COLORS: Record<string, string> = {
  black: "#000000",
  white: "#ffffff",
  red: "#ff0000",
  green: "#008000",
  blue: "#0000ff",
  yellow: "#ffff00",
  orange: "#ffa500",
  purple: "#800080",
  gray: "#808080",
  grey: "#808080",
  silver: "#c0c0c0",
  maroon: "#800000",
  navy: "#000080",
  teal: "#008080",
  olive: "#808000",
  lime: "#00ff00",
  aqua: "#00ffff",
  cyan: "#00ffff",
  fuchsia: "#ff00ff",
  magenta: "#ff00ff",
};

function clampByte(n: number): number {
  return Math.max(0, Math.min(255, Math.round(n)));
}

/**
 * Normalize a CSS color to canonical lowercase `#rrggbb`, or `undefined` when
 * it isn't one of the forms sites actually use. Alpha is dropped — the color is
 * only ever used as a tint over the app's own surface.
 */
export function normalizeCssColor(raw: string | undefined): string | undefined {
  if (!raw) return undefined;
  const value = raw.trim().toLowerCase();

  const named = NAMED_COLORS[value];
  if (named) return named;

  const hex = /^#([0-9a-f]{3,8})$/.exec(value)?.[1];
  if (hex) {
    if (hex.length === 3 || hex.length === 4) {
      const [r, g, b] = [...hex.slice(0, 3)];
      return `#${r}${r}${g}${g}${b}${b}`;
    }
    if (hex.length === 6 || hex.length === 8) return `#${hex.slice(0, 6)}`;
    return undefined;
  }

  const rgb = /^rgba?\(([^)]+)\)$/.exec(value)?.[1];
  if (rgb) {
    const parts = rgb.split(/[\s,/]+/).filter(Boolean).slice(0, 3);
    if (parts.length !== 3) return undefined;
    const bytes = parts.map((p) => {
      const n = parseFloat(p);
      if (!Number.isFinite(n)) return NaN;
      return clampByte(p.endsWith("%") ? (n / 100) * 255 : n);
    });
    if (bytes.some(Number.isNaN)) return undefined;
    return `#${bytes.map((b) => b.toString(16).padStart(2, "0")).join("")}`;
  }

  return undefined;
}

// ── JSON-LD helpers (file-private) ────────────────────────────────────────────

type JsonLdResult = {
  title?: string;
  description?: string;
  thumbnailUrl?: string;
};

function ldString(v: unknown): string | undefined {
  if (typeof v === "string" && v.trim()) return v.trim();
  if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
    return v[0].trim();
  }
  return undefined;
}

function ldImage(v: unknown): string | undefined {
  if (
    typeof v === "string" &&
    (v.startsWith("http://") || v.startsWith("https://"))
  ) return v;
  if (Array.isArray(v)) return ldImage(v[0]);
  if (v && typeof v === "object") {
    const o = v as Record<string, unknown>;
    return ldImage(o.url ?? o.contentUrl);
  }
  return undefined;
}

function ldExtractNode(data: unknown): JsonLdResult {
  if (Array.isArray(data)) {
    for (const item of data) {
      const r = ldExtractNode(item);
      if (r.title || r.thumbnailUrl) return r;
    }
    return {};
  }
  if (!data || typeof data !== "object") return {};
  const o = data as Record<string, unknown>;
  if (o["@graph"]) return ldExtractNode(o["@graph"]);
  return {
    title: ldString(o.name ?? o.headline),
    description: ldString(o.description),
    thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
  };
}

/**
 * Parse every `<script type="application/ld+json">` block and return the first
 * node that yields a title or image. Handles `@graph`, arrays, and the common
 * `image` shapes (string, string[], ImageObject).
 */
export function extractJsonLd(html: string): JsonLdResult {
  const pattern =
    /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
  let match: RegExpExecArray | null;
  while ((match = pattern.exec(html)) !== null) {
    try {
      const result = ldExtractNode(JSON.parse(match[1]));
      if (result.title || result.thumbnailUrl) return result;
    } catch { /* invalid JSON — skip */ }
  }
  return {};
}

/**
 * Return the `src` of the first `<img>` whose declared width or height is at
 * least `minSize` pixels (default 200). Skips data URIs. Resolves relative URLs.
 */
export function extractLargeImage(
  html: string,
  baseUrl: string,
  minSize = 200,
): string | undefined {
  const imgPattern = /<img[^>]+>/gi;
  let match: RegExpExecArray | null;
  while ((match = imgPattern.exec(html)) !== null) {
    const tag = match[0];
    const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
    if (!src || src.startsWith("data:")) continue;
    const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
    const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
    if (w >= minSize && h >= minSize) {
      try {
        return new URL(src, baseUrl).toString();
      } catch {
        continue;
      }
    }
  }
  return undefined;
}

/**
 * Collect all `<link rel="icon">` / `<link rel="apple-touch-icon">` tags, rank
 * them by declared size (largest wins), and return the best resolved URL.
 * Falls back to the first match when no `sizes` attribute is present.
 *
 * SVG icons are ranked above everything: they carry no `sizes` attribute, so
 * they'd otherwise score 0 and lose to a 16×16 PNG, yet they scale cleanly to
 * whatever size the placeholder renders them at.
 */
export function extractBestIcon(
  html: string,
  baseUrl: string,
): string | undefined {
  const linkRe = /<link[^>]+>/gi;
  const relRe = /\brel=["']([^"']+)["']/i;
  const hrefRe = /\bhref=["']([^"']+)["']/i;
  const sizesRe = /\bsizes=["']([^"']+)["']/i;

  const candidates: { href: string; area: number }[] = [];

  let m: RegExpExecArray | null;
  while ((m = linkRe.exec(html)) !== null) {
    const tag = m[0];
    const rel = relRe.exec(tag)?.[1] ?? "";
    if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
    const href = hrefRe.exec(tag)?.[1];
    if (!href) continue;
    const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
    const sm = sizesStr.match(/(\d+)x(\d+)/i);
    const isSvg = /\.svg(\?|$)/i.test(href) ||
      /\btype=["']image\/svg\+xml["']/i.test(tag);
    const area = isSvg
      ? Number.MAX_SAFE_INTEGER
      : sm
      ? parseInt(sm[1]) * parseInt(sm[2])
      : 0;
    try {
      candidates.push({ href: new URL(href, baseUrl).toString(), area });
    } catch {
      continue;
    }
  }

  if (candidates.length === 0) return undefined;
  candidates.sort((a, b) => b.area - a.area);
  return candidates[0].href;
}

/**
 * Return the `src` of the first `<img>` that looks like content rather than UI
 * chrome. Skips SVGs, data URIs, and images whose filename matches common
 * icon/logo/nav patterns (logo, icon, sprite, favicon, avatar, banner, etc.).
 * Resolves relative and protocol-relative URLs.
 */
const UI_IMAGE_KEYWORDS = new Set([
  "logo",
  "icon",
  "sprite",
  "favicon",
  "avatar",
  "banner",
  "header",
  "nav",
  "menu",
  "cart",
  "search",
  "tracking",
  "pixel",
  "bg",
  "background",
]);

function isUiImage(src: string): boolean {
  if (/\.svg(\?|$)/i.test(src)) return true;
  const filename = src.split("?")[0].split("/").pop() ?? "";
  const baseName = filename.replace(/\.[^.]+$/, ""); // strip extension
  // Split on common filename separators (-, _, .) and check each token
  return baseName.toLowerCase().split(/[-_.]/).some((t) =>
    UI_IMAGE_KEYWORDS.has(t)
  );
}

/**
 * Return the `src` of the first `<img>` that looks like content rather than UI
 * chrome. Skips SVGs, data URIs, and images whose filename tokens match common
 * icon/logo/nav patterns (logo, icon, sprite, etc.).
 * Resolves relative and protocol-relative URLs.
 */
export function extractFirstContentImage(
  html: string,
  baseUrl: string,
): string | undefined {
  const imgPattern = /<img[^>]+>/gi;
  let match: RegExpExecArray | null;
  while ((match = imgPattern.exec(html)) !== null) {
    const tag = match[0];
    const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
    if (!src || src.startsWith("data:")) continue;
    if (isUiImage(src)) continue;
    try {
      return new URL(src, baseUrl).toString();
    } catch {
      continue;
    }
  }
  return undefined;
}

/**
 * Extract `href` from the first `<link rel="…">` whose rel contains `relFragment`,
 * resolved to an absolute URL using `baseUrl`.
 */
export function extractLinkHref(
  html: string,
  relFragment: string,
  baseUrl: string,
): string | undefined {
  const escaped = relFragment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const patterns = [
    new RegExp(
      `<link[^>]+rel=["'][^"']*${escaped}[^"']*["'][^>]+href=["']([^"']+)["']`,
      "i",
    ),
    new RegExp(
      `<link[^>]+href=["']([^"']+)["'][^>]+rel=["'][^"']*${escaped}[^"']*["']`,
      "i",
    ),
  ];
  for (const pattern of patterns) {
    const match = html.match(pattern);
    if (match) {
      try {
        return new URL(match[1], baseUrl).toString();
      } catch {
        return undefined;
      }
    }
  }
  return undefined;
}

export function isValidHttpUrl(raw: string): boolean {
  try {
    const u = new URL(raw);
    if (u.protocol !== "http:" && u.protocol !== "https:") return false;
    // This site's own pages are fine even on a private host (e.g. localhost in
    // development): selfProvider reads them from the DB, no HTTP involved.
    // Every actual fetch is checked again, after DNS, in fetchWithTimeout.
    return !isPrivateHostname(u) || isSelfUrl(u);
  } catch {
    return false;
  }
}

export interface FetchRichContentResult {
  /** False when the page couldn't be reached at all, as opposed to reached and
   * found to carry no metadata. Callers that already hold good metadata must
   * not overwrite it with a failure stub. */
  ok: boolean;
  content?: RichContent;
}

/**
 * Fetch metadata for `url`, reporting whether the fetch itself succeeded.
 *
 * On failure it still yields a minimal stub so a *new* dump has something
 * displayable, but `ok: false` lets callers with existing metadata keep it.
 */
export async function tryFetchRichContent(
  url: string,
): Promise<FetchRichContentResult> {
  try {
    const provider = providers.find((p) => p.matches(url))!;
    return { ok: true, content: await provider.fetch(url) };
  } catch (err) {
    console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err);
    try {
      return {
        ok: false,
        content: {
          type: "generic",
          url,
          siteName: new URL(url).hostname.replace(/^www\./, ""),
        },
      };
    } catch {
      return { ok: false };
    }
  }
}

/** Metadata for `url`, or a minimal stub when it can't be fetched. */
export async function fetchRichContent(
  url: string,
): Promise<RichContent | undefined> {
  return (await tryFetchRichContent(url)).content;
}