All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 49s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
758 lines
23 KiB
TypeScript
758 lines
23 KiB
TypeScript
import type { RichContent } from "../model/interfaces.ts";
|
||
import { youtubeProvider } from "./providers/youtube.ts";
|
||
import { bandcampProvider } from "./providers/bandcamp.ts";
|
||
import { soundcloudProvider } from "./providers/soundcloud.ts";
|
||
import { selfProvider } from "./providers/self.ts";
|
||
import { genericProvider } from "./providers/generic.ts";
|
||
import {
|
||
isPrivateHostname,
|
||
resolvePublicAddress,
|
||
} from "../lib/public-address.ts";
|
||
import { isSelfUrl } from "../lib/self-url.ts";
|
||
|
||
export interface RichContentProvider {
|
||
name: string;
|
||
matches(url: string): boolean;
|
||
fetch(url: string): Promise<RichContent>;
|
||
}
|
||
|
||
/**
|
||
* Register providers in priority order. The first match wins.
|
||
* `selfProvider` resolves gerbeur URLs directly from the DB (no HTTP round-trip).
|
||
* `genericProvider` must stay last — it always matches.
|
||
*/
|
||
const providers: RichContentProvider[] = [
|
||
youtubeProvider,
|
||
bandcampProvider,
|
||
soundcloudProvider,
|
||
selfProvider,
|
||
genericProvider,
|
||
];
|
||
|
||
// Shared utilities exported for use by providers
|
||
|
||
const FETCH_HEADERS = {
|
||
"User-Agent":
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
|
||
"Accept":
|
||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||
};
|
||
|
||
const MAX_REDIRECTS = 5;
|
||
|
||
/**
|
||
* Fetch `url` via a `curl --insecure` subprocess and return a minimal Response.
|
||
* Used as a last resort when Deno's rustls rejects the server's TLS certificate
|
||
* due to an unsupported algorithm (e.g. UnsupportedSignatureAlgorithm).
|
||
* Returns null if curl is unavailable or exits non-zero.
|
||
*
|
||
* `address` pins the connection to the IP already checked by
|
||
* resolvePublicAddress, and curl doesn't follow redirects itself: a 3xx comes
|
||
* back with its Location so fetchWithTimeout can check the next hop too.
|
||
*/
|
||
async function fetchViaCurl(
|
||
url: URL,
|
||
address: string,
|
||
timeoutMs: number,
|
||
): Promise<Response | null> {
|
||
const tmpPath = await Deno.makeTempFile();
|
||
const port = url.port || (url.protocol === "https:" ? "443" : "80");
|
||
const pinned = address.includes(":") ? `[${address}]` : address;
|
||
try {
|
||
const { code, stdout } = await new Deno.Command("curl", {
|
||
args: [
|
||
"--silent",
|
||
"--insecure",
|
||
"--proto",
|
||
"=http,https",
|
||
"--resolve",
|
||
`${url.hostname.replace(/^\[(.*)\]$/, "$1")}:${port}:${pinned}`,
|
||
"--max-time",
|
||
String(Math.ceil(timeoutMs / 1000)),
|
||
"--max-filesize",
|
||
String(2 * MAX_PAGE_BYTES),
|
||
"--user-agent",
|
||
FETCH_HEADERS["User-Agent"],
|
||
"--header",
|
||
`Accept: ${FETCH_HEADERS["Accept"]}`,
|
||
"--header",
|
||
`Accept-Language: ${FETCH_HEADERS["Accept-Language"]}`,
|
||
"--output",
|
||
tmpPath,
|
||
"--write-out",
|
||
"%{http_code}\n%{redirect_url}\n%{content_type}",
|
||
url.href,
|
||
],
|
||
stdout: "piped",
|
||
stderr: "null",
|
||
}).output();
|
||
|
||
if (code !== 0) return null;
|
||
|
||
const [status, location, contentType = ""] = new TextDecoder()
|
||
.decode(stdout).split("\n").map((l) => l.trim());
|
||
const statusCode = Number(status);
|
||
if (!(statusCode >= 200 && statusCode <= 599)) return null;
|
||
const headers = new Headers();
|
||
if (location) headers.set("location", location);
|
||
if (contentType) headers.set("content-type", contentType);
|
||
|
||
const bytes = await Deno.readFile(tmpPath);
|
||
if (!contentType.startsWith("text/")) {
|
||
return new Response(bytes, { status: statusCode, headers });
|
||
}
|
||
// Decode using the charset declared in the Content-Type header so that
|
||
// pages served in ISO-8859-1, windows-1252, etc. are read correctly.
|
||
const charset = /charset=([\w-]+)/i.exec(contentType)?.[1] ?? "utf-8";
|
||
let bodyText: string;
|
||
try {
|
||
bodyText = new TextDecoder(charset, { fatal: false }).decode(bytes);
|
||
} catch {
|
||
bodyText = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
||
}
|
||
headers.set("content-type", contentType.replace(/;.*$/, "; charset=utf-8"));
|
||
return new Response(bodyText, { status: statusCode, headers });
|
||
} catch {
|
||
return null;
|
||
} finally {
|
||
await Deno.remove(tmpPath).catch(() => {});
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Most bytes of a fetched page that are read. Metadata lives in the <head>,
|
||
* and a "page" past this is a download, not something to scrape.
|
||
*/
|
||
export const MAX_PAGE_BYTES = 4 * 1024 * 1024;
|
||
|
||
/**
|
||
* Hands `body` on, and clears the request's deadline once it has been read to
|
||
* the end or cancelled. Until then the deadline still applies: a server that
|
||
* answers at once and then drips its body can't hold the request open.
|
||
*/
|
||
function withDeadline(
|
||
body: ReadableStream<Uint8Array>,
|
||
timer: ReturnType<typeof setTimeout>,
|
||
): ReadableStream<Uint8Array> {
|
||
const reader = body.getReader();
|
||
return new ReadableStream({
|
||
async pull(controller) {
|
||
try {
|
||
const { done, value } = await reader.read();
|
||
if (done) {
|
||
clearTimeout(timer);
|
||
controller.close();
|
||
} else {
|
||
controller.enqueue(value);
|
||
}
|
||
} catch (err) {
|
||
clearTimeout(timer);
|
||
controller.error(err);
|
||
}
|
||
},
|
||
cancel(reason) {
|
||
clearTimeout(timer);
|
||
return reader.cancel(reason);
|
||
},
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Reads a response's body as text, stopping at `maxBytes` (the rest is
|
||
* cancelled, never downloaded).
|
||
*/
|
||
export async function readTextCapped(
|
||
res: Response,
|
||
maxBytes = MAX_PAGE_BYTES,
|
||
): Promise<string> {
|
||
if (!res.body) return "";
|
||
const reader = res.body.getReader();
|
||
const chunks: Uint8Array[] = [];
|
||
let size = 0;
|
||
try {
|
||
while (size < maxBytes) {
|
||
const { done, value } = await reader.read();
|
||
if (done) break;
|
||
chunks.push(value);
|
||
size += value.byteLength;
|
||
}
|
||
} finally {
|
||
await reader.cancel().catch(() => {});
|
||
}
|
||
const bytes = new Uint8Array(Math.min(size, maxBytes));
|
||
let offset = 0;
|
||
for (const chunk of chunks) {
|
||
const part = chunk.subarray(0, bytes.byteLength - offset);
|
||
bytes.set(part, offset);
|
||
offset += part.byteLength;
|
||
if (offset >= bytes.byteLength) break;
|
||
}
|
||
return new TextDecoder().decode(bytes);
|
||
}
|
||
|
||
/** One request, no redirects followed. The timeout covers the body too. */
|
||
async function fetchOnce(
|
||
url: URL,
|
||
address: string,
|
||
timeoutMs: number,
|
||
): Promise<Response> {
|
||
const controller = new AbortController();
|
||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||
let res: Response;
|
||
try {
|
||
res = await fetch(url, {
|
||
signal: controller.signal,
|
||
headers: FETCH_HEADERS,
|
||
redirect: "manual",
|
||
});
|
||
} catch (err) {
|
||
clearTimeout(timer);
|
||
if (!(err instanceof TypeError && err.message.includes("certificate"))) {
|
||
throw err;
|
||
}
|
||
|
||
// Deno/rustls rejected the server's TLS certificate (expired, self-signed,
|
||
// or an unsupported signature algorithm) and offers no way to override
|
||
// verification programmatically. Fall back to a `curl --insecure`
|
||
// subprocess, which uses its own TLS stack.
|
||
const curlRes = await fetchViaCurl(url, address, timeoutMs);
|
||
if (curlRes) return curlRes;
|
||
|
||
throw err;
|
||
}
|
||
if (!res.body) {
|
||
clearTimeout(timer);
|
||
return res;
|
||
}
|
||
return new Response(withDeadline(res.body, timer), {
|
||
status: res.status,
|
||
statusText: res.statusText,
|
||
headers: res.headers,
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Fetch a user-supplied URL. Every hop — the URL itself and each redirect —
|
||
* must resolve to a public address (see resolvePublicAddress), so a link or a
|
||
* redirect can't point the server at its own network.
|
||
*/
|
||
export async function fetchWithTimeout(
|
||
url: string,
|
||
timeoutMs = 5000,
|
||
): Promise<Response> {
|
||
let current = new URL(url);
|
||
for (let hop = 0;; hop++) {
|
||
if (current.protocol !== "http:" && current.protocol !== "https:") {
|
||
throw new TypeError(`Unsupported protocol ${current.protocol}`);
|
||
}
|
||
const address = await resolvePublicAddress(current);
|
||
const res = await fetchOnce(current, address, timeoutMs);
|
||
const location = res.headers.get("location");
|
||
if (res.status < 300 || res.status >= 400 || !location) return res;
|
||
|
||
await res.body?.cancel();
|
||
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
|
||
current = new URL(location, current);
|
||
}
|
||
}
|
||
|
||
function decodeHtmlEntities(str: string): string {
|
||
return str
|
||
.replace(/&/gi, "&")
|
||
.replace(/</gi, "<")
|
||
.replace(/>/gi, ">")
|
||
.replace(/"/gi, '"')
|
||
.replace(/'/gi, "'")
|
||
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
|
||
.replace(
|
||
/&#x([0-9a-f]+);/gi,
|
||
(_, hex) => String.fromCodePoint(parseInt(hex, 16)),
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Find the first `<meta>` tag whose `attr` equals `value` and return its
|
||
* `content` (either attribute ordering). Each tag is matched in isolation so
|
||
* the `content` capture can never span across tag boundaries.
|
||
*/
|
||
function extractMetaContent(
|
||
html: string,
|
||
attr: string,
|
||
value: string,
|
||
): string | undefined {
|
||
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||
const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
|
||
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
|
||
const tagRe = /<meta\b[^>]*>/gi;
|
||
let m: RegExpExecArray | null;
|
||
while ((m = tagRe.exec(html)) !== null) {
|
||
const tag = m[0];
|
||
if (!keyRe.test(tag)) continue;
|
||
const content = contentRe.exec(tag);
|
||
if (content) return decodeHtmlEntities(content[2]);
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
export function extractOgTag(
|
||
html: string,
|
||
tag: string,
|
||
): string | undefined {
|
||
return extractMetaContent(html, "property", `og:${tag}`);
|
||
}
|
||
|
||
/** Extract content from `<meta name="…" content="…">` (both attribute orderings). */
|
||
export function extractMetaName(
|
||
html: string,
|
||
name: string,
|
||
): string | undefined {
|
||
return extractMetaContent(html, "name", name);
|
||
}
|
||
|
||
/** Extract the text content of the `<title>` element. */
|
||
export function extractPageTitle(html: string): string | undefined {
|
||
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||
return match ? decodeHtmlEntities(match[1].trim()) : undefined;
|
||
}
|
||
|
||
// ── Brand color helpers ───────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Find the page's `theme-color`.
|
||
*
|
||
* Sites routinely ship the tag twice, scoped by `media`, and often list the
|
||
* dark-scheme one first — so unlike `extractMetaName` this can't just take the
|
||
* first match. Prefer the unscoped tag, then the light-scheme one, then any.
|
||
*/
|
||
export function extractThemeColor(html: string): string | undefined {
|
||
const tagRe = /<meta\b[^>]*>/gi;
|
||
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
|
||
const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
|
||
let unscoped: string | undefined;
|
||
let light: string | undefined;
|
||
let any: string | undefined;
|
||
|
||
let m: RegExpExecArray | null;
|
||
while ((m = tagRe.exec(html)) !== null) {
|
||
const tag = m[0];
|
||
if (!/\bname=["']theme-color["']/i.test(tag)) continue;
|
||
const content = contentRe.exec(tag)?.[2];
|
||
if (!content) continue;
|
||
const media = mediaRe.exec(tag)?.[2];
|
||
if (!media) unscoped ??= content;
|
||
else if (/light/i.test(media)) light ??= content;
|
||
any ??= content;
|
||
}
|
||
return unscoped ?? light ?? any;
|
||
}
|
||
|
||
/** Extract the `color` of `<link rel="mask-icon">` (Safari pinned tabs). */
|
||
export function extractMaskIconColor(html: string): string | undefined {
|
||
const linkRe = /<link[^>]+>/gi;
|
||
let m: RegExpExecArray | null;
|
||
while ((m = linkRe.exec(html)) !== null) {
|
||
const tag = m[0];
|
||
if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
|
||
const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
|
||
if (color) return color;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
// The handful of named colors that actually show up in `theme-color`. The full
|
||
// 148-name table isn't worth carrying for the long tail.
|
||
const NAMED_COLORS: Record<string, string> = {
|
||
black: "#000000",
|
||
white: "#ffffff",
|
||
red: "#ff0000",
|
||
green: "#008000",
|
||
blue: "#0000ff",
|
||
yellow: "#ffff00",
|
||
orange: "#ffa500",
|
||
purple: "#800080",
|
||
gray: "#808080",
|
||
grey: "#808080",
|
||
silver: "#c0c0c0",
|
||
maroon: "#800000",
|
||
navy: "#000080",
|
||
teal: "#008080",
|
||
olive: "#808000",
|
||
lime: "#00ff00",
|
||
aqua: "#00ffff",
|
||
cyan: "#00ffff",
|
||
fuchsia: "#ff00ff",
|
||
magenta: "#ff00ff",
|
||
};
|
||
|
||
function clampByte(n: number): number {
|
||
return Math.max(0, Math.min(255, Math.round(n)));
|
||
}
|
||
|
||
/**
|
||
* Normalize a CSS color to canonical lowercase `#rrggbb`, or `undefined` when
|
||
* it isn't one of the forms sites actually use. Alpha is dropped — the color is
|
||
* only ever used as a tint over the app's own surface.
|
||
*/
|
||
export function normalizeCssColor(raw: string | undefined): string | undefined {
|
||
if (!raw) return undefined;
|
||
const value = raw.trim().toLowerCase();
|
||
|
||
const named = NAMED_COLORS[value];
|
||
if (named) return named;
|
||
|
||
const hex = /^#([0-9a-f]{3,8})$/.exec(value)?.[1];
|
||
if (hex) {
|
||
if (hex.length === 3 || hex.length === 4) {
|
||
const [r, g, b] = [...hex.slice(0, 3)];
|
||
return `#${r}${r}${g}${g}${b}${b}`;
|
||
}
|
||
if (hex.length === 6 || hex.length === 8) return `#${hex.slice(0, 6)}`;
|
||
return undefined;
|
||
}
|
||
|
||
const rgb = /^rgba?\(([^)]+)\)$/.exec(value)?.[1];
|
||
if (rgb) {
|
||
const parts = rgb.split(/[\s,/]+/).filter(Boolean).slice(0, 3);
|
||
if (parts.length !== 3) return undefined;
|
||
const bytes = parts.map((p) => {
|
||
const n = parseFloat(p);
|
||
if (!Number.isFinite(n)) return NaN;
|
||
return clampByte(p.endsWith("%") ? (n / 100) * 255 : n);
|
||
});
|
||
if (bytes.some(Number.isNaN)) return undefined;
|
||
return `#${bytes.map((b) => b.toString(16).padStart(2, "0")).join("")}`;
|
||
}
|
||
|
||
return undefined;
|
||
}
|
||
|
||
// ── JSON-LD helpers (file-private) ────────────────────────────────────────────
|
||
|
||
type JsonLdResult = {
|
||
title?: string;
|
||
description?: string;
|
||
thumbnailUrl?: string;
|
||
};
|
||
|
||
function ldString(v: unknown): string | undefined {
|
||
if (typeof v === "string" && v.trim()) return v.trim();
|
||
if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
|
||
return v[0].trim();
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
function ldImage(v: unknown): string | undefined {
|
||
if (
|
||
typeof v === "string" &&
|
||
(v.startsWith("http://") || v.startsWith("https://"))
|
||
) return v;
|
||
if (Array.isArray(v)) return ldImage(v[0]);
|
||
if (v && typeof v === "object") {
|
||
const o = v as Record<string, unknown>;
|
||
return ldImage(o.url ?? o.contentUrl);
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
function ldExtractNode(data: unknown): JsonLdResult {
|
||
if (Array.isArray(data)) {
|
||
for (const item of data) {
|
||
const r = ldExtractNode(item);
|
||
if (r.title || r.thumbnailUrl) return r;
|
||
}
|
||
return {};
|
||
}
|
||
if (!data || typeof data !== "object") return {};
|
||
const o = data as Record<string, unknown>;
|
||
if (o["@graph"]) return ldExtractNode(o["@graph"]);
|
||
return {
|
||
title: ldString(o.name ?? o.headline),
|
||
description: ldString(o.description),
|
||
thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Parse every `<script type="application/ld+json">` block and return the first
|
||
* node that yields a title or image. Handles `@graph`, arrays, and the common
|
||
* `image` shapes (string, string[], ImageObject).
|
||
*/
|
||
export function extractJsonLd(html: string): JsonLdResult {
|
||
const pattern =
|
||
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
||
let match: RegExpExecArray | null;
|
||
while ((match = pattern.exec(html)) !== null) {
|
||
try {
|
||
const result = ldExtractNode(JSON.parse(match[1]));
|
||
if (result.title || result.thumbnailUrl) return result;
|
||
} catch { /* invalid JSON — skip */ }
|
||
}
|
||
return {};
|
||
}
|
||
|
||
/**
|
||
* Return the `src` of the first `<img>` whose declared width or height is at
|
||
* least `minSize` pixels (default 200). Skips data URIs. Resolves relative URLs.
|
||
*/
|
||
export function extractLargeImage(
|
||
html: string,
|
||
baseUrl: string,
|
||
minSize = 200,
|
||
): string | undefined {
|
||
const imgPattern = /<img[^>]+>/gi;
|
||
let match: RegExpExecArray | null;
|
||
while ((match = imgPattern.exec(html)) !== null) {
|
||
const tag = match[0];
|
||
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
|
||
if (!src || src.startsWith("data:")) continue;
|
||
const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
|
||
const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
|
||
if (w >= minSize && h >= minSize) {
|
||
try {
|
||
return new URL(src, baseUrl).toString();
|
||
} catch {
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/**
|
||
* Collect all `<link rel="icon">` / `<link rel="apple-touch-icon">` tags, rank
|
||
* them by declared size (largest wins), and return the best resolved URL.
|
||
* Falls back to the first match when no `sizes` attribute is present.
|
||
*
|
||
* SVG icons are ranked above everything: they carry no `sizes` attribute, so
|
||
* they'd otherwise score 0 and lose to a 16×16 PNG, yet they scale cleanly to
|
||
* whatever size the placeholder renders them at.
|
||
*/
|
||
export function extractBestIcon(
|
||
html: string,
|
||
baseUrl: string,
|
||
): string | undefined {
|
||
const linkRe = /<link[^>]+>/gi;
|
||
const relRe = /\brel=["']([^"']+)["']/i;
|
||
const hrefRe = /\bhref=["']([^"']+)["']/i;
|
||
const sizesRe = /\bsizes=["']([^"']+)["']/i;
|
||
|
||
const candidates: { href: string; area: number }[] = [];
|
||
|
||
let m: RegExpExecArray | null;
|
||
while ((m = linkRe.exec(html)) !== null) {
|
||
const tag = m[0];
|
||
const rel = relRe.exec(tag)?.[1] ?? "";
|
||
if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
|
||
const href = hrefRe.exec(tag)?.[1];
|
||
if (!href) continue;
|
||
const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
|
||
const sm = sizesStr.match(/(\d+)x(\d+)/i);
|
||
const isSvg = /\.svg(\?|$)/i.test(href) ||
|
||
/\btype=["']image\/svg\+xml["']/i.test(tag);
|
||
const area = isSvg
|
||
? Number.MAX_SAFE_INTEGER
|
||
: sm
|
||
? parseInt(sm[1]) * parseInt(sm[2])
|
||
: 0;
|
||
try {
|
||
candidates.push({ href: new URL(href, baseUrl).toString(), area });
|
||
} catch {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
if (candidates.length === 0) return undefined;
|
||
candidates.sort((a, b) => b.area - a.area);
|
||
return candidates[0].href;
|
||
}
|
||
|
||
/**
|
||
* Return the `src` of the first `<img>` that looks like content rather than UI
|
||
* chrome. Skips SVGs, data URIs, and images whose filename matches common
|
||
* icon/logo/nav patterns (logo, icon, sprite, favicon, avatar, banner, etc.).
|
||
* Resolves relative and protocol-relative URLs.
|
||
*/
|
||
const UI_IMAGE_KEYWORDS = new Set([
|
||
"logo",
|
||
"icon",
|
||
"sprite",
|
||
"favicon",
|
||
"avatar",
|
||
"banner",
|
||
"header",
|
||
"nav",
|
||
"menu",
|
||
"cart",
|
||
"search",
|
||
"tracking",
|
||
"pixel",
|
||
"bg",
|
||
"background",
|
||
]);
|
||
|
||
function isUiImage(src: string): boolean {
|
||
if (/\.svg(\?|$)/i.test(src)) return true;
|
||
const filename = src.split("?")[0].split("/").pop() ?? "";
|
||
const baseName = filename.replace(/\.[^.]+$/, ""); // strip extension
|
||
// Split on common filename separators (-, _, .) and check each token
|
||
return baseName.toLowerCase().split(/[-_.]/).some((t) =>
|
||
UI_IMAGE_KEYWORDS.has(t)
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Return the `src` of the first `<img>` that looks like content rather than UI
|
||
* chrome. Skips SVGs, data URIs, and images whose filename tokens match common
|
||
* icon/logo/nav patterns (logo, icon, sprite, etc.).
|
||
* Resolves relative and protocol-relative URLs.
|
||
*/
|
||
export function extractFirstContentImage(
|
||
html: string,
|
||
baseUrl: string,
|
||
): string | undefined {
|
||
const imgPattern = /<img[^>]+>/gi;
|
||
let match: RegExpExecArray | null;
|
||
while ((match = imgPattern.exec(html)) !== null) {
|
||
const tag = match[0];
|
||
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
|
||
if (!src || src.startsWith("data:")) continue;
|
||
if (isUiImage(src)) continue;
|
||
try {
|
||
return new URL(src, baseUrl).toString();
|
||
} catch {
|
||
continue;
|
||
}
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/**
|
||
* Extract `href` from the first `<link rel="…">` whose rel contains `relFragment`,
|
||
* resolved to an absolute URL using `baseUrl`.
|
||
*/
|
||
export function extractLinkHref(
|
||
html: string,
|
||
relFragment: string,
|
||
baseUrl: string,
|
||
): string | undefined {
|
||
const escaped = relFragment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||
const patterns = [
|
||
new RegExp(
|
||
`<link[^>]+rel=["'][^"']*${escaped}[^"']*["'][^>]+href=["']([^"']+)["']`,
|
||
"i",
|
||
),
|
||
new RegExp(
|
||
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["'][^"']*${escaped}[^"']*["']`,
|
||
"i",
|
||
),
|
||
];
|
||
for (const pattern of patterns) {
|
||
const match = html.match(pattern);
|
||
if (match) {
|
||
try {
|
||
return new URL(match[1], baseUrl).toString();
|
||
} catch {
|
||
return undefined;
|
||
}
|
||
}
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
export function isValidHttpUrl(raw: string): boolean {
|
||
try {
|
||
const u = new URL(raw);
|
||
if (u.protocol !== "http:" && u.protocol !== "https:") return false;
|
||
// This site's own pages are fine even on a private host (e.g. localhost in
|
||
// development): selfProvider reads them from the DB, no HTTP involved.
|
||
// Every actual fetch is checked again, after DNS, in fetchWithTimeout.
|
||
return !isPrivateHostname(u) || isSelfUrl(u);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
export interface FetchRichContentResult {
|
||
/** False when the page couldn't be reached at all, as opposed to reached and
|
||
* found to carry no metadata. Callers that already hold good metadata must
|
||
* not overwrite it with a failure stub. */
|
||
ok: boolean;
|
||
content?: RichContent;
|
||
}
|
||
|
||
/**
|
||
* Fetch metadata for `url`, reporting whether the fetch itself succeeded.
|
||
*
|
||
* On failure it still yields a minimal stub so a *new* dump has something
|
||
* displayable, but `ok: false` lets callers with existing metadata keep it.
|
||
*/
|
||
// Metadata fetched in the last few minutes, by URL. The create form previews a
|
||
// link and then posts it; without this the page was fetched twice, and the
|
||
// second fetch was the slow part of posting. Only successes are kept, and a
|
||
// metadata refresh (`fresh`) always goes to the page.
|
||
const RECENT_TTL_MS = 5 * 60 * 1000;
|
||
const RECENT_MAX = 200;
|
||
const recent = new Map<
|
||
string,
|
||
{ at: number; result: Promise<FetchRichContentResult> }
|
||
>();
|
||
|
||
/** Forget every cached result (tests). */
|
||
export function clearRichContentCache(): void {
|
||
recent.clear();
|
||
}
|
||
|
||
export function tryFetchRichContent(
|
||
url: string,
|
||
{ fresh = false }: { fresh?: boolean } = {},
|
||
): Promise<FetchRichContentResult> {
|
||
const hit = recent.get(url);
|
||
if (!fresh && hit && Date.now() - hit.at < RECENT_TTL_MS) return hit.result;
|
||
recent.delete(url);
|
||
|
||
// Stored while in flight too, so a preview and a post racing share a fetch.
|
||
const result = fetchRichContentUncached(url);
|
||
if (recent.size >= RECENT_MAX) {
|
||
const oldest = recent.keys().next().value;
|
||
if (oldest !== undefined) recent.delete(oldest);
|
||
}
|
||
const entry = { at: Date.now(), result };
|
||
recent.set(url, entry);
|
||
result.then(({ ok }) => {
|
||
if (!ok && recent.get(url) === entry) recent.delete(url);
|
||
});
|
||
return result;
|
||
}
|
||
|
||
async function fetchRichContentUncached(
|
||
url: string,
|
||
): Promise<FetchRichContentResult> {
|
||
try {
|
||
const provider = providers.find((p) => p.matches(url))!;
|
||
return { ok: true, content: await provider.fetch(url) };
|
||
} catch (err) {
|
||
console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err);
|
||
try {
|
||
return {
|
||
ok: false,
|
||
content: {
|
||
type: "generic",
|
||
url,
|
||
siteName: new URL(url).hostname.replace(/^www\./, ""),
|
||
},
|
||
};
|
||
} catch {
|
||
return { ok: false };
|
||
}
|
||
}
|
||
}
|
||
|
||
/** Metadata for `url`, or a minimal stub when it can't be fetched. */
|
||
export async function fetchRichContent(
|
||
url: string,
|
||
): Promise<RichContent | undefined> {
|
||
return (await tryFetchRichContent(url)).content;
|
||
}
|