Files
gerbeur/api/services/rich-content-service.ts
khannurien de5abeceb2
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 49s
v3: a performance pass over both halves — indexed queries, streamed files, cached previews, and feeds that stop re-rendering on every tick — a review turned up whole-file reads, unindexed scans and render storms; each fix comes with a test. Database: migration 0016 adds the indexes the hot queries lacked — dumps by created_at, (user_id, created_at) and slug, votes by (user_id, created_at), playlists by slug, users by lower(username) for @mentions, and the child side of every cascading foreign key (comment replies, a user's comments, follows both ways, chat messages, invites, reset tokens); the feeds, dump and playlist pages, upvotes and follower fan-out all plan as index searches now, with no temp-table sort. SQLite runs in WAL with synchronous=NORMAL and a busy timeout, and a transaction() helper wraps the multi-row writes (a dump and its categories, category sets, backlinks, attachment links, playlist reorders, follower notifications) so they commit once instead of once a row. Lists no longer run a COUNT(*) per page: they fetch limit + 1 rows and hasMore comes from the extra one; total stays only on search, the one place it's shown. Files: /api/files read the whole upload (up to 50 MB) for every range request a player makes while seeking; api/lib/serve-file.ts streams the requested bytes from disk, with an ETag and 304s, and serves avatars, attachments, covers and thumbnails too. File and thumbnail URLs carry the file's version (size and last change), so the current one is cached for a year and anything else revalidates — a same-size replacement used to stay stale, and a video's still was cached immutable forever under a URL that never changed. Feeds showed image dumps at full size: /api/thumbnails now makes a 960px preview with ffmpeg (GIFs and unreadable images go out as they are), one run per output however many cards ask at once, and ffmpeg is probed once per process instead of spawned per request. Uploads stream to disk; replacing a file writes beside it and renames over, a hard link keeping the old one for rollback instead of holding both in memory; generated stills and previews go when the file is replaced or the dump deleted, and a custom thumbnail with its dump. Outbound: fetchWithTimeout's deadline now covers the body — a server that answered at once and then dripped could hold a create open indefinitely — and pages are read to 4 MB at most; metadata is cached five minutes so the create form's preview and the post that follows share one fetch (refresh-metadata still goes to the page); proxied images sit in a 32 MB LRU. Broadcasts serialize once rather than per socket, and index.html is kept in memory until the file changes. Frontend: currentTime lived in PlayerContext, so every card re-rendered — and re-parsed its Markdown — four times a second while anything played; the playhead has its own PlayerTimeContext now. The socket context is split three ways (feed, chat via useWSChat, presence via useWSPresence), so typing, presence and chat messages leave the feeds alone. DumpCard, JournalCard and Markdown are memoized; card images use the server preview and load lazily; the feed cache is written when the page is left rather than after every pause in scrolling; the theme context value is memoized.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-27 16:35:51 +00:00

758 lines
23 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import type { RichContent } from "../model/interfaces.ts";
import { youtubeProvider } from "./providers/youtube.ts";
import { bandcampProvider } from "./providers/bandcamp.ts";
import { soundcloudProvider } from "./providers/soundcloud.ts";
import { selfProvider } from "./providers/self.ts";
import { genericProvider } from "./providers/generic.ts";
import {
isPrivateHostname,
resolvePublicAddress,
} from "../lib/public-address.ts";
import { isSelfUrl } from "../lib/self-url.ts";
export interface RichContentProvider {
name: string;
matches(url: string): boolean;
fetch(url: string): Promise<RichContent>;
}
/**
* Register providers in priority order. The first match wins.
* `selfProvider` resolves gerbeur URLs directly from the DB (no HTTP round-trip).
* `genericProvider` must stay last — it always matches.
*/
const providers: RichContentProvider[] = [
youtubeProvider,
bandcampProvider,
soundcloudProvider,
selfProvider,
genericProvider,
];
// Shared utilities exported for use by providers
const FETCH_HEADERS = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
};
const MAX_REDIRECTS = 5;
/**
* Fetch `url` via a `curl --insecure` subprocess and return a minimal Response.
* Used as a last resort when Deno's rustls rejects the server's TLS certificate
* due to an unsupported algorithm (e.g. UnsupportedSignatureAlgorithm).
* Returns null if curl is unavailable or exits non-zero.
*
* `address` pins the connection to the IP already checked by
* resolvePublicAddress, and curl doesn't follow redirects itself: a 3xx comes
* back with its Location so fetchWithTimeout can check the next hop too.
*/
async function fetchViaCurl(
url: URL,
address: string,
timeoutMs: number,
): Promise<Response | null> {
const tmpPath = await Deno.makeTempFile();
const port = url.port || (url.protocol === "https:" ? "443" : "80");
const pinned = address.includes(":") ? `[${address}]` : address;
try {
const { code, stdout } = await new Deno.Command("curl", {
args: [
"--silent",
"--insecure",
"--proto",
"=http,https",
"--resolve",
`${url.hostname.replace(/^\[(.*)\]$/, "$1")}:${port}:${pinned}`,
"--max-time",
String(Math.ceil(timeoutMs / 1000)),
"--max-filesize",
String(2 * MAX_PAGE_BYTES),
"--user-agent",
FETCH_HEADERS["User-Agent"],
"--header",
`Accept: ${FETCH_HEADERS["Accept"]}`,
"--header",
`Accept-Language: ${FETCH_HEADERS["Accept-Language"]}`,
"--output",
tmpPath,
"--write-out",
"%{http_code}\n%{redirect_url}\n%{content_type}",
url.href,
],
stdout: "piped",
stderr: "null",
}).output();
if (code !== 0) return null;
const [status, location, contentType = ""] = new TextDecoder()
.decode(stdout).split("\n").map((l) => l.trim());
const statusCode = Number(status);
if (!(statusCode >= 200 && statusCode <= 599)) return null;
const headers = new Headers();
if (location) headers.set("location", location);
if (contentType) headers.set("content-type", contentType);
const bytes = await Deno.readFile(tmpPath);
if (!contentType.startsWith("text/")) {
return new Response(bytes, { status: statusCode, headers });
}
// Decode using the charset declared in the Content-Type header so that
// pages served in ISO-8859-1, windows-1252, etc. are read correctly.
const charset = /charset=([\w-]+)/i.exec(contentType)?.[1] ?? "utf-8";
let bodyText: string;
try {
bodyText = new TextDecoder(charset, { fatal: false }).decode(bytes);
} catch {
bodyText = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
}
headers.set("content-type", contentType.replace(/;.*$/, "; charset=utf-8"));
return new Response(bodyText, { status: statusCode, headers });
} catch {
return null;
} finally {
await Deno.remove(tmpPath).catch(() => {});
}
}
/**
* Most bytes of a fetched page that are read. Metadata lives in the <head>,
* and a "page" past this is a download, not something to scrape.
*/
export const MAX_PAGE_BYTES = 4 * 1024 * 1024;
/**
* Hands `body` on, and clears the request's deadline once it has been read to
* the end or cancelled. Until then the deadline still applies: a server that
* answers at once and then drips its body can't hold the request open.
*/
function withDeadline(
body: ReadableStream<Uint8Array>,
timer: ReturnType<typeof setTimeout>,
): ReadableStream<Uint8Array> {
const reader = body.getReader();
return new ReadableStream({
async pull(controller) {
try {
const { done, value } = await reader.read();
if (done) {
clearTimeout(timer);
controller.close();
} else {
controller.enqueue(value);
}
} catch (err) {
clearTimeout(timer);
controller.error(err);
}
},
cancel(reason) {
clearTimeout(timer);
return reader.cancel(reason);
},
});
}
/**
* Reads a response's body as text, stopping at `maxBytes` (the rest is
* cancelled, never downloaded).
*/
export async function readTextCapped(
res: Response,
maxBytes = MAX_PAGE_BYTES,
): Promise<string> {
if (!res.body) return "";
const reader = res.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
while (size < maxBytes) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
size += value.byteLength;
}
} finally {
await reader.cancel().catch(() => {});
}
const bytes = new Uint8Array(Math.min(size, maxBytes));
let offset = 0;
for (const chunk of chunks) {
const part = chunk.subarray(0, bytes.byteLength - offset);
bytes.set(part, offset);
offset += part.byteLength;
if (offset >= bytes.byteLength) break;
}
return new TextDecoder().decode(bytes);
}
/** One request, no redirects followed. The timeout covers the body too. */
async function fetchOnce(
url: URL,
address: string,
timeoutMs: number,
): Promise<Response> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
let res: Response;
try {
res = await fetch(url, {
signal: controller.signal,
headers: FETCH_HEADERS,
redirect: "manual",
});
} catch (err) {
clearTimeout(timer);
if (!(err instanceof TypeError && err.message.includes("certificate"))) {
throw err;
}
// Deno/rustls rejected the server's TLS certificate (expired, self-signed,
// or an unsupported signature algorithm) and offers no way to override
// verification programmatically. Fall back to a `curl --insecure`
// subprocess, which uses its own TLS stack.
const curlRes = await fetchViaCurl(url, address, timeoutMs);
if (curlRes) return curlRes;
throw err;
}
if (!res.body) {
clearTimeout(timer);
return res;
}
return new Response(withDeadline(res.body, timer), {
status: res.status,
statusText: res.statusText,
headers: res.headers,
});
}
/**
* Fetch a user-supplied URL. Every hop — the URL itself and each redirect —
* must resolve to a public address (see resolvePublicAddress), so a link or a
* redirect can't point the server at its own network.
*/
export async function fetchWithTimeout(
url: string,
timeoutMs = 5000,
): Promise<Response> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) return res;
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
}
}
function decodeHtmlEntities(str: string): string {
return str
.replace(/&amp;/gi, "&")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">")
.replace(/&quot;/gi, '"')
.replace(/&apos;/gi, "'")
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
.replace(
/&#x([0-9a-f]+);/gi,
(_, hex) => String.fromCodePoint(parseInt(hex, 16)),
);
}
/**
* Find the first `<meta>` tag whose `attr` equals `value` and return its
* `content` (either attribute ordering). Each tag is matched in isolation so
* the `content` capture can never span across tag boundaries.
*/
function extractMetaContent(
html: string,
attr: string,
value: string,
): string | undefined {
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
const tagRe = /<meta\b[^>]*>/gi;
let m: RegExpExecArray | null;
while ((m = tagRe.exec(html)) !== null) {
const tag = m[0];
if (!keyRe.test(tag)) continue;
const content = contentRe.exec(tag);
if (content) return decodeHtmlEntities(content[2]);
}
return undefined;
}
export function extractOgTag(
html: string,
tag: string,
): string | undefined {
return extractMetaContent(html, "property", `og:${tag}`);
}
/** Extract content from `<meta name="…" content="…">` (both attribute orderings). */
export function extractMetaName(
html: string,
name: string,
): string | undefined {
return extractMetaContent(html, "name", name);
}
/** Extract the text content of the `<title>` element. */
export function extractPageTitle(html: string): string | undefined {
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
return match ? decodeHtmlEntities(match[1].trim()) : undefined;
}
// ── Brand color helpers ───────────────────────────────────────────────────────
/**
* Find the page's `theme-color`.
*
* Sites routinely ship the tag twice, scoped by `media`, and often list the
* dark-scheme one first — so unlike `extractMetaName` this can't just take the
* first match. Prefer the unscoped tag, then the light-scheme one, then any.
*/
export function extractThemeColor(html: string): string | undefined {
const tagRe = /<meta\b[^>]*>/gi;
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
let unscoped: string | undefined;
let light: string | undefined;
let any: string | undefined;
let m: RegExpExecArray | null;
while ((m = tagRe.exec(html)) !== null) {
const tag = m[0];
if (!/\bname=["']theme-color["']/i.test(tag)) continue;
const content = contentRe.exec(tag)?.[2];
if (!content) continue;
const media = mediaRe.exec(tag)?.[2];
if (!media) unscoped ??= content;
else if (/light/i.test(media)) light ??= content;
any ??= content;
}
return unscoped ?? light ?? any;
}
/** Extract the `color` of `<link rel="mask-icon">` (Safari pinned tabs). */
export function extractMaskIconColor(html: string): string | undefined {
const linkRe = /<link[^>]+>/gi;
let m: RegExpExecArray | null;
while ((m = linkRe.exec(html)) !== null) {
const tag = m[0];
if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
if (color) return color;
}
return undefined;
}
// The handful of named colors that actually show up in `theme-color`. The full
// 148-name table isn't worth carrying for the long tail.
const NAMED_COLORS: Record<string, string> = {
black: "#000000",
white: "#ffffff",
red: "#ff0000",
green: "#008000",
blue: "#0000ff",
yellow: "#ffff00",
orange: "#ffa500",
purple: "#800080",
gray: "#808080",
grey: "#808080",
silver: "#c0c0c0",
maroon: "#800000",
navy: "#000080",
teal: "#008080",
olive: "#808000",
lime: "#00ff00",
aqua: "#00ffff",
cyan: "#00ffff",
fuchsia: "#ff00ff",
magenta: "#ff00ff",
};
function clampByte(n: number): number {
return Math.max(0, Math.min(255, Math.round(n)));
}
/**
* Normalize a CSS color to canonical lowercase `#rrggbb`, or `undefined` when
* it isn't one of the forms sites actually use. Alpha is dropped — the color is
* only ever used as a tint over the app's own surface.
*/
export function normalizeCssColor(raw: string | undefined): string | undefined {
if (!raw) return undefined;
const value = raw.trim().toLowerCase();
const named = NAMED_COLORS[value];
if (named) return named;
const hex = /^#([0-9a-f]{3,8})$/.exec(value)?.[1];
if (hex) {
if (hex.length === 3 || hex.length === 4) {
const [r, g, b] = [...hex.slice(0, 3)];
return `#${r}${r}${g}${g}${b}${b}`;
}
if (hex.length === 6 || hex.length === 8) return `#${hex.slice(0, 6)}`;
return undefined;
}
const rgb = /^rgba?\(([^)]+)\)$/.exec(value)?.[1];
if (rgb) {
const parts = rgb.split(/[\s,/]+/).filter(Boolean).slice(0, 3);
if (parts.length !== 3) return undefined;
const bytes = parts.map((p) => {
const n = parseFloat(p);
if (!Number.isFinite(n)) return NaN;
return clampByte(p.endsWith("%") ? (n / 100) * 255 : n);
});
if (bytes.some(Number.isNaN)) return undefined;
return `#${bytes.map((b) => b.toString(16).padStart(2, "0")).join("")}`;
}
return undefined;
}
// ── JSON-LD helpers (file-private) ────────────────────────────────────────────
type JsonLdResult = {
title?: string;
description?: string;
thumbnailUrl?: string;
};
function ldString(v: unknown): string | undefined {
if (typeof v === "string" && v.trim()) return v.trim();
if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
return v[0].trim();
}
return undefined;
}
function ldImage(v: unknown): string | undefined {
if (
typeof v === "string" &&
(v.startsWith("http://") || v.startsWith("https://"))
) return v;
if (Array.isArray(v)) return ldImage(v[0]);
if (v && typeof v === "object") {
const o = v as Record<string, unknown>;
return ldImage(o.url ?? o.contentUrl);
}
return undefined;
}
function ldExtractNode(data: unknown): JsonLdResult {
if (Array.isArray(data)) {
for (const item of data) {
const r = ldExtractNode(item);
if (r.title || r.thumbnailUrl) return r;
}
return {};
}
if (!data || typeof data !== "object") return {};
const o = data as Record<string, unknown>;
if (o["@graph"]) return ldExtractNode(o["@graph"]);
return {
title: ldString(o.name ?? o.headline),
description: ldString(o.description),
thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
};
}
/**
* Parse every `<script type="application/ld+json">` block and return the first
* node that yields a title or image. Handles `@graph`, arrays, and the common
* `image` shapes (string, string[], ImageObject).
*/
export function extractJsonLd(html: string): JsonLdResult {
const pattern =
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
let match: RegExpExecArray | null;
while ((match = pattern.exec(html)) !== null) {
try {
const result = ldExtractNode(JSON.parse(match[1]));
if (result.title || result.thumbnailUrl) return result;
} catch { /* invalid JSON — skip */ }
}
return {};
}
/**
* Return the `src` of the first `<img>` whose declared width or height is at
* least `minSize` pixels (default 200). Skips data URIs. Resolves relative URLs.
*/
export function extractLargeImage(
html: string,
baseUrl: string,
minSize = 200,
): string | undefined {
const imgPattern = /<img[^>]+>/gi;
let match: RegExpExecArray | null;
while ((match = imgPattern.exec(html)) !== null) {
const tag = match[0];
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
if (!src || src.startsWith("data:")) continue;
const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
if (w >= minSize && h >= minSize) {
try {
return new URL(src, baseUrl).toString();
} catch {
continue;
}
}
}
return undefined;
}
/**
* Collect all `<link rel="icon">` / `<link rel="apple-touch-icon">` tags, rank
* them by declared size (largest wins), and return the best resolved URL.
* Falls back to the first match when no `sizes` attribute is present.
*
* SVG icons are ranked above everything: they carry no `sizes` attribute, so
* they'd otherwise score 0 and lose to a 16×16 PNG, yet they scale cleanly to
* whatever size the placeholder renders them at.
*/
export function extractBestIcon(
html: string,
baseUrl: string,
): string | undefined {
const linkRe = /<link[^>]+>/gi;
const relRe = /\brel=["']([^"']+)["']/i;
const hrefRe = /\bhref=["']([^"']+)["']/i;
const sizesRe = /\bsizes=["']([^"']+)["']/i;
const candidates: { href: string; area: number }[] = [];
let m: RegExpExecArray | null;
while ((m = linkRe.exec(html)) !== null) {
const tag = m[0];
const rel = relRe.exec(tag)?.[1] ?? "";
if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
const href = hrefRe.exec(tag)?.[1];
if (!href) continue;
const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
const sm = sizesStr.match(/(\d+)x(\d+)/i);
const isSvg = /\.svg(\?|$)/i.test(href) ||
/\btype=["']image\/svg\+xml["']/i.test(tag);
const area = isSvg
? Number.MAX_SAFE_INTEGER
: sm
? parseInt(sm[1]) * parseInt(sm[2])
: 0;
try {
candidates.push({ href: new URL(href, baseUrl).toString(), area });
} catch {
continue;
}
}
if (candidates.length === 0) return undefined;
candidates.sort((a, b) => b.area - a.area);
return candidates[0].href;
}
/**
* Return the `src` of the first `<img>` that looks like content rather than UI
* chrome. Skips SVGs, data URIs, and images whose filename matches common
* icon/logo/nav patterns (logo, icon, sprite, favicon, avatar, banner, etc.).
* Resolves relative and protocol-relative URLs.
*/
const UI_IMAGE_KEYWORDS = new Set([
"logo",
"icon",
"sprite",
"favicon",
"avatar",
"banner",
"header",
"nav",
"menu",
"cart",
"search",
"tracking",
"pixel",
"bg",
"background",
]);
function isUiImage(src: string): boolean {
if (/\.svg(\?|$)/i.test(src)) return true;
const filename = src.split("?")[0].split("/").pop() ?? "";
const baseName = filename.replace(/\.[^.]+$/, ""); // strip extension
// Split on common filename separators (-, _, .) and check each token
return baseName.toLowerCase().split(/[-_.]/).some((t) =>
UI_IMAGE_KEYWORDS.has(t)
);
}
/**
* Return the `src` of the first `<img>` that looks like content rather than UI
* chrome. Skips SVGs, data URIs, and images whose filename tokens match common
* icon/logo/nav patterns (logo, icon, sprite, etc.).
* Resolves relative and protocol-relative URLs.
*/
export function extractFirstContentImage(
html: string,
baseUrl: string,
): string | undefined {
const imgPattern = /<img[^>]+>/gi;
let match: RegExpExecArray | null;
while ((match = imgPattern.exec(html)) !== null) {
const tag = match[0];
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
if (!src || src.startsWith("data:")) continue;
if (isUiImage(src)) continue;
try {
return new URL(src, baseUrl).toString();
} catch {
continue;
}
}
return undefined;
}
/**
* Extract `href` from the first `<link rel="…">` whose rel contains `relFragment`,
* resolved to an absolute URL using `baseUrl`.
*/
export function extractLinkHref(
html: string,
relFragment: string,
baseUrl: string,
): string | undefined {
const escaped = relFragment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const patterns = [
new RegExp(
`<link[^>]+rel=["'][^"']*${escaped}[^"']*["'][^>]+href=["']([^"']+)["']`,
"i",
),
new RegExp(
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["'][^"']*${escaped}[^"']*["']`,
"i",
),
];
for (const pattern of patterns) {
const match = html.match(pattern);
if (match) {
try {
return new URL(match[1], baseUrl).toString();
} catch {
return undefined;
}
}
}
return undefined;
}
export function isValidHttpUrl(raw: string): boolean {
try {
const u = new URL(raw);
if (u.protocol !== "http:" && u.protocol !== "https:") return false;
// This site's own pages are fine even on a private host (e.g. localhost in
// development): selfProvider reads them from the DB, no HTTP involved.
// Every actual fetch is checked again, after DNS, in fetchWithTimeout.
return !isPrivateHostname(u) || isSelfUrl(u);
} catch {
return false;
}
}
export interface FetchRichContentResult {
/** False when the page couldn't be reached at all, as opposed to reached and
* found to carry no metadata. Callers that already hold good metadata must
* not overwrite it with a failure stub. */
ok: boolean;
content?: RichContent;
}
/**
* Fetch metadata for `url`, reporting whether the fetch itself succeeded.
*
* On failure it still yields a minimal stub so a *new* dump has something
* displayable, but `ok: false` lets callers with existing metadata keep it.
*/
// Metadata fetched in the last few minutes, by URL. The create form previews a
// link and then posts it; without this the page was fetched twice, and the
// second fetch was the slow part of posting. Only successes are kept, and a
// metadata refresh (`fresh`) always goes to the page.
const RECENT_TTL_MS = 5 * 60 * 1000;
const RECENT_MAX = 200;
const recent = new Map<
string,
{ at: number; result: Promise<FetchRichContentResult> }
>();
/** Forget every cached result (tests). */
export function clearRichContentCache(): void {
recent.clear();
}
export function tryFetchRichContent(
url: string,
{ fresh = false }: { fresh?: boolean } = {},
): Promise<FetchRichContentResult> {
const hit = recent.get(url);
if (!fresh && hit && Date.now() - hit.at < RECENT_TTL_MS) return hit.result;
recent.delete(url);
// Stored while in flight too, so a preview and a post racing share a fetch.
const result = fetchRichContentUncached(url);
if (recent.size >= RECENT_MAX) {
const oldest = recent.keys().next().value;
if (oldest !== undefined) recent.delete(oldest);
}
const entry = { at: Date.now(), result };
recent.set(url, entry);
result.then(({ ok }) => {
if (!ok && recent.get(url) === entry) recent.delete(url);
});
return result;
}
async function fetchRichContentUncached(
url: string,
): Promise<FetchRichContentResult> {
try {
const provider = providers.find((p) => p.matches(url))!;
return { ok: true, content: await provider.fetch(url) };
} catch (err) {
console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err);
try {
return {
ok: false,
content: {
type: "generic",
url,
siteName: new URL(url).hostname.replace(/^www\./, ""),
},
};
} catch {
return { ok: false };
}
}
}
/** Metadata for `url`, or a minimal stub when it can't be fetched. */
export async function fetchRichContent(
url: string,
): Promise<RichContent | undefined> {
return (await tryFetchRichContent(url)).content;
}