/** * Fetches a link's preview through the provider that knows its site, and * reuses it for a few minutes. `dump-service.ts` calls it when a link is * posted, changed or refreshed, and `GET /api/preview` when the create form * previews one. * * @module */ import type { RichContent } from "../../shared/dump.ts"; import type { RichContentProvider } from "../remote/providers/provider.ts"; import { youtubeProvider } from "../remote/providers/youtube.ts"; import { bandcampProvider } from "../remote/providers/bandcamp.ts"; import { soundcloudProvider } from "../remote/providers/soundcloud.ts"; import { selfProvider } from "../remote/providers/self.ts"; import { genericProvider } from "../remote/providers/generic.ts"; /** * Lists the providers in priority order: the first that matches a URL reads * it. `selfProvider` reads this site's own pages from the database, and * `genericProvider` must stay last, since it matches every URL. */ const providers: RichContentProvider[] = [ youtubeProvider, bandcampProvider, soundcloudProvider, selfProvider, genericProvider, ]; /** * Returned by {@link tryFetchRichContent} to `GET /api/preview`, which reports * `ok` as `reached`, and to `refreshDumpMetadata`, which gives up when it is * false. */ export interface FetchRichContentResult { /** * False when the page couldn't be reached at all, as opposed to reached and * found to carry no metadata. A caller that already holds metadata must not * overwrite it with the failure stub. */ ok: boolean; /** * On failure, a stub holding the URL and its host name. Missing only when * the URL doesn't parse. */ content?: RichContent; } /** How long a fetched preview is reused. */ const RECENT_TTL_MS = 5 * 60 * 1000; /** How many previews are kept. The oldest written goes first. */ const RECENT_MAX = 200; /** * Holds the previews fetched in the last few minutes, by URL. The create form * previews a link and then posts it, and without this the second fetch of the * page was the slow part of posting. Only successes are kept. */ const recent = new Map< string, { at: number; result: Promise } >(); /** * Forgets every cached preview. The tests' `stubNetwork` calls it, since * previews fetched from the previous stubbed network no longer apply. */ export function clearRichContentCache(): void { recent.clear(); } /** * Fetches the preview of `url`, and says whether the page could be reached. A * preview fetched in the last five minutes is reused unless `fresh` is set, as * a metadata refresh does. It never rejects: a failure resolves to a stub. */ export function tryFetchRichContent( url: string, { fresh = false }: { fresh?: boolean } = {}, ): Promise { const hit = recent.get(url); if (!fresh && hit && Date.now() - hit.at < RECENT_TTL_MS) return hit.result; recent.delete(url); // Stored while in flight too, so a preview and a post racing share a fetch. const result = fetchRichContentUncached(url); if (recent.size >= RECENT_MAX) { const oldest = recent.keys().next().value; if (oldest !== undefined) recent.delete(oldest); } const entry = { at: Date.now(), result }; recent.set(url, entry); result.then(({ ok }) => { if (!ok && recent.get(url) === entry) recent.delete(url); }); return result; } /** * Fetches the preview of `url` through the first provider that matches, or * makes the failure stub when the provider rejects. */ async function fetchRichContentUncached( url: string, ): Promise { try { const provider = providers.find((p) => p.matches(url))!; return { ok: true, content: await provider.fetch(url) }; } catch (err) { console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err); try { return { ok: false, content: { type: "generic", url, siteName: new URL(url).hostname.replace(/^www\./, ""), }, }; } catch { return { ok: false }; } } } /** * Returns the preview of `url`, or a stub holding its host name when the page * can't be reached. `createUrlDump` and `updateDump` use it for a new link, * which has no earlier preview to keep. */ export async function fetchRichContent( url: string, ): Promise { return (await tryFetchRichContent(url)).content; }