/** * Fetches a link's preview through the provider that knows its site, and * reuses it for a few minutes. `dump-service.ts` calls it when a link is * posted, changed or refreshed, and `GET /api/remote/preview` when the create form * previews one. * * @module */ import type { RichContent } from "../../shared/dump.ts"; import { siteHost } from "../../shared/url.ts"; import { TtlCache } from "../lib/ttl-cache.ts"; import type { RichContentProvider } from "../remote/providers/provider.ts"; import { youtubeProvider } from "../remote/providers/youtube.ts"; import { bandcampProvider } from "../remote/providers/bandcamp.ts"; import { soundcloudProvider } from "../remote/providers/soundcloud.ts"; import { selfProvider } from "../remote/providers/self.ts"; import { genericProvider } from "../remote/providers/generic.ts"; /** * Lists the providers in priority order: the first that matches a URL reads * it. `selfProvider` reads this site's own pages from the database, and * `genericProvider` must stay last, since it matches every URL. */ const providers: RichContentProvider[] = [ youtubeProvider, bandcampProvider, soundcloudProvider, selfProvider, genericProvider, ]; /** * Returned by {@link tryFetchRichContent} to `GET /api/remote/preview`, which reports * `ok` as `reached`, and to `refreshDumpMetadata`, which gives up when it is * false. */ export interface FetchRichContentResult { /** * False when the page couldn't be reached at all, as opposed to reached and * found to carry no metadata. A caller that already holds metadata must not * overwrite it with the failure stub. */ ok: boolean; /** * On failure, a stub holding the URL and its host name. Missing only when * the URL doesn't parse. */ content?: RichContent; } /** * Makes posting a link fast after the create form previewed it, by keeping the * previews fetched in the last five minutes, by URL. Without it, the second * fetch of the page was the slow part of posting. Only successes are kept. */ const recent = new TtlCache>(5 * 60_000, 200); /** * Forgets every cached preview. The tests' `stubNetwork` calls it, since * previews fetched from the previous stubbed network no longer apply. */ export function clearRichContentCache(): void { recent.clear(); } /** * Fetches the preview of `url`, and says whether the page could be reached. A * preview fetched in the last five minutes is reused unless `fresh` is set, as * a metadata refresh does. It never rejects: a failure resolves to a stub. */ export function tryFetchRichContent( url: string, { fresh = false }: { fresh?: boolean } = {}, ): Promise { const hit = fresh ? undefined : recent.get(url); if (hit) return hit; // Stored while in flight too, so a preview and a post racing share a fetch. const result = recent.set(url, fetchRichContentUncached(url)); result.then(({ ok }) => { if (!ok) recent.delete(url, result); }); return result; } /** The longest title a preview keeps, in characters. */ const MAX_TITLE_LENGTH = 300; /** * The longest description a preview keeps, in characters: it is stored with * the dump, searched and sent with every feed, and some pages put a whole * article in theirs. */ const MAX_DESCRIPTION_LENGTH = 1000; /** * Returns `text` with its whitespace collapsed, cut to `max` characters with * an ellipsis; undefined when nothing is left. */ function tidyText( text: string | undefined, max: number, ): string | undefined { const collapsed = text?.replace(/\s+/g, " ").trim(); if (!collapsed) return undefined; const chars = [...collapsed]; return chars.length > max ? chars.slice(0, max - 1).join("").trimEnd() + "…" : collapsed; } /** Returns `url` if it is an absolute http(s) URL, and undefined otherwise. */ function httpUrl(url: string | undefined): string | undefined { if (!url) return undefined; try { const { protocol } = new URL(url); return protocol === "http:" || protocol === "https:" ? url : undefined; } catch { return undefined; } } /** * Applies the rules every preview follows, whichever provider made it: text * on one line and of bounded length, and only http(s) URLs for the images and * the player, which the frontend loads. */ function tidy(content: RichContent): RichContent { const tidied: RichContent = { ...content, title: tidyText(content.title, MAX_TITLE_LENGTH), description: tidyText(content.description, MAX_DESCRIPTION_LENGTH), siteName: tidyText(content.siteName, MAX_TITLE_LENGTH), thumbnailUrl: httpUrl(content.thumbnailUrl), faviconUrl: httpUrl(content.faviconUrl), embedUrl: httpUrl(content.embedUrl), }; // Stored as JSON: a field left undefined would only be dropped there. for (const key of Object.keys(tidied) as (keyof RichContent)[]) { if (tidied[key] === undefined) delete tidied[key]; } return tidied; } /** * Fetches the preview of `url` through the first provider that matches, or * makes the failure stub when the provider rejects. */ async function fetchRichContentUncached( url: string, ): Promise { try { const provider = providers.find((p) => p.matches(url))!; return { ok: true, content: tidy(await provider.fetch(url)) }; } catch (err) { console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err); const siteName = siteHost(url); return siteName ? { ok: false, content: { type: "generic", url, siteName } } : { ok: false }; } } /** * Returns the preview of `url`, or a stub holding its host name when the page * can't be reached. `createUrlDump` and `updateDump` use it for a new link, * which has no earlier preview to keep. */ export async function fetchRichContent( url: string, ): Promise { return (await tryFetchRichContent(url)).content; }