136 lines
4.3 KiB
TypeScript
136 lines
4.3 KiB
TypeScript
/**
|
|
* Fetches a link's preview through the provider that knows its site, and
|
|
* reuses it for a few minutes. `dump-service.ts` calls it when a link is
|
|
* posted, changed or refreshed, and `GET /api/preview` when the create form
|
|
* previews one.
|
|
*
|
|
* @module
|
|
*/
|
|
|
|
import type { RichContent } from "../../shared/dump.ts";
|
|
import type { RichContentProvider } from "../remote/providers/provider.ts";
|
|
import { youtubeProvider } from "../remote/providers/youtube.ts";
|
|
import { bandcampProvider } from "../remote/providers/bandcamp.ts";
|
|
import { soundcloudProvider } from "../remote/providers/soundcloud.ts";
|
|
import { selfProvider } from "../remote/providers/self.ts";
|
|
import { genericProvider } from "../remote/providers/generic.ts";
|
|
|
|
/**
|
|
* Lists the providers in priority order: the first that matches a URL reads
|
|
* it. `selfProvider` reads this site's own pages from the database, and
|
|
* `genericProvider` must stay last, since it matches every URL.
|
|
*/
|
|
const providers: RichContentProvider[] = [
|
|
youtubeProvider,
|
|
bandcampProvider,
|
|
soundcloudProvider,
|
|
selfProvider,
|
|
genericProvider,
|
|
];
|
|
|
|
/**
|
|
* Returned by {@link tryFetchRichContent} to `GET /api/preview`, which reports
|
|
* `ok` as `reached`, and to `refreshDumpMetadata`, which gives up when it is
|
|
* false.
|
|
*/
|
|
export interface FetchRichContentResult {
|
|
/**
|
|
* False when the page couldn't be reached at all, as opposed to reached and
|
|
* found to carry no metadata. A caller that already holds metadata must not
|
|
* overwrite it with the failure stub.
|
|
*/
|
|
ok: boolean;
|
|
/**
|
|
* On failure, a stub holding the URL and its host name. Missing only when
|
|
* the URL doesn't parse.
|
|
*/
|
|
content?: RichContent;
|
|
}
|
|
|
|
/** How long a fetched preview is reused. */
|
|
const RECENT_TTL_MS = 5 * 60 * 1000;
|
|
/** How many previews are kept. The oldest written goes first. */
|
|
const RECENT_MAX = 200;
|
|
/**
|
|
* Holds the previews fetched in the last few minutes, by URL. The create form
|
|
* previews a link and then posts it, and without this the second fetch of the
|
|
* page was the slow part of posting. Only successes are kept.
|
|
*/
|
|
const recent = new Map<
|
|
string,
|
|
{ at: number; result: Promise<FetchRichContentResult> }
|
|
>();
|
|
|
|
/**
|
|
* Forgets every cached preview. The tests' `stubNetwork` calls it, since
|
|
* previews fetched from the previous stubbed network no longer apply.
|
|
*/
|
|
export function clearRichContentCache(): void {
|
|
recent.clear();
|
|
}
|
|
|
|
/**
|
|
* Fetches the preview of `url`, and says whether the page could be reached. A
|
|
* preview fetched in the last five minutes is reused unless `fresh` is set, as
|
|
* a metadata refresh does. It never rejects: a failure resolves to a stub.
|
|
*/
|
|
export function tryFetchRichContent(
|
|
url: string,
|
|
{ fresh = false }: { fresh?: boolean } = {},
|
|
): Promise<FetchRichContentResult> {
|
|
const hit = recent.get(url);
|
|
if (!fresh && hit && Date.now() - hit.at < RECENT_TTL_MS) return hit.result;
|
|
recent.delete(url);
|
|
|
|
// Stored while in flight too, so a preview and a post racing share a fetch.
|
|
const result = fetchRichContentUncached(url);
|
|
if (recent.size >= RECENT_MAX) {
|
|
const oldest = recent.keys().next().value;
|
|
if (oldest !== undefined) recent.delete(oldest);
|
|
}
|
|
const entry = { at: Date.now(), result };
|
|
recent.set(url, entry);
|
|
result.then(({ ok }) => {
|
|
if (!ok && recent.get(url) === entry) recent.delete(url);
|
|
});
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Fetches the preview of `url` through the first provider that matches, or
|
|
* makes the failure stub when the provider rejects.
|
|
*/
|
|
async function fetchRichContentUncached(
|
|
url: string,
|
|
): Promise<FetchRichContentResult> {
|
|
try {
|
|
const provider = providers.find((p) => p.matches(url))!;
|
|
return { ok: true, content: await provider.fetch(url) };
|
|
} catch (err) {
|
|
console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err);
|
|
try {
|
|
return {
|
|
ok: false,
|
|
content: {
|
|
type: "generic",
|
|
url,
|
|
siteName: new URL(url).hostname.replace(/^www\./, ""),
|
|
},
|
|
};
|
|
} catch {
|
|
return { ok: false };
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns the preview of `url`, or a stub holding its host name when the page
|
|
* can't be reached. `createUrlDump` and `updateDump` use it for a new link,
|
|
* which has no earlier preview to keep.
|
|
*/
|
|
export async function fetchRichContent(
|
|
url: string,
|
|
): Promise<RichContent | undefined> {
|
|
return (await tryFetchRichContent(url)).content;
|
|
}
|