gerbeur/api/services/rich-content-service.ts

/**
 * Fetches a link's preview through the provider that knows its site, and
 * reuses it for a few minutes. `dump-service.ts` calls it when a link is
 * posted, changed or refreshed, and `GET /api/remote/preview` when the create form
 * previews one.
 *
 * @module
 */

import type { RichContent } from "../../shared/dump.ts";
import { siteHost } from "../../shared/url.ts";
import { TtlCache } from "../lib/ttl-cache.ts";
import type { RichContentProvider } from "../remote/providers/provider.ts";
import { youtubeProvider } from "../remote/providers/youtube.ts";
import { bandcampProvider } from "../remote/providers/bandcamp.ts";
import { soundcloudProvider } from "../remote/providers/soundcloud.ts";
import { selfProvider } from "../remote/providers/self.ts";
import { genericProvider } from "../remote/providers/generic.ts";

/**
 * Lists the providers in priority order: the first that matches a URL reads
 * it. `selfProvider` reads this site's own pages from the database, and
 * `genericProvider` must stay last, since it matches every URL.
 */
const providers: RichContentProvider[] = [
  youtubeProvider,
  bandcampProvider,
  soundcloudProvider,
  selfProvider,
  genericProvider,
];

/**
 * Returned by {@link tryFetchRichContent} to `GET /api/remote/preview`, which reports
 * `ok` as `reached`, and to `refreshDumpMetadata`, which gives up when it is
 * false.
 */
export interface FetchRichContentResult {
  /**
   * False when the page couldn't be reached at all, as opposed to reached and
   * found to carry no metadata. A caller that already holds metadata must not
   * overwrite it with the failure stub.
   */
  ok: boolean;
  /**
   * On failure, a stub holding the URL and its host name. Missing only when
   * the URL doesn't parse.
   */
  content?: RichContent;
}

/**
 * Makes posting a link fast after the create form previewed it, by keeping the
 * previews fetched in the last five minutes, by URL. Without it, the second
 * fetch of the page was the slow part of posting. Only successes are kept.
 */
const recent = new TtlCache<Promise<FetchRichContentResult>>(5 * 60_000, 200);

/**
 * Forgets every cached preview. The tests' `stubNetwork` calls it, since
 * previews fetched from the previous stubbed network no longer apply.
 */
export function clearRichContentCache(): void {
  recent.clear();
}

/**
 * Fetches the preview of `url`, and says whether the page could be reached. A
 * preview fetched in the last five minutes is reused unless `fresh` is set, as
 * a metadata refresh does. It never rejects: a failure resolves to a stub.
 */
export function tryFetchRichContent(
  url: string,
  { fresh = false }: { fresh?: boolean } = {},
): Promise<FetchRichContentResult> {
  const hit = fresh ? undefined : recent.get(url);
  if (hit) return hit;

  // Stored while in flight too, so a preview and a post racing share a fetch.
  const result = recent.set(url, fetchRichContentUncached(url));
  result.then(({ ok }) => {
    if (!ok) recent.delete(url, result);
  });
  return result;
}

/** The longest title a preview keeps, in characters. */
const MAX_TITLE_LENGTH = 300;

/**
 * The longest description a preview keeps, in characters: it is stored with
 * the dump, searched and sent with every feed, and some pages put a whole
 * article in theirs.
 */
const MAX_DESCRIPTION_LENGTH = 1000;

/**
 * Returns `text` with its whitespace collapsed, cut to `max` characters with
 * an ellipsis; undefined when nothing is left.
 */
function tidyText(
  text: string | undefined,
  max: number,
): string | undefined {
  const collapsed = text?.replace(/\s+/g, " ").trim();
  if (!collapsed) return undefined;
  const chars = [...collapsed];
  return chars.length > max
    ? chars.slice(0, max - 1).join("").trimEnd() + "…"
    : collapsed;
}

/** Returns `url` if it is an absolute http(s) URL, and undefined otherwise. */
function httpUrl(url: string | undefined): string | undefined {
  if (!url) return undefined;
  try {
    const { protocol } = new URL(url);
    return protocol === "http:" || protocol === "https:" ? url : undefined;
  } catch {
    return undefined;
  }
}

/**
 * Applies the rules every preview follows, whichever provider made it: text
 * on one line and of bounded length, and only http(s) URLs for the images and
 * the player, which the frontend loads.
 */
function tidy(content: RichContent): RichContent {
  const tidied: RichContent = {
    ...content,
    title: tidyText(content.title, MAX_TITLE_LENGTH),
    description: tidyText(content.description, MAX_DESCRIPTION_LENGTH),
    siteName: tidyText(content.siteName, MAX_TITLE_LENGTH),
    thumbnailUrl: httpUrl(content.thumbnailUrl),
    faviconUrl: httpUrl(content.faviconUrl),
    embedUrl: httpUrl(content.embedUrl),
  };
  // Stored as JSON: a field left undefined would only be dropped there.
  for (const key of Object.keys(tidied) as (keyof RichContent)[]) {
    if (tidied[key] === undefined) delete tidied[key];
  }
  return tidied;
}

/**
 * Fetches the preview of `url` through the first provider that matches, or
 * makes the failure stub when the provider rejects.
 */
async function fetchRichContentUncached(
  url: string,
): Promise<FetchRichContentResult> {
  try {
    const provider = providers.find((p) => p.matches(url))!;
    return { ok: true, content: tidy(await provider.fetch(url)) };
  } catch (err) {
    console.error(`[rich-content] Failed to fetch metadata for ${url}:`, err);
    const siteName = siteHost(url);
    return siteName
      ? { ok: false, content: { type: "generic", url, siteName } }
      : { ok: false };
  }
}

/**
 * Returns the preview of `url`, or a stub holding its host name when the page
 * can't be reached. `createUrlDump` and `updateDump` use it for a new link,
 * which has no earlier preview to keep.
 */
export async function fetchRichContent(
  url: string,
): Promise<RichContent | undefined> {
  return (await tryFetchRichContent(url)).content;
}