gerbeur/api/lib/canonical-url.ts

/**
 * Reduces a URL to a key shared by every link to the same page or video.
 * `dump-service` stores it in `dumps.url_canonical` when a
 * dump is posted or edited, and `GET /api/dumps/by-url` matches on it
 * to warn the create form of a repost. The key is never fetched or shown, so it
 * may be lossy: it forces `https`, drops `www.`, tracking parameters and
 * timestamps.
 *
 * Migration 0013 backfilled existing rows with a frozen copy of these rules.
 * Changing them needs a new backfill migration, or old rows keep old keys.
 *
 * @module
 */

const TRACKING_PARAMS = new Set([
  "fbclid",
  "gclid",
  "dclid",
  "msclkid",
  "twclid",
  "yclid",
  "mc_cid",
  "mc_eid",
  "igshid",
  "igsh",
  "si",
  "spm",
  "ref_src",
  "ref_url",
  "_ga",
  "_gl",
  "__twitter_impression",
]);

function isTrackingParam(key: string): boolean {
  const k = key.toLowerCase();
  return k.startsWith("utm_") || TRACKING_PARAMS.has(k);
}

/** YouTube's hosts, without their "www.". */
const YOUTUBE_HOSTS = new Set([
  "youtube.com",
  "m.youtube.com",
  "music.youtube.com",
  "youtube-nocookie.com",
]);

/**
 * Returns `u`'s host in lower case and without "www.", when it is one of
 * YouTube's (youtu.be aside), and null otherwise. The YouTube preview provider
 * uses it too, so both agree on what a YouTube link is.
 */
export function youtubeHost(u: URL): string | null {
  const host = u.hostname.toLowerCase().replace(/^www\./, "");
  return YOUTUBE_HOSTS.has(host) ? host : null;
}

/**
 * Returns the id of the video a YouTube URL points at, in any of the shapes
 * people paste (`youtu.be/ID`, `/watch?v=ID`, `/embed/ID`, `/shorts/ID`,
 * `/live/ID`), and null for any other URL. `host` is lower case, without
 * "www.". The canonical key drops the timestamp, since the same video linked
 * at 2:30 is still a repost; the YouTube provider reads it separately.
 */
export function youtubeVideoId(
  host: string,
  pathname: string,
  params: URLSearchParams,
): string | null {
  if (host === "youtu.be") return pathname.split("/")[1] || null;
  if (!YOUTUBE_HOSTS.has(host)) return null;
  if (pathname === "/watch") return params.get("v");
  if (/^\/(embed|shorts|live)\//.test(pathname)) {
    return pathname.split("/")[2] || null;
  }
  return null;
}

/**
 * Returns the canonical key of `raw`, or null when it isn't an http(s) URL. A
 * YouTube video or playlist becomes one fixed URL whatever shape it came in.
 */
export function canonicalizeUrl(raw: string): string | null {
  let u: URL;
  try {
    u = new URL(raw);
  } catch {
    return null;
  }
  if (u.protocol !== "http:" && u.protocol !== "https:") return null;

  const host = u.hostname.toLowerCase().replace(/^www\./, "");
  if (!host) return null;

  const videoId = youtubeVideoId(host, u.pathname, u.searchParams);
  if (videoId) return `https://youtube.com/watch?v=${videoId}`;

  const listId = YOUTUBE_HOSTS.has(host) && u.pathname === "/playlist"
    ? u.searchParams.get("list")
    : null;
  if (listId) return `https://youtube.com/playlist?list=${listId}`;

  // Trailing slashes are dropped; "/" itself becomes "".
  const path = u.pathname.replace(/\/+$/, "");

  const params = [...u.searchParams.entries()]
    .filter(([key]) => !isTrackingParam(key))
    .sort(([a, av], [b, bv]) => a.localeCompare(b) || av.localeCompare(bv));
  const query = new URLSearchParams(params).toString();

  // A "#section" anchor points into the same page and is dropped, but "#/route"
  // and "#!/route" are distinct pages of a hash-routed app.
  const hash = /^#!?\//.test(u.hash) ? u.hash : "";

  return `https://${host}${path}${query ? `?${query}` : ""}${hash}`;
}