/** * Reduces a URL to a key shared by every link to the same page or video. * `dump-service` stores it in `dumps.url_canonical` when a * dump is posted or edited, and `GET /api/dumps/by-url` matches on it * to warn the create form of a repost. The key is never fetched or shown, so it * may be lossy: it forces `https`, drops `www.`, tracking parameters and * timestamps. * * Migration 0013 backfilled existing rows with a frozen copy of these rules. * Changing them needs a new backfill migration, or old rows keep old keys. * * @module */ const TRACKING_PARAMS = new Set([ "fbclid", "gclid", "dclid", "msclkid", "twclid", "yclid", "mc_cid", "mc_eid", "igshid", "igsh", "si", "spm", "ref_src", "ref_url", "_ga", "_gl", "__twitter_impression", ]); function isTrackingParam(key: string): boolean { const k = key.toLowerCase(); return k.startsWith("utm_") || TRACKING_PARAMS.has(k); } /** YouTube's hosts, without their "www.". */ const YOUTUBE_HOSTS = new Set([ "youtube.com", "m.youtube.com", "music.youtube.com", "youtube-nocookie.com", ]); /** * Returns `u`'s host in lower case and without "www.", when it is one of * YouTube's (youtu.be aside), and null otherwise. The YouTube preview provider * uses it too, so both agree on what a YouTube link is. */ export function youtubeHost(u: URL): string | null { const host = u.hostname.toLowerCase().replace(/^www\./, ""); return YOUTUBE_HOSTS.has(host) ? host : null; } /** * Returns the id of the video a YouTube URL points at, in any of the shapes * people paste (`youtu.be/ID`, `/watch?v=ID`, `/embed/ID`, `/shorts/ID`, * `/live/ID`), and null for any other URL. `host` is lower case, without * "www.". The canonical key drops the timestamp, since the same video linked * at 2:30 is still a repost; the YouTube provider reads it separately. */ export function youtubeVideoId( host: string, pathname: string, params: URLSearchParams, ): string | null { if (host === "youtu.be") return pathname.split("/")[1] || null; if (!YOUTUBE_HOSTS.has(host)) return null; if (pathname === "/watch") return params.get("v"); if (/^\/(embed|shorts|live)\//.test(pathname)) { return pathname.split("/")[2] || null; } return null; } /** * Returns the canonical key of `raw`, or null when it isn't an http(s) URL. A * YouTube video or playlist becomes one fixed URL whatever shape it came in. */ export function canonicalizeUrl(raw: string): string | null { let u: URL; try { u = new URL(raw); } catch { return null; } if (u.protocol !== "http:" && u.protocol !== "https:") return null; const host = u.hostname.toLowerCase().replace(/^www\./, ""); if (!host) return null; const videoId = youtubeVideoId(host, u.pathname, u.searchParams); if (videoId) return `https://youtube.com/watch?v=${videoId}`; const listId = YOUTUBE_HOSTS.has(host) && u.pathname === "/playlist" ? u.searchParams.get("list") : null; if (listId) return `https://youtube.com/playlist?list=${listId}`; // Trailing slashes are dropped; "/" itself becomes "". const path = u.pathname.replace(/\/+$/, ""); const params = [...u.searchParams.entries()] .filter(([key]) => !isTrackingParam(key)) .sort(([a, av], [b, bv]) => a.localeCompare(b) || av.localeCompare(bv)); const query = new URLSearchParams(params).toString(); // A "#section" anchor points into the same page and is dropped, but "#/route" // and "#!/route" are distinct pages of a hash-routed app. const hash = /^#!?\//.test(u.hash) ? u.hash : ""; return `https://${host}${path}${query ? `?${query}` : ""}${hash}`; }