/** * Canonical form of a URL, used *only* to recognise that two links point at the * same thing — the duplicate check the create form runs while it fetches a * preview. It is a lookup key, never something we display or fetch: `dumps.url` * keeps the exact string the poster submitted. * * Because it is only ever compared against other canonical forms, it may be * lossy in ways a real URL never could be — it forces `https`, drops `www.`, * and throws away share/tracking parameters and timestamps, so * `http://www.example.com/a/?utm_source=x` and `https://example.com/a` collapse * to one key. * * Stored in `dumps.url_canonical`. Migration 0013 carries a frozen copy of this * logic to backfill existing rows; changing the rules here therefore needs a * new backfill migration, or old rows keep keys computed under the old rules. */ const TRACKING_PARAMS = new Set([ "fbclid", "gclid", "dclid", "msclkid", "twclid", "yclid", "mc_cid", "mc_eid", "igshid", "igsh", "si", "spm", "ref_src", "ref_url", "_ga", "_gl", "__twitter_impression", ]); function isTrackingParam(key: string): boolean { const k = key.toLowerCase(); return k.startsWith("utm_") || TRACKING_PARAMS.has(k); } // Hosts already stripped of a leading "www.". const YOUTUBE_HOSTS = new Set([ "youtube.com", "m.youtube.com", "music.youtube.com", "youtube-nocookie.com", ]); /** * The video a YouTube URL points at, in any of the shapes people paste * (`youtu.be/ID`, `/watch?v=ID`, `/embed/ID`, `/shorts/ID`, `/live/ID`). * Timestamps are deliberately ignored: the same video linked at 2:30 is still * the same video for duplicate purposes. */ function youtubeVideoId( host: string, pathname: string, params: URLSearchParams, ): string | null { if (host === "youtu.be") return pathname.split("/")[1] || null; if (!YOUTUBE_HOSTS.has(host)) return null; if (pathname === "/watch") return params.get("v"); if (/^\/(embed|shorts|live)\//.test(pathname)) { return pathname.split("/")[2] || null; } return null; } export function canonicalizeUrl(raw: string): string | null { let u: URL; try { u = new URL(raw); } catch { return null; } if (u.protocol !== "http:" && u.protocol !== "https:") return null; const host = u.hostname.toLowerCase().replace(/^www\./, ""); if (!host) return null; const videoId = youtubeVideoId(host, u.pathname, u.searchParams); if (videoId) return `https://youtube.com/watch?v=${videoId}`; const listId = YOUTUBE_HOSTS.has(host) && u.pathname === "/playlist" ? u.searchParams.get("list") : null; if (listId) return `https://youtube.com/playlist?list=${listId}`; // A single trailing slash is never meaningful; "/" itself becomes "". const path = u.pathname.replace(/\/+$/, ""); const params = [...u.searchParams.entries()] .filter(([key]) => !isTrackingParam(key)) .sort(([a, av], [b, bv]) => a.localeCompare(b) || av.localeCompare(bv)); const query = new URLSearchParams(params).toString(); // A bare "#section" anchor points into the same page, so it is dropped — // but "#/route" and "#!/route" address distinct pages of a hash-routed app. const hash = /^#!?\//.test(u.hash) ? u.hash : ""; return `https://${host}${path}${query ? `?${query}` : ""}${hash}`; }