/** * Lets the create form warn that a link was already dumped. It adds * `dumps.url_canonical`, the key the hint looks a URL up by, and fills it for * every URL dump already posted. * * It fills the key from each stored URL and fetches nothing. Its helpers are a * copy of lib/canonical-url.ts as it stood when this shipped: importing the * live one would give two databases migrated at different times keys made by * different rules. A change to lib/canonical-url.ts therefore needs a new * migration that fills the keys again. * * @module */ import type { DatabaseSync } from "node:sqlite"; const TRACKING_PARAMS = new Set([ "fbclid", "gclid", "dclid", "msclkid", "twclid", "yclid", "mc_cid", "mc_eid", "igshid", "igsh", "si", "spm", "ref_src", "ref_url", "_ga", "_gl", "__twitter_impression", ]); function isTrackingParam(key: string): boolean { const k = key.toLowerCase(); return k.startsWith("utm_") || TRACKING_PARAMS.has(k); } const YOUTUBE_HOSTS = new Set([ "youtube.com", "m.youtube.com", "music.youtube.com", "youtube-nocookie.com", ]); function youtubeVideoId( host: string, pathname: string, params: URLSearchParams, ): string | null { if (host === "youtu.be") return pathname.split("/")[1] || null; if (!YOUTUBE_HOSTS.has(host)) return null; if (pathname === "/watch") return params.get("v"); if (/^\/(embed|shorts|live)\//.test(pathname)) { return pathname.split("/")[2] || null; } return null; } function canonicalizeUrl(raw: string): string | null { let u: URL; try { u = new URL(raw); } catch { return null; } if (u.protocol !== "http:" && u.protocol !== "https:") return null; const host = u.hostname.toLowerCase().replace(/^www\./, ""); if (!host) return null; const videoId = youtubeVideoId(host, u.pathname, u.searchParams); if (videoId) return `https://youtube.com/watch?v=${videoId}`; const listId = YOUTUBE_HOSTS.has(host) && u.pathname === "/playlist" ? u.searchParams.get("list") : null; if (listId) return `https://youtube.com/playlist?list=${listId}`; const path = u.pathname.replace(/\/+$/, ""); const params = [...u.searchParams.entries()] .filter(([key]) => !isTrackingParam(key)) .sort(([a, av], [b, bv]) => a.localeCompare(b) || av.localeCompare(bv)); const query = new URLSearchParams(params).toString(); const hash = /^#!?\//.test(u.hash) ? u.hash : ""; return `https://${host}${path}${query ? `?${query}` : ""}${hash}`; } /** * Run once by `runMigrations`. It fills only the keys still NULL, and leaves * NULL a URL that isn't http(s). */ export function up(db: DatabaseSync): void { const columns = db.prepare(`PRAGMA table_info(dumps);`).all() as { name: string; }[]; if (!columns.some((c) => c.name === "url_canonical")) { db.exec(`ALTER TABLE dumps ADD COLUMN url_canonical TEXT;`); } db.exec( `CREATE INDEX IF NOT EXISTS idx_dumps_url_canonical ON dumps(url_canonical);`, ); const rows = db.prepare( `SELECT id, url FROM dumps WHERE url IS NOT NULL AND url_canonical IS NULL;`, ).all() as { id: string; url: string }[]; const update = db.prepare( `UPDATE dumps SET url_canonical = ? WHERE id = ?;`, ); for (const row of rows) { const canonical = canonicalizeUrl(row.url); if (canonical) update.run(canonical, row.id); } }