125 lines
3.5 KiB
TypeScript
125 lines
3.5 KiB
TypeScript
/**
|
|
* Adds `dumps.url_canonical` — the lookup key behind the "already dumped?"
|
|
* check on the create form — and backfills it for every existing URL dump.
|
|
*
|
|
* Purely local: it re-derives the key from the URL already stored on each row
|
|
* and makes no network calls.
|
|
*
|
|
* The helpers below are a deliberate frozen copy of `api/lib/canonical-url.ts`
|
|
* as it stood when this migration shipped, not an import of it. A migration
|
|
* runs exactly once per database, so importing the live version would mean two
|
|
* databases migrating at different times end up with keys computed under
|
|
* different rules. Improving the shared canonicalizer therefore calls for a new
|
|
* backfill migration rather than an edit here.
|
|
*
|
|
* Idempotent: the column is only added when missing and only rows whose key is
|
|
* still NULL are touched, so a fresh database built from schema.sql is a no-op.
|
|
*
|
|
* @module
|
|
*/
|
|
|
|
import type { DatabaseSync } from "node:sqlite";
|
|
|
|
const TRACKING_PARAMS = new Set([
|
|
"fbclid",
|
|
"gclid",
|
|
"dclid",
|
|
"msclkid",
|
|
"twclid",
|
|
"yclid",
|
|
"mc_cid",
|
|
"mc_eid",
|
|
"igshid",
|
|
"igsh",
|
|
"si",
|
|
"spm",
|
|
"ref_src",
|
|
"ref_url",
|
|
"_ga",
|
|
"_gl",
|
|
"__twitter_impression",
|
|
]);
|
|
|
|
function isTrackingParam(key: string): boolean {
|
|
const k = key.toLowerCase();
|
|
return k.startsWith("utm_") || TRACKING_PARAMS.has(k);
|
|
}
|
|
|
|
const YOUTUBE_HOSTS = new Set([
|
|
"youtube.com",
|
|
"m.youtube.com",
|
|
"music.youtube.com",
|
|
"youtube-nocookie.com",
|
|
]);
|
|
|
|
function youtubeVideoId(
|
|
host: string,
|
|
pathname: string,
|
|
params: URLSearchParams,
|
|
): string | null {
|
|
if (host === "youtu.be") return pathname.split("/")[1] || null;
|
|
if (!YOUTUBE_HOSTS.has(host)) return null;
|
|
if (pathname === "/watch") return params.get("v");
|
|
if (/^\/(embed|shorts|live)\//.test(pathname)) {
|
|
return pathname.split("/")[2] || null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function canonicalizeUrl(raw: string): string | null {
|
|
let u: URL;
|
|
try {
|
|
u = new URL(raw);
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
|
|
|
const host = u.hostname.toLowerCase().replace(/^www\./, "");
|
|
if (!host) return null;
|
|
|
|
const videoId = youtubeVideoId(host, u.pathname, u.searchParams);
|
|
if (videoId) return `https://youtube.com/watch?v=${videoId}`;
|
|
|
|
const listId = YOUTUBE_HOSTS.has(host) && u.pathname === "/playlist"
|
|
? u.searchParams.get("list")
|
|
: null;
|
|
if (listId) return `https://youtube.com/playlist?list=${listId}`;
|
|
|
|
const path = u.pathname.replace(/\/+$/, "");
|
|
|
|
const params = [...u.searchParams.entries()]
|
|
.filter(([key]) => !isTrackingParam(key))
|
|
.sort(([a, av], [b, bv]) => a.localeCompare(b) || av.localeCompare(bv));
|
|
const query = new URLSearchParams(params).toString();
|
|
|
|
const hash = /^#!?\//.test(u.hash) ? u.hash : "";
|
|
|
|
return `https://${host}${path}${query ? `?${query}` : ""}${hash}`;
|
|
}
|
|
|
|
/** Applies this migration to `db`. */
|
|
export function up(db: DatabaseSync): void {
|
|
const columns = db.prepare(`PRAGMA table_info(dumps);`).all() as {
|
|
name: string;
|
|
}[];
|
|
if (!columns.some((c) => c.name === "url_canonical")) {
|
|
db.exec(`ALTER TABLE dumps ADD COLUMN url_canonical TEXT;`);
|
|
}
|
|
db.exec(
|
|
`CREATE INDEX IF NOT EXISTS idx_dumps_url_canonical ON dumps(url_canonical);`,
|
|
);
|
|
|
|
const rows = db.prepare(
|
|
`SELECT id, url FROM dumps WHERE url IS NOT NULL AND url_canonical IS NULL;`,
|
|
).all() as { id: string; url: string }[];
|
|
|
|
const update = db.prepare(
|
|
`UPDATE dumps SET url_canonical = ? WHERE id = ?;`,
|
|
);
|
|
for (const row of rows) {
|
|
const canonical = canonicalizeUrl(row.url);
|
|
if (canonical) update.run(canonical, row.id);
|
|
}
|
|
}
|