import "./setup.ts"; import { assertEquals } from "@std/assert"; import { canonicalizeUrl } from "../lib/canonical-url.ts"; import { up as up0013 } from "../db/migrations/0013_dump_url_canonical.ts"; import { DatabaseSync } from "node:sqlite"; Deno.test("canonicalizeUrl: scheme, www and trailing slash collapse", () => { const want = "https://example.com/a"; for ( const url of [ "http://example.com/a", "https://www.example.com/a", "https://example.com/a/", "https://EXAMPLE.com/a//", ] ) { assertEquals(canonicalizeUrl(url), want, url); } assertEquals(canonicalizeUrl("https://example.com/"), "https://example.com"); }); Deno.test("canonicalizeUrl: tracking parameters dropped, the rest sorted", () => { assertEquals( canonicalizeUrl( "https://example.com/p?utm_source=x&b=2&fbclid=y&a=1&UTM_Medium=z&si=q", ), "https://example.com/p?a=1&b=2", ); assertEquals( canonicalizeUrl("https://example.com/p?b=1&a=2"), canonicalizeUrl("https://example.com/p?a=2&b=1"), ); }); Deno.test("canonicalizeUrl: section anchors dropped, hash routes kept", () => { assertEquals( canonicalizeUrl("https://example.com/doc#intro"), "https://example.com/doc", ); assertEquals( canonicalizeUrl("https://app.example.com/#/inbox"), "https://app.example.com#/inbox", ); assertEquals( canonicalizeUrl("https://app.example.com/#!/inbox"), "https://app.example.com#!/inbox", ); }); Deno.test("canonicalizeUrl: every YouTube video shape is one key", () => { const want = "https://youtube.com/watch?v=dQw4w9WgXcQ"; for ( const url of [ "https://youtu.be/dQw4w9WgXcQ", "https://youtu.be/dQw4w9WgXcQ?t=42", "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL1&t=1m", "https://m.youtube.com/watch?v=dQw4w9WgXcQ", "https://music.youtube.com/watch?v=dQw4w9WgXcQ", "https://www.youtube.com/embed/dQw4w9WgXcQ", "https://youtube.com/shorts/dQw4w9WgXcQ", "https://youtube.com/live/dQw4w9WgXcQ", "https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ", ] ) { assertEquals(canonicalizeUrl(url), want, url); } assertEquals( canonicalizeUrl("https://www.youtube.com/playlist?list=PLx&si=abc"), "https://youtube.com/playlist?list=PLx", ); }); Deno.test("canonicalizeUrl: non-http and unparseable input", () => { assertEquals(canonicalizeUrl("ftp://example.com/a"), null); assertEquals(canonicalizeUrl("javascript:alert(1)"), null); assertEquals(canonicalizeUrl("not a url"), null); assertEquals(canonicalizeUrl(""), null); }); // Migration 0013 carries a frozen copy of the canonicalizer to backfill old // rows. Until the rules change on purpose (with a new backfill migration), the // two must agree, or rows from before and after the migration won't match. Deno.test("migration 0013 backfills the same keys canonicalizeUrl computes", () => { const urls = [ "http://www.example.com/a/?utm_source=x&b=2&a=1#top", "https://youtu.be/abc123?t=5", "https://www.youtube.com/playlist?list=PLx", "https://app.example.com/#/route", ]; const db = new DatabaseSync(":memory:"); db.exec(`CREATE TABLE dumps (id TEXT PRIMARY KEY, url TEXT);`); urls.forEach((url, i) => db.prepare(`INSERT INTO dumps (id, url) VALUES (?, ?);`).run(String(i), url) ); up0013(db); up0013(db); // idempotent const rows = db.prepare(`SELECT id, url_canonical FROM dumps ORDER BY id;`) .all() as { id: string; url_canonical: string }[]; assertEquals( rows.map((r) => r.url_canonical), urls.map(canonicalizeUrl), ); db.close(); });