120 lines
3.6 KiB
TypeScript
120 lines
3.6 KiB
TypeScript
/**
|
|
* Spots reposts and links to known dumps, by reducing a URL to a key shared by
|
|
* every link to the same page or video. `dump-service` stores it in
|
|
* `dumps.url_canonical` when a dump is posted or edited, `GET
|
|
* /api/dumps/by-url` matches on it to warn the create form of a repost, and
|
|
* backlink-service compares links by it. The key is never fetched or shown, so
|
|
* it may be lossy: it forces `https`, drops `www.`, tracking parameters and
|
|
* timestamps.
|
|
*
|
|
* Migration 0013 backfilled existing rows with a frozen copy of these rules.
|
|
* Changing them needs a new backfill migration, or old rows keep old keys.
|
|
*
|
|
* @module
|
|
*/
|
|
|
|
import { siteHostOf } from "../../shared/url.ts";
|
|
|
|
const TRACKING_PARAMS = new Set([
|
|
"fbclid",
|
|
"gclid",
|
|
"dclid",
|
|
"msclkid",
|
|
"twclid",
|
|
"yclid",
|
|
"mc_cid",
|
|
"mc_eid",
|
|
"igshid",
|
|
"igsh",
|
|
"si",
|
|
"spm",
|
|
"ref_src",
|
|
"ref_url",
|
|
"_ga",
|
|
"_gl",
|
|
"__twitter_impression",
|
|
]);
|
|
|
|
function isTrackingParam(key: string): boolean {
|
|
const k = key.toLowerCase();
|
|
return k.startsWith("utm_") || TRACKING_PARAMS.has(k);
|
|
}
|
|
|
|
/** YouTube's hosts, without their "www.". */
|
|
const YOUTUBE_HOSTS = new Set([
|
|
"youtube.com",
|
|
"m.youtube.com",
|
|
"music.youtube.com",
|
|
"youtube-nocookie.com",
|
|
]);
|
|
|
|
/**
|
|
* Returns `u`'s host in lower case and without "www.", when it is one of
|
|
* YouTube's (youtu.be aside), and null otherwise. The YouTube preview provider
|
|
* uses it too, so both agree on what a YouTube link is.
|
|
*/
|
|
export function youtubeHost(u: URL): string | null {
|
|
const host = siteHostOf(u);
|
|
return YOUTUBE_HOSTS.has(host) ? host : null;
|
|
}
|
|
|
|
/**
|
|
* Returns the id of the video a YouTube URL points at, in any of the shapes
|
|
* people paste (`youtu.be/ID`, `/watch?v=ID`, `/embed/ID`, `/shorts/ID`,
|
|
* `/live/ID`), and null for any other URL. `host` is lower case, without
|
|
* "www.". The canonical key drops the timestamp, since the same video linked
|
|
* at 2:30 is still a repost; the YouTube provider reads it separately.
|
|
*/
|
|
export function youtubeVideoId(
|
|
host: string,
|
|
pathname: string,
|
|
params: URLSearchParams,
|
|
): string | null {
|
|
if (host === "youtu.be") return pathname.split("/")[1] || null;
|
|
if (!YOUTUBE_HOSTS.has(host)) return null;
|
|
if (pathname === "/watch") return params.get("v");
|
|
if (/^\/(embed|shorts|live)\//.test(pathname)) {
|
|
return pathname.split("/")[2] || null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Returns the canonical key of `raw`, or null when it isn't an http(s) URL. A
|
|
* YouTube video or playlist becomes one fixed URL whatever shape it came in.
|
|
*/
|
|
export function canonicalizeUrl(raw: string): string | null {
|
|
let u: URL;
|
|
try {
|
|
u = new URL(raw);
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
|
|
|
const host = siteHostOf(u);
|
|
if (!host) return null;
|
|
|
|
const videoId = youtubeVideoId(host, u.pathname, u.searchParams);
|
|
if (videoId) return `https://youtube.com/watch?v=${videoId}`;
|
|
|
|
const listId = YOUTUBE_HOSTS.has(host) && u.pathname === "/playlist"
|
|
? u.searchParams.get("list")
|
|
: null;
|
|
if (listId) return `https://youtube.com/playlist?list=${listId}`;
|
|
|
|
// Trailing slashes are dropped; "/" itself becomes "".
|
|
const path = u.pathname.replace(/\/+$/, "");
|
|
|
|
const params = [...u.searchParams.entries()]
|
|
.filter(([key]) => !isTrackingParam(key))
|
|
.sort(([a, av], [b, bv]) => a.localeCompare(b) || av.localeCompare(bv));
|
|
const query = new URLSearchParams(params).toString();
|
|
|
|
// A "#section" anchor points into the same page and is dropped, but "#/route"
|
|
// and "#!/route" are distinct pages of a hash-routed app.
|
|
const hash = /^#!?\//.test(u.hash) ? u.hash : "";
|
|
|
|
return `https://${host}${path}${query ? `?${query}` : ""}${hash}`;
|
|
}
|