vincent/gerbeur
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
api/remote/providers/bandcamp.ts
View File

@@ -1,15 +1,35 @@
/**
* Previews Bandcamp links from their page's OG tags, for
* `rich-content-service.ts`.
* Previews Bandcamp links from their page, for `rich-content-service.ts`.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage } from "../fetch.ts";
import { BANDCAMP_HOST_RE } from "../bandcamp-tralbum.ts";
import { genericPreview } from "./generic.ts";
/**
* Returns the embed player's URL a page's `og:video` names, over https; only
* a player on bandcamp.com counts, since the frontend frames it.
*/
function embeddedPlayerUrl(raw: string | undefined): string | undefined {
if (!raw) return undefined;
try {
const url = new URL(raw);
if (
(url.protocol !== "https:" && url.protocol !== "http:") ||
url.hostname !== "bandcamp.com" ||
!url.pathname.startsWith("/EmbeddedPlayer/")
) return undefined;
url.protocol = "https:";
return url.href;
} catch {
return undefined;
}
}
/**
* Claims bandcamp.com and its subdomains, where artists have their pages, and
@@ -29,24 +49,16 @@ export const bandcampProvider: RichContentProvider = {
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
const fetched = await fetchPage(url);
if (fetched.html === undefined) {
return { type: "bandcamp", siteName: "Bandcamp", url };
}
const html = await readTextCapped(res);
const page = parsePage(fetched.html, fetched.url);
return {
...genericPreview(page, url, fetched.url),
type: "bandcamp",
siteName: "Bandcamp",
url,
title: extractOgTag(html, "title"),
description: extractOgTag(html, "description"),
thumbnailUrl: extractOgTag(html, "image"),
embedUrl: extractOgTag(html, "video") ?? undefined,
embedUrl: embeddedPlayerUrl(page.meta("og:video")),
};
},
};

176
api/remote/providers/generic.ts
View File

@@ -1,6 +1,8 @@
/**
* Previews any page no other provider claims, for `rich-content-service.ts`,
* from whatever metadata its HTML carries.
* from whatever metadata its HTML carries. Its reading of a page,
* {@link genericPreview}, is also the base of the Bandcamp, SoundCloud and
* YouTube channel previews.
*
* @module
*/
@@ -9,25 +11,97 @@ import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { siteHost } from "../../../shared/url.ts";
import {
extractBestIcon,
extractFirstContentImage,
extractJsonLd,
extractLargeImage,
extractMaskIconColor,
extractMetaName,
extractOgTag,
extractPageTitle,
extractThemeColor,
absoluteHttpUrl,
normalizeCssColor,
type Page,
parsePage,
} from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { fetchPage } from "../fetch.ts";
/** Returns the path of `url` without its trailing slashes, `/` for the root. */
function pathOf(url: string): string {
return new URL(url).pathname.replace(/\/+$/, "") || "/";
}
/**
* Matches every URL, so it comes last in the service's list. Each field falls
* back from OG tags to Twitter tags, JSON-LD and the markup itself. The icon
* and brand color feed the placeholder the frontend draws when there is no
* thumbnail.
* Strips a trailing ` | Site`, ` – Site` or the like from a `<title>`, which
* names the page and then its site; the site is in the preview's badge.
*/
function withoutSiteSuffix(title: string, siteName: string): string {
const escaped = siteName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const suffix = new RegExp(`\\s+[|–—\\-·•]\\s+${escaped}\\s*$`, "i");
return title.replace(suffix, "") || title;
}
/**
* Returns the preview of the page `url` that `page` read, served from
* `finalUrl` after redirects. Each field falls back from OG tags to Twitter
* tags, JSON-LD and the markup itself. The icon and brand color feed the
* placeholder the frontend draws when there is no thumbnail.
*/
export function genericPreview(
page: Page,
url: string,
finalUrl: string,
): RichContent {
// An og:url naming the home page from another page means the OG tags are
// the site's defaults, not this page's: their title and image are skipped.
// Any other og:url is the page's canonical address, often with a slug the
// link left out, and its tags are the page's.
const ogUrl = page.meta("og:url");
let useOg = true;
try {
useOg = !ogUrl || pathOf(new URL(ogUrl, finalUrl).href) !== "/" ||
pathOf(finalUrl) === "/";
} catch { /* an og:url that doesn't parse names no other page */ }
const ogSiteName = page.meta("og:site_name");
const pageTitle = page.title && ogSiteName
? withoutSiteSuffix(page.title, ogSiteName)
: page.title;
const title = (useOg ? page.meta("og:title") : undefined) ??
page.meta("twitter:title") ??
page.jsonLd.title ??
pageTitle;
const description = page.meta("og:description") ??
page.meta("twitter:description") ??
page.jsonLd.description ??
page.meta("description");
// Never the favicon: a 16×16 icon cropped into a thumbnail looks broken,
// and the placeholder drawn instead shows the icon properly. A tag naming
// no usable URL falls through to the next.
const thumbnailUrl =
(useOg ? absoluteHttpUrl(page.meta("og:image"), finalUrl) : undefined) ??
absoluteHttpUrl(page.meta("twitter:image"), finalUrl) ??
page.jsonLd.thumbnailUrl ??
page.largeImage ??
page.firstContentImage;
// `/favicon.ico` is a guess. When it doesn't load, the placeholder shows
// the site's initial.
const faviconUrl = page.bestIcon ?? `${new URL(finalUrl).origin}/favicon.ico`;
const accentColor = normalizeCssColor(
page.themeColor ??
page.meta("msapplication-TileColor") ??
page.maskIconColor,
);
return {
type: "generic",
url,
title,
description,
thumbnailUrl,
faviconUrl,
accentColor,
siteName: ogSiteName ?? siteHost(url),
};
}
/** Matches every URL, so it comes last in the service's list. */
export const genericProvider: RichContentProvider = {
name: "generic",
@@ -36,72 +110,12 @@ export const genericProvider: RichContentProvider = {
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "generic", url };
}
const html = await readTextCapped(res);
const ld = extractJsonLd(html);
// An og:url naming another page (often the home page) means the OG tags
// are the site's defaults, not this page's: its title and image are
// skipped.
const ogUrl = extractOgTag(html, "url");
const useOg = !ogUrl || (() => {
try {
const ogPath = new URL(ogUrl).pathname.replace(/\/+$/, "") || "/";
const pagePath = new URL(url).pathname.replace(/\/+$/, "") || "/";
return ogPath === pagePath;
} catch {
return true;
}
})();
const title = (useOg ? extractOgTag(html, "title") : undefined) ??
extractMetaName(html, "twitter:title") ??
ld.title ??
extractPageTitle(html);
const siteName = extractOgTag(html, "site_name") ??
siteHost(url);
const description = extractOgTag(html, "description") ??
extractMetaName(html, "twitter:description") ??
ld.description ??
extractMetaName(html, "description");
// Never the favicon: a 16×16 icon cropped into a thumbnail looks broken,
// and the placeholder drawn instead shows the icon properly.
const thumbnailUrl = (useOg ? extractOgTag(html, "image") : undefined) ??
extractMetaName(html, "twitter:image") ??
ld.thumbnailUrl ??
extractLargeImage(html, url) ??
extractFirstContentImage(html, url);
// `/favicon.ico` is a guess. When it doesn't load, the placeholder shows
// the site's initial.
const faviconUrl = extractBestIcon(html, url) ??
`${new URL(url).origin}/favicon.ico`;
const accentColor = normalizeCssColor(
extractThemeColor(html) ??
extractMetaName(html, "msapplication-TileColor") ??
extractMaskIconColor(html),
);
return {
type: "generic",
const fetched = await fetchPage(url);
if (fetched.html === undefined) return { type: "generic", url };
return genericPreview(
parsePage(fetched.html, fetched.url),
url,
title,
description,
thumbnailUrl,
faviconUrl,
accentColor,
siteName,
};
fetched.url,
);
},
};

7
api/remote/providers/provider.ts
View File

@@ -18,9 +18,10 @@ export interface RichContentProvider {
matches(url: string): boolean;
/**
* Returns the preview of `url`, which ends up in the dump's `rich_content` or
* the create form's preview. It rejects when the page can't be fetched, and
* the service then falls back to a stub. A page that isn't HTML yields a
* preview without metadata.
* the create form's preview. It rejects when the page can't be fetched or
* answers with an error status, and the service then falls back to a stub.
* A page that isn't HTML yields a preview without metadata. The service
* tidies what it returns, so a provider leaves text as the page wrote it.
*/
fetch(url: string): Promise<RichContent>;
}

5
api/remote/providers/self.ts
View File

@@ -1,7 +1,8 @@
/**
* Previews links to this site's own dump, user and playlist pages, as
* `parsePagePath` reads them, from the database, for `rich-content-service.ts`. No request goes out, so this works
* even when the site runs on a private host that `fetchWithTimeout` refuses.
* `parsePagePath` reads them, from the database, for
* `rich-content-service.ts`. No request goes out, so this works even when the
* site runs on a private host that `fetchWithTimeout` refuses.
*
* @module
*/

66
api/remote/providers/soundcloud.ts
View File

@@ -1,50 +1,72 @@
/**
* Previews SoundCloud links from their page's OG tags, for
* `rich-content-service.ts`.
* Previews SoundCloud links from their page, for `rich-content-service.ts`.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage } from "../fetch.ts";
import { genericPreview } from "./generic.ts";
/** SoundCloud's own hosts, the short links' `on.` included. */
const SOUNDCLOUD_HOSTS = new Set([
"soundcloud.com",
"www.soundcloud.com",
"m.soundcloud.com",
"on.soundcloud.com",
]);
/**
* Claims soundcloud.com only, not www. or m.soundcloud.com, and builds the
* embed player's URL from the link itself.
* Returns the soundcloud.com address of the page, which the embed player
* takes: its `og:url`, or else the URL it was served from. A short link only
* redirects, so it is never the answer; tracking parameters are dropped.
*/
function pageAddress(ogUrl: string | undefined, finalUrl: string): string {
for (const raw of [ogUrl, finalUrl]) {
if (!raw) continue;
try {
const url = new URL(raw);
if (
!SOUNDCLOUD_HOSTS.has(url.hostname) ||
url.hostname === "on.soundcloud.com"
) continue;
return `https://soundcloud.com${url.pathname}`;
} catch {
continue;
}
}
return finalUrl;
}
/**
* Claims SoundCloud's hosts, short links included, and builds the embed
* player's URL from the page's address.
*/
export const soundcloudProvider: RichContentProvider = {
name: "soundcloud",
matches(url: string): boolean {
try {
return new URL(url).hostname === "soundcloud.com";
return SOUNDCLOUD_HOSTS.has(new URL(url).hostname);
} catch {
return false;
}
},
async fetch(url: string): Promise<RichContent> {
const res = await fetchWithTimeout(url);
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "soundcloud", siteName: "SoundCloud", url };
}
const html = await readTextCapped(res);
const fetched = await fetchPage(url);
const page = fetched.html === undefined
? undefined
: parsePage(fetched.html, fetched.url);
const address = pageAddress(page?.meta("og:url"), fetched.url);
return {
...(page ? genericPreview(page, url, fetched.url) : { url }),
type: "soundcloud",
siteName: "SoundCloud",
url,
title: extractOgTag(html, "title"),
description: extractOgTag(html, "description"),
thumbnailUrl: extractOgTag(html, "image"),
embedUrl: `https://w.soundcloud.com/player/?url=${
encodeURIComponent(url)
encodeURIComponent(address)
}&visual=true&auto_play=false`,
};
},

30
api/remote/providers/youtube.ts
View File

@@ -1,17 +1,18 @@
/**
* Previews YouTube videos, playlists and channels, for
* `rich-content-service.ts`. Videos and playlists are read from YouTube's
* oEmbed endpoint, channels from their page's OG tags.
* oEmbed endpoint, channels from their page.
*
* @module
*/
import type { RichContent } from "../../../shared/dump.ts";
import type { RichContentProvider } from "./provider.ts";
import { extractOgTag } from "../html.ts";
import { fetchWithTimeout, readTextCapped } from "../fetch.ts";
import { parsePage } from "../html.ts";
import { fetchPage, fetchWithTimeout } from "../fetch.ts";
import { youtubeHost, youtubeVideoId } from "../../lib/canonical-url.ts";
import { siteHostOf } from "../../../shared/url.ts";
import { genericPreview } from "./generic.ts";
/** Returns the id of the video `url` points at, in any shape YouTube uses. */
function extractVideoId(url: string): string | null {
@@ -123,27 +124,17 @@ export const youtubeProvider: RichContentProvider = {
if (channelPath && !videoId) {
// oEmbed doesn't take channel URLs.
let title: string | undefined;
let thumbnailUrl: string | undefined;
let channel: RichContent | undefined;
try {
const res = await fetchWithTimeout(url);
if (res.ok) {
const html = await readTextCapped(res);
title = extractOgTag(html, "title");
thumbnailUrl = extractOgTag(html, "image");
} else {
await res.body?.cancel();
const fetched = await fetchPage(url);
if (fetched.html !== undefined) {
const page = parsePage(fetched.html, fetched.url);
channel = genericPreview(page, url, fetched.url);
}
} catch {
// YouTube unreachable: the preview goes without.
}
return {
type: "youtube",
siteName: "YouTube",
url,
title,
thumbnailUrl,
};
return { ...channel, type: "youtube", siteName: "YouTube", url };
}
if (listId && !videoId) {
@@ -172,7 +163,6 @@ export const youtubeProvider: RichContentProvider = {
type: "youtube",
siteName: "YouTube",
url,
videoId: videoId!,
title,
thumbnailUrl,
embedUrl: `https://www.youtube.com/embed/${videoId}?${embedParams}`,