vincent/gerbeur
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4: link previews are read once, in one place, and no longer take an error page, a relative image or a mask icon for the page's — each provider fetched its page and scanned it with regular expressions of its own, the generic one some fifteen times, and Bandcamp and SoundCloud read only OG tags. fetchPage (remote/fetch.ts) now fetches every page, follows redirects to the address it was served from, and refuses an error status, so a 404 or a bot challenge is unreachable rather than a preview titled "Just a moment...", and a refresh no longer overwrites good metadata with it. parsePage (remote/html.ts) reads it in one linear pass with a real attribute reader: unquoted values, a > inside quotes, og: tags under name as well as property, and nothing from comments or scripts. Every URL a page names is resolved against where it was served from, so a relative og:image works; entities are decoded in one pass, &eacute; and &rsquo; included, and one that names no character no longer fails the preview (bandcamp-tralbum.ts shares the decoder). The mask icon is no longer the favicon, JSON-LD prefers the page's node over the site's, tracking pixels and lazy placeholders aren't thumbnails, an og:url with a slug keeps the OG tags, and a <title> loses its site suffix. genericPreview is the base of the Bandcamp, SoundCloud and YouTube channel previews; SoundCloud takes www., m. and on.soundcloud.com short links, embedding the page they lead to, and Bandcamp keeps only its own player. The service tidies every preview: text on one line, titles and descriptions capped, only http(s) images and players. RichContent.type is now RichContentType, and the unused videoId is gone.

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-10-09 20:46:32 +00:00
parent b4936cccf7
commit 2c3ba6ad11
22 changed files with 1111 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
api/remote/fetch.ts
View File

@@ -175,7 +175,7 @@ function declaredCharset(contentType: string, bytes: Uint8Array): string {
/**
* Reads the body of `res` as text, up to `maxBytes`, in the charset the page
* declares; the rest is cancelled, never downloaded. The providers and the
* declares; the rest is cancelled, never downloaded. {@link fetchPage} and the
* Bandcamp and YouTube resolvers read pages through it.
*/
export async function readTextCapped(
@@ -235,6 +235,33 @@ async function fetchOnce(
});
}
/**
* Fetches `url` as {@link fetchWithTimeout} does, and also returns the URL the
* last hop answered from, which {@link fetchPage} resolves the page's links
* against.
*/
async function fetchFollowing(
url: string,
timeoutMs: number,
): Promise<{ res: Response; url: URL }> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) {
return { res, url: current };
}
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
}
}
/**
* Fetches a user-supplied URL, following up to five redirects, each of which
* must resolve to a public address too. `timeoutMs` bounds each hop, body
@@ -249,20 +276,41 @@ export async function fetchWithTimeout(
url: string,
timeoutMs = 5000,
): Promise<Response> {
let current = new URL(url);
for (let hop = 0;; hop++) {
if (current.protocol !== "http:" && current.protocol !== "https:") {
throw new TypeError(`Unsupported protocol ${current.protocol}`);
}
const address = await resolvePublicAddress(current);
const res = await fetchOnce(current, address, timeoutMs);
const location = res.headers.get("location");
if (res.status < 300 || res.status >= 400 || !location) return res;
return (await fetchFollowing(url, timeoutMs)).res;
}
/**
* Returned by {@link fetchPage} to the link preview providers.
*/
export interface FetchedPage {
/**
* The URL the page was served from, after redirects: the base its relative
* links resolve against.
*/
url: string;
/** The page's text, up to {@link MAX_PAGE_BYTES}; missing when it isn't HTML. */
html?: string;
}
/**
* Fetches a page for its link preview, through {@link fetchWithTimeout}. A
* response that isn't HTML is cancelled unread.
*
* @throws {Error} When the page answers with an error status, so that an error
* page never becomes a preview; and as {@link fetchWithTimeout} does.
*/
export async function fetchPage(url: string): Promise<FetchedPage> {
const { res, url: finalUrl } = await fetchFollowing(url, 5000);
if (!res.ok) {
await res.body?.cancel();
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
current = new URL(location, current);
throw new Error(`${finalUrl} answered ${res.status}`);
}
const contentType = res.headers.get("content-type") ?? "";
if (!/^(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
await res.body?.cancel();
return { url: finalUrl.href };
}
return { url: finalUrl.href, html: await readTextCapped(res) };
}
/**