v3: api/ gets one layout rule and keeps to it, and whatever crosses the wire is defined once, in shared/ — api/model/interfaces.ts and src/model.ts declared the same 40 types by hand and had drifted: the error envelope, email required on one side and optional on the other, Invitee/InviteTreeEntry and LinkPreview/UrlPreviewResponse for one shape each. The validation limits, the role permissions, the chat length cap, the @mention pattern and the Bandcamp and YouTube player responses were copied too, behind "keep in sync" comments. shared/ now holds them once, one file per domain (api, category, chat, comment, dump, follow, mention, notification, permissions, player, playlist, search, user, validation, ws), and imports nothing from either half. The server sends Dates; the client reads Wire<T>, the same type with every Date an ISO string, and turns it back with src/utils/shared/deserialize.ts. APIErrorCode becomes a const object, since the frontend's erasableSyntaxOnly rejects an enum. The shared types showed two real gaps: the avatar upload and GET /api/users/me return the full self user and the frontend read them as public users, and a stored session may predate media tokens, so the client keeps it as its own Session type. The backend rule: each folder holds one kind of module, each file is named after its domain. routes/ parse and answer; services/ hold the logic and every SQL query (lib/auth.ts's token check moves to user-service); remote/ is the only place that fetches from other sites — the guarded fetch and the HTML scraping out of rich-content-service, public-address, the Bandcamp and YouTube resolvers, the link providers, the image proxy (fetch, checks and cache, out of routes/preview.ts) and the Twemoji glyphs (out of lib/site-icons.ts). model/ holds per table a row type, its guard and its mappers; validators/ the request guards; db/connection.ts the handle and transaction(), db/startup.ts the purges and default admin that db.ts ran on import, called at the same moment. Tests mirror the source one file per module, test/<layer>/<module>.test.ts, with shared/'s under shared/test/ (deno task test runs both); the test that covered the bandcamp and soundcloud providers together is two now, and the backend can() has its own, so 202 tests. deno lint src/ is clean too: window.* becomes globalThis.*, and test bodies and act() callbacks that never awaited lose their async. The README states the rule. No behaviour changes; the catalogues only move their source references.
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 50s
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 50s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
@@ -1,20 +1,10 @@
|
||||
import type { RichContent } from "../model/interfaces.ts";
|
||||
import { youtubeProvider } from "./providers/youtube.ts";
|
||||
import { bandcampProvider } from "./providers/bandcamp.ts";
|
||||
import { soundcloudProvider } from "./providers/soundcloud.ts";
|
||||
import { selfProvider } from "./providers/self.ts";
|
||||
import { genericProvider } from "./providers/generic.ts";
|
||||
import {
|
||||
isPrivateHostname,
|
||||
resolvePublicAddress,
|
||||
} from "../lib/public-address.ts";
|
||||
import { isSelfUrl } from "../lib/self-url.ts";
|
||||
|
||||
export interface RichContentProvider {
|
||||
name: string;
|
||||
matches(url: string): boolean;
|
||||
fetch(url: string): Promise<RichContent>;
|
||||
}
|
||||
import type { RichContent } from "../../shared/dump.ts";
|
||||
import type { RichContentProvider } from "../remote/providers/provider.ts";
|
||||
import { youtubeProvider } from "../remote/providers/youtube.ts";
|
||||
import { bandcampProvider } from "../remote/providers/bandcamp.ts";
|
||||
import { soundcloudProvider } from "../remote/providers/soundcloud.ts";
|
||||
import { selfProvider } from "../remote/providers/self.ts";
|
||||
import { genericProvider } from "../remote/providers/generic.ts";
|
||||
|
||||
/**
|
||||
* Register providers in priority order. The first match wins.
|
||||
@@ -29,651 +19,6 @@ const providers: RichContentProvider[] = [
|
||||
genericProvider,
|
||||
];
|
||||
|
||||
// Shared utilities exported for use by providers
|
||||
|
||||
const FETCH_HEADERS = {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
|
||||
"Accept":
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
};
|
||||
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
/**
|
||||
* Fetch `url` via a `curl --insecure` subprocess and return a minimal Response.
|
||||
* Used as a last resort when Deno's rustls rejects the server's TLS certificate
|
||||
* due to an unsupported algorithm (e.g. UnsupportedSignatureAlgorithm).
|
||||
* Returns null if curl is unavailable or exits non-zero.
|
||||
*
|
||||
* `address` pins the connection to the IP already checked by
|
||||
* resolvePublicAddress, and curl doesn't follow redirects itself: a 3xx comes
|
||||
* back with its Location so fetchWithTimeout can check the next hop too.
|
||||
*/
|
||||
async function fetchViaCurl(
|
||||
url: URL,
|
||||
address: string,
|
||||
timeoutMs: number,
|
||||
): Promise<Response | null> {
|
||||
const tmpPath = await Deno.makeTempFile();
|
||||
const port = url.port || (url.protocol === "https:" ? "443" : "80");
|
||||
const pinned = address.includes(":") ? `[${address}]` : address;
|
||||
try {
|
||||
const { code, stdout } = await new Deno.Command("curl", {
|
||||
args: [
|
||||
"--silent",
|
||||
"--insecure",
|
||||
"--proto",
|
||||
"=http,https",
|
||||
"--resolve",
|
||||
`${url.hostname.replace(/^\[(.*)\]$/, "$1")}:${port}:${pinned}`,
|
||||
"--max-time",
|
||||
String(Math.ceil(timeoutMs / 1000)),
|
||||
"--max-filesize",
|
||||
String(2 * MAX_PAGE_BYTES),
|
||||
"--user-agent",
|
||||
FETCH_HEADERS["User-Agent"],
|
||||
"--header",
|
||||
`Accept: ${FETCH_HEADERS["Accept"]}`,
|
||||
"--header",
|
||||
`Accept-Language: ${FETCH_HEADERS["Accept-Language"]}`,
|
||||
"--output",
|
||||
tmpPath,
|
||||
"--write-out",
|
||||
"%{http_code}\n%{redirect_url}\n%{content_type}",
|
||||
url.href,
|
||||
],
|
||||
stdout: "piped",
|
||||
stderr: "null",
|
||||
}).output();
|
||||
|
||||
if (code !== 0) return null;
|
||||
|
||||
const [status, location, contentType = ""] = new TextDecoder()
|
||||
.decode(stdout).split("\n").map((l) => l.trim());
|
||||
const statusCode = Number(status);
|
||||
if (!(statusCode >= 200 && statusCode <= 599)) return null;
|
||||
const headers = new Headers();
|
||||
if (location) headers.set("location", location);
|
||||
if (contentType) headers.set("content-type", contentType);
|
||||
|
||||
const bytes = await Deno.readFile(tmpPath);
|
||||
if (!contentType.startsWith("text/")) {
|
||||
return new Response(bytes, { status: statusCode, headers });
|
||||
}
|
||||
// Decode using the charset declared in the Content-Type header so that
|
||||
// pages served in ISO-8859-1, windows-1252, etc. are read correctly.
|
||||
const charset = /charset=([\w-]+)/i.exec(contentType)?.[1] ?? "utf-8";
|
||||
let bodyText: string;
|
||||
try {
|
||||
bodyText = new TextDecoder(charset, { fatal: false }).decode(bytes);
|
||||
} catch {
|
||||
bodyText = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
||||
}
|
||||
headers.set("content-type", contentType.replace(/;.*$/, "; charset=utf-8"));
|
||||
return new Response(bodyText, { status: statusCode, headers });
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
await Deno.remove(tmpPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Most bytes of a fetched page that are read. Metadata lives in the <head>,
|
||||
* and a "page" past this is a download, not something to scrape.
|
||||
*/
|
||||
export const MAX_PAGE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Hands `body` on, and clears the request's deadline once it has been read to
|
||||
* the end or cancelled. Until then the deadline still applies: a server that
|
||||
* answers at once and then drips its body can't hold the request open.
|
||||
*/
|
||||
function withDeadline(
|
||||
body: ReadableStream<Uint8Array>,
|
||||
timer: ReturnType<typeof setTimeout>,
|
||||
): ReadableStream<Uint8Array> {
|
||||
const reader = body.getReader();
|
||||
return new ReadableStream({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
clearTimeout(timer);
|
||||
controller.close();
|
||||
} else {
|
||||
controller.enqueue(value);
|
||||
}
|
||||
} catch (err) {
|
||||
clearTimeout(timer);
|
||||
controller.error(err);
|
||||
}
|
||||
},
|
||||
cancel(reason) {
|
||||
clearTimeout(timer);
|
||||
return reader.cancel(reason);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a response's body as text, stopping at `maxBytes` (the rest is
|
||||
* cancelled, never downloaded).
|
||||
*/
|
||||
export async function readTextCapped(
|
||||
res: Response,
|
||||
maxBytes = MAX_PAGE_BYTES,
|
||||
): Promise<string> {
|
||||
if (!res.body) return "";
|
||||
const reader = res.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
while (size < maxBytes) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
size += value.byteLength;
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
}
|
||||
const bytes = new Uint8Array(Math.min(size, maxBytes));
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
const part = chunk.subarray(0, bytes.byteLength - offset);
|
||||
bytes.set(part, offset);
|
||||
offset += part.byteLength;
|
||||
if (offset >= bytes.byteLength) break;
|
||||
}
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
|
||||
/** One request, no redirects followed. The timeout covers the body too. */
|
||||
async function fetchOnce(
|
||||
url: URL,
|
||||
address: string,
|
||||
timeoutMs: number,
|
||||
): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: FETCH_HEADERS,
|
||||
redirect: "manual",
|
||||
});
|
||||
} catch (err) {
|
||||
clearTimeout(timer);
|
||||
if (!(err instanceof TypeError && err.message.includes("certificate"))) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Deno/rustls rejected the server's TLS certificate (expired, self-signed,
|
||||
// or an unsupported signature algorithm) and offers no way to override
|
||||
// verification programmatically. Fall back to a `curl --insecure`
|
||||
// subprocess, which uses its own TLS stack.
|
||||
const curlRes = await fetchViaCurl(url, address, timeoutMs);
|
||||
if (curlRes) return curlRes;
|
||||
|
||||
throw err;
|
||||
}
|
||||
if (!res.body) {
|
||||
clearTimeout(timer);
|
||||
return res;
|
||||
}
|
||||
return new Response(withDeadline(res.body, timer), {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
headers: res.headers,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a user-supplied URL. Every hop — the URL itself and each redirect —
|
||||
* must resolve to a public address (see resolvePublicAddress), so a link or a
|
||||
* redirect can't point the server at its own network.
|
||||
*/
|
||||
export async function fetchWithTimeout(
|
||||
url: string,
|
||||
timeoutMs = 5000,
|
||||
): Promise<Response> {
|
||||
let current = new URL(url);
|
||||
for (let hop = 0;; hop++) {
|
||||
if (current.protocol !== "http:" && current.protocol !== "https:") {
|
||||
throw new TypeError(`Unsupported protocol ${current.protocol}`);
|
||||
}
|
||||
const address = await resolvePublicAddress(current);
|
||||
const res = await fetchOnce(current, address, timeoutMs);
|
||||
const location = res.headers.get("location");
|
||||
if (res.status < 300 || res.status >= 400 || !location) return res;
|
||||
|
||||
await res.body?.cancel();
|
||||
if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
|
||||
current = new URL(location, current);
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(str: string): string {
|
||||
return str
|
||||
.replace(/&/gi, "&")
|
||||
.replace(/</gi, "<")
|
||||
.replace(/>/gi, ">")
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/'/gi, "'")
|
||||
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
|
||||
.replace(
|
||||
/&#x([0-9a-f]+);/gi,
|
||||
(_, hex) => String.fromCodePoint(parseInt(hex, 16)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the first `<meta>` tag whose `attr` equals `value` and return its
|
||||
* `content` (either attribute ordering). Each tag is matched in isolation so
|
||||
* the `content` capture can never span across tag boundaries.
|
||||
*/
|
||||
function extractMetaContent(
|
||||
html: string,
|
||||
attr: string,
|
||||
value: string,
|
||||
): string | undefined {
|
||||
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const keyRe = new RegExp(`\\b${attr}=["']${escaped}["']`, "i");
|
||||
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
|
||||
const tagRe = /<meta\b[^>]*>/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = tagRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
if (!keyRe.test(tag)) continue;
|
||||
const content = contentRe.exec(tag);
|
||||
if (content) return decodeHtmlEntities(content[2]);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function extractOgTag(
|
||||
html: string,
|
||||
tag: string,
|
||||
): string | undefined {
|
||||
return extractMetaContent(html, "property", `og:${tag}`);
|
||||
}
|
||||
|
||||
/** Extract content from `<meta name="…" content="…">` (both attribute orderings). */
|
||||
export function extractMetaName(
|
||||
html: string,
|
||||
name: string,
|
||||
): string | undefined {
|
||||
return extractMetaContent(html, "name", name);
|
||||
}
|
||||
|
||||
/** Extract the text content of the `<title>` element. */
|
||||
export function extractPageTitle(html: string): string | undefined {
|
||||
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
return match ? decodeHtmlEntities(match[1].trim()) : undefined;
|
||||
}
|
||||
|
||||
// ── Brand color helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Find the page's `theme-color`.
|
||||
*
|
||||
* Sites routinely ship the tag twice, scoped by `media`, and often list the
|
||||
* dark-scheme one first — so unlike `extractMetaName` this can't just take the
|
||||
* first match. Prefer the unscoped tag, then the light-scheme one, then any.
|
||||
*/
|
||||
export function extractThemeColor(html: string): string | undefined {
|
||||
const tagRe = /<meta\b[^>]*>/gi;
|
||||
const contentRe = /\bcontent=(["'])([\s\S]*?)\1/i;
|
||||
const mediaRe = /\bmedia=(["'])([\s\S]*?)\1/i;
|
||||
let unscoped: string | undefined;
|
||||
let light: string | undefined;
|
||||
let any: string | undefined;
|
||||
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = tagRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
if (!/\bname=["']theme-color["']/i.test(tag)) continue;
|
||||
const content = contentRe.exec(tag)?.[2];
|
||||
if (!content) continue;
|
||||
const media = mediaRe.exec(tag)?.[2];
|
||||
if (!media) unscoped ??= content;
|
||||
else if (/light/i.test(media)) light ??= content;
|
||||
any ??= content;
|
||||
}
|
||||
return unscoped ?? light ?? any;
|
||||
}
|
||||
|
||||
/** Extract the `color` of `<link rel="mask-icon">` (Safari pinned tabs). */
|
||||
export function extractMaskIconColor(html: string): string | undefined {
|
||||
const linkRe = /<link[^>]+>/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = linkRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
if (!/\brel=["'][^"']*mask-icon[^"']*["']/i.test(tag)) continue;
|
||||
const color = /\bcolor=(["'])([\s\S]*?)\1/i.exec(tag)?.[2];
|
||||
if (color) return color;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// The handful of named colors that actually show up in `theme-color`. The full
|
||||
// 148-name table isn't worth carrying for the long tail.
|
||||
const NAMED_COLORS: Record<string, string> = {
|
||||
black: "#000000",
|
||||
white: "#ffffff",
|
||||
red: "#ff0000",
|
||||
green: "#008000",
|
||||
blue: "#0000ff",
|
||||
yellow: "#ffff00",
|
||||
orange: "#ffa500",
|
||||
purple: "#800080",
|
||||
gray: "#808080",
|
||||
grey: "#808080",
|
||||
silver: "#c0c0c0",
|
||||
maroon: "#800000",
|
||||
navy: "#000080",
|
||||
teal: "#008080",
|
||||
olive: "#808000",
|
||||
lime: "#00ff00",
|
||||
aqua: "#00ffff",
|
||||
cyan: "#00ffff",
|
||||
fuchsia: "#ff00ff",
|
||||
magenta: "#ff00ff",
|
||||
};
|
||||
|
||||
function clampByte(n: number): number {
|
||||
return Math.max(0, Math.min(255, Math.round(n)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a CSS color to canonical lowercase `#rrggbb`, or `undefined` when
|
||||
* it isn't one of the forms sites actually use. Alpha is dropped — the color is
|
||||
* only ever used as a tint over the app's own surface.
|
||||
*/
|
||||
export function normalizeCssColor(raw: string | undefined): string | undefined {
|
||||
if (!raw) return undefined;
|
||||
const value = raw.trim().toLowerCase();
|
||||
|
||||
const named = NAMED_COLORS[value];
|
||||
if (named) return named;
|
||||
|
||||
const hex = /^#([0-9a-f]{3,8})$/.exec(value)?.[1];
|
||||
if (hex) {
|
||||
if (hex.length === 3 || hex.length === 4) {
|
||||
const [r, g, b] = [...hex.slice(0, 3)];
|
||||
return `#${r}${r}${g}${g}${b}${b}`;
|
||||
}
|
||||
if (hex.length === 6 || hex.length === 8) return `#${hex.slice(0, 6)}`;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const rgb = /^rgba?\(([^)]+)\)$/.exec(value)?.[1];
|
||||
if (rgb) {
|
||||
const parts = rgb.split(/[\s,/]+/).filter(Boolean).slice(0, 3);
|
||||
if (parts.length !== 3) return undefined;
|
||||
const bytes = parts.map((p) => {
|
||||
const n = parseFloat(p);
|
||||
if (!Number.isFinite(n)) return NaN;
|
||||
return clampByte(p.endsWith("%") ? (n / 100) * 255 : n);
|
||||
});
|
||||
if (bytes.some(Number.isNaN)) return undefined;
|
||||
return `#${bytes.map((b) => b.toString(16).padStart(2, "0")).join("")}`;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// ── JSON-LD helpers (file-private) ────────────────────────────────────────────
|
||||
|
||||
type JsonLdResult = {
|
||||
title?: string;
|
||||
description?: string;
|
||||
thumbnailUrl?: string;
|
||||
};
|
||||
|
||||
function ldString(v: unknown): string | undefined {
|
||||
if (typeof v === "string" && v.trim()) return v.trim();
|
||||
if (Array.isArray(v) && typeof v[0] === "string" && v[0].trim()) {
|
||||
return v[0].trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function ldImage(v: unknown): string | undefined {
|
||||
if (
|
||||
typeof v === "string" &&
|
||||
(v.startsWith("http://") || v.startsWith("https://"))
|
||||
) return v;
|
||||
if (Array.isArray(v)) return ldImage(v[0]);
|
||||
if (v && typeof v === "object") {
|
||||
const o = v as Record<string, unknown>;
|
||||
return ldImage(o.url ?? o.contentUrl);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function ldExtractNode(data: unknown): JsonLdResult {
|
||||
if (Array.isArray(data)) {
|
||||
for (const item of data) {
|
||||
const r = ldExtractNode(item);
|
||||
if (r.title || r.thumbnailUrl) return r;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (!data || typeof data !== "object") return {};
|
||||
const o = data as Record<string, unknown>;
|
||||
if (o["@graph"]) return ldExtractNode(o["@graph"]);
|
||||
return {
|
||||
title: ldString(o.name ?? o.headline),
|
||||
description: ldString(o.description),
|
||||
thumbnailUrl: ldImage(o.image ?? o.thumbnailUrl ?? o.thumbnail),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse every `<script type="application/ld+json">` block and return the first
|
||||
* node that yields a title or image. Handles `@graph`, arrays, and the common
|
||||
* `image` shapes (string, string[], ImageObject).
|
||||
*/
|
||||
export function extractJsonLd(html: string): JsonLdResult {
|
||||
const pattern =
|
||||
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = pattern.exec(html)) !== null) {
|
||||
try {
|
||||
const result = ldExtractNode(JSON.parse(match[1]));
|
||||
if (result.title || result.thumbnailUrl) return result;
|
||||
} catch { /* invalid JSON — skip */ }
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the `src` of the first `<img>` whose declared width or height is at
|
||||
* least `minSize` pixels (default 200). Skips data URIs. Resolves relative URLs.
|
||||
*/
|
||||
export function extractLargeImage(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
minSize = 200,
|
||||
): string | undefined {
|
||||
const imgPattern = /<img[^>]+>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = imgPattern.exec(html)) !== null) {
|
||||
const tag = match[0];
|
||||
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
|
||||
if (!src || src.startsWith("data:")) continue;
|
||||
const w = parseInt(/\bwidth=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
|
||||
const h = parseInt(/\bheight=["']?(\d+)/i.exec(tag)?.[1] ?? "0");
|
||||
if (w >= minSize && h >= minSize) {
|
||||
try {
|
||||
return new URL(src, baseUrl).toString();
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect all `<link rel="icon">` / `<link rel="apple-touch-icon">` tags, rank
|
||||
* them by declared size (largest wins), and return the best resolved URL.
|
||||
* Falls back to the first match when no `sizes` attribute is present.
|
||||
*
|
||||
* SVG icons are ranked above everything: they carry no `sizes` attribute, so
|
||||
* they'd otherwise score 0 and lose to a 16×16 PNG, yet they scale cleanly to
|
||||
* whatever size the placeholder renders them at.
|
||||
*/
|
||||
export function extractBestIcon(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
): string | undefined {
|
||||
const linkRe = /<link[^>]+>/gi;
|
||||
const relRe = /\brel=["']([^"']+)["']/i;
|
||||
const hrefRe = /\bhref=["']([^"']+)["']/i;
|
||||
const sizesRe = /\bsizes=["']([^"']+)["']/i;
|
||||
|
||||
const candidates: { href: string; area: number }[] = [];
|
||||
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = linkRe.exec(html)) !== null) {
|
||||
const tag = m[0];
|
||||
const rel = relRe.exec(tag)?.[1] ?? "";
|
||||
if (!/\bicon\b/i.test(rel) && !/apple-touch-icon/i.test(rel)) continue;
|
||||
const href = hrefRe.exec(tag)?.[1];
|
||||
if (!href) continue;
|
||||
const sizesStr = sizesRe.exec(tag)?.[1] ?? "";
|
||||
const sm = sizesStr.match(/(\d+)x(\d+)/i);
|
||||
const isSvg = /\.svg(\?|$)/i.test(href) ||
|
||||
/\btype=["']image\/svg\+xml["']/i.test(tag);
|
||||
const area = isSvg
|
||||
? Number.MAX_SAFE_INTEGER
|
||||
: sm
|
||||
? parseInt(sm[1]) * parseInt(sm[2])
|
||||
: 0;
|
||||
try {
|
||||
candidates.push({ href: new URL(href, baseUrl).toString(), area });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (candidates.length === 0) return undefined;
|
||||
candidates.sort((a, b) => b.area - a.area);
|
||||
return candidates[0].href;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the `src` of the first `<img>` that looks like content rather than UI
|
||||
* chrome. Skips SVGs, data URIs, and images whose filename matches common
|
||||
* icon/logo/nav patterns (logo, icon, sprite, favicon, avatar, banner, etc.).
|
||||
* Resolves relative and protocol-relative URLs.
|
||||
*/
|
||||
const UI_IMAGE_KEYWORDS = new Set([
|
||||
"logo",
|
||||
"icon",
|
||||
"sprite",
|
||||
"favicon",
|
||||
"avatar",
|
||||
"banner",
|
||||
"header",
|
||||
"nav",
|
||||
"menu",
|
||||
"cart",
|
||||
"search",
|
||||
"tracking",
|
||||
"pixel",
|
||||
"bg",
|
||||
"background",
|
||||
]);
|
||||
|
||||
function isUiImage(src: string): boolean {
|
||||
if (/\.svg(\?|$)/i.test(src)) return true;
|
||||
const filename = src.split("?")[0].split("/").pop() ?? "";
|
||||
const baseName = filename.replace(/\.[^.]+$/, ""); // strip extension
|
||||
// Split on common filename separators (-, _, .) and check each token
|
||||
return baseName.toLowerCase().split(/[-_.]/).some((t) =>
|
||||
UI_IMAGE_KEYWORDS.has(t)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the `src` of the first `<img>` that looks like content rather than UI
|
||||
* chrome. Skips SVGs, data URIs, and images whose filename tokens match common
|
||||
* icon/logo/nav patterns (logo, icon, sprite, etc.).
|
||||
* Resolves relative and protocol-relative URLs.
|
||||
*/
|
||||
export function extractFirstContentImage(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
): string | undefined {
|
||||
const imgPattern = /<img[^>]+>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = imgPattern.exec(html)) !== null) {
|
||||
const tag = match[0];
|
||||
const src = /\bsrc=["']([^"']+)["']/i.exec(tag)?.[1];
|
||||
if (!src || src.startsWith("data:")) continue;
|
||||
if (isUiImage(src)) continue;
|
||||
try {
|
||||
return new URL(src, baseUrl).toString();
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract `href` from the first `<link rel="…">` whose rel contains `relFragment`,
|
||||
* resolved to an absolute URL using `baseUrl`.
|
||||
*/
|
||||
export function extractLinkHref(
|
||||
html: string,
|
||||
relFragment: string,
|
||||
baseUrl: string,
|
||||
): string | undefined {
|
||||
const escaped = relFragment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const patterns = [
|
||||
new RegExp(
|
||||
`<link[^>]+rel=["'][^"']*${escaped}[^"']*["'][^>]+href=["']([^"']+)["']`,
|
||||
"i",
|
||||
),
|
||||
new RegExp(
|
||||
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["'][^"']*${escaped}[^"']*["']`,
|
||||
"i",
|
||||
),
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
const match = html.match(pattern);
|
||||
if (match) {
|
||||
try {
|
||||
return new URL(match[1], baseUrl).toString();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function isValidHttpUrl(raw: string): boolean {
|
||||
try {
|
||||
const u = new URL(raw);
|
||||
if (u.protocol !== "http:" && u.protocol !== "https:") return false;
|
||||
// This site's own pages are fine even on a private host (e.g. localhost in
|
||||
// development): selfProvider reads them from the DB, no HTTP involved.
|
||||
// Every actual fetch is checked again, after DNS, in fetchWithTimeout.
|
||||
return !isPrivateHostname(u) || isSelfUrl(u);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface FetchRichContentResult {
|
||||
/** False when the page couldn't be reached at all, as opposed to reached and
|
||||
* found to carry no metadata. Callers that already hold good metadata must
|
||||
|
||||
Reference in New Issue
Block a user