gerbeur/api/remote/fetch.ts

/**
 * Fetches pages and images from other sites for link previews, the image proxy
 * and the Bandcamp resolver, without letting a user-supplied URL reach the
 * server's own network. Each hop must resolve to a public address, the whole
 * exchange has a deadline, and pages are read up to a cap.
 *
 * @module
 */

import {
  bareHostname,
  isPrivateHostname,
  resolvePublicAddress,
} from "./public-address.ts";
import { isSelfUrl } from "../lib/self-url.ts";

const FETCH_HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
  "Accept":
    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
};

const MAX_REDIRECTS = 5;

/**
 * Fetches `url` with `curl --insecure`, for a site whose TLS certificate Deno
 * rejects. Returns a Response carrying only the `Location` and `Content-Type`
 * headers, or null when curl is missing or fails. The connection is pinned to
 * `address`, the one {@link resolvePublicAddress} checked, and redirects come
 * back unfollowed so that {@link fetchWithTimeout} checks each hop.
 */
async function fetchViaCurl(
  url: URL,
  address: string,
  timeoutMs: number,
): Promise<Response | null> {
  const tmpPath = await Deno.makeTempFile();
  const port = url.port || (url.protocol === "https:" ? "443" : "80");
  const pinned = address.includes(":") ? `[${address}]` : address;
  try {
    const { code, stdout } = await new Deno.Command("curl", {
      args: [
        "--silent",
        "--insecure",
        "--proto",
        "=http,https",
        "--resolve",
        `${bareHostname(url)}:${port}:${pinned}`,
        "--max-time",
        String(Math.ceil(timeoutMs / 1000)),
        // curl fails on a larger file rather than stopping at a cap, so this is
        // looser than MAX_PAGE_BYTES, where readTextCapped still cuts.
        "--max-filesize",
        String(2 * MAX_PAGE_BYTES),
        "--user-agent",
        FETCH_HEADERS["User-Agent"],
        "--header",
        `Accept: ${FETCH_HEADERS["Accept"]}`,
        "--header",
        `Accept-Language: ${FETCH_HEADERS["Accept-Language"]}`,
        "--output",
        tmpPath,
        "--write-out",
        "%{http_code}\n%{redirect_url}\n%{content_type}",
        url.href,
      ],
      stdout: "piped",
      stderr: "null",
    }).output();

    if (code !== 0) return null;

    const [status, location, contentType = ""] = new TextDecoder()
      .decode(stdout).split("\n").map((l) => l.trim());
    const statusCode = Number(status);
    if (!(statusCode >= 200 && statusCode <= 599)) return null;
    const headers = new Headers();
    if (location) headers.set("location", location);
    if (contentType) headers.set("content-type", contentType);

    const bytes = await Deno.readFile(tmpPath);
    return new Response(bytes, { status: statusCode, headers });
  } catch {
    return null;
  } finally {
    await Deno.remove(tmpPath).catch(() => {});
  }
}

/**
 * The default cap of {@link readTextCapped}. The metadata previews read is in
 * the `<head>`, and a page past this is a download.
 */
export const MAX_PAGE_BYTES = 4 * 1024 * 1024;

/**
 * Wraps `body` so that the request's deadline `timer` is cleared only once the
 * body is read to the end or cancelled. A server that answers at once and then
 * drips its body is still cut off.
 */
function withDeadline(
  body: ReadableStream<Uint8Array>,
  timer: ReturnType<typeof setTimeout>,
): ReadableStream<Uint8Array> {
  const reader = body.getReader();
  return new ReadableStream({
    async pull(controller) {
      try {
        const { done, value } = await reader.read();
        if (done) {
          clearTimeout(timer);
          controller.close();
        } else {
          controller.enqueue(value);
        }
      } catch (err) {
        clearTimeout(timer);
        controller.error(err);
      }
    },
    cancel(reason) {
      clearTimeout(timer);
      return reader.cancel(reason);
    },
  });
}

/**
 * Reads the body of `res` up to `maxBytes`; the rest is cancelled, never
 * downloaded. The image proxy reads through it, and {@link readTextCapped}.
 */
export async function readBytesCapped(
  res: Response,
  maxBytes: number,
): Promise<Uint8Array> {
  if (!res.body) return new Uint8Array();
  const reader = res.body.getReader();
  const chunks: Uint8Array[] = [];
  let size = 0;
  try {
    while (size < maxBytes) {
      const { done, value } = await reader.read();
      if (done) break;
      chunks.push(value);
      size += value.byteLength;
    }
  } finally {
    await reader.cancel().catch(() => {});
  }
  const bytes = new Uint8Array(Math.min(size, maxBytes));
  let offset = 0;
  for (const chunk of chunks) {
    const part = chunk.subarray(0, bytes.byteLength - offset);
    bytes.set(part, offset);
    offset += part.byteLength;
    if (offset >= bytes.byteLength) break;
  }
  return bytes;
}

/**
 * Returns the charset a page declares, in its `Content-Type` header or else in
 * a `<meta>` near its top, as HTML allows; UTF-8 when it declares none.
 */
function declaredCharset(contentType: string, bytes: Uint8Array): string {
  const fromHeader = /charset=["']?([\w-]+)/i.exec(contentType)?.[1];
  if (fromHeader) return fromHeader;
  // Latin-1 maps every byte to a character, so the ASCII of the tag reads
  // right whatever the page's encoding.
  const head = new TextDecoder("latin1").decode(bytes.subarray(0, 1024));
  return /<meta[^>]+charset=["']?([\w-]+)/i.exec(head)?.[1] ?? "utf-8";
}

/**
 * Reads the body of `res` as text, up to `maxBytes`, in the charset the page
 * declares; the rest is cancelled, never downloaded. {@link fetchPage} and the
 * Bandcamp and YouTube resolvers read pages through it.
 */
export async function readTextCapped(
  res: Response,
  maxBytes = MAX_PAGE_BYTES,
): Promise<string> {
  const bytes = await readBytesCapped(res, maxBytes);
  const charset = declaredCharset(res.headers.get("content-type") ?? "", bytes);
  try {
    return new TextDecoder(charset).decode(bytes);
  } catch {
    // TextDecoder throws on a label it doesn't know rather than guess.
    return new TextDecoder().decode(bytes);
  }
}

/**
 * Makes one request to `url`, following no redirects, with a deadline that
 * covers the body too.
 */
async function fetchOnce(
  url: URL,
  address: string,
  timeoutMs: number,
): Promise<Response> {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  let res: Response;
  try {
    res = await fetch(url, {
      signal: controller.signal,
      headers: FETCH_HEADERS,
      redirect: "manual",
    });
  } catch (err) {
    clearTimeout(timer);
    if (!(err instanceof TypeError && err.message.includes("certificate"))) {
      throw err;
    }

    // Deno can't be told to skip certificate checks for one request, so a
    // site with an expired, self-signed or unusually signed certificate is
    // fetched with curl instead.
    const curlRes = await fetchViaCurl(url, address, timeoutMs);
    if (curlRes) return curlRes;

    throw err;
  }
  if (!res.body) {
    clearTimeout(timer);
    return res;
  }
  return new Response(withDeadline(res.body, timer), {
    status: res.status,
    statusText: res.statusText,
    headers: res.headers,
  });
}

/**
 * Fetches `url` as {@link fetchWithTimeout} does, and also returns the URL the
 * last hop answered from, which {@link fetchPage} resolves the page's links
 * against.
 */
async function fetchFollowing(
  url: string,
  timeoutMs: number,
): Promise<{ res: Response; url: URL }> {
  let current = new URL(url);
  for (let hop = 0;; hop++) {
    if (current.protocol !== "http:" && current.protocol !== "https:") {
      throw new TypeError(`Unsupported protocol ${current.protocol}`);
    }
    const address = await resolvePublicAddress(current);
    const res = await fetchOnce(current, address, timeoutMs);
    const location = res.headers.get("location");
    if (res.status < 300 || res.status >= 400 || !location) {
      return { res, url: current };
    }

    await res.body?.cancel();
    if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
    current = new URL(location, current);
  }
}

/**
 * Fetches a user-supplied URL, following up to five redirects, each of which
 * must resolve to a public address too. `timeoutMs` bounds each hop, body
 * included. The caller must read or cancel the body, or its connection stays
 * open.
 *
 * @throws {TypeError} When a hop isn't http(s), there are too many redirects,
 * or the request fails.
 * @throws {Error} When a hop's host is private or can't be resolved.
 */
export async function fetchWithTimeout(
  url: string,
  timeoutMs = 5000,
): Promise<Response> {
  return (await fetchFollowing(url, timeoutMs)).res;
}

/**
 * Returned by {@link fetchPage} to the link preview providers.
 */
export interface FetchedPage {
  /**
   * The URL the page was served from, after redirects: the base its relative
   * links resolve against.
   */
  url: string;
  /** The page's text, up to {@link MAX_PAGE_BYTES}; missing when it isn't HTML. */
  html?: string;
}

/**
 * Fetches a page for its link preview, through {@link fetchWithTimeout}. A
 * response that isn't HTML is cancelled unread.
 *
 * @throws {Error} When the page answers with an error status, so that an error
 * page never becomes a preview; and as {@link fetchWithTimeout} does.
 */
export async function fetchPage(url: string): Promise<FetchedPage> {
  const { res, url: finalUrl } = await fetchFollowing(url, 5000);
  if (!res.ok) {
    await res.body?.cancel();
    throw new Error(`${finalUrl} answered ${res.status}`);
  }
  const contentType = res.headers.get("content-type") ?? "";
  if (!/^(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
    await res.body?.cancel();
    return { url: finalUrl.href };
  }
  return { url: finalUrl.href, html: await readTextCapped(res) };
}

/**
 * Checks whether `raw` is an http(s) URL on a host that isn't private, or a
 * link to one of this site's own pages. Routes and services check a URL with it
 * before storing, previewing or proxying it; it does no DNS lookup, which
 * {@link fetchWithTimeout} does.
 */
export function isValidHttpUrl(raw: string): boolean {
  try {
    const u = new URL(raw);
    if (u.protocol !== "http:" && u.protocol !== "https:") return false;
    // This site may run on a private host (localhost in development), and the
    // self provider reads its pages from the database without fetching them.
    return !isPrivateHostname(u) || isSelfUrl(u);
  } catch {
    return false;
  }
}