gerbeur/api/remote/fetch.ts

/**
 * The guarded fetch every request to another site goes through: each hop must
 * resolve to a public address, the whole exchange has a deadline, and pages are
 * read up to a cap.
 *
 * @module
 */

import { isPrivateHostname, resolvePublicAddress } from "./public-address.ts";
import { isSelfUrl } from "../lib/self-url.ts";

const FETCH_HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
  "Accept":
    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
};

const MAX_REDIRECTS = 5;

/**
 * Fetches `url` through a `curl --insecure` subprocess and returns a minimal
 * Response.
 * Used as a last resort when Deno's rustls rejects the server's TLS certificate
 * due to an unsupported algorithm (e.g. UnsupportedSignatureAlgorithm).
 * Returns null if curl is unavailable or exits non-zero.
 *
 * `address` pins the connection to the IP already checked by
 * resolvePublicAddress, and curl doesn't follow redirects itself: a 3xx comes
 * back with its Location so fetchWithTimeout can check the next hop too.
 */
async function fetchViaCurl(
  url: URL,
  address: string,
  timeoutMs: number,
): Promise<Response | null> {
  const tmpPath = await Deno.makeTempFile();
  const port = url.port || (url.protocol === "https:" ? "443" : "80");
  const pinned = address.includes(":") ? `[${address}]` : address;
  try {
    const { code, stdout } = await new Deno.Command("curl", {
      args: [
        "--silent",
        "--insecure",
        "--proto",
        "=http,https",
        "--resolve",
        `${url.hostname.replace(/^\[(.*)\]$/, "$1")}:${port}:${pinned}`,
        "--max-time",
        String(Math.ceil(timeoutMs / 1000)),
        "--max-filesize",
        String(2 * MAX_PAGE_BYTES),
        "--user-agent",
        FETCH_HEADERS["User-Agent"],
        "--header",
        `Accept: ${FETCH_HEADERS["Accept"]}`,
        "--header",
        `Accept-Language: ${FETCH_HEADERS["Accept-Language"]}`,
        "--output",
        tmpPath,
        "--write-out",
        "%{http_code}\n%{redirect_url}\n%{content_type}",
        url.href,
      ],
      stdout: "piped",
      stderr: "null",
    }).output();

    if (code !== 0) return null;

    const [status, location, contentType = ""] = new TextDecoder()
      .decode(stdout).split("\n").map((l) => l.trim());
    const statusCode = Number(status);
    if (!(statusCode >= 200 && statusCode <= 599)) return null;
    const headers = new Headers();
    if (location) headers.set("location", location);
    if (contentType) headers.set("content-type", contentType);

    const bytes = await Deno.readFile(tmpPath);
    if (!contentType.startsWith("text/")) {
      return new Response(bytes, { status: statusCode, headers });
    }
    // Decode using the charset declared in the Content-Type header so that
    // pages served in ISO-8859-1, windows-1252, etc. are read correctly.
    const charset = /charset=([\w-]+)/i.exec(contentType)?.[1] ?? "utf-8";
    let bodyText: string;
    try {
      bodyText = new TextDecoder(charset, { fatal: false }).decode(bytes);
    } catch {
      bodyText = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
    }
    headers.set("content-type", contentType.replace(/;.*$/, "; charset=utf-8"));
    return new Response(bodyText, { status: statusCode, headers });
  } catch {
    return null;
  } finally {
    await Deno.remove(tmpPath).catch(() => {});
  }
}

/**
 * The most bytes of a fetched page that are read. Metadata lives in the <head>,
 * and a "page" past this is a download, not something to scrape.
 */
export const MAX_PAGE_BYTES = 4 * 1024 * 1024;

/**
 * Hands `body` on, and clears the request's deadline once it has been read to
 * the end or cancelled. Until then the deadline still applies: a server that
 * answers at once and then drips its body can't hold the request open.
 */
function withDeadline(
  body: ReadableStream<Uint8Array>,
  timer: ReturnType<typeof setTimeout>,
): ReadableStream<Uint8Array> {
  const reader = body.getReader();
  return new ReadableStream({
    async pull(controller) {
      try {
        const { done, value } = await reader.read();
        if (done) {
          clearTimeout(timer);
          controller.close();
        } else {
          controller.enqueue(value);
        }
      } catch (err) {
        clearTimeout(timer);
        controller.error(err);
      }
    },
    cancel(reason) {
      clearTimeout(timer);
      return reader.cancel(reason);
    },
  });
}

/**
 * Reads a response's body as text, stopping at `maxBytes` (the rest is
 * cancelled, never downloaded).
 */
export async function readTextCapped(
  res: Response,
  maxBytes = MAX_PAGE_BYTES,
): Promise<string> {
  if (!res.body) return "";
  const reader = res.body.getReader();
  const chunks: Uint8Array[] = [];
  let size = 0;
  try {
    while (size < maxBytes) {
      const { done, value } = await reader.read();
      if (done) break;
      chunks.push(value);
      size += value.byteLength;
    }
  } finally {
    await reader.cancel().catch(() => {});
  }
  const bytes = new Uint8Array(Math.min(size, maxBytes));
  let offset = 0;
  for (const chunk of chunks) {
    const part = chunk.subarray(0, bytes.byteLength - offset);
    bytes.set(part, offset);
    offset += part.byteLength;
    if (offset >= bytes.byteLength) break;
  }
  return new TextDecoder().decode(bytes);
}

/** Makes one request, following no redirects. The timeout covers the body too. */
async function fetchOnce(
  url: URL,
  address: string,
  timeoutMs: number,
): Promise<Response> {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  let res: Response;
  try {
    res = await fetch(url, {
      signal: controller.signal,
      headers: FETCH_HEADERS,
      redirect: "manual",
    });
  } catch (err) {
    clearTimeout(timer);
    if (!(err instanceof TypeError && err.message.includes("certificate"))) {
      throw err;
    }

    // Deno/rustls rejected the server's TLS certificate (expired, self-signed,
    // or an unsupported signature algorithm) and offers no way to override
    // verification programmatically. Fall back to a `curl --insecure`
    // subprocess, which uses its own TLS stack.
    const curlRes = await fetchViaCurl(url, address, timeoutMs);
    if (curlRes) return curlRes;

    throw err;
  }
  if (!res.body) {
    clearTimeout(timer);
    return res;
  }
  return new Response(withDeadline(res.body, timer), {
    status: res.status,
    statusText: res.statusText,
    headers: res.headers,
  });
}

/**
 * Fetches a user-supplied URL, following up to five redirects. Every hop — the
 * URL itself and each redirect — must resolve to a public address (see
 * {@link resolvePublicAddress}), so a link or a redirect can't point the
 * server at its own network.
 *
 * @throws {TypeError} When a hop isn't http(s), there are too many redirects,
 * or the request fails.
 * @throws {Error} When a hop's host is private or can't be resolved.
 */
export async function fetchWithTimeout(
  url: string,
  timeoutMs = 5000,
): Promise<Response> {
  let current = new URL(url);
  for (let hop = 0;; hop++) {
    if (current.protocol !== "http:" && current.protocol !== "https:") {
      throw new TypeError(`Unsupported protocol ${current.protocol}`);
    }
    const address = await resolvePublicAddress(current);
    const res = await fetchOnce(current, address, timeoutMs);
    const location = res.headers.get("location");
    if (res.status < 300 || res.status >= 400 || !location) return res;

    await res.body?.cancel();
    if (hop >= MAX_REDIRECTS) throw new TypeError("Too many redirects");
    current = new URL(location, current);
  }
}

/**
 * Checks whether `raw` is an http(s) URL a dump may link to: not on a private
 * host, unless it is one of this site's own pages.
 */
export function isValidHttpUrl(raw: string): boolean {
  try {
    const u = new URL(raw);
    if (u.protocol !== "http:" && u.protocol !== "https:") return false;
    // This site's own pages are fine even on a private host (e.g. localhost in
    // development): selfProvider reads them from the DB, no HTTP involved.
    // Every actual fetch is checked again, after DNS, in fetchWithTimeout.
    return !isPrivateHostname(u) || isSelfUrl(u);
  } catch {
    return false;
  }
}