gerbeur/api/test/remote/fetch.test.ts

import { html, stubNetwork } from "../helpers.ts";
import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert";
import {
  fetchPage,
  fetchWithTimeout,
  isValidHttpUrl,
  readTextCapped,
} from "../../remote/fetch.ts";

Deno.test("isValidHttpUrl: http(s) to public hosts, or our own pages", () => {
  assert(isValidHttpUrl("https://example.com/x"));
  assert(isValidHttpUrl("http://gerbeur.test/dumps/abc"));
  assertFalse(isValidHttpUrl("http://localhost:8000/"));
  assertFalse(isValidHttpUrl("http://10.0.0.1/"));
  assertFalse(isValidHttpUrl("file:///etc/passwd"));
  assertFalse(isValidHttpUrl("nope"));
});

Deno.test("fetchWithTimeout: the deadline covers a body that never ends", async () => {
  // Headers at once, then a trickle that never finishes: the read must fail
  // at the deadline rather than hold the request open.
  using _net = stubNetwork((_url, init) =>
    new Response(
      new ReadableStream({
        start(controller) {
          controller.enqueue(new TextEncoder().encode("<html>"));
          init?.signal?.addEventListener(
            "abort",
            () => controller.error(new DOMException("aborted", "AbortError")),
          );
        },
      }),
      { headers: { "content-type": "text/html" } },
    )
  );
  const res = await fetchWithTimeout("https://example.com/slow", 50);
  await assertRejects(() => res.text());
});

Deno.test("readTextCapped: stops reading at the cap", async () => {
  let pulled = 0;
  const body = new ReadableStream<Uint8Array>({
    pull(controller) {
      pulled++;
      controller.enqueue(new TextEncoder().encode("abcdefghij"));
    },
  });
  const text = await readTextCapped(new Response(body), 25);
  assertEquals(text, "abcdefghijabcdefghijabcde");
  assert(pulled < 10);
  assertEquals(await readTextCapped(new Response("short")), "short");
});

Deno.test("fetchWithTimeout: follows redirects, checking every hop", async () => {
  using net = stubNetwork((url) => {
    if (url.pathname === "/start") {
      return new Response(null, { status: 302, headers: { location: "/end" } });
    }
    if (url.pathname === "/to-private") {
      return new Response(null, {
        status: 301,
        headers: { location: "http://169.254.169.254/latest/meta-data" },
      });
    }
    if (url.pathname === "/loop") {
      return new Response(null, {
        status: 302,
        headers: { location: "/loop" },
      });
    }
    return new Response("done");
  });
  const res = await fetchWithTimeout("https://example.com/start");
  assertEquals(await res.text(), "done");
  assertEquals(net.calls.map((u) => u.pathname), ["/start", "/end"]);

  await assertRejects(
    () => fetchWithTimeout("https://example.com/to-private"),
    Error,
    "private",
  );
  await assertRejects(
    () => fetchWithTimeout("https://example.com/loop"),
    TypeError,
    "Too many redirects",
  );
  await assertRejects(
    () => fetchWithTimeout("ftp://example.com/"),
    TypeError,
    "Unsupported protocol",
  );
});

Deno.test("readTextCapped: decodes the charset the header or a <meta> declares", async () => {
  const cafe = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // "café" in Latin-1
  const page = (contentType: string, head = "") =>
    new Response(
      new Uint8Array([...new TextEncoder().encode(head), ...cafe]),
      { headers: { "content-type": contentType } },
    );
  assertEquals(
    await readTextCapped(page("text/html; charset=ISO-8859-1")),
    "café",
  );
  assertEquals(
    await readTextCapped(page("text/html", '<meta charset="windows-1252">')),
    '<meta charset="windows-1252">café',
  );
  assertEquals(
    await readTextCapped(page("text/html; charset=nonsense")),
    "caf<61>",
  );
});

Deno.test("fetchPage: the page and where it was served from", async () => {
  using _net = stubNetwork((url) => {
    if (url.pathname === "/moved") {
      return new Response(null, {
        status: 301,
        headers: { location: "https://www.example.com/here" },
      });
    }
    if (url.pathname === "/gone") return html("<title>Gone</title>", 410);
    if (url.pathname === "/data") {
      return new Response("{}", {
        headers: { "content-type": "application/json" },
      });
    }
    return html("<title>Here</title>");
  });
  assertEquals(await fetchPage("https://example.com/moved"), {
    url: "https://www.example.com/here",
    html: "<title>Here</title>",
  });
  assertEquals(await fetchPage("https://example.com/data"), {
    url: "https://example.com/data",
  });
  await assertRejects(
    () => fetchPage("https://example.com/gone"),
    Error,
    "410",
  );
});