import { html, stubNetwork } from "../helpers.ts"; import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert"; import { fetchPage, fetchWithTimeout, isValidHttpUrl, readTextCapped, } from "../../remote/fetch.ts"; Deno.test("isValidHttpUrl: http(s) to public hosts, or our own pages", () => { assert(isValidHttpUrl("https://example.com/x")); assert(isValidHttpUrl("http://gerbeur.test/dumps/abc")); assertFalse(isValidHttpUrl("http://localhost:8000/")); assertFalse(isValidHttpUrl("http://10.0.0.1/")); assertFalse(isValidHttpUrl("file:///etc/passwd")); assertFalse(isValidHttpUrl("nope")); }); Deno.test("fetchWithTimeout: the deadline covers a body that never ends", async () => { // Headers at once, then a trickle that never finishes: the read must fail // at the deadline rather than hold the request open. using _net = stubNetwork((_url, init) => new Response( new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode("")); init?.signal?.addEventListener( "abort", () => controller.error(new DOMException("aborted", "AbortError")), ); }, }), { headers: { "content-type": "text/html" } }, ) ); const res = await fetchWithTimeout("https://example.com/slow", 50); await assertRejects(() => res.text()); }); Deno.test("readTextCapped: stops reading at the cap", async () => { let pulled = 0; const body = new ReadableStream({ pull(controller) { pulled++; controller.enqueue(new TextEncoder().encode("abcdefghij")); }, }); const text = await readTextCapped(new Response(body), 25); assertEquals(text, "abcdefghijabcdefghijabcde"); assert(pulled < 10); assertEquals(await readTextCapped(new Response("short")), "short"); }); Deno.test("fetchWithTimeout: follows redirects, checking every hop", async () => { using net = stubNetwork((url) => { if (url.pathname === "/start") { return new Response(null, { status: 302, headers: { location: "/end" } }); } if (url.pathname === "/to-private") { return new Response(null, { status: 301, headers: { location: "http://169.254.169.254/latest/meta-data" }, }); } if (url.pathname === "/loop") { return new Response(null, { status: 302, headers: { location: "/loop" }, }); } return new Response("done"); }); const res = await fetchWithTimeout("https://example.com/start"); assertEquals(await res.text(), "done"); assertEquals(net.calls.map((u) => u.pathname), ["/start", "/end"]); await assertRejects( () => fetchWithTimeout("https://example.com/to-private"), Error, "private", ); await assertRejects( () => fetchWithTimeout("https://example.com/loop"), TypeError, "Too many redirects", ); await assertRejects( () => fetchWithTimeout("ftp://example.com/"), TypeError, "Unsupported protocol", ); }); Deno.test("readTextCapped: decodes the charset the header or a declares", async () => { const cafe = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // "café" in Latin-1 const page = (contentType: string, head = "") => new Response( new Uint8Array([...new TextEncoder().encode(head), ...cafe]), { headers: { "content-type": contentType } }, ); assertEquals( await readTextCapped(page("text/html; charset=ISO-8859-1")), "café", ); assertEquals( await readTextCapped(page("text/html", '')), 'café', ); assertEquals( await readTextCapped(page("text/html; charset=nonsense")), "caf�", ); }); Deno.test("fetchPage: the page and where it was served from", async () => { using _net = stubNetwork((url) => { if (url.pathname === "/moved") { return new Response(null, { status: 301, headers: { location: "https://www.example.com/here" }, }); } if (url.pathname === "/gone") return html("Gone", 410); if (url.pathname === "/data") { return new Response("{}", { headers: { "content-type": "application/json" }, }); } return html("Here"); }); assertEquals(await fetchPage("https://example.com/moved"), { url: "https://www.example.com/here", html: "Here", }); assertEquals(await fetchPage("https://example.com/data"), { url: "https://example.com/data", }); await assertRejects( () => fetchPage("https://example.com/gone"), Error, "410", ); });