dev/third_party/pi/packages/durable/test/env-line-scan.test.ts

import { describe, expect, it } from "vitest";
import { StreamDecoder } from "../src/env/decode.ts";
import { LineScanner } from "../src/env/line-scan.ts";

function random(seed: number): () => number {
	let state = seed >>> 0;
	return () => {
		state = (state + 0x6d2b79f5) >>> 0;
		let t = state;
		t = Math.imul(t ^ (t >>> 15), t | 1);
		t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
		return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
	};
}

// Newlines, ASCII, a byte-order mark, valid multi-byte sequences, and bytes that form invalid or truncated sequences.
const PIECES = [
	[0x0a],
	[0x0a],
	[0x61],
	[0x62, 0x63],
	[0xef, 0xbb, 0xbf],
	[0xc3, 0xa9],
	[0xe2, 0x82, 0xac],
	[0xf0, 0x9f, 0x98, 0x80],
	[0xe2, 0x82],
	[0xff],
	[0x80],
	[0xf0, 0x9f],
	[0x0d, 0x0a],
];

function randomFile(next: () => number): Uint8Array {
	const bytes: number[] = [];
	const pieces = Math.floor(next() * 60);
	for (let index = 0; index < pieces; index++) bytes.push(...PIECES[Math.floor(next() * PIECES.length)]!);
	return Uint8Array.from(bytes);
}

const utf8 = (text: string): number => new TextEncoder().encode(text).length;

describe("StreamDecoder", () => {
	// Node's streaming TextDecoder with BOM handling dropped this U+FEFF, which follows an invalid sequence.
	it("keeps a U+FEFF that does not start the stream", () => {
		const bytes = Uint8Array.from([0xe2, 0x82, 0xef, 0xbb, 0xbf, 0x61]);
		const decoder = new StreamDecoder();
		const text = [...bytes].map((byte) => decoder.decode(Uint8Array.of(byte))).join("") + decoder.decode();
		expect(text).toBe(new TextDecoder().decode(bytes));
		expect(text).toBe("\ufffd\ufeffa");
	});

	it("decodes any chunking like decoding the whole stream", () => {
		for (let seed = 1; seed <= 5000; seed++) {
			const next = random(seed);
			const bytes = randomFile(next);
			const decoder = new StreamDecoder();
			let text = "";
			for (let offset = 0; offset < bytes.length; ) {
				const size = 1 + Math.floor(next() * 7);
				text += decoder.decode(bytes.subarray(offset, offset + size));
				offset += size;
			}
			expect(text + decoder.decode(), `seed ${seed}`).toBe(new TextDecoder().decode(bytes));
		}
	});
});

describe("LineScanner", () => {
	it("agrees with decoding and splitting the whole file", () => {
		for (let seed = 1; seed <= 5000; seed++) {
			const next = random(seed);
			const file = randomFile(next);
			const lines = new TextDecoder().decode(file).split("\n");
			const startLine = Math.floor(next() * (lines.length + 2));
			const endLine = next() < 0.3 ? undefined : startLine + 1 + Math.floor(next() * (lines.length + 1));
			const scanner = new LineScanner(startLine, endLine);
			for (let offset = 0; offset < file.length; ) {
				const size = 1 + Math.floor(next() * 7);
				scanner.push(file.subarray(offset, offset + size));
				offset += size;
			}
			const scan = scanner.finish();
			const message = `seed ${seed}`;
			expect(scan.newlines, message).toBe(lines.length - 1);
			const selected = lines.slice(startLine, endLine);
			const decode = (from: number, to: number) =>
				new TextDecoder("utf-8", { ignoreBOM: from > 0 }).decode(file.subarray(from, to));
			expect(decode(scan.start, scan.end), message).toBe(selected.join("\n"));
			expect(scan.selectedBytes, message).toBe(utf8(selected.join("\n")));
			if (startLine < lines.length) {
				expect(decode(scan.start, scan.firstLineEnd), message).toBe(lines[startLine]);
				expect(scan.firstLineBytes, message).toBe(utf8(lines[startLine]!));
				const lastLine = Math.min(endLine ?? lines.length, lines.length) - 1;
				const lastEnd = lastLine + 1 < lines.length ? file.indexOf(0x0a, scan.lastLineStart) : file.length;
				expect(decode(scan.lastLineStart, lastEnd), message).toBe(lines[lastLine]);
			} else {
				expect(scan, message).toMatchObject({ start: file.length, end: file.length, selectedBytes: 0 });
			}
		}
	});

	it("rejects empty or invalid ranges", () => {
		expect(() => new LineScanner(2, 2)).toThrow(RangeError);
		expect(() => new LineScanner(-1)).toThrow(RangeError);
		expect(() => new LineScanner(1.5)).toThrow(RangeError);
	});
});