citation-evidence/src/shared/text/normalize.test.ts

import { describe, expect, it } from "vitest";
import { NORMALIZE_VERSION, normalize } from "./normalize.js";

describe("normalize (NORMALIZE_VERSION=1)", () => {
  it("returns the version constant alongside the text", () => {
    const out = normalize("hello");
    expect(out.version).toBe(NORMALIZE_VERSION);
    expect(out.text).toBe("hello");
  });

  it("applies Unicode NFC composition", () => {
    // "é" decomposed (e + combining acute) vs precomposed.
    const decomposed = "café";
    const precomposed = "café";
    expect(normalize(decomposed).text).toBe(precomposed);
  });

  it("normalizes CRLF and CR line endings to LF", () => {
    expect(normalize("a\r\nb\rc").text).toBe("a\nb\nc");
  });

  it("collapses horizontal whitespace runs to a single space", () => {
    expect(normalize("a    b\t\tc d").text).toBe("a b c d");
  });

  it("preserves paragraph boundaries but collapses 3+ blank lines to one", () => {
    const input = "para one\n\n\n\npara two\n\npara three";
    expect(normalize(input).text).toBe("para one\n\npara two\n\npara three");
  });

  it("strips soft hyphens (German line-broken word reassembly)", () => {
    // German "Donau­dampf­schiff" line-broken with soft hyphens.
    expect(normalize("Donau­dampf­schiff").text).toBe(
      "Donaudampfschiff",
    );
  });

  it("strips soft hyphens that span a newline ('word-\\nfragment' → 'wordfragment')", () => {
    expect(normalize("word­\nfragment").text).toBe("wordfragment");
  });

  it("does not mangle ligatures (preserves the round-trip)", () => {
    // The ligature "fi" (U+FB01) is left as-is — NFC does NOT decompose it.
    // Test documents that current behavior so a future change is intentional.
    expect(normalize("efficient").text).toBe("efficient");
  });

  it("handles a mixed-whitespace paragraph realistically", () => {
    const input = "  First   line  \r\n   Second line.\r\n\r\n\r\nNext para.  ";
    expect(normalize(input).text).toBe("First line\nSecond line.\n\nNext para.");
  });

  it("returns an empty string for whitespace-only input", () => {
    expect(normalize("   \n\n  \t  ").text).toBe("");
  });
});