Add Intl.Segmenter modes: intl-word and grapheme

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project are documented in this file.
 The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and the project adheres to [Semantic Versioning](https://semver.org/).
 
+## [Unreleased]
+
+### Added
+- `intl-word` and `grapheme` modes via `Intl.Segmenter` (with a `locale`
+  option): locale-aware word diffs for unspaced scripts (Japanese, Chinese,
+  Thai) and cluster-safe character diffs (ZWJ emoji, combining sequences).
+  `refine` drops `intl-word` pairs to grapheme granularity. (#15)
+
 ## [1.1.0] - 2026-08-31
 
 ### Added

README.md

@@ -70,13 +70,24 @@ Returns `DiffEntry[]` — the shortest edit script between `a` and `b`.
 
 | option | type | default | description |
 |---|---|---|---|
-| `mode` | `'word' \| 'char' \| 'line'` | `'word'` | tokenization granularity |
+| `mode` | `'word' \| 'char' \| 'line' \| 'intl-word' \| 'grapheme'` | `'word'` | tokenization granularity |
+| `locale` | `string \| string[]` | runtime locale | BCP 47 locale(s) for the `Intl.Segmenter` modes |
 | `refine` | `boolean` | `false` | re-diff each delete/insert pair one level finer (`line`→word, `word`→char), e.g. `quick`→`quicker` reports just `+er` |
 | `heuristic` | `boolean` | `false` | cap the search cost like git does, keeping pathological inputs fast (the 227 ms worst case below drops to ~8 ms, +8% edit-script size); output stays identical to exact mode while the edit distance is small |
 
 - `word` — runs of Unicode letters/digits/underscore, whitespace runs, symbol runs
 - `char` — individual code points (surrogate-pair safe)
 - `line` — lines with their terminators attached
+- `intl-word` — locale-aware words via `Intl.Segmenter`: splits unspaced scripts (Japanese, Chinese, Thai) that `word` mode sees as one token
+
+  ```ts
+  diff('私は猫が好きです', '私は犬が好きです', { mode: 'intl-word', locale: 'ja' });
+  // equal '私は' · delete '猫' · insert '犬' · equal 'が好きです'
+  ```
+
+- `grapheme` — grapheme clusters via `Intl.Segmenter`: ZWJ emoji (👨‍👩‍👧) and combining sequences stay whole where `char` mode would split code points
+
+The `Intl.Segmenter` modes are opt-in because they're slower than the scanner modes; they throw a clear `TypeError` on runtimes without `Intl.Segmenter` (Node < 16, older browsers).
 
 ### `diffRanges(a, b, options?)`
 

src/index.ts

@@ -7,8 +7,16 @@ import { pushEntry, type DiffEntry, type DiffOperation } from './entries.ts';
 export { tokenize, type DiffMode, type DiffEntry, type DiffOperation };
 
 export interface DiffOptions {
-  /** Tokenization granularity. Defaults to 'word'. */
+  /**
+   * Tokenization granularity. Defaults to 'word'. The scanner modes
+   * ('word' | 'char' | 'line') are the fastest; 'intl-word' and 'grapheme'
+   * use Intl.Segmenter for locale-aware word boundaries (unspaced scripts
+   * like Japanese/Chinese/Thai) and cluster-safe characters (ZWJ emoji,
+   * combining sequences).
+   */
   mode?: DiffMode;
+  /** BCP 47 locale(s) for the Intl.Segmenter modes. Defaults to the runtime locale. */
+  locale?: string | string[];
   /**
    * Re-diffs each delete/insert pair one granularity finer ('line' pairs by
    * word, 'word' pairs by char), so replacing "quick" with "quicker" reports
@@ -45,13 +53,28 @@ export function diff(a: string, b: string, options: DiffOptions = {}): DiffEntry
   }
   const mode = options.mode ?? 'word';
   const heuristic = options.heuristic === true;
-  const entries = mode === 'char' ? diffChars(a, b, heuristic) : diffScanned(a, b, mode, heuristic);
-  if (options.refine === true && mode !== 'char') {
-    return refineEntries(entries, mode === 'line' ? 'word' : 'char', heuristic);
+  let entries: DiffEntry[];
+  if (mode === 'char') {
+    entries = diffChars(a, b, heuristic);
+  } else if (mode === 'intl-word' || mode === 'grapheme') {
+    entries = diffTokens(tokenize(a, mode, options.locale), tokenize(b, mode, options.locale), { heuristic });
+  } else {
+    entries = diffScanned(a, b, mode, heuristic);
+  }
+  const finer = REFINE_TARGET[mode];
+  if (options.refine === true && finer !== undefined) {
+    return refineEntries(entries, finer, heuristic, options.locale);
   }
   return entries;
 }
 
+/** Which granularity a refine pass drops to; char/grapheme are already finest. */
+const REFINE_TARGET: Partial<Record<DiffMode, DiffMode>> = {
+  line: 'word',
+  word: 'char',
+  'intl-word': 'grapheme',
+};
+
 /**
  * A changed region as code-unit offsets into the inputs:
  * a[aStart, aEnd) was replaced by b[bStart, bEnd). Either side (but never
@@ -95,13 +118,15 @@ export function diffRanges(a: string, b: string, options: DiffOptions = {}): Dif
 }
 
 /** Re-diffs adjacent delete/insert pairs at a finer granularity. */
-function refineEntries(entries: DiffEntry[], finerMode: DiffMode, heuristic: boolean): DiffEntry[] {
+function refineEntries(
+  entries: DiffEntry[], finerMode: DiffMode, heuristic: boolean, locale?: string | string[],
+): DiffEntry[] {
   const out: DiffEntry[] = [];
   for (let i = 0; i < entries.length; i++) {
     const entry = entries[i];
     const next = entries[i + 1];
     if (entry.operation === 'delete' && next !== undefined && next.operation === 'insert') {
-      for (const sub of diff(entry.text, next.text, { mode: finerMode, heuristic })) {
+      for (const sub of diff(entry.text, next.text, { mode: finerMode, heuristic, locale })) {
         pushEntry(out, sub.operation, sub.text);
       }
       i++;

test/segmenter.test.ts

@@ -0,0 +1,90 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { diff, tokenize, type DiffEntry } from '../src/index.ts';
+
+function joinSide(entries: DiffEntry[], skip: 'insert' | 'delete'): string {
+  let s = '';
+  for (const e of entries) if (e.operation !== skip) s += e.text;
+  return s;
+}
+
+test('tokenize: intl-word segments unspaced Japanese into words', () => {
+  assert.deepEqual(
+    tokenize('私は猫が好きです', 'intl-word', 'ja'),
+    ['私', 'は', '猫', 'が', '好き', 'です'],
+  );
+});
+
+test('tokenize: grapheme keeps ZWJ emoji and combining sequences whole', () => {
+  assert.deepEqual(tokenize('a👨‍👩‍👧b', 'grapheme'), ['a', '👨‍👩‍👧', 'b']);
+  assert.deepEqual(tokenize('', 'grapheme'), []);
+  assert.deepEqual(tokenize('', 'intl-word'), []);
+});
+
+test('intl-word: Japanese diff isolates the changed word (regular word mode cannot)', () => {
+  // Regular word mode sees one opaque letter-run, so everything changes.
+  const coarse = diff('私は猫が好きです', '私は犬が好きです');
+  assert.deepEqual(coarse, [
+    { operation: 'delete', text: '私は猫が好きです' },
+    { operation: 'insert', text: '私は犬が好きです' },
+  ]);
+  // intl-word isolates 猫 -> 犬.
+  assert.deepEqual(diff('私は猫が好きです', '私は犬が好きです', { mode: 'intl-word', locale: 'ja' }), [
+    { operation: 'equal', text: '私は' },
+    { operation: 'delete', text: '猫' },
+    { operation: 'insert', text: '犬' },
+    { operation: 'equal', text: 'が好きです' },
+  ]);
+});
+
+test('grapheme: ZWJ emoji replaced as one cluster (char mode splits code points)', () => {
+  const a = 'x👨‍👩‍👧y';
+  const b = 'x👨‍👩‍👦y';
+  assert.deepEqual(diff(a, b, { mode: 'grapheme' }), [
+    { operation: 'equal', text: 'x' },
+    { operation: 'delete', text: '👨‍👩‍👧' },
+    { operation: 'insert', text: '👨‍👩‍👦' },
+    { operation: 'equal', text: 'y' },
+  ]);
+});
+
+test('intl-word: refine drops to grapheme granularity', () => {
+  const entries = diff('color', 'colour', { mode: 'intl-word', refine: true });
+  assert.deepEqual(entries, [
+    { operation: 'equal', text: 'colo' },
+    { operation: 'insert', text: 'u' },
+    { operation: 'equal', text: 'r' },
+  ]);
+});
+
+test('segmenter modes: round-trip fuzz with mixed scripts', () => {
+  let state = 20270101 >>> 0;
+  const rng = () => {
+    state = (state * 1664525 + 1013904223) >>> 0;
+    return state / 0x100000000;
+  };
+  const pieces = ['猫', '好き', 'a', ' ', '👨‍👩‍👧', '한글', '\n', '.', 'देवनागरी'];
+  const make = () => {
+    const len = Math.floor(rng() * 25);
+    let s = '';
+    for (let i = 0; i < len; i++) s += pieces[Math.floor(rng() * pieces.length)];
+    return s;
+  };
+  for (let iter = 0; iter < 100; iter++) {
+    const a = make();
+    const b = make();
+    for (const mode of ['intl-word', 'grapheme'] as const) {
+      const entries = diff(a, b, { mode });
+      assert.equal(joinSide(entries, 'insert'), a, `a mismatch mode=${mode}`);
+      assert.equal(joinSide(entries, 'delete'), b, `b mismatch mode=${mode}`);
+    }
+  }
+});
+
+test('segmenter modes: heuristic flag composes', () => {
+  const a = '猫'.repeat(200) + '好き'.repeat(200);
+  const b = '犬'.repeat(180) + '嫌い'.repeat(180);
+  const entries = diff(a, b, { mode: 'grapheme', heuristic: true });
+  assert.equal(joinSide(entries, 'insert'), a);
+  assert.equal(joinSide(entries, 'delete'), b);
+});

tsconfig.json

@@ -3,7 +3,7 @@
     "target": "ES2020",
     "module": "ESNext",
     "moduleResolution": "bundler",
-    "lib": ["ES2020"],
+    "lib": ["ES2022"],
     "strict": true,
     "noEmit": true,
     "allowImportingTsExtensions": true,