diff --git a/docs/release_log.rst b/docs/release_log.rst index 84b774e..260b6fd 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -1,5 +1,9 @@ Release Log =========== +* 1.4.0 - Unreleased + + - Fix the ``"Lastname, Firstname"`` comma format not being recognized when the input uses the Arabic comma ``،`` (U+060C, the standard comma in Arabic/Persian/Urdu text) or the fullwidth CJK comma ``，`` (U+FF0C) instead of the ASCII comma: both variants now also split the format and no longer leak into the parsed output (closes #265) + * 1.3.1 - July 11, 2026 - Fix invisible Unicode bidirectional control characters (LRM/RLM/ALM, the embedding/override marks, and the isolates U+2066–U+2069) surviving parsing and sticking to ``first``/``last``/etc., so a copy-pasted right-to-left name silently failed equality and dedup. They are now stripped in preprocessing like emoji; disable via ``CONSTANTS.regexes.bidi = False`` (closes #266) diff --git a/nameparser/config/__init__.py b/nameparser/config/__init__.py index c455405..35ab9fd 100644 --- a/nameparser/config/__init__.py +++ b/nameparser/config/__init__.py @@ -590,8 +590,9 @@ class Constants: ``None`` (no additional splitting beyond the standard comma split). Note: setting this to ``","`` or ``", "`` has no additional effect — - the full name is already split on bare commas first, and each resulting - part is stripped of surrounding whitespace before this step runs. + the full name is already split on comma characters first (including the + Arabic ``،`` and fullwidth ``，`` variants), and each resulting part is + stripped of surrounding whitespace before this step runs. The delimiter is only applied to parts once they've been identified as a suffix group, so it never leaks into a first- or middle-name part. For diff --git a/nameparser/config/regexes.py b/nameparser/config/regexes.py index 6ff0b6b..792e826 100644 --- a/nameparser/config/regexes.py +++ b/nameparser/config/regexes.py @@ -29,6 +29,10 @@ "bidi": re_bidi, "phd": re.compile(r'\s(ph\.?\s+d\.?)', re.I), "space_before_comma": re.compile(r'\s+,'), + # ASCII comma plus its Arabic (U+060C) and fullwidth CJK (U+FF0C) + # counterparts, used to split "Last, First" format and to strip a + # trailing comma before parsing (#265). + "commas": re.compile(r'[,،，]'), "east_slavic_patronymic": re.compile( r'(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$', re.I, diff --git a/nameparser/parser.py b/nameparser/parser.py index 8059818..81fd625 100644 --- a/nameparser/parser.py +++ b/nameparser/parser.py @@ -923,7 +923,7 @@ def _apply_full_name(self, value: str | bytes, *, stacklevel: int) -> None: def collapse_whitespace(self, string: str) -> str: # collapse multiple spaces into single space string = self.C.regexes.spaces.sub(" ", string.strip()) - if string.endswith(","): + if string and self.C.regexes.commas.fullmatch(string[-1]): string = string[:-1] return string @@ -1195,8 +1195,14 @@ def parse_full_name(self) -> None: self._full_name = self.collapse_whitespace(self._full_name) - # break up full_name by commas - parts = [x.strip() for x in self._full_name.split(",")] + # break up full_name by commas. A missing "commas" key in a custom + # regexes dict falls back to RegexTupleManager's EMPTY_REGEX, whose + # .split() matches between every character rather than not + # splitting at all -- guard against that so a custom regexes dict + # that omits "commas" disables the comma split instead of shattering + # the name into single characters. + commas = self.C.regexes.commas + parts = [x.strip() for x in (commas.split(self._full_name) if commas.pattern else [self._full_name])] self._had_comma = len(parts) > 1 log.debug("full_name: %s", self._full_name) diff --git a/tests/test_comma_variants.py b/tests/test_comma_variants.py new file mode 100644 index 0000000..7e1a74e --- /dev/null +++ b/tests/test_comma_variants.py @@ -0,0 +1,47 @@ +from nameparser import HumanName +from nameparser.config import Constants +from nameparser.config.regexes import REGEXES + +from tests.base import HumanNameTestBase + + +class HumanNameCommaVariantsTests(HumanNameTestBase): + """Non-ASCII comma characters should split "Last, First" the same as ',' (#265).""" + + def test_arabic_comma_splits_lastname_format(self) -> None: + hn = HumanName("سلمان، محمد") + self.m(hn.first, "محمد", hn) + self.m(hn.last, "سلمان", hn) + + def test_fullwidth_comma_splits_lastname_format(self) -> None: + hn = HumanName("Smith，John") + self.m(hn.first, "John", hn) + self.m(hn.last, "Smith", hn) + + def test_arabic_comma_does_not_pollute_output(self) -> None: + hn = HumanName("سلمان، محمد") + self.assertNotIn("،", hn.last) + self.assertNotIn("،", str(hn)) + + def test_trailing_arabic_comma_stripped(self) -> None: + # matches ASCII behavior: a single word with a trailing comma has + # nothing after the comma, so it's a bare name, not "Last," + hn = HumanName("سلمان،") + self.m(hn.first, "سلمان", hn) + + def test_custom_regexes_without_commas_key_does_not_shatter_name(self) -> None: + # A custom regexes dict that omits "commas" entirely must not fall + # back to RegexTupleManager's EMPTY_REGEX default for splitting -- + # re.compile('').split(...) matches between every character, which + # explodes any name into single-char pieces instead of leaving it + # unsplit (the EMPTY_REGEX convention elsewhere in this codebase + # means "feature disabled", not "split on every character"). + # With comma splitting disabled, "Smith, John" is tokenized like any + # other no-comma input (word tokenizing drops the punctuation), + # yielding a plain first/last pair -- not the inverted "Last, First" + # reading, and definitely not single-character pieces. + custom = {k: v for k, v in REGEXES.items() if k != 'commas'} + c = Constants(regexes=custom) + hn = HumanName("Smith, John", constants=c) + self.m(hn.first, "Smith", hn) + self.m(hn.last, "John", hn)