From baf8fe68d860233d853c450f53c2ad082bd80e3c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 23 Sep 2026 22:39:04 -0700 Subject: [PATCH 1/9] fix(#459): case repair lays a letter mask, and writes a dotted credential and a numeral in capitals A capitalization_exceptions value is now a case mask -- the key's own letters and digits, each in the case it takes -- laid over the word as written, so one 'phd' entry repairs phd to PhD and ph.d. to Ph.D., and repair never adds or drops a character ('john smith iii.' keeps its period). A mask's own punctuation marks where its letters are joined and is never written: a single letter the writer split off beside a full stop is an initial and is capitalized where the mask keeps that letter inside a run ('p.h.d.' gives 'P.H.D.'), while a mask that spells the letter alone leaves it as the mask says ('h.c' keeps 'h.c.'). Lexicon raises ValueError on a value that spells anything else, at every entry point, the v1 shim's snapshot included; that is a decided exception to the shim's never-raise rule, recorded in AGENTS.md, since v1 substituted such a value and repair now only recases. The shipped map is phd/bsc/msc. md, ii, iii and iv leave it: a SUFFIX md is a listed acronym, and a SUFFIX roman numeral is now written in capitals by shape (vi through x included), on a hand copy of the pipeline's roman pattern pinned by test_regex_sync, since layering forbids _render the import. A suffix read as a credential by its dotted shape (shape:acronym, now defined in _types so _render can read it) repairs as a listed acronym does. A word the parse put in a NAME role repairs as a name word: 'iv smith' gives 'Iv Smith'. rules.md#R4 states the invariant and pins each clause; the rules corpus is regenerated and the ledger rows its new lines reach are re-recorded. No role field moves; the differential gate exits 0 at every baseline. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 +- docs/design/rules.md | 64 +++++- nameparser/_config_shim.py | 14 +- nameparser/_lexicon.py | 44 +++- nameparser/_pipeline/_state.py | 32 +-- nameparser/_render.py | 182 +++++++++++++--- nameparser/_types.py | 10 + nameparser/config/capitalization.py | 23 +- tests/test_capitalization.py | 58 +++-- tests/test_constants.py | 6 +- tests/test_output_format.py | 2 +- tests/v2/test_config_shim.py | 12 + tests/v2/test_facade.py | 2 +- tests/v2/test_ledger_guards.py | 41 +++- tests/v2/test_lexicon.py | 129 ++++++++++- tests/v2/test_properties.py | 15 +- tests/v2/test_regex_sync.py | 3 + tests/v2/test_render.py | 302 +++++++++++++++++++++++++- tools/differential/corpus_rules.jsonl | 22 ++ 19 files changed, 867 insertions(+), 96 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e4f01427..1b2d28c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -319,7 +319,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, the R1 entry pass `suffix_entries` re-run over the forced state so a suffix value's entries follow its own commas, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. **Between raise and silence sits the construction-time `UserWarning`**, for a gap that is statically decidable, harmless to SOME deliberate caller, and indistinguishable-from-working for everyone else: the segmenterless activation (#337 — `parser_for(locales.JA)` without a segmenter behaved exactly like a working parser minus the feature) warns rather than raises because the inert JA registration is itself a pinned property, and a warning is filterable by the caller who wants exactly that. The message must carry every applicable remedy and no inapplicable one (the `ja_segmenter` hint fires only when a Japanese script is among the dead ones). Test fuzzers that legitimately construct such configs suppress the warning by MESSAGE, never by category — a blanket `UserWarning` ignore would mask the next construction diagnostic (`_quiet_parser` in `tests/v2/test_properties.py` is the pattern). -- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Seven exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), `conjunctions_ambiguous = CONJUNCTIONS_AMBIGUOUS ∩ conjunctions` (#383/#479 behavior with no v1 manager of its own either, so the one v1 knob that reaches it is deleting the conjunction — which turns the marking off, `test_snapshot_removing_a_conjunction_turns_the_marker_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. +- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Seven exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), `conjunctions_ambiguous = CONJUNCTIONS_AMBIGUOUS ∩ conjunctions` (#383/#479 behavior with no v1 manager of its own either, so the one v1 knob that reaches it is deleting the conjunction — which turns the marking off, `test_snapshot_removing_a_conjunction_turns_the_marker_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. A `capitalization_exceptions` value that does not spell its key's letters raises `ValueError` at snapshot rather than being translated (#459), because v1 substituted such a value for the word and 2.4 repair only recases, so there is no v1 behavior to reproduce; pinned by `test_a_mismatched_exception_value_raises_at_the_first_parse`. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Every pipeline stage is one module holding one public function of the same name** (`_tokenize.py`/`tokenize`), with the exceptions named where they stand — `_group.py`'s `marker_run_length`, a shared predicate (mechanisms.md#ONE-PREDICATE-PER-QUESTION), and since #511 `_post_rules.py`'s `suffix_entries`, the R1 entry pass as a function, because `Parser.revise` runs that one pass over a forced-role sub-parse and a stage's tail cannot be called on its own — **and its module docstring declares the contract in three labelled lines**: `Consumes:` what it takes from `ParseState`, `Produces:` what it hands back, `Reads:` which `Policy`/`Lexicon` fields it consults. `Reads:` is the load-bearing one — it makes "which stage do I touch for this feature?" a grep rather than a read-through. The authority for the stage set is `_pipeline/__init__.py`'s `STAGES` (eight, and not public API) with the field-ownership map in `ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`; NOT rules.md, which is implementation-free by its own preamble and whose `implemented:` names modules honoring a rule rather than stages. A `_pipeline/` module that is not a stage says so in its first line instead (`_assemble.py`: "Not a stage: …", omitting `Reads:` because it consults neither), so absence of the three lines is a claim about the module rather than an oversight. Provenance: §5 of the 2026-07-11 conventions spec, recorded here 2026-08-16. - **A claim about WHICH STAGE or WHICH LAYER does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. That single claim then shipped SIX times across three correction rounds, which is the part worth internalizing: **when a mechanism claim turns out wrong, sweep for where else you wrote it, and sweep again at the END of the change over the words the change itself just added.** Prose density here means one idea lives in a docstring, a case note, a release-log entry and this file at once; the implementer working against a wrong mechanism is the person most likely to restate it; and rewriting a mechanism claim is writing one, so the correction earns the same one-command check as the original — two of the six instances were fresh errors introduced by the sentence fixing the previous one. **Adjacency is the trap.** The claim that feels already-known is the one about the neighbouring stage or the neighbouring layer: one comment block in `tests/v2/test_facade_cases.py` got the exception type, the raising layer, the skip mechanism, the count of skipped rows, and which row was blocked all wrong at once — every one a claim about `_config_shim` versus real 1.4.0, written from reasoning, in the file whose whole job is translating between them. What finally held was not better prose but moving the claim into a test (`_CORE_ONLY_IDS`), which cannot be wrong the way a sentence can. diff --git a/docs/design/rules.md b/docs/design/rules.md index 876c6ae4..43d9c8d0 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2150,7 +2150,7 @@ R4. Rationale: case repair is a display concern, applied only on request and never destructively. Case repair returns a repaired copy and never mutates the parse. Where it acts at all — R5 decides where — the copy honors the - casing a vocabulary entry records (Ph.D.) and the Mac/Mc + casing a vocabulary entry records (PhD, BSc) and the Mac/Mc convention (McDonald), not only ordinary word-by-word casing, and a part whose every word is particle vocabulary is repaired as ordinary name words, since none of them is doing a particle's @@ -2168,6 +2168,25 @@ R4. Rationale: case repair is a display concern, applied only on vocabulary does not know, was read as no generation and keeps its lowercase there like any other connective — the third part of a comma form is the shape that puts one there. + Repair changes case and nothing else: the repaired word is the + word as written, recased. A vocabulary entry records casing as a + mask — its word's letters, each in the case it takes — and repair + lays it over the word as the writer punctuated it, so the one + entry for phd repairs phd to PhD and ph.d. to Ph.D.; the mask + applies wherever the word stands, whatever the parse read it as. + A single letter split off alone beside a full stop is written as + an initial and takes capitals whatever the mask says there — but + only where the mask itself writes that letter inside a run of two + or more letters: the writer split a chunk the mask keeps + together, so the split-off letter is an initial. A run is of + LETTERS only, so a digit ends one exactly as a full stop does. A + run of two or more letters beside a full stop takes the mask's + case letter by letter and is never an initial. Where the mask + writes the letter as a run of one too, there is no split to + repair, and the mask's own case stands. So ph.d. is Ph.D. under + the phd mask, and p.h.d. — three initials, not the mask's one run + 'PhD' — is P.H.D.; but h.c. under a caller's own h.c mask, which + spells each letter alone, stays h.c. A name already written the way repair would write it comes back unchanged, measured by repair's own conventions rather than by the bearer's. A spelling written in a @@ -2175,18 +2194,47 @@ R4. Rationale: case repair is a display concern, applied only on nothing in the text marks it as a choice; where the text does mark one, R5 defers to it. A credential acronym the exceptions map does not carry is an initialism, so a single-case word the parse - put in the suffix role from the acronym vocabulary repairs to its + put in the suffix role from the acronym vocabulary, or read as a + credential by its dotted shape alone (S3), repairs to its all-caps spelling rather than a title-cased one, and that repair outranks the Mac/Mc convention where a word fits both (MCSE, not - McSe); a word in that vocabulary that parsed as an ordinary name - word repairs as that name word, and a suffix word that is neither - an acronym nor an exceptions-map entry -- the generational `jr`, - `sr` -- keeps its title case. + McSe). A roman numeral the parse put in the suffix role is + written in capitals the way a generation is written, whether or + not the vocabulary lists it (VI as well as III). Those two + classes read the role and the mask does not, so a word of either + class that parsed as an ordinary name word repairs as that name + word unless the exceptions map carries it (qc mp gives Qc MP, + phd smith gives PhD Smith), and a suffix word that is neither an + acronym, a numeral nor an exceptions-map entry -- the + generational `jr`, `sr` -- keeps its title case. "juan mcdonald" → capitalized="Juan McDonald" "Juan McDonald" → capitalized_forced="Juan McDonald" "ANH DO" → capitalized="Anh Do" "anh van do" → capitalized="Anh Van Do" - "john smith phd" → capitalized="John Smith Ph.D." + "john smith phd" → capitalized="John Smith PhD" + "john smith ph.d." → capitalized="John Smith Ph.D." + "JOHN SMITH PH.D." → capitalized="John Smith Ph.D." + "john smith p.h.d." → capitalized="John Smith P.H.D." + "john smith ph. d." → capitalized="John Smith Ph. D." + "john smith bsc" → capitalized="John Smith BSc" + "Dr. med. univ. Margit Popp, MSc" → capitalized_forced="Dr. Med. Univ. Margit Popp MSc" + "MSc Dr. med. univ." → capitalized_forced="MSc Dr. Med. Univ." + "Md Abdul Karim" → capitalized_forced="Md Abdul Karim" + "john smith md" → capitalized="John Smith MD" + "john smith m.d." → capitalized="John Smith M.D." + "john smith iii." → capitalized="John Smith III." + "john smith x.y.z." → capitalized="John Smith X.Y.Z." + "John Smith R.A.I." → capitalized_forced="John Smith R.A.I." + "john doe q.c." → capitalized="John Doe Q.C." + "Jack X.Y.Z." → capitalized_forced="Jack X.y.z." · boundary + "qc mp" → capitalized="Qc MP" · boundary + "john smith ed" → capitalized="John Smith ED" + "john smith vi" → capitalized="John Smith VI" + "john smith ix" → capitalized="John Smith IX" + "john smith iii" → capitalized="John Smith III" + "john smith, v" → capitalized="John Smith V" + "john smith xi" → capitalized="John Smith Xi" · boundary + "john smith, vi" → capitalized="Vi John Smith" · boundary "john smith mba" → capitalized="John Smith MBA" "john smith mcse" → capitalized="John Smith MCSE" "john smith jr" → capitalized="John Smith Jr" · boundary @@ -2237,7 +2285,7 @@ R4. Rationale: case repair is a display concern, applied only on both of the parsed name's views: a middle revised to "e-f" repairs to "E-F" as the parsed name does, where splicing the same text in gives "e-F". - history: decisions.md#R4 · interacts: R2, R3, R5 · implemented: nameparser/_render.py + history: decisions.md#R4 · interacts: R2, R3, R5, S3 · implemented: nameparser/_render.py, nameparser/_lexicon.py R5. Rationale: mixed case is evidence that the writer cased the name deliberately, and a repair cannot tell a deliberate spelling from diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 601dd5d0..390483a9 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -1093,7 +1093,19 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: honorific_tails=GLUED_HONORIFICS & suffix_words, # TupleManager is dict[str, object] (v1 parity: values were # never statically str-typed); every real entry is a str, - # same assumption _DelimiterManager's sentinel lookup makes + # same assumption _DelimiterManager's sentinel lookup makes. + # NOT translated: the one DECIDED exception to this + # method's never-raise rule (#459, decisions.md#R4 and + # #3-0-reevaluations) is a value that does not spell its + # key's letters, which raises ValueError from Lexicon + # here, at the first parse -- v1 substituted such a value + # for the word, 2.4 repair only recases, so there is no + # v1 behavior left to reproduce, and no such value was + # found in the tracker, the docs or any test. A value that + # merely carries punctuation is NOT such an exception -- + # it is a legal mask (the punctuation marks which letters + # are joined and is never written into the word), so it + # passes through untranslated and unwarned like any other. capitalization_exceptions=tuple( sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] ) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 74241cf1..07ea3d4b 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -376,6 +376,10 @@ def _normset( return frozenset(normalized) +def _alnum(text: str) -> str: + return "".join(c for c in text if c.isalnum()) + + def _normpairs( raw: Mapping[str, str] | Iterable[tuple[str, str]], ) -> tuple[tuple[str, str], ...]: @@ -427,6 +431,36 @@ def _normpairs( f"empty (lowercase + strip full stops/whitespace leaves " f"nothing)" ) + # A value is a case MASK (#459): the key's own letters and + # digits recased. Its punctuation marks where its letters are + # JOINED -- letters written side by side are one run, and any + # character that is not a letter between them (a full stop, a + # space, a digit) ends the run, so the mask's own case stands + # for a letter it spells alone ('h.c' on 'h.c.' stays 'h.c.'). + # It is never written into the word: case repair keeps the + # WRITER's own punctuation, not the mask's. Compared through + # the same fold as the key, which keeps interior periods + # ('Ph.D.' is stored 'ph.d') and composes NFC, so both sides + # are read alike. A raise, not a warning: a mismatched value + # used to be SUBSTITUTED for the word, and rules.md#R4: + # "Repair changes case and nothing else", so there is no + # reading of one that repair can honor. + if _alnum(_normalize(v)) != _alnum(normalized_key): + upper = normalized_key.upper() + # The offered spelling must itself construct: .upper() can + # change a letter's COUNT ('straße' -> 'STRASSE', ß -> SS), + # which would fail the very check above if pasted. Falling + # back to the key unchanged -- an identity mask, always + # legal -- keeps the offer actionable in every case rather + # than only the common one. + offered = upper if _alnum(_normalize(upper)) == _alnum( + normalized_key) else normalized_key + raise ValueError( + f"capitalization_exceptions value {v!r} for key {k!r} " + f"does not spell the key's letters: a value is a case " + f"mask, the key's own letters recased -- e.g. " + f"capitalization_exceptions=(({normalized_key!r}, " + f"{offered!r}),)") # capitalized() looks words up one at a time (the _WORD regex # never yields spaces), so a multi-word key is unreachable. # interior whitespace test; split() covers all Unicode whitespace @@ -566,8 +600,14 @@ class Lexicon: #: vocabulary is. Full default list: #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`. honorific_tails: frozenset[str] = frozenset() - #: Lowercase word -> exact-cased replacement used by capitalized() - #: ("phd" -> "Ph.D."). Pair-valued: change it with + #: Lowercase word -> case mask used by capitalized(): the word's + #: own letters and digits, each in the case it takes ("phd" -> + #: "PhD"), laid over the word as written, so the one entry + #: repairs "ph.d." to "Ph.D.". The mask's own punctuation marks + #: where its letters are joined into one run versus split apart, + #: and is never written into the word -- repair keeps the writer's + #: own punctuation. A value that does not spell the key's letters + #: raises ValueError. Pair-valued: change it with #: dataclasses.replace(), not add()/remove(); read it as a mapping #: via capitalization_exceptions_map. Full default mapping: #: :data:`~nameparser.config.capitalization.CAPITALIZATION_EXCEPTIONS`. diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 870a7ef1..968d9844 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -17,7 +17,8 @@ from nameparser._lexicon import Lexicon from nameparser._policy import Policy -from nameparser._types import AmbiguityKind, Role, Segmenter, Span +from nameparser._types import (SHAPE_ACRONYM_TAG, AmbiguityKind, + Role, Segmenter, Span) # The comma characters (ASCII/Arabic/fullwidth, #265). Shared here so @@ -64,20 +65,21 @@ class WorkToken: #: cannot reach the other way). _NEVER_FLIPPED = frozenset({"vocab:bound-given", "initial"}) -#: The by-shape half of #289/#516's ambiguous credential class: a -#: token classify admits to `vocab:suffix-ambiguous`'s READING by -#: SHAPE rather than by the listed vocabulary -#: (`Policy.unlisted_dotted_suffixes` is the first emitter; -#: `Policy.unlisted_caps_suffixes` is the second, and classify writes -#: the tag from both branches). One constant, not a string literal at -#: each site, because the readers that must tell a by-shape member -#: apart from a listed one -- `_pieces.peel_trailing` and -#: `_pieces.listed_lean`, which `segment_suffix_reading` asks through, -#: so the reading it decides is second-hand -- cannot afford to spell -#: it several ways and have one of them typo silently past the -#: others. The two sites that want EITHER half read -#: `_AMBIGUOUS_CREDENTIAL_TAGS` below rather than this constant. -SHAPE_ACRONYM_TAG = "shape:acronym" +# SHAPE_ACRONYM_TAG, imported from _types above -- where it lives +# since #459, because case repair reads it too and _render may not +# import the pipeline -- is the by-shape half of #289/#516's +# ambiguous credential class: a token classify admits to +# `vocab:suffix-ambiguous`'s READING by SHAPE rather than by the +# listed vocabulary (`Policy.unlisted_dotted_suffixes` is the first +# emitter; `Policy.unlisted_caps_suffixes` is the second, and +# classify writes the tag from both branches). One constant, not a +# string literal at each site, because the readers that must tell a +# by-shape member apart from a listed one -- `_pieces.peel_trailing` +# and `_pieces.listed_lean`, which `segment_suffix_reading` asks +# through, so the reading it decides is second-hand -- cannot afford +# to spell it several ways and have one of them typo silently past +# the others. The two sites that want EITHER half read +# `_AMBIGUOUS_CREDENTIAL_TAGS` below rather than this constant. #: The MEMBERSHIP half of the same class: classify's tag for a token #: the ambiguous credential vocabulary claims, by listing diff --git a/nameparser/_render.py b/nameparser/_render.py index 66783384..fcb9b3d2 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -1,10 +1,10 @@ """Rendering for the 2.0 API: ParsedName -> display strings. Layering: imports nameparser._types, and nameparser._lexicon for -Lexicon.default() (capitalized() with lexicon=None) and _normalize -(enforced by tests/v2/test_layering.py). Parsing code never imports -this module; ParsedName's rendering methods delegate here via -call-time imports. +Lexicon.default() (capitalized() with lexicon=None), _normalize and +FULL_STOPS (enforced by tests/v2/test_layering.py). Parsing code +never imports this module; ParsedName's rendering methods delegate +here via call-time imports. Malformed str.format specs beyond unknown keys (positional fields, bad conversions) surface the raw str.format error; only unknown KEYS @@ -14,10 +14,11 @@ import re -from nameparser._lexicon import Lexicon, _normalize -from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, - UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, - Ambiguity, ParsedName, Role, Token) +from nameparser._lexicon import FULL_STOPS, Lexicon, _normalize +from nameparser._types import (FOLDED_TAG, SHAPE_ACRONYM_TAG, + UNCLASSIFIED_TAG, UNJOINED_CONJUNCTION_TAG, + UNJOINED_TAG, Ambiguity, ParsedName, Role, + Token) _SPACES = re.compile(r"\s+") _SPACE_BEFORE_COMMA = re.compile(r"\s+,") @@ -87,6 +88,17 @@ # still not worth the import layering forbids. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") +# v1 regexes.py "roman_numeral" -- the pipeline's _vocab._ROMAN, +# copied by hand because layering forbids this module the import +# (the reason _INITIAL above is a copy too) and pinned against config +# by tests/v2/test_regex_sync.py. Its one reader is _cap_word's +# numeral clause, which asks a RENDERING question of a word the parse +# already put in the suffix role -- how a numeral is written -- and +# not the parse's own question of whether the word is a suffix, so +# the role is honored rather than re-derived +# (mechanisms.md#RENDER-HONORS-THE-PARSE). +_ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I) + def _reads_as_conjunction(word: str, lex: Lexicon) -> bool: """v1's is_conjunction, asked only where the CALLER supplies no @@ -210,10 +222,103 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str return _format_spec(spec, values, "initials", _INITIALS_KEYS) +def _letter_run_ge2(text: str) -> list[bool]: + """One flag per alphanumeric character of `text`, in the order + _apply_mask below walks them: True where that character is a + LETTER with a letter immediately before or after it WITHIN + `text` -- a full stop or a digit breaks a run, exactly as either + breaks one in the word _apply_mask is casing. A digit's own flag + is always False; the initial-beside-a-stop exception only ever + concerns letters.""" + n = len(text) + flags: list[bool] = [] + for i, c in enumerate(text): + if not c.isalnum(): + continue + if not c.isalpha(): + flags.append(False) + continue + prev_letter = i > 0 and text[i - 1].isalpha() + next_letter = i + 1 < n and text[i + 1].isalpha() + flags.append(prev_letter or next_letter) + return flags + + +def _apply_mask(word: str, mask: str) -> str | None: + """rules.md#R4's mask: `word` with each letter or digit recased to + the case of the mask's alphanumeric in the same position, and + every other character kept where the writer put it -- 'ph.d.' + under 'PhD' is 'Ph.D.'. None where the two alphanumeric counts + differ. The lookup key makes that rare and not impossible: the + key is the word NFC-composed, so a word written in decomposed + hangul spells one syllable in two letters and still finds a + one-letter key. The caller falls through to its next clause + rather than guess. + + One exception the mask does not decide: a SINGLE LETTER split off + alone beside a full stop (FULL_STOPS, not the ASCII period alone + -- #322) is written in capitals whatever the mask says there -- + but only where the MASK ALSO writes that letter inside a run of + two or more letters: the writer split a chunk the mask keeps + together, so the split-off letter is an initial. 'p.h.d.' under + 'PhD' is 'P.H.D.': p, h and d all sit inside the mask's ONE run, + 'PhD', so each, written alone beside a stop, is an initial and is + capitalized. Where the mask spells the letter alone too -- a + caller's own 'h.c' mask on 'h.c.' -- there is no split to repair, + and the mask's own case stands unchanged. A run of two or more + letters beside a full stop ('sc' in 'b.sc.') is never an initial + either way and always takes the mask's case letter by letter. + + Casing goes through the WHOLE word (word.lower()/word.upper()) + rather than per character, when both have the same length as + `word`: a per-character str.lower()/str.upper() call is + context-free and gets some letters wrong that the whole-word form + gets right -- a medial sigma where Greek wants a final one, for + one. A mask letter is read as upper when it is not lower + (`not letter.islower()`), so a titlecase letter (Unicode category + Lt) reads as upper rather than lower -- a known limit: no + per-character titlecase mapping is attempted.""" + mask_chars = [c for c in mask if c.isalnum()] + if len(mask_chars) != sum(1 for c in word if c.isalnum()): + return None + mask_run = _letter_run_ge2(mask) + lowered = word.lower() + uppered = word.upper() + same_length = len(lowered) == len(word) and len(uppered) == len(word) + n = len(word) + out: list[str] = [] + at = 0 + for i, c in enumerate(word): + if not c.isalnum(): + out.append(c) + continue + mask_char = mask_chars[at] + in_mask_run = mask_run[at] + at += 1 + if c.isalpha(): + prev_letter = i > 0 and word[i - 1].isalpha() + next_letter = i + 1 < n and word[i + 1].isalpha() + beside_stop = (i > 0 and word[i - 1] in FULL_STOPS) or ( + i + 1 < n and word[i + 1] in FULL_STOPS) + if (not prev_letter and not next_letter and beside_stop + and in_mask_run): + out.append(uppered[i] if same_length else c.upper()) + continue + if mask_char.islower(): + out.append(lowered[i] if same_length else c.lower()) + else: + out.append(uppered[i] if same_length else c.upper()) + return "".join(out) + + def _cap_word(word: str, role: Role, tags: frozenset[str], lex: Lexicon) -> str: - # v1 cap_word order: particle/conjunction rule first, then the - # exceptions map, then Mac/Mc, then str.capitalize + # Clause order: the particle/connective arm, then the exceptions + # map as a mask, then the acronym clause (listed or by shape), + # then the numeral clause, then Mac/Mc, then str.capitalize. v1's + # cap_word had the first, second, fifth and sixth in this order; + # the mask ahead of the acronym clause is what gives bsc 'BSc' + # though bsc is a listed acronym too. normalized = _normalize(word) # rules.md#R4: "a part whose every word is particle vocabulary is # repaired as ordinary name words, since none of them is doing a @@ -318,25 +423,46 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], and _reads_as_conjunction(word, lex))): return word.lower() # v1 cap_word tries the edge-stripped form, then the period-free - # form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map) + # form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map). The + # value found is a MASK, not a replacement (#459): it recases the + # word as the writer punctuated it, so 'Ph.D.' under 'PhD' stays + # 'Ph.D.' and 'phd' becomes 'PhD'. v1 substituted the value, which + # is how 'md' became 'M.D.' and 'iii.' lost its period; + # rules.md#R4: "Repair changes case and nothing else". Role-free, + # as the map always was: an entry is the caller saying how a word + # is written wherever it stands. for key in (normalized, normalized.replace(".", "")): - exception = lex.capitalization_exceptions_map.get(key) - if exception is not None: - return exception + mask = lex.capitalization_exceptions_map.get(key) + if mask is not None: + masked = _apply_mask(word, mask) + if masked is not None: + return masked # A credential acronym the exceptions map doesn't carry (mba, jd, - # qc, mp, ...) is an initialism, not a word to title-case: a one- - # case name repairs to the acronym's caps instead of 'Mba' (#459). - # The exceptions map is consulted first and holds the entries that - # spell differently -- md -> M.D. and phd -> Ph.D. (the generational - # ii/iii/iv are suffix_words, not acronyms, and ride the map because - # str.capitalize() would give 'Ii'). The all-caps default is the - # right call for an initialism; its cost is that an acronym - # conventionally written mixed-case (bsc, msc) reads all-caps here - # (BSc -> BSC under force) rather than mixed, which the letter-mask - # design deferred to #459 is meant to recover. Gated on the SUFFIX - # role so a word that is a family name only happens to be in the - # vocabulary (anh van DO) still repairs as an ordinary name word. - if role is Role.SUFFIX and normalized.replace(".", "") in lex.suffix_acronyms: + # qc, and md since #459 took it out of the map) is an initialism, + # not a word to title-case: a one-case name repairs to the + # acronym's caps instead of 'Mba' (#459). So is a word classify + # admitted to the credential class by its dotted SHAPE alone + # (SHAPE_ACRONYM_TAG; rules.md#S3's unlisted 'x.y.z.'), which has + # no vocabulary entry to be listed in. The mask above is asked + # first, which is what keeps a conventionally mixed-case acronym as + # it is written (bsc -> BSc) though it is listed here too. Gated + # on the SUFFIX role so a word that is a family name only happens + # to be in the vocabulary (anh van DO) still repairs as an ordinary + # name word -- #459's given-role half, decided: repair follows the + # role the parse chose ('qc mp' -> 'Qc MP'). + if role is Role.SUFFIX and ( + normalized.replace(".", "") in lex.suffix_acronyms + or SHAPE_ACRONYM_TAG in tags): + return word.upper() + # A roman numeral the parse put in the suffix role is written in + # capitals whether or not the vocabulary lists it: 'vi' through + # 'x' carry no vocabulary tag and title-cased to 'Vi'/'Ix' until + # #459, while 'ii'/'iii'/'iv' rode the exceptions map, which is + # why they left it. Suffix-gated for the acronym clause's reason + # -- 'Vi' is a given name -- and it is also what writes a + # generational 'i' the connective arm's `generation` guard let + # through ('Carod i' forced -> 'Carod I'). + if role is Role.SUFFIX and _ROMAN.match(normalized): return word.upper() if _MAC.match(word): return _MAC.sub( @@ -365,6 +491,8 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, touched unless force=True; the gate reads the joined token texts (not render() output -- the case gate stays decoupled from spec formatting and the #254 collapse). + Repair changes case and nothing else: an exceptions-map value is + a mask recasing the word as written (#459), never a replacement. The repair reads token TAGS as well as texts: a part whose every word is particle vocabulary is repaired as ordinary name words, and the mark saying so comes from the pipeline, as does the diff --git a/nameparser/_types.py b/nameparser/_types.py index de7d4386..f2b8907d 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -151,6 +151,16 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: not the signal either -- an ordinary parsed name word carries none. UNCLASSIFIED_TAG = "vocab:unclassified" +#: The by-shape half of #289/#516's ambiguous credential class: the +#: tag classify writes on a token it admits to the credential reading +#: by its SHAPE rather than by listed vocabulary. Defined here, at +#: the bottom of the graph, because a render view reads it as well +#: as the pipeline: case repair writes such a suffix in capitals as +#: it does a listed acronym (#459), and _render may not import +#: _pipeline. _pipeline/_state.py re-exports it beside the +#: membership half and says why the stages need ONE constant. +SHAPE_ACRONYM_TAG = "shape:acronym" + #: The one-element tag set its two producers stamp, built once. _UNCLASSIFIED = frozenset({UNCLASSIFIED_TAG}) diff --git a/nameparser/config/capitalization.py b/nameparser/config/capitalization.py index 6973c7d4..4e431918 100644 --- a/nameparser/config/capitalization.py +++ b/nameparser/config/capitalization.py @@ -1,16 +1,25 @@ from nameparser.config._invariants import assert_normalized CAPITALIZATION_EXCEPTIONS = { - 'ii': 'II', - 'iii': 'III', - 'iv': 'IV', - 'md': 'M.D.', - 'phd': 'Ph.D.', + 'bsc': 'BSc', + 'msc': 'MSc', + 'phd': 'PhD', } """ -Any pieces that are not capitalized by capitalizing the first letter. +Words whose case ``str.capitalize()`` gets wrong, each mapped to a +case MASK: the key's own letters and digits, each in the case it +should take. Case repair lays the mask over the word as it was +written and keeps every other character where the writer put it, so +``'phd': 'PhD'`` repairs ``phd`` to ``PhD`` and ``ph.d.`` to +``Ph.D.``. The mask's own punctuation marks where its letters are +joined into one run versus split apart, and is never written into +the word -- repair keeps the writer's own punctuation. A value that +does not spell its key's letters is a ``ValueError`` when a +``Lexicon`` is built from it. A credential acronym written all in +capitals and a roman numeral need no entry: case repair writes a +suffix of either kind in capitals by itself. """ -# Keys only -- the values are the exact-cased replacements, cased on purpose. +# Keys only -- the values are case masks, cased on purpose. assert_normalized("CAPITALIZATION_EXCEPTIONS", CAPITALIZATION_EXCEPTIONS) diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index 59253f34..bd81f226 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -5,6 +5,7 @@ import pytest from nameparser import HumanName +from nameparser.config import Constants from tests.base import HumanNameTestBase @@ -90,22 +91,27 @@ def test_capitalize_empty_suffix_produces_no_spurious_tokens(self) -> None: def test_capitalize_single_suffix_still_works(self) -> None: hn = HumanName('JOHN DOE PHD') hn.capitalize() - self.assertEqual(hn.suffix_list, ['Ph.D.']) + self.assertEqual(hn.suffix_list, ['PhD']) def test_capitalize_multiple_suffixes_still_split_correctly(self) -> None: hn = HumanName('JOHN DOE PHD MD') hn.capitalize() # The split this guards is capitalize() giving each word its own - # exception form rather than title-casing the run, and that is - # untouched. The two words are ONE entry since #436 -- the writer - # spaced them, so they render with a space -- and one entry is one - # suffix_list element. A deliberate deviation from 1.4.0, which - # inserted a comma into a run the writer had spaced. - self.assertEqual(hn.suffix_list, ['Ph.D. M.D.']) + # repair rather than title-casing the run, and that is + # untouched: PHD by the exceptions map's mask, MD by the acronym + # clause since #459 took md out of the map. The two words are + # ONE entry since #436 -- the writer spaced them, so they render + # with a space -- and one entry is one suffix_list element. A + # deliberate deviation from 1.4.0, which inserted a comma into a + # run the writer had spaced, and gave 'Ph.D.', 'M.D.' besides. + self.assertEqual(hn.suffix_list, ['PhD MD']) def test_capitalize_suffix_acronym_with_dots(self) -> None: - # Suffixes already written with dots (e.g. "M.D.") should capitalize - # to their exception form, not title-case to "M.d." (issue #141) + # Suffixes already written with dots (e.g. "M.D.") keep them and + # do not title-case to "M.d." (issue #141). Through 2.3 the + # exceptions map gave this string by SUBSTITUTING 'M.D.'; since + # #459 md is a listed acronym the acronym clause writes in + # capitals, recasing the word as written. hn = HumanName('GREGORY HOUSE M.D.') hn.capitalize() self.assertEqual(hn.suffix, 'M.D.') @@ -124,17 +130,43 @@ def test_capitalize_suffix_acronym_is_all_caps(self) -> None: hn.capitalize() self.m(str(hn), expect, hn) - # The exceptions map's five keep their special casing; the new - # all-caps path must not shadow them (#459). + # The exceptions map is asked before the all-caps acronym clause, + # and since #459 it holds case MASKS: bsc and msc are listed + # acronyms too, and the mask is what keeps them mixed-case where + # the acronym clause would give 'BSC'. md left the map and reads + # 'MD' by the acronym clause; 1.4.0 through 2.3.0 gave 'M.D.' and + # 'Ph.D.', the map substituting its value for the word. def test_capitalize_exceptions_still_win_over_acronyms(self) -> None: for src, expect in [ - ('john smith md', 'John Smith M.D.'), - ('john smith phd', 'John Smith Ph.D.'), + ('john smith md', 'John Smith MD'), + ('john smith phd', 'John Smith PhD'), + ('john smith ph.d.', 'John Smith Ph.D.'), + ('john smith bsc', 'John Smith BSc'), + ('JOHN SMITH MSC', 'John Smith MSc'), ]: hn = HumanName(src) hn.capitalize() self.m(str(hn), expect, hn) + # #459: a capitalization_exceptions value is a case MASK -- the + # key's own letters recased -- so a v1 Constants carrying any other + # value raises at the first parse, where the snapshot builds the + # Lexicon. A DECIDED exception to the shim's never-raise rule + # (decisions.md#R4, and #3-0-reevaluations): 1.4.0 substituted + # such a value for the word, repair now only recases, and no such + # value was found in the tracker, the docs or any test. + def test_a_mismatched_exception_value_raises_at_the_first_parse( + self, + ) -> None: + c = Constants(capitalization_exceptions={'jr': 'Junior'}) + with pytest.raises(ValueError, + match="does not spell the key's letters"): + HumanName('john smith jr', constants=c) + ok = Constants(capitalization_exceptions={'jr': 'JR'}) + hn = HumanName('john smith jr', constants=ok) + hn.capitalize() + self.m(str(hn), 'John Smith JR', hn) + # A word in the acronym vocabulary that parses as a family name # still repairs as an ordinary name word, not an acronym (#459). def test_capitalize_family_name_in_acronym_vocab_stays_title_case(self) -> None: diff --git a/tests/test_constants.py b/tests/test_constants.py index a8ebe82d..4788277d 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -749,9 +749,9 @@ def test_tuplemanager_delattr_removes_dict_entry(self) -> None: ``del tm['key']`` are the same operation. """ c = Constants() - self.assertIn('ii', c.capitalization_exceptions) - del c.capitalization_exceptions.ii # type: ignore[attr-defined] - self.assertNotIn('ii', c.capitalization_exceptions) + self.assertIn('phd', c.capitalization_exceptions) + del c.capitalization_exceptions.phd # type: ignore[attr-defined] + self.assertNotIn('phd', c.capitalization_exceptions) def test_assigning_iterable_to_set_attr_wraps_and_normalizes(self) -> None: """Assigning a plain iterable to a set field wraps it in a SetManager. diff --git a/tests/test_output_format.py b/tests/test_output_format.py index 08e37a27..64762dc8 100644 --- a/tests/test_output_format.py +++ b/tests/test_output_format.py @@ -28,7 +28,7 @@ def test_capitalize_name_constants_attribute(self) -> None: c = Constants() c.capitalize_name = True hn = HumanName("bob v. de la macdole-eisenhower phd", constants=c) - self.assertEqual(str(hn), "Bob V. de la MacDole-Eisenhower Ph.D.") + self.assertEqual(str(hn), "Bob V. de la MacDole-Eisenhower PhD") def test_force_mixed_case_capitalization_constants_attribute(self) -> None: c = Constants() diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index f6f53c8f..585f60e3 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -375,6 +375,18 @@ def test_v14_pickle_restores_and_parses_warning_free() -> None: HumanName("Jane Roe", constants=c) +def test_a_non_str_exception_value_raises_typeerror_at_the_first_parse( +) -> None: + """The shim does not translate or otherwise intercept a + capitalization_exceptions value -- a caller's non-str entry + (v1's TupleManager stores dict[str, object], never statically + str-typed) reaches Lexicon's own TypeError unchanged, at the + first parse where the snapshot is built.""" + c = Constants(capitalization_exceptions={'phd': 42}) + with pytest.raises(TypeError, match="str -> str"): + HumanName("john smith phd", constants=c) + + def test_snapshot_keeps_a_multi_word_first_name_title() -> None: # v1 looks first_name_titles up on the joined title string, so a # multi-word entry is reachable with only its WORDS in titles. diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index e8b38f21..3c5b7c72 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -363,7 +363,7 @@ def test_render_default_setters_validate() -> None: def test_capitalize_gate_and_force() -> None: n = HumanName("bob v. de la macdole-eisenhower phd") n.capitalize() - assert str(n) == "Bob V. de la MacDole-Eisenhower Ph.D." + assert str(n) == "Bob V. de la MacDole-Eisenhower PhD" m = HumanName("Shirley Maclaine") # mixed case: untouched m.capitalize() assert str(m) == "Shirley Maclaine" diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index c52de55d..f76a1638 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -3358,8 +3358,12 @@ def _claim(rule: dict) -> _Claim: # 2026-09-22, #397 follow-up: 363 -> 364, the one new comma # name the M2 deviation example adds. Reach again, verified # name by name. + # 2026-09-23, #459: 364 -> 366, the two comma-written numeral + # lines rules.md#R4 gained ('john smith, v', 'john smith, vi'); + # its forced 'Dr. med. univ. Margit Popp, MSc' line was already + # in corpus_issues.jsonl. Reach, verified name by name. "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(364, ('given', 'suffix', 'title'), 'e3bf2a2b8de9', None), + _Claim(366, ('given', 'suffix', 'title'), '0b1a15dd9b78', None), "fix(comma-family) a comma followed only by titles keeps the given/family split": _Claim(2, ('family', 'given'), "5bd9c6d96c38", None), "fix(comma-family) a comma followed only by titles keeps the given/family split, the C1 example": @@ -3415,8 +3419,12 @@ def _claim(rule: dict) -> _Claim: # name as the rule above and for the same reason. # 2026-09-22, #397 follow-up: 363 -> 364, the same one new # comma name as the rule above and for the same reason. + # 2026-09-23, #459: 364 -> 366, the two comma-written numeral + # lines rules.md#R4 gained ('john smith, v', 'john smith, vi'); + # its forced 'Dr. med. univ. Margit Popp, MSc' line was already + # in corpus_issues.jsonl. Reach, verified name by name. "fix(comma-precomma-family) pre-comma run reads as family, not given": - _Claim(364, ('family', 'given'), 'e3bf2a2b8de9', None), + _Claim(366, ('family', 'given'), '0b1a15dd9b78', None), # 2026-09-20, #397: retitled in place, reach and digest # unchanged -- the rule keeps 'Carod i', which the landing # leaves byte-identical. @@ -3582,8 +3590,10 @@ def _claim(rule: dict) -> _Claim: # and pins the shape alone. "fix(suffix-routing) a two-token name ending in the suffix word jr keeps it in `suffix`": _Claim(6, ('family', 'suffix'), "dd3fc23d90a1", None), + # 2026-09-23, #459: 2 -> 3, 'qc mp', rules.md#R4's name-role + # boundary line. Reach, verified name by name. "fix(suffix-routing) a two-token name ending in a credential acronym keeps it in `suffix`": - _Claim(2, ('family', 'suffix'), "ed72c9672214", None), + _Claim(3, ('family', 'suffix'), "07c5470c399e", None), "fix(suffix-routing) the dotted M.A. spelling reads as a credential (ma-do)": _Claim(1, ('family', 'suffix'), "17379620526b", None), # #484's six `_initials` rules. Four of them reach far more @@ -3677,8 +3687,10 @@ def _claim(rule: dict) -> _Claim: # name and neither widened. Verified name by name. "fix(initials-per-word) a particle chain inside a name part initials each word (facade, since 2.0.0)": _Claim(112, ('_initials',), 'b3b3b696a56e', ('DEFAULT',)), + # 2026-09-23, #459: 18 -> 19, 'john smith ph. d.', rules.md#R4's + # two-token line. Reach, verified name by name. "fix(initials-per-word) the Ph. D. merge initials each word (facade, since 2.0.0)": - _Claim(18, ('_initials',), "f67d8ebddd56", ('DEFAULT',)), + _Claim(19, ('_initials',), "adfdec5a9e13", ('DEFAULT',)), # The 2.3 title-run bundle's five rules, last in every # ledger. All five are anchored on NAMES, so the reach IS the # mover list: 2 names for the run keying, 1 for the esq drop, @@ -3958,8 +3970,10 @@ def _claim(rule: dict) -> _Claim: # strings and the corpora are one set. "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), + # 2026-09-23, #459: 2 -> 3, 'qc mp', rules.md#R4's name-role + # boundary line. Same digest in all three 2.x ledgers. "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": - _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), + _Claim(3, ('_ambiguities',), "6d5341766f53", None), # The join clause's own rule, added by the #518 review round: # hoisting the clause out from under O5's field-deciding # guards reached two corpus names, both a peeled title in @@ -4474,8 +4488,10 @@ def _claim(rule: dict) -> _Claim: # strings and the corpora are one set. "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), + # 2026-09-23, #459: 2 -> 3, 'qc mp', rules.md#R4's name-role + # boundary line. Same digest in all three 2.x ledgers. "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": - _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), + _Claim(3, ('_ambiguities',), "6d5341766f53", None), # The join clause's own rule, added by the #518 review round: # hoisting the clause out from under O5's field-deciding # guards reached two corpus names, both a peeled title in @@ -4767,8 +4783,10 @@ def _claim(rule: dict) -> _Claim: # strings and the corpora are one set. "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), + # 2026-09-23, #459: 2 -> 3, 'qc mp', rules.md#R4's name-role + # boundary line. Same digest in all three 2.x ledgers. "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": - _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), + _Claim(3, ('_ambiguities',), "6d5341766f53", None), # The join clause's own rule, added by the #518 review round: # hoisting the clause out from under O5's field-deciding # guards reached two corpus names, both a peeled title in @@ -6736,7 +6754,11 @@ class _Excluded(NamedTuple): #: revisit the day one does. _EXCLUSION_EFFECT: dict[str, _Excluded] = { "(?i)^(?!\\s*ph\\.)(?![^\\s,]+\\s*,\\s*ph\\.\\s*d\\.\\s*$)(?![\\u0000-\\u024f]*\\b(?:jr|sr|ii|iii|iv)\\.?\\s+ph\\.\\s*d\\.\\s*$)[\\u0000-\\u024f]*\\bph\\.\\s*d\\.\\s*$": - _Excluded(3, "5a12a8117651", + _Excluded(6, "69491e3986b1", + # 2026-09-23, #459: 3 -> 6 captures, rules.md#R4's + # three new trailing Ph. D. spellings + # ('john smith ph.d.', 'JOHN SMITH PH.D.', + # 'john smith ph. d.'); absorbed_by unchanged. # fix(comma-precomma-family) JOINED this tuple in #372, # it did not replace anything: it claims the {given, # family} readings, which it legitimately describes for a @@ -6772,7 +6794,8 @@ class _Excluded(NamedTuple): # # Decided rather than absorbed. The rule's regex is # the bare fragment `\bph\. d\.`, which reaches all - # 18 corpus spellings while explaining the two + # 19 corpus spellings (18 until #459's 'john smith + # ph. d.' line, 2026-09-23) while explaining the two # LEADING ones it is named for; a trailing 'Ph. D.' # is what this entry protects, and the exclusion # refuses those names before any rule is consulted, diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index afaffc99..293d11cb 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -12,6 +12,7 @@ _title_key, ) from nameparser._policy import Script, _SCRIPT_RANGES +from nameparser.config import Constants from nameparser.config.suffixes import SUFFIX_ACRONYMS, SUFFIX_WORDS @@ -31,10 +32,10 @@ def test_default_sources_v1_vocabulary() -> None: # the policy is 'e' and not 'y' (decisions.md#P3): a bare E initial is # common, where 'y' between two surnames is the commonest Hispanic compound assert "e" in lex.conjunctions_ambiguous and "y" not in lex.conjunctions_ambiguous - # v1's CAPITALIZATION_EXCEPTIONS maps 'phd' -> 'Ph.D.' (verbatim, not - # normalized -- only keys are lowercased/period-stripped at - # construction, values pass through unchanged). - assert lex.capitalization_exceptions_map["phd"] == "Ph.D." + # CAPITALIZATION_EXCEPTIONS maps 'phd' -> 'PhD', a case mask since + # #459 (verbatim, not normalized -- only keys are lowercased and + # period-stripped at construction; values pass through unchanged). + assert lex.capitalization_exceptions_map["phd"] == "PhD" # maiden markers source from the same data-module pattern (#274); # non-colliding Cyrillic entries live in the default per the locales # design's sorting rule, and both ё/е spellings are listed because @@ -111,8 +112,9 @@ def test_entries_normalizing_to_empty_raise() -> None: def test_colliding_exception_keys_dedupe_last_wins() -> None: # 'Phd.' and 'phd' collide under edge-period normalization - lex = Lexicon(capitalization_exceptions=(("Phd.", "A"), ("phd", "B"))) - assert lex.capitalization_exceptions == (("phd", "B"),) + lex = Lexicon(capitalization_exceptions=(("Phd.", "PHD"), + ("phd", "PhD"))) + assert lex.capitalization_exceptions == (("phd", "PhD"),) rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) @@ -122,6 +124,121 @@ def test_lexicon_rejects_non_str_exception_values() -> None: Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] +def _unpickled_with(value: object) -> Lexicon: + state = Lexicon.default().__getstate__() + state["capitalization_exceptions"] = value + restored = Lexicon.__new__(Lexicon) + restored.__setstate__(state) + return restored + + +# Every entry point that builds the pair field, because a check on one +# member of a family belongs on all of it (AGENTS.md): construction, +# replace(), the right-biased union, unpickling, and the v1 shim. +_MASK_ENTRY_POINTS: list[Callable[[object], object]] = [ + lambda v: Lexicon(capitalization_exceptions=v), # type: ignore[arg-type] + lambda v: dataclasses.replace(Lexicon.default(), + capitalization_exceptions=v), # type: ignore[arg-type] + lambda v: Lexicon.default() | Lexicon(capitalization_exceptions=v), # type: ignore[arg-type] + _unpickled_with, + lambda v: Constants(capitalization_exceptions=v)._snapshot(), +] + + +@pytest.mark.parametrize("build", _MASK_ENTRY_POINTS) +@pytest.mark.parametrize("key, value", [ + ("jr", "Junior"), ("phd", "PhDs"), ("phd", ""), ("md", "MB"), + ("phd", "D.Ph."), ("2nd", "3ND"), +]) +def test_an_exception_value_that_does_not_spell_its_key_raises( + build: Callable[[object], object], key: str, value: str) -> None: + """#459: a capitalization_exceptions value is a case MASK, the + key's own letters recased. Case repair lays it over the word as + written, so a value spelling anything else -- a replacement + ('Junior'), a letter too many or too few, the right letters in + the wrong order, a DIGIT changed ('2nd' -> '3ND') -- has no + reading, and raises where it is built.""" + with pytest.raises(ValueError, + match="does not spell the key's letters") as caught: + build(((key, value),)) + assert repr(key) in str(caught.value) + assert repr(value) in str(caught.value) + + +def test_a_mask_may_recase_a_digit_key_unchanged() -> None: + """#459 review: the mask compares ALPHANUMERICS, not letters alone + (_alnum, not the old letters-only _letters), so a digit is part of + what a value must spell -- '2nd' accepts '2ND' ('2' unchanged, + letters recased) and rejects '3ND' above.""" + lex = Lexicon(capitalization_exceptions=(("2nd", "2ND"),)) + assert lex.capitalization_exceptions_map["2nd"] == "2ND" + + +def test_a_decomposed_value_is_compared_nfc_composed() -> None: + """Both the key and the value fold through _normalize, which NFC- + composes non-ASCII text (#459's decomposed-hangul note applies + here too): a value written in decomposed form is accepted against + a key written composed, since both sides read the same letters + once composed.""" + decomposed = unicodedata.normalize("NFD", "CAFÉ") + assert decomposed != "CAFÉ" # the draw actually decomposed something + lex = Lexicon(capitalization_exceptions=(("café", decomposed),)) + assert lex.capitalization_exceptions_map["café"] == decomposed + + +@pytest.mark.parametrize("key, value", [ + ("phd", "Ph.D."), ("md", "M.D."), ("hc", "h.c"), ("ph.d", "PhD"), + ("phd", "P.H.D."), ("md", "M. D."), ("dphil", "DPhil"), +]) +def test_a_value_may_carry_punctuation_that_marks_its_joins( + key: str, value: str) -> None: + """#459 review: a value's punctuation is not ignored -- it marks + which of the key's letters are JOINED (one run) versus split, and + _apply_mask's initial rule reads that structure -- but it is never + WRITTEN into the word, so there is nothing here for a warning to + catch. Every one of these constructs silently.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + lex = Lexicon(capitalization_exceptions=((key, value),)) + assert tuple(lex.capitalization_exceptions_map.values()) == (value,) + + +def test_the_mask_error_offers_a_spelling_that_works_and_type_checks( +) -> None: + """The message hands the reader code to paste, so the code has to + run and to type-check. The field is annotated with what it STORES, + a tuple of pairs, so a dict literal there is an arg-type error -- + AGENTS.md's rule for actionable messages. The offered spelling is + the offending KEY recased upper, not a fixed example, so the two + constructions below use the spelling the 'jr' key's own error + offers, verbatim; mypy checks this file.""" + offered = "capitalization_exceptions=(('jr', 'JR'),)" + with pytest.raises(ValueError) as caught: + Lexicon(capitalization_exceptions=(("jr", "Junior"),)) + assert offered in str(caught.value) + assert Lexicon(capitalization_exceptions=(('jr', 'JR'),)) \ + .capitalization_exceptions_map == {"jr": "JR"} + lexicon, _policy, _render = Constants( + capitalization_exceptions=(('jr', 'JR'),))._snapshot() + assert lexicon.capitalization_exceptions_map == {"jr": "JR"} + + +def test_the_mask_error_falls_back_when_upper_would_not_itself_validate( +) -> None: + """#459 review: .upper() can change a letter's COUNT -- + 'straße'.upper() is 'STRASSE', ß growing to two letters -- so + offering it verbatim would raise if pasted. The offer falls back + to the key unchanged (an identity mask, always legal) whenever + upper() would not itself spell the key's own letters.""" + with pytest.raises(ValueError) as caught: + Lexicon(capitalization_exceptions=(("straße", "STRASSE"),)) + offered = "capitalization_exceptions=(('straße', 'straße'),)" + assert offered in str(caught.value) + # pasted, the offered spelling constructs + assert Lexicon(capitalization_exceptions=(('straße', 'straße'),)) \ + .capitalization_exceptions_map == {"straße": "straße"} + + def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index fd310e6f..a69dec1e 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -703,13 +703,26 @@ def _fix_invariants(**fields: frozenset[str]) -> dict[str, frozenset[str]]: _SET_FIELDS = _VOCAB_FIELDS +@st.composite +def _recased(draw: st.DrawFn, word: str) -> str: + """A capitalization_exceptions value for `word`: its own letters, + each drawn upper or lower. Any other value is a ValueError at + construction (#459's case mask), so an independent draw would + turn every fuzzed Lexicon holding a pair into a raise.""" + flips = draw(st.lists(st.booleans(), min_size=len(word), + max_size=len(word))) + return "".join(c.upper() if flip else c.lower() + for c, flip in zip(word, flips)) + + @st.composite def _lexicons(draw: st.DrawFn) -> Lexicon: fields = {name: draw(st.frozensets( _TITLE_VOCAB if name == "given_name_titles" else _VOCAB, max_size=5)) for name in _SET_FIELDS} - caps = draw(st.lists(st.tuples(_VOCAB, _VOCAB), max_size=3)) + keys = draw(st.lists(_VOCAB, max_size=3)) + caps = [(key, draw(_recased(key))) for key in keys] return Lexicon(capitalization_exceptions=tuple(caps), **_fix_invariants(**fields)) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 2721fa29..86f9f077 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -129,6 +129,9 @@ def test_initial_copies_agree_with_each_other_and_config() -> None: ("_render", "_SPACE_BEFORE_COMMA"): "space_before_comma", ("_render", "_MAC"): "mac", ("_render", "_WORD"): "word", + # the numeral clause's copy of _vocab._ROMAN (#459); layering + # forbids _render the import, so it is pinned here instead + ("_render", "_ROMAN"): "roman_numeral", ("_vocab", "_PERIOD_NOT_AT_END"): "period_not_at_end", # Deliberately NOT a straight copy -- pinned by the dedicated tests # above, which assert the documented RELATIONSHIP instead: diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 87b84e77..b0438b16 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,7 +1,12 @@ +import dataclasses +import unicodedata +import warnings + import pytest from nameparser import FAMILY_FIRST, HumanName, Parser, Policy, parse from nameparser._lexicon import Lexicon +from nameparser.config import Constants from nameparser._render import _collapse, render from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, UNJOINED_TAG, Ambiguity, AmbiguityKind, ParsedName, Role, @@ -527,7 +532,7 @@ def test_capitalized_all_lower_input_v1_parity() -> None: assert out.given == "Bob" assert out.middle == "V." assert out.family == "de la MacDole-Eisenhower" # particles stay lower - assert out.suffix == "Ph.D." # exceptions map, verbatim + assert out.suffix == "PhD" # exceptions map, a mask # same spans, new texts (provenance is a documented non-invariant) assert [t.span for t in out.tokens] == [t.span for t in _lowercase_mac().tokens] @@ -562,6 +567,301 @@ def test_capitalized_with_explicit_lexicon() -> None: assert out.suffix == "Phd" +def test_a_mask_recases_the_word_as_the_writer_punctuated_it() -> None: + """rules.md#R4's mask (#459): an exceptions-map value is its key's + letters in the case each takes, laid over the word as written -- + one entry covers every punctuation of the word, and repair adds + and removes nothing.""" + for text, suffix in (("john smith phd", "PhD"), + ("john smith ph.d.", "Ph.D."), + ("JOHN SMITH PH.D.", "Ph.D."), + ("john smith bsc", "BSc"), + ("JOHN SMITH MSC", "MSc")): + assert parse(text).capitalized().suffix == suffix, text + # two tokens, and 'ph.' is no key: nothing to mask, nothing moves + assert str(parse("john smith ph. d.").capitalized()) \ + == "John Smith Ph. D." + # the mask is asked BEFORE the acronym clause, which would give + # 'BSC' -- bsc is a listed acronym too + assert "bsc" in Lexicon.default().suffix_acronyms + # forced: a mixed-case corpus name R5 would otherwise hold back + assert str(parse("Dr. med. univ. Margit Popp, MSc").capitalized( + force=True)) == "Dr. Med. Univ. Margit Popp MSc" + + +def test_a_mask_applies_whatever_role_the_word_took() -> None: + """The map stays role-free (#459): an entry is the caller saying + how a word is written, wherever it stands. The acronym and + numeral clauses read the role; the mask does not.""" + assert str(parse("phd smith").capitalized()) == "PhD Smith" + assert str(parse("john phd smith").capitalized()) == "John PhD Smith" + assert str(parse("MSc Dr. med. univ.").capitalized(force=True)) \ + == "MSc Dr. Med. Univ." + + +def test_a_word_that_left_the_map_repairs_by_the_role_it_took() -> None: + """md, ii, iii and iv left the exceptions map (#459). A suffix md + is a listed acronym and a suffix numeral a numeral, so the acronym + and numeral clauses write them in capitals -- keeping whatever + punctuation the writer used, which the map's substitution did + not. A word the parse put in a NAME role is repaired as a name + word whatever vocabulary holds it (decisions.md#R4's given-role + half), which is also how the abbreviated Mohammed is written.""" + assert str(parse("john smith md").capitalized()) == "John Smith MD" + assert str(parse("john smith m.d.").capitalized()) == "John Smith M.D." + assert str(parse("john smith iii").capitalized()) == "John Smith III" + assert str(parse("john smith iii.").capitalized()) == "John Smith III." + assert str(parse("Andrew Perkins (M.D)").capitalized(force=True)) \ + == "Andrew Perkins M.D" + assert str(parse("Md Abdul Karim").capitalized(force=True)) \ + == "Md Abdul Karim" + assert parse("iv smith").given == "iv" + assert str(parse("iv smith").capitalized()) == "Iv Smith" + # the other name-role numeral shape: a numeral in a name role + # repairs as the name word the parse read it as, decided + assert str(parse("john iii smith").capitalized()) == "John Iii Smith" + + +def test_a_credential_read_by_its_shape_repairs_to_capitals() -> None: + """rules.md#R4, #516's by-shape half (#459): a suffix classify + admitted to the credential class by its dotted shape alone + carries SHAPE_ACRONYM_TAG and no vocabulary entry, and repairs as + a listed acronym does.""" + assert str(parse("john smith x.y.z.").capitalized()) \ + == "John Smith X.Y.Z." + assert str(parse("John Smith R.A.I.").capitalized(force=True)) \ + == "John Smith R.A.I." + # the same shape read as the FAMILY -- no words to spare -- is a + # name word + assert parse("Jack X.Y.Z.").family == "X.Y.Z." + assert str(parse("Jack X.Y.Z.").capitalized(force=True)) \ + == "Jack X.y.z." + # and with the dotted-shape switch off it is family by position + off = Parser(policy=Policy(unlisted_dotted_suffixes=False)) + assert str(off.capitalized(off.parse("john smith x.y.z."))) \ + == "John Smith X.y.z." + # the opt-in all-caps half writes the same tag + caps = Parser(policy=Policy(unlisted_caps_suffixes=True)) + assert str(caps.capitalized(caps.parse("John Smith XYZ"), + force=True)) == "John Smith XYZ" + assert str(parse("John Smith XYZ").capitalized(force=True)) \ + == "John Smith Xyz" + + +def test_a_suffix_numeral_repairs_to_capitals_by_its_shape() -> None: + """rules.md#R4 (#459): vi through x carry no vocabulary tag and + title-cased to 'Vi'/'Ix'; the numeral clause reads the roman + shape of a SUFFIX-roled word. xi and up are no suffix to the + parse (its roman shape stops at x), and a numeral after a family + comma is the given name -- parse limits both, so both repair as + the name words they were read as.""" + for text, suffix in (("john smith vi", "VI"), + ("john smith vii", "VII"), + ("john smith viii", "VIII"), + ("john smith ix", "IX"), + ("john smith x", "X"), + ("john smith, v", "V")): + name = parse(text) + assert name.suffix == text.split()[-1], text + assert name.capitalized().suffix == suffix, text + assert parse("john smith xi").family == "xi" + assert str(parse("john smith xi").capitalized()) == "John Smith Xi" + assert parse("john smith, vi").given == "vi" + assert str(parse("john smith, vi").capitalized()) == "Vi John Smith" + # the generation guard's other half: a generational 'i' skips the + # connective arm, and the numeral clause is what writes it + assert str(parse("Carod i").capitalized(force=True)) == "Carod I" + # the clause reads no vocabulary, so an empty lexicon repairs too + assert _pn("john smith vi", [ + Token("john", Span(0, 4), Role.GIVEN), + Token("smith", Span(5, 10), Role.FAMILY), + Token("vi", Span(11, 13), Role.SUFFIX), + ]).capitalized(Lexicon.empty()).suffix == "VI" + + +def test_the_mask_keeps_every_non_letter_and_declines_a_miscount() -> None: + from nameparser._render import _apply_mask + assert _apply_mask("ph.d.", "PhD") == "Ph.D." + assert _apply_mask("PHD", "PhD") == "PhD" + assert _apply_mask("bsc", "BSc") == "BSc" + # The lookup key is NFC-composed and the word is not: decomposed + # hangul spells one syllable in two letters, so the counts differ + # and the applier declines rather than guess -- and the repair + # falls through to the next clause instead of raising. + decomposed = unicodedata.normalize("NFD", "씨") + assert len(decomposed) == 2 + assert _apply_mask(decomposed, "씨") is None + lex = dataclasses.replace(Lexicon.default(), + capitalization_exceptions=(("씨", "씨"),)) + p = Parser(lexicon=lex) + name = p.parse(unicodedata.normalize("NFD", "John Smith 씨")) + assert p.capitalized(name, force=True).suffix == decomposed + + +def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( +) -> None: + """rules.md#R4 (#459 review, narrowed): a letter written alone + beside a full stop is an initial ONLY where the MASK writes that + same letter inside a run of two or more letters -- the writer + split a chunk the mask keeps together, so each split piece is an + initial. Where the mask spells the letter alone too, there is no + split to repair, and the mask's own case stands unchanged -- a + caller's own 'h.c' mask on 'h.c.' stays 'h.c.'. A run of two or + more letters beside a full stop ('sc' in 'b.sc.') is never an + initial either way.""" + from nameparser._render import _apply_mask + for word, mask, expected in ( + ("p.h.d.", "PhD", "P.H.D."), + ("b.sc.", "BSc", "B.Sc."), + ("h.c.", "h.c", "h.c."), # mask ALSO spells each letter alone + ("y", "y", "y"), # no full stop at all + (".a", "a", ".a"), # single-letter mask never overrides + ("a.", "a", "a."), + ("2b.", "2b", "2b."), # 'b' has no LETTER neighbor in '2b' + # a full stop on one side only, under a multi-letter mask: + # 'a' splits from the mask's 'Abc' run and is forced + # upper; 'b' and 'c' sit next to each other in the WORD + # (not lone) and take the mask's own lowercase case. Weak + # as a mutation pin -- the mask's own 'A' is ALREADY + # upper, so mask-driven casing alone (override skipped) + # gives the same answer; the next two rows use a mask + # with no letter already capitalized, so only the + # override, not the mask's own case, can produce the 'A'. + ("a.bc", "Abc", "A.bc"), + # stop on the NEXT side, at the mask's run START: 'a' + # splits from the mask's one run 'abc' and is forced + # upper; 'b'/'c' are not lone in the word and take the + # mask's own (here lowercase) case + ("a.bc", "abc", "A.bc"), + # stop on the PREVIOUS side, at the mask's run END: 'c' + # splits from the same one-run mask and is forced upper; + # 'a'/'b' are not lone and stay lowercase + ("ab.c", "abc", "ab.C"), + # a digit breaks a letter run on BOTH sides: 'x2bc' is two + # runs, {x} and {bc} (the digit itself gets no run, and + # is not a letter, so it never triggers the override + # either). 'x' is adjacent to the DIGIT '2' -- not a full + # stop -- so it is never even beside-a-stop and keeps the + # mask's lowercase; 'b' and 'c' are each lone in the word + # AND inside the mask's {bc} run, so both are forced upper + ("x2b.c", "x2bc", "x2B.C"), + # the fullwidth stop U+FF0E is a FULL_STOPS member, so the + # word-side adjacency test reads it like an ASCII period + # -- but _render._WORD ("(\\w|\\.)+") only ever yields the + # ASCII period inside a token's text, so a word carrying + # this stop reaches _apply_mask only through a direct + # call, never through _cap_text's normal tokenizing + ("a．bc", "abc", "A．bc"), + ): + assert _apply_mask(word, mask) == expected, (word, mask) + for text, suffix in (("john smith b.s.c.", "B.S.C."), + ("JOHN SMITH B.S.C.", "B.S.C."), + ("john smith m.s.c.", "M.S.C."), + ("john smith p.h.d.", "P.H.D.")): + assert parse(text).capitalized().suffix == suffix, text + hn = HumanName(text) + hn.capitalize() + assert hn.suffix == suffix, text + # the boundary the rule draws: a two-letter run beside a period + # is not lone, and still takes the mask's case + assert parse("john smith b.sc.").capitalized().suffix == "B.Sc." + # end to end: a caller's own mask that ALSO spells each letter + # alone (honoris causa -- "Dr. h.c.") leaves the split-looking + # word unchanged rather than forcing capitals nobody asked for + lex = dataclasses.replace(Lexicon.default(), + capitalization_exceptions=(("h.c", "h.c"),)) + p = Parser(lexicon=lex) + assert str(p.capitalized(p.parse("dr. h.c. hans meier"), + force=True)) == "Dr. h.c. Hans Meier" + + +def test_a_masks_punctuation_marks_its_joins_and_is_never_written() -> None: + """#459 review: a value's punctuation is never written into the + word -- 'md' repairs to 'MD' and 'm.d.' to 'M.D.' under the + ('md','M.D.') mask, identically to what the plain 'MD' mask + gives -- but it is not IGNORED either: it marks which of the + mask's letters are one run versus split apart, and the + lone-initial clause reads that structure. Two masks that differ + ONLY in punctuation, 'h.c' and 'hc', therefore give DIFFERENT + output on the same split-looking word: the writer wrote 'h.c.', + and whether that reads as one abbreviation or two initials + depends on how the mask itself joins the letters.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + lex = dataclasses.replace( + Lexicon.default(), + capitalization_exceptions=(("md", "M.D."),)) + assert str(Parser(lexicon=lex).capitalized( + Parser(lexicon=lex).parse("john smith md"))) == "John Smith MD" + assert str(Parser(lexicon=lex).capitalized( + Parser(lexicon=lex).parse("john smith m.d."))) \ + == "John Smith M.D." + # the facade twin: warning-free at the first parse (the shim's + # Lexicon snapshot is built lazily, so this is where it would fire) + c = Constants(capitalization_exceptions={'md': 'M.D.'}) + with warnings.catch_warnings(): + warnings.simplefilter("error") + hn = HumanName("john smith md", constants=c) + hn.capitalize() + assert str(hn) == "John Smith MD" + with warnings.catch_warnings(): + warnings.simplefilter("error") + joined = dataclasses.replace( + Lexicon.default(), capitalization_exceptions=(("hc", "h.c"),)) + split = dataclasses.replace( + Lexicon.default(), capitalization_exceptions=(("hc", "hc"),)) + pj, ps = Parser(lexicon=joined), Parser(lexicon=split) + assert str(pj.capitalized(pj.parse("dr. h.c. hans meier"), + force=True)) == "Dr. h.c. Hans Meier" + assert str(ps.capitalized(ps.parse("dr. h.c. hans meier"), + force=True)) == "Dr. H.C. Hans Meier" + + +def test_a_mask_recases_a_digit_key_unchanged() -> None: + """#459 review: the mask walks ALPHANUMERICS, not letters alone, + so a digit in the word is carried through unchanged (it has no + case) while the surrounding letters still take the mask's case. + The clause reads no vocabulary (like the roman-numeral test + above), so a synthetic token proves it without depending on how + '2nd' happens to parse on its own.""" + lex = dataclasses.replace(Lexicon.default(), + capitalization_exceptions=(("2nd", "2ND"),)) + assert _pn("2nd", [ + Token("2nd", Span(0, 3), Role.GIVEN), + ]).capitalized(lex, force=True).given == "2ND" + + +def test_a_mask_cases_through_the_whole_word_for_context_sensitive_letters( +) -> None: + """#459 review: casing goes through word.lower()/word.upper() + when both keep the word's length, not a per-character + str.lower()/str.upper() call, which is context-free and gets some + letters wrong that the whole-word form gets right -- a Greek + medial sigma where a FINAL one belongs, here. A titlecase mask + letter reads as upper rather than lower (a known limit: no + per-character titlecase mapping is attempted, so a digraph's + OWN titlecase spelling is not reproduced -- only the Greek case + is pinned here, since it recovers the correct written form).""" + from nameparser._render import _apply_mask + assert _apply_mask("ΚΟΣ", "Κος") \ + == "Κος" # 'ΚΟΣ' under 'Κος' -> 'Κος' (final sigma) + # the FALLBACK side of the same branch: 'straße'.upper() is + # 'STRASSE' (ß grows to two letters), longer than the 6-char + # word, so same_length is False and this exercises the + # per-character c.upper()/c.lower() path rather than the + # whole-word indexed lookup -- ß keeps its own lowercase form + # under the mask's lowercase letter there (c.lower() on 'ß' is + # 'ß', not the two-letter 'ss' casefold() would give) + assert _apply_mask("straße", "STRAßE") == "STRAßE" + # end to end, through a custom Lexicon: a per-character + # c.lower()/c.upper() walk gave 'Κοσ' here (medial sigma), wrong + lex = dataclasses.replace(Lexicon.default(), + capitalization_exceptions=(("κος", "Κος"),)) + assert _pn("ΚΟΣ", [ + Token("ΚΟΣ", Span(0, 3), Role.GIVEN), + ]).capitalized(lex, force=True).given == "Κος" + + def test_capitalized_lowers_the_words_the_parse_tagged_conjunction() -> None: # #458: whether a word is the conjunction or an initial is # classify's decision, recorded as the tag; repair honors the tag diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 4f6745ad..daa90f5e 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -65,6 +65,7 @@ "Dr. Smith née Jones" "Dr. Smith, John" "Dr. abdul salam" +"Dr. med. univ. Margit Popp, MSc" "Duke of Edinburgh" "Esq. Smith" "Freiherr von Berg MA" @@ -86,12 +87,14 @@ "JOHN PROF. MA" "JOHN QUINCY SMITH I" "JOHN SMITH MA" +"JOHN SMITH PH.D." "JUAN GARCIA Y LOPEZ" "Jack MA" "Jack Ma" "Jack Ma." "Jack Wei Ma" "Jack X.Y.I." +"Jack X.Y.Z." "Jane (née Jones) Smith" "Jane Doe (nee Smith MA)" "Jane Doe (nee Smith Ma)" @@ -149,6 +152,7 @@ "John Smith Prof. Dr." "John Smith Prof. Jr." "John Smith Q.W.E.R.T." +"John Smith R.A.I." "John Smith Sir" "John Smith X.Y.Z." "John Smith XYZ" @@ -206,6 +210,7 @@ "Juan y Garcia née Jones" "Lord Chancellor" "MD DDS" +"MSc Dr. med. univ." "Mari' Aube'" "Maria Kowalska (z domu Nowak)" "Maria Kowalska (z domu)" @@ -217,6 +222,7 @@ "Mary Jane King." "Mary Smith née Jones Prof." "Mc Donald" +"Md Abdul Karim" "Mesnil de" "Morse, Det. Insp. Jane" "Mr. Jack Jill" @@ -325,12 +331,27 @@ "de la Vega y Santos Juan" "de los Santos" "ibn Awf abdul Rahman" +"john doe q.c." "john e smith" +"john smith bsc" +"john smith ed" +"john smith iii" +"john smith iii." +"john smith ix" "john smith jr" +"john smith m.d." "john smith mba" "john smith mcse" +"john smith md" +"john smith p.h.d." +"john smith ph. d." +"john smith ph.d." "john smith phd" +"john smith vi" "john smith x.y.z." +"john smith xi" +"john smith, v" +"john smith, vi" "john van der berg ma" "jose e maria santos" "juan de la vega" @@ -338,6 +359,7 @@ "juan mcdonald" "mohamad ali smith" "née Jones" +"qc mp" "van Berg Jan de" "van Gogh" "van der Berg, abdul née Jones" From fd76c27bf1196cbfd8682520c02082ece1139406 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 23 Sep 2026 22:56:05 -0700 Subject: [PATCH 2/9] fix(#492): a suffix is no evidence about how the name was cased R5 repairs only a name written wholly in one case, reading mixed case as the writer's deliberate choice. The gate read every token, so a suffix written the way a suffix is written -- 'III', 'PhD', 'Jr.' -- held a lowercase name back: 'juan garcia III' came back untouched in every release, and v1's test for it had been a known failure since 2012 (the Google Code tracker's issue 22). The gate text is now every token but the SUFFIX-roled ones. Titles stay in -- not as evidence about the name, but because title repair is still plain str.capitalize() and cannot yet be trusted to act on one ('Dr. juan garcia' is left alone) -- and so does every name word, nickname and maiden name ('Juan garcia III' is left alone). Force is unchanged. The parser's own one-case reading still counts the suffix (P3, S2), so 'john e jones III' repairs to 'John e Jones III' where 'john e jones iii' gives 'John E Jones III'; that split is decided and recorded. The xfail retires under a name that no longer says KNOWN_FAILURE, and rules.md#R5 states the sentence and pins both boundaries. Co-Authored-By: Claude Fable 5.1 --- docs/design/rules.md | 21 +++++++++++---- nameparser/_facade.py | 3 ++- nameparser/_render.py | 29 +++++++++++++-------- nameparser/_types.py | 5 ++-- tests/test_capitalization.py | 20 ++++++++++++--- tests/v2/test_render.py | 37 +++++++++++++++++++++++++++ tools/differential/corpus_rules.jsonl | 6 +++++ 7 files changed, 99 insertions(+), 22 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index 43d9c8d0..45543765 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2295,15 +2295,26 @@ R5. Rationale: mixed case is evidence that the writer cased the name could not be bothered — so repair proceeds there, which is a choice about how to act absent evidence rather than a claim that nothing can be lost by it. - Case repair acts only on a name written entirely in one case. A - name written in more than one case is kept as it was written, - and whether that casing is right does not enter into it, unless - repair was asked for anyway. + Case repair acts only on a name written entirely in one case. + The suffixes are left out of that test: a suffix says nothing + about how the writer cased the name — III, PhD, Jr. are written + that way whatever the name's case — so the test reads every word + the parse did not put in the suffix role. A title is not left + out, and a cased title still holds repair back. A name whose + words outside the suffix are written in more than one case is + kept as it was written, and whether that casing is right does + not enter into it, unless repair was asked for anyway. "juan mcdonald" → capitalized="Juan McDonald" "SHIRLEY MACLAINE" → capitalized="Shirley MacLaine" "Shirley Maclaine" → capitalized="Shirley Maclaine" "Shirley Maclaine" → capitalized_forced="Shirley MacLaine" · boundary - history: decisions.md#R5 · interacts: R4 · implemented: nameparser/_render.py + "juan garcia III" → capitalized="Juan Garcia III" + "juan garcia PhD" → capitalized="Juan Garcia PhD" + "JUAN GARCIA Jr." → capitalized="Juan Garcia Jr." + "dr. juan garcia III" → capitalized="Dr. Juan Garcia III" + "Dr. juan garcia" → capitalized="Dr. juan garcia" · boundary + "Juan garcia III" → capitalized="Juan garcia III" · boundary + history: decisions.md#R5 · interacts: R4, P3, S2 · implemented: nameparser/_render.py ## Construction & configuration diagnostics (D) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index e22e60cc..fce2b72c 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -271,7 +271,8 @@ def capitalize(self, force: bool | None = None) -> None: """Re-capitalize the current parse against the bound lexicon. force=None reads the bound Constants' render default (force_mixed_case_capitalization); the core's capitalized() - implements the single-case gate (v1 parity) -- not + implements the one-case gate (rules.md#R5, suffixes left out + since #492) -- not re-implemented here.""" self._resolve() if force is None: diff --git a/nameparser/_render.py b/nameparser/_render.py index fcb9b3d2..7f05890b 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -487,10 +487,11 @@ def _cap_text(text: str, role: Role, tags: frozenset[str], def capitalized(name: ParsedName, lexicon: Lexicon | None, *, force: bool) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token - texts. Gate (v1 parity): only single-case input is - touched unless force=True; the gate reads the joined token texts - (not render() output -- the case gate stays decoupled from spec - formatting and the #254 collapse). + texts. Gate: only a name whose words outside the suffix are + written in one case is touched unless force=True; the gate reads + the joined texts of every token not roled SUFFIX (#492) -- not + render() output, so it stays decoupled from spec formatting and + the #254 collapse. Repair changes case and nothing else: an exceptions-map value is a mask recasing the word as written (#459), never a replacement. The repair reads token TAGS as well as texts: a part whose every @@ -507,18 +508,26 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, 'de y' keeps the 'y' lowercase, as the parse does and as 1.4.0 did. Parser.revise() is the edit that classifies the value, and gives 'De La' (rules.md#R4's Accepted boundary). - Idempotent: without force, a capitalized result is mixed-case and - the gate returns it unchanged; with force, every _cap_word rule is - a fixpoint on its own output.""" + Idempotent: every _cap_word rule, and the hyphen rule in + _cap_text, is a fixpoint on its own output, so a repaired name + comes back unchanged whether or not the gate admits it again + (a name whose non-suffix words are caseless, 'Kim Minjun' in + hangul with a 'phd', is admitted every time).""" if lexicon is not None and not isinstance(lexicon, Lexicon): # eager, before the gate: a garbage argument must not become a # silent no-op on mixed-case input or a deep AttributeError raise TypeError(f"lexicon must be a Lexicon or None, got {lexicon!r}") lex = Lexicon.default() if lexicon is None else lexicon - joined = " ".join(t.text for t in name.tokens) # rules.md#R5: "case repair acts only on a name written entirely - # in one case" - if not force and joined not in (joined.upper(), joined.lower()): + # in one case" -- and "the suffixes are left out of that test": a + # credential or a generation written the way one is written + # ('III', 'PhD', 'Jr.') says nothing about how the writer cased + # the NAME (#492). Titles stay in because title repair is not yet + # trusted to act on a cased title (decisions.md#R5 names the three + # titles that showed why). + gate = " ".join(t.text for t in name.tokens + if t.role is not Role.SUFFIX) + if not force and gate not in (gate.upper(), gate.lower()): return name new_tokens = tuple( Token(_cap_text(t.text, t.role, t.tags, lex), t.span, t.role, t.tags) diff --git a/nameparser/_types.py b/nameparser/_types.py index f2b8907d..f331de31 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -1115,7 +1115,8 @@ def capitalized(self, lexicon: Lexicon | None = None, *, token texts. Needs a lexicon for capitalization_exceptions and particle rules; None uses the DEFAULT lexicon -- if this name came from a custom Parser, pass its lexicon or use - Parser.capitalized. force=False preserves mixed-case input - (v1 parity). Idempotent.""" + Parser.capitalized. force=False preserves a name whose words + outside the suffix are written in more than one case + (rules.md#R5). Idempotent.""" import nameparser._render as _render return _render.capitalized(self, lexicon, force=force) diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index bd81f226..1926f4ac 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -16,13 +16,25 @@ def test_capitalization_exception_for_III(self) -> None: hn.capitalize() self.m(str(hn), 'Juan Q. Xavier Velasquez y Garcia III', hn) - # FIXME: this test does not pass due to a known issue - # http://code.google.com/p/python-nameparser/issues/detail?id=22 - @pytest.mark.xfail(reason="#492") - def test_capitalization_exception_for_already_capitalized_III_KNOWN_FAILURE(self) -> None: + # A known failure since 2012 (the Google Code tracker's issue 22) + # until #492: the one-case gate read the SUFFIX as evidence that + # the writer cased the whole name, so 'III' held the lowercase + # name back. The gate now leaves the suffixes out (rules.md#R5): + # a generation written the way one is written says nothing about + # how the name was cased. The name words still count, and so + # does a title -- the two boundary rows. + def test_capitalization_exception_for_already_capitalized_III( + self, + ) -> None: hn = HumanName('juan garcia III') hn.capitalize() self.m(str(hn), 'Juan Garcia III', hn) + mixed = HumanName('Juan garcia III') + mixed.capitalize() + self.m(str(mixed), 'Juan garcia III', mixed) + titled = HumanName('Dr. juan garcia') + titled.capitalize() + self.m(str(titled), 'Dr. juan garcia', titled) def test_capitalize_title(self) -> None: hn = HumanName('lt. gen. john a. kenneth doe iv') diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index b0438b16..c63cc995 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -554,6 +554,43 @@ def test_capitalized_preserves_mixed_case_unless_forced() -> None: assert pn.capitalized(force=True).family == "MacLaine" +def test_the_gate_leaves_the_suffixes_out() -> None: + """rules.md#R5 (#492): a credential or a generation written the + way one is written ('III', 'PhD', 'Jr.') is no evidence about how + the NAME was cased, so the one-case test reads every token but + the SUFFIX-roled ones. Titles stay in -- a cased title still holds + repair back -- and so does every name word.""" + for text, repaired in (("juan garcia III", "Juan Garcia III"), + ("juan garcia PhD", "Juan Garcia PhD"), + ("JUAN GARCIA Jr.", "Juan Garcia Jr."), + ("dr. juan garcia III", "Dr. Juan Garcia III"), + ("JUAN GARCIA iii", "Juan Garcia III")): + assert str(parse(text).capitalized()) == repaired, text + for untouched in ("Dr. juan garcia", "DR. juan garcia III", + "Juan garcia III", "Juan Garcia iii", + # NICKNAME and MAIDEN tokens stay IN the gate + # text -- only SUFFIX is excluded, so a cased + # nickname or maiden word still holds repair + # back (a mutant that also filtered those + # roles survived the whole suite otherwise) + "jane doe nee SMITH III", "juan garcia (Bob)"): + name = parse(untouched) + assert name.capitalized() == name, untouched + # a caseless name with a MIXED-CASE suffix: the old gate counted + # the suffix, read the whole joined text as mixed case and + # refused; the new gate excludes it, and the non-suffix text has + # no case at all (trivially one-case), so only the suffix repairs + assert str(parse("김민준 Phd").capitalized()) == "민준 김 PhD" + # no non-suffix token at all -- a synthetic name only, since every + # parse names somebody (rules.md#H4): the empty text is one-case + only_suffixes = _pn("phd md", [ + Token("phd", Span(0, 3), Role.SUFFIX), + Token("md", Span(4, 6), Role.SUFFIX), + ]) + assert [t.text for t in only_suffixes.capitalized().tokens] \ + == ["PhD", "MD"] + + def test_capitalized_is_idempotent() -> None: once = _lowercase_mac().capitalized() assert once.capitalized() == once diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index daa90f5e..9659cf65 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -65,6 +65,7 @@ "Dr. Smith née Jones" "Dr. Smith, John" "Dr. abdul salam" +"Dr. juan garcia" "Dr. med. univ. Margit Popp, MSc" "Duke of Edinburgh" "Esq. Smith" @@ -88,6 +89,7 @@ "JOHN QUINCY SMITH I" "JOHN SMITH MA" "JOHN SMITH PH.D." +"JUAN GARCIA Jr." "JUAN GARCIA Y LOPEZ" "Jack MA" "Jack Ma" @@ -204,6 +206,7 @@ "Juan de" "Juan de la Vega" "Juan de y" +"Juan garcia III" "Juan van der" "Juan y Eva Garcia" "Juan y Garcia" @@ -330,6 +333,7 @@ "de la Vega" "de la Vega y Santos Juan" "de los Santos" +"dr. juan garcia III" "ibn Awf abdul Rahman" "john doe q.c." "john e smith" @@ -355,6 +359,8 @@ "john van der berg ma" "jose e maria santos" "juan de la vega" +"juan garcia III" +"juan garcia PhD" "juan garcia y lopez" "juan mcdonald" "mohamad ali smith" From 20cf32f676637ce8a0ee69e975bf7026fed7cff6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 23 Sep 2026 23:18:39 -0700 Subject: [PATCH 3/9] fix(#478): a connective inside a hyphenated surname keeps its lowercase 'jose ortega-y-gasset' and 'JOSE ORTEGA-Y-GASSET' both repair to 'Jose Ortega-y-Gasset', where 2.2 and 2.3 gave 'Ortega-Y-Gasset': inside one hyphenated word, a part that is connective vocabulary with a worded part on each side keeps its lowercase, the hyphens joining the name around it as the spaced connective would. An edge part is ordinary name text, #458's answer kept -- 'juan e-f smith' still gives 'Juan E-F Smith' -- and the rule reads position and vocabulary, as the particle arm already does per sub-word, never re-deriving the connective-versus-initial class from shape. Only connective vocabulary is read, so the Maori 'Te Awanui-a-Rangi Black' is not reached, and a period-marked part is an initial there as it is in spaced text: 'j.-e.-p. dupont' keeps 'J.-E.-P.'. rules.md#R4 states the rule, pins its edges, and narrows the Accepted sentence that said a vocabulary word inside a longer word is never a conjunction. Co-Authored-By: Claude Fable 5.1 --- docs/design/rules.md | 29 ++++++++++++--- nameparser/_render.py | 34 ++++++++++++++++- tests/test_capitalization.py | 38 ++++++++++++------- tests/v2/test_render.py | 53 +++++++++++++++++++++++++++ tools/differential/corpus_rules.jsonl | 10 +++++ 5 files changed, 145 insertions(+), 19 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index 45543765..31a71b8e 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2207,6 +2207,13 @@ R4. Rationale: case repair is a display concern, applied only on phd smith gives PhD Smith), and a suffix word that is neither an acronym, a numeral nor an exceptions-map entry -- the generational `jr`, `sr` -- keeps its title case. + Inside a hyphenated word, a part that is connective vocabulary + with a worded part on each side of it keeps its lowercase, the + hyphens joining the name around it as the spaced connective would + (Ortega-y-Gasset); at either end of the hyphenated word it is + repaired as ordinary name text. A single letter marked with a + period is read as an initial there, as the parse reads it, never + as the connective. "juan mcdonald" → capitalized="Juan McDonald" "Juan McDonald" → capitalized_forced="Juan McDonald" "ANH DO" → capitalized="Anh Do" @@ -2243,6 +2250,16 @@ R4. Rationale: case repair is a display concern, applied only on "Smith, John, and" → capitalized_forced="John Smith and" "Doe, Jane, and Jr." → capitalized_forced="Jane Doe and Jr." "juan de la vega" → capitalized="Juan de la Vega" · boundary + "jose ortega-y-gasset" → capitalized="Jose Ortega-y-Gasset" + "JOSE ORTEGA-Y-GASSET" → capitalized="Jose Ortega-y-Gasset" + "maria silva-e-sousa" → capitalized="Maria Silva-e-Sousa" + "Jose Ortega-Y-Gasset" → capitalized_forced="Jose Ortega-y-Gasset" + "juan e-f smith" → capitalized="Juan E-F Smith" · boundary + "juan y-garcia" → capitalized="Juan Y-Garcia" · boundary + "donovan mcnabb-smith" → capitalized="Donovan McNabb-Smith" + "maria da-silva" → capitalized="Maria da-Silva" + "juan y-garcia-lopez" → capitalized="Juan Y-Garcia-Lopez" · boundary + "j.-e.-p. dupont" → capitalized_forced="J.-E.-P. Dupont" · boundary Accepted: the all-particle clause reaches a part the parser read. A field spliced in as raw text after the parse carries no reading of its own, so a family set that way to "de la" stays lowercase @@ -2261,10 +2278,12 @@ R4. Rationale: case repair is a display concern, applied only on rather than per field: the conjunction carve-out reaches a word the parse read as a conjunction, and where the parse read nothing at all it reaches what the vocabulary says. A word of that - vocabulary standing inside a longer written word is not a - conjunction, because the parse read that word as one ordinary - name word — but a field spliced in as raw text was read by - nobody, so repair asks the vocabulary and a family set to "de y" + vocabulary standing at either end of a longer written word is + not a conjunction there, because the hyphen clause above reaches + only a worded part with a worded part on each side, and an edge + part is outside that reach — but a field spliced in as raw + text was read by nobody, so repair asks the vocabulary and a + family set to "de y" keeps its "y" lowercase. Whether a word is the conjunction or an initial is a property of the word, which a vocabulary can answer; whether a particle is acting as a particle is a property of the @@ -2285,7 +2304,7 @@ R4. Rationale: case repair is a display concern, applied only on both of the parsed name's views: a middle revised to "e-f" repairs to "E-F" as the parsed name does, where splicing the same text in gives "e-F". - history: decisions.md#R4 · interacts: R2, R3, R5, S3 · implemented: nameparser/_render.py, nameparser/_lexicon.py + history: decisions.md#R4 · interacts: P3, R2, R3, R5, S3 · implemented: nameparser/_render.py, nameparser/_lexicon.py R5. Rationale: mixed case is evidence that the writer cased the name deliberately, and a repair cannot tell a deliberate spelling from diff --git a/nameparser/_render.py b/nameparser/_render.py index 7f05890b..1a12a29b 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -479,7 +479,39 @@ def _cap_text(text: str, role: Role, tags: frozenset[str], # vocabulary asked per word: the parse would have made one token # per word of that text, so this is the granularity its answer # would have had. - return _WORD.sub(lambda m: _cap_word(m.group(0), role, tags, lex), text) + def cap(match: re.Match[str]) -> str: + return _cap_word(match.group(0), role, tags, lex) + + if "-" not in text: + return _WORD.sub(cap, text) + parts = text.split("-") + named = [at for at, part in enumerate(parts) if _WORD.search(part)] + if len(named) < 3: + return _WORD.sub(cap, text) + # rules.md#R4: "Inside a hyphenated word, a part that is + # connective vocabulary with a worded part on each side of it + # keeps its lowercase" -- the hyphens are the writer joining the + # name around it, as the spaced connective would (#478). Position + # and vocabulary both come from the whole token, which is why + # this sits here and not in _cap_word, whose word has lost its + # neighbours. It does NOT re-derive the conjunction-versus- + # initial class from a word's CASE, the thing #458 removed: an + # EDGE part has a part on one side only and stays ordinary name + # text, so 'juan e-f smith' keeps 'E-F'. A 'part' is one holding + # a word, so a doubled or trailing hyphen ('md-phd-') supplies no + # neighbour, and the two-part compound ('mcnabb-smith') never + # gets past the count above. The period is the one mark classify + # itself reads as an initial: a single letter marked with a + # period is read as an initial there, as the parse reads it, + # never as the connective, so 'j.-e.-p. dupont' keeps 'E.' while + # the multi-letter 'und.' in 'hans smith-und.-jones' still lowers. + first, last = named[0], named[-1] + return "-".join( + part.lower() + if (first < at < last and not re.fullmatch(r"\w\.", part) + and _normalize(part) in lex.conjunctions) + else _WORD.sub(cap, part) + for at, part in enumerate(parts)) # rules.md#R4: "case repair returns a repaired copy and never mutates diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index 1926f4ac..0ad05e0d 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -415,9 +415,11 @@ def test_an_assigned_field_keeps_v1_conjunction_repair(self) -> None: # an initial, not the Italian conjunction ('middle', 'e.', 'John E. Smith'), # v1 asks per WORD of the assigned text, so the - # conjunction inside a hyphenated word IS lowered here - # -- the opposite of the parsed reading pinned below, - # and the difference is that one carries a reading + # conjunction ENDING a hyphenated word IS lowered here + # -- the opposite of the parsed reading, where an edge + # part is ordinary name text (pinned below), and the + # difference is that one carries a reading. An + # interior link is lowered on both paths since #478. ('last', 'smith-y', 'John Smith-y')): hn = HumanName('john smith') setattr(hn, field, value) @@ -440,22 +442,32 @@ def test_a_conjunction_inside_a_longer_token_is_a_name_word(self) -> None: uppered.capitalize() self.m(str(uppered), 'Juan E-F Smith', uppered) - # The same shape on a real name, which is what the release note - # cites: Ortega y Gasset is routinely hyphenated in catalogues, and - # `y` is conjunction vocabulary. Before #458 the two spellings - # repaired to 'Jose Ortega-y-Gasset' and 'Jose Ortega-Y-Gasset' - # (measured on the pre-#458 tree). The SPACED form is the contrast - # and is untouched -- there `y` is a token of its own and IS the - # conjunction, so it keeps the lowercase Spanish convention. - def test_a_hyphenated_compound_surname_capitalizes_its_conjunction( + # #478: Ortega y Gasset is routinely hyphenated in catalogues, and + # inside one hyphenated word a connective with a part on each side + # keeps its lowercase -- the hyphens are the writer joining the name + # around it, as the spaced connective does (rules.md#R4). This + # reverses #458's answer for the INTERIOR position ('Jose + # Ortega-Y-Gasset' at 2.2.0 and 2.3.0) and keeps it at the edges, + # where a part has a neighbour on one side only and is ordinary + # name text ('juan e-f smith' above, 'juan y-garcia' here). Both + # one-case spellings now agree; 1.4.0 re-decided per word and gave + # 'Jose Ortega-y-Gasset' lowered but 'Jose Ortega-Y-Gasset' upper + # (measured on the released wheel), so the all-caps half is a + # parity break, #479's one-case precedent. The SPACED form is the + # contrast and was never in question -- there `y` is a token of its + # own and the parse tags it the conjunction. + def test_a_hyphenated_compound_surname_keeps_its_link_lowercase( self, ) -> None: lowered = HumanName('jose ortega-y-gasset') lowered.capitalize(force=True) - self.m(str(lowered), 'Jose Ortega-Y-Gasset', lowered) + self.m(str(lowered), 'Jose Ortega-y-Gasset', lowered) uppered = HumanName('JOSE ORTEGA-Y-GASSET') uppered.capitalize() - self.m(str(uppered), 'Jose Ortega-Y-Gasset', uppered) + self.m(str(uppered), 'Jose Ortega-y-Gasset', uppered) + edge = HumanName('juan y-garcia') + edge.capitalize() + self.m(str(edge), 'Juan Y-Garcia', edge) spaced = HumanName('jose ortega y gasset') spaced.capitalize(force=True) self.m(str(spaced), 'Jose Ortega y Gasset', spaced) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index c63cc995..ca2f6c55 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -951,6 +951,59 @@ def test_capitalized_lowers_the_words_the_parse_tagged_conjunction() -> None: assert hyphenated.capitalized(force=True).middle == "E-F" +def test_a_link_inside_a_hyphenated_word_keeps_its_lowercase() -> None: + """rules.md#R4 (#478): inside one hyphenated word, a part that is + connective vocabulary with a worded part on each side of it keeps + its lowercase, in every role. At either END it is ordinary name + text -- #458's answer, kept -- and a word outside the connective + vocabulary is never reached, the Maori 'a' among them. A single + letter marked with a period is an initial there too, never the + connective -- a multi-letter word marked with a period is not.""" + for text, repaired in ( + ("jose ortega-y-gasset", "Jose Ortega-y-Gasset"), + ("JOSE ORTEGA-Y-GASSET", "Jose Ortega-y-Gasset"), + ("maria silva-e-sousa", "Maria Silva-e-Sousa"), + ("mary-e-smith", "Mary-e-Smith"), + ("john smith-and-jones", "John Smith-and-Jones"), + # edges + ("juan e-f smith", "Juan E-F Smith"), + ("juan y-garcia", "Juan Y-Garcia"), + ("jose ortega-y-", "Jose Ortega-Y-"), + ("jose -y-gasset", "Jose -Y-Gasset"), + # no connective, and the particle arm's own per-word answer + ("donovan mcnabb-smith", "Donovan McNabb-Smith"), + ("maria da-silva", "Maria da-Silva"), + # three worded parts on the ambiguous side of `first < at + # < last`, so a mutant dropping that bound cannot pass + ("juan y-garcia-lopez", "Juan Y-Garcia-Lopez"), + ("juan garcia-lopez-y", "Juan Garcia-Lopez-Y")): + assert str(parse(text).capitalized()) == repaired, text + # mixed case is R5's: untouched unless forced + mixed = parse("Jose Ortega-Y-Gasset") + assert mixed.capitalized() == mixed + assert str(mixed.capitalized(force=True)) == "Jose Ortega-y-Gasset" + # not connective vocabulary, so not reached (decisions.md#R4) + assert "a" not in Lexicon.default().conjunctions + assert str(parse("Te Awanui-a-Rangi Black").capitalized( + force=True)) == "Te Awanui-A-Rangi Black" + # a single letter marked with a period is an initial there, + # exactly as in spaced text, never the connective -- `_normalize` + # strips the period, so a naive vocabulary check alone would read + # 'e.'/'y.' as the connective and lower it + assert str(parse("j.-e.-p. dupont").capitalized( + force=True)) == "J.-E.-P. Dupont" + assert str(parse("J.-Y.-M. COUSTEAU").capitalized()) == \ + "J.-Y.-M. Cousteau" + # a MULTI-letter word marked with a period is not an initial, and + # stays reachable as the connective, the same as its spaced + # reading ('hans smith und. jones' tags 'und.' a conjunction) + assert str(parse("hans smith-und.-jones").capitalized( + force=True)) == "Hans Smith-und.-Jones" + # every role, not only FAMILY above + assert str(parse("smith, jose ortega-y-gasset").capitalized( + force=True)) == "Jose Ortega-y-Gasset Smith" + + def test_case_repair_falls_back_for_text_the_parse_never_read() -> None: """A token carrying UNCLASSIFIED_TAG holds raw text no parse read, so there is no decision to honor and case repair -- which is handed diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 9659cf65..e85bfac6 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -89,6 +89,7 @@ "JOHN QUINCY SMITH I" "JOHN SMITH MA" "JOHN SMITH PH.D." +"JOSE ORTEGA-Y-GASSET" "JUAN GARCIA Jr." "JUAN GARCIA Y LOPEZ" "Jack MA" @@ -194,6 +195,7 @@ "Jong, Anke de" "Jong, Piet de" "Jose E Maria Santos" +"Jose Ortega-Y-Gasset" "Jose e Maria Santos" "Josep Carod i Rovira" "Josep Lluis Carod i III" @@ -333,8 +335,10 @@ "de la Vega" "de la Vega y Santos Juan" "de los Santos" +"donovan mcnabb-smith" "dr. juan garcia III" "ibn Awf abdul Rahman" +"j.-e.-p. dupont" "john doe q.c." "john e smith" "john smith bsc" @@ -358,11 +362,17 @@ "john smith, vi" "john van der berg ma" "jose e maria santos" +"jose ortega-y-gasset" "juan de la vega" +"juan e-f smith" "juan garcia III" "juan garcia PhD" "juan garcia y lopez" "juan mcdonald" +"juan y-garcia" +"juan y-garcia-lopez" +"maria da-silva" +"maria silva-e-sousa" "mohamad ali smith" "née Jones" "qc mp" From 491e402aa792c2c12390ea80c63a51736386b31f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 23 Sep 2026 23:32:17 -0700 Subject: [PATCH 4/9] test(#459/#492/#478): case repair changes case and nothing else rules.md#R4's invariant, held over every name a caller has written -- the deduped differential corpus glob plus the case table's texts -- on both surfaces, plain and forced, under the shipped lexicon and one with the exceptions map emptied: a repaired token equals its input under str.casefold(). Compared token by token on the core and list by list on the facade, never through a rendered string. The recorded control is the parent, where the map substituted its value and the walk fails on 257 repairs; the live control puts the substitution back through the mask applier and must be seen. The hypothesis walk that now draws masks gets the same assertion rather than a second fuzzer. Under a second for the new walk. Co-Authored-By: Claude Fable 5.1 --- tests/v2/test_properties.py | 117 ++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index a69dec1e..cc841f76 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -31,6 +31,7 @@ from nameparser._types import (UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, AmbiguityKind, ParsedName, Role, Token) +from .cases import CASES from .conftest import differential_corpus _ALPHABET = st.sampled_from( @@ -953,6 +954,18 @@ def test_any_valid_config_still_parses_totally( assert isinstance(str(parsed), str) assert isinstance(parsed.capitalized().given, str) assert isinstance(parsed.initials(), str) + # rules.md#R4 under configuration: what this buys is COVERAGE of + # the non-mask clauses (shape-acronym, numeral, plain word) across + # drawn vocabularies and policies -- it falsifies on e.g. a clause + # that strips a period -- not DETECTION of a mask substitution. + # _recased draws each mask as the key's own letters recased, so + # under this fuzzer every masked word already differs from its + # mask only in case (`_apply_mask` returning `mask` verbatim still + # passes here); the shipped-map walk at the end of this module is + # what covers substitution. + repaired = parser.capitalized(parsed, force=True) + assert ([t.text.casefold() for t in repaired.tokens] + == [t.text.casefold() for t in parsed.tokens]) @given(_lexicons(), _policies()) @@ -2460,3 +2473,107 @@ def test_no_birth_name_word_reads_as_a_word_of_the_current_name() -> None: assert not failures, ( f"{len(failures)} birth-name word(s) read as a word of the " f"current name:\n" + "\n".join(failures[:10])) + + +# --- #459/#492/#478: case repair changes case and nothing else ------ +# Not a grid, and deliberately not joined to one: the three grids +# above generate connective shapes under variant lexicons, and this +# invariant is about every name a caller has actually written -- the +# deduped corpus glob plus the case table's texts -- under the two +# lexicons that decide what the exceptions map can do, the shipped one +# and one with the map emptied. The fuzzed half rides the existing +# hypothesis walk instead (test_any_valid_config_still_parses_totally), +# which already draws masks. Measured 2026-09-23 on py3.11: 1606 texts, +# and the whole walk -- two lexicons, both surfaces, plain and forced +# -- under a second (`--durations`). + +_CASE_ONLY_TEXTS = list(dict.fromkeys( + [*_FORK_CORPUS, *(case.text for case in CASES)])) +_FACADE_LISTS = ("title_list", "first_list", "middle_list", "last_list", + "suffix_list", "nickname_list", "maiden_list") + + +def _case_only_violations() -> list[str]: + """Every repaired token, and every repaired facade list element, + that differs from what it repaired by more than case. Compared + token by token on the core and list by list on the facade, never + through a rendered string, so render spacing cannot hide or fake + a difference.""" + emptied = dataclasses.replace(Lexicon.default(), + capitalization_exceptions=()) + configs = (("default", Parser(), Constants()), + ("map emptied", Parser(lexicon=emptied), + Constants(capitalization_exceptions={}))) + out: list[str] = [] + for label, parser, constants in configs: + for text in _CASE_ONLY_TEXTS: + name = parser.parse(text) + for force in (False, True): + repaired = parser.capitalized(name, force=force) + for was, now in zip(name.tokens, repaired.tokens, + strict=True): + if was.text.casefold() != now.text.casefold(): + out.append( + f"[core, {label}, force={force}] {text!r}: " + f"{was.text!r} -> {now.text!r}") + human = HumanName(text, constants=constants) + before = {attr: getattr(human, attr) for attr in _FACADE_LISTS} + # capitalize() mutates in place, so the forced call repairs + # the plain call's output -- still compared against the + # lists as parsed, which casefold equality makes transitive + for force in (False, True): + human.capitalize(force=force) + for attr in _FACADE_LISTS: + after = getattr(human, attr) + if ([s.casefold() for s in after] + != [s.casefold() for s in before[attr]]): + out.append( + f"[facade, {label}, force={force}] {text!r} " + f"{attr}: {before[attr]!r} -> {after!r}") + return out + + +def test_case_repair_changes_case_and_nothing_else() -> None: + """rules.md#R4 read as an invariant: a repaired token equals its + input under str.casefold(), on both surfaces, plain and forced, + under the shipped lexicon and under one whose exceptions map is + empty. casefold(), not a character-for-character check, is the + comparator because the rule was never stated against Python's own + casing tables: 'ss'/'SS' both fold to 'ss' though 'ß'.upper() is + 'SS' (two characters for one), and the roman-numeral shape under + re.I admits the dotless 'ı', which lower()s to itself but + upper()s to 'I' -- both are casing this walk accepts, not length + changes rules.md#R4 promises against. + + Recorded negative control, measured 2026-09-23 over this walk: at + the parent 4d0680e6, where the exceptions map SUBSTITUTED its + value ('md' -> 'M.D.', 'iii.' -> 'III'), this fails on 257 + repairs, every one under the shipped lexicon and none with the map + emptied. The live control is the test below. + """ + failures = _case_only_violations() + assert not failures, ( + f"{len(failures)} repair(s) changed more than case:\n" + + "\n".join(failures[:10])) + + +def test_the_case_only_walk_can_fail( + monkeypatch: pytest.MonkeyPatch) -> None: + """The walk above must be able to report a violation, or its + silence proves nothing. Putting the pre-#459 substitution back -- + the mask applier returning the map's value verbatim -- has to be + seen. Measured 2026-09-23, it fails on 16 repairs: 8 token repairs + counted on both surfaces (core and facade), the same way the + parent's 257 is counted -- the five texts that write Ph.D. with + its periods, in either case, including the all-initials spelling + p.h.d., which the mask 'PhD' would strip of them. Far fewer than + the parent's 257 because the shipped map no longer holds a value + spelled unlike its key's common spellings; the count is a floor on + the walk's reach, not a target. + """ + import nameparser._render as render_module + monkeypatch.setattr(render_module, "_apply_mask", + lambda word, mask: mask) + failures = _case_only_violations() + assert failures, "the case-only walk cannot see a substitution" + assert any("'Ph.D.' -> 'PhD'" in line for line in failures), failures From d4a26b33a85dbe3fde5f22bf5163ba78ce4e5009 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 24 Sep 2026 00:12:51 -0700 Subject: [PATCH 5/9] docs(#459/#492/#478): the decisions, the guides and the release notes decisions.md#R4 records the masks and why a replacement string is no longer capitalization; the check (letters and digits, the offered key, a decomposed word finding a composed key); the decided raise through the v1 shim; the given-role decision and what it reaches (iv smith -> Iv Smith, Option A); the mask's punctuation marking its joins and the split-off-initial rule, with the two drafts that were wrong; the DECLINED construction-time warning and shim translation, written in review and dropped when their premise was measured false; the invariant's Unicode boundary (ss, dotless i); the numerals and the xi parse limit; the shape acronyms; the hyphen rule with its initial guard and Maori non-reach; and the movers with a recipe that states its per-sub-word attribution. Two Excluded blocks keep md/ii/iii/iv out of the map and 'a' out of the connectives. decisions.md#R5 records the suffix-free gate, why titles stay evidence, the split from the parser's own one-case reading (now pinned in test_render), and its population; short dated bullets under the entries that said #492 stayed deferred say it landed, and the triage entry's standing marker counts are amended in place after the #489 precedent. The 3-0 list carries the shim raise. mechanisms.md's RENDER-HONORS-THE-PARSE loses a passage stale since #461. customize.rst, migrate.rst and usage.rst teach masks, the gate sentence and what a spliced field falls back to; AGENTS.md's sites that taught the dotted output are corrected; the release log amends the #459 acronym bullet in place and adds four bullets, every count a dated snapshot over the parent's corpus with its recipe named. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 8 +++--- docs/customize.rst | 39 +++++++++++++++++++++------- docs/design/decisions.md | 52 ++++++++++++++++++++++++++++++++++--- docs/design/mechanisms.md | 2 +- docs/migrate.rst | 12 ++++++--- docs/release_log.rst | 12 +++++++-- docs/usage.rst | 12 +++++++-- nameparser/_config_shim.py | 2 +- nameparser/_lexicon.py | 14 +++++----- nameparser/_render.py | 6 +++-- tests/v2/test_properties.py | 13 ++++++---- tests/v2/test_render.py | 21 +++++++++++++++ 12 files changed, 155 insertions(+), 38 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1b2d28c1..7ddee687 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -269,7 +269,7 @@ The library has two layers: `nameparser/config/` (data) and `nameparser/parser.p ### Configuration layer (`nameparser/config/`) -Most modules define a `frozenset` of known name pieces; `capitalization.py` and `regexes.py` define dicts. The SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed by configuring an object — a private `Constants` for `HumanName`, a `Lexicon` for the 2.0 API — never by editing the constant. A union of a `frozenset` with a set literal is still a `frozenset` (`TITLES`, `PARTICLES`), so the derived sets are frozen too. `CONSTANTS`/`Constants` still hand out mutable `SetManager`s; the freeze is on the module set constants they copy from. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the split-default hazard the freeze closes is still live for it, measured on 2.2: `CAPITALIZATION_EXCEPTIONS['phd'] = 'PhD'` reaches a freshly built `Constants` and neither the cached `Lexicon.default()` nor the shared `CONSTANTS`. Same advice — configure the object (`constants.capitalization_exceptions[...]`, or `dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. +Most modules define a `frozenset` of known name pieces; `capitalization.py` and `regexes.py` define dicts. The SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed by configuring an object — a private `Constants` for `HumanName`, a `Lexicon` for the 2.0 API — never by editing the constant. A union of a `frozenset` with a set literal is still a `frozenset` (`TITLES`, `PARTICLES`), so the derived sets are frozen too. `CONSTANTS`/`Constants` still hand out mutable `SetManager`s; the freeze is on the module set constants they copy from. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the split-default hazard the freeze closes is still live for it, measured on 2.2 and re-measured 2026-09-23 with the key below once `'PhD'` became the shipped value: `CAPITALIZATION_EXCEPTIONS['dphil'] = 'DPhil'` reaches a freshly built `Constants` and neither the cached `Lexicon.default()` nor the shared `CONSTANTS`. Same advice — configure the object (`constants.capitalization_exceptions[...]`, or `dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. - `titles.py` — `TITLES` (prenominals) and `GIVEN_NAME_TITLES` (e.g. "Sir", which treat the following name as given, not family) - `suffixes.py` — `SUFFIX_ACRONYMS` (with periods, e.g. "M.D.") and `SUFFIX_WORDS` (e.g. "Jr."), plus `GLUED_HONORIFICS` (#308), the subset of `SUFFIX_WORDS` the peel may split off the END of a name token — a separate, harsher set, since the glued position has no writer-drawn boundary to lean on @@ -278,7 +278,7 @@ Most modules define a `frozenset` of known name pieces; `capitalization.py` and - `conjunctions.py` — `CONJUNCTIONS` (e.g. "and", "of") used to chain multi-word titles and to join name parts (rules.md#P3), plus `CONJUNCTIONS_AMBIGUOUS` (#383/#479), the subset of single cased letters that read as an INITIAL in a name written wholly in one case — a marker set with no v1 `Constants` attribute of its own, so the only v1 knob that reaches it is deleting the conjunction - `maiden_markers.py` — `MAIDEN_MARKERS` (e.g. "née", "geb.") routing the following name to `maiden` - `surnames.py` — `KOREAN_SURNAMES`, the census list the 2.0 API splits unspaced hangul on (#271). With `maiden_markers.py` it is one of the two data modules `Constants` has **no** attribute for: both reach the parse only through `Constants._snapshot()` → `Lexicon`, so the v1 surface stays frozen and there is no v1 knob to turn either off (the opt-out is the 2.0 `Policy`) -- `capitalization.py` — `CAPITALIZATION_EXCEPTIONS` mapping (e.g. `{'phd': 'Ph.D.'}`) +- `capitalization.py` — `CAPITALIZATION_EXCEPTIONS` mapping of case MASKS (e.g. `{'phd': 'PhD'}`): each value is its key's own letters and digits recased, laid over the word as written, so `ph.d.` repairs to `Ph.D.` and repair never writes the value's own punctuation into the word (#459); `Lexicon` raises `ValueError` on a value that spells anything else - `regexes.py` — dict of compiled regular expressions (wrapped in `RegexTupleManager` by `Constants`) `config/__init__.py` wraps everything into `SetManager` and `TupleManager` instances inside a `Constants` class. A module-level singleton `CONSTANTS` is shared across all `HumanName` instances by default. @@ -319,7 +319,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, the R1 entry pass `suffix_entries` re-run over the forced state so a suffix value's entries follow its own commas, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. **Between raise and silence sits the construction-time `UserWarning`**, for a gap that is statically decidable, harmless to SOME deliberate caller, and indistinguishable-from-working for everyone else: the segmenterless activation (#337 — `parser_for(locales.JA)` without a segmenter behaved exactly like a working parser minus the feature) warns rather than raises because the inert JA registration is itself a pinned property, and a warning is filterable by the caller who wants exactly that. The message must carry every applicable remedy and no inapplicable one (the `ja_segmenter` hint fires only when a Japanese script is among the dead ones). Test fuzzers that legitimately construct such configs suppress the warning by MESSAGE, never by category — a blanket `UserWarning` ignore would mask the next construction diagnostic (`_quiet_parser` in `tests/v2/test_properties.py` is the pattern). -- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Seven exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), `conjunctions_ambiguous = CONJUNCTIONS_AMBIGUOUS ∩ conjunctions` (#383/#479 behavior with no v1 manager of its own either, so the one v1 knob that reaches it is deleting the conjunction — which turns the marking off, `test_snapshot_removing_a_conjunction_turns_the_marker_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. A `capitalization_exceptions` value that does not spell its key's letters raises `ValueError` at snapshot rather than being translated (#459), because v1 substituted such a value for the word and 2.4 repair only recases, so there is no v1 behavior to reproduce; pinned by `test_a_mismatched_exception_value_raises_at_the_first_parse`. +- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Seven exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), `conjunctions_ambiguous = CONJUNCTIONS_AMBIGUOUS ∩ conjunctions` (#383/#479 behavior with no v1 manager of its own either, so the one v1 knob that reaches it is deleting the conjunction — which turns the marking off, `test_snapshot_removing_a_conjunction_turns_the_marker_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. A `capitalization_exceptions` value that does not spell its key's letters and digits raises `ValueError` at snapshot rather than being translated (#459), because v1 substituted such a value for the word and 2.4 repair only recases, so there is no v1 behavior to reproduce; pinned by `test_a_mismatched_exception_value_raises_at_the_first_parse`. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Every pipeline stage is one module holding one public function of the same name** (`_tokenize.py`/`tokenize`), with the exceptions named where they stand — `_group.py`'s `marker_run_length`, a shared predicate (mechanisms.md#ONE-PREDICATE-PER-QUESTION), and since #511 `_post_rules.py`'s `suffix_entries`, the R1 entry pass as a function, because `Parser.revise` runs that one pass over a forced-role sub-parse and a stage's tail cannot be called on its own — **and its module docstring declares the contract in three labelled lines**: `Consumes:` what it takes from `ParseState`, `Produces:` what it hands back, `Reads:` which `Policy`/`Lexicon` fields it consults. `Reads:` is the load-bearing one — it makes "which stage do I touch for this feature?" a grep rather than a read-through. The authority for the stage set is `_pipeline/__init__.py`'s `STAGES` (eight, and not public API) with the field-ownership map in `ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`; NOT rules.md, which is implementation-free by its own preamble and whose `implemented:` names modules honoring a rule rather than stages. A `_pipeline/` module that is not a stage says so in its first line instead (`_assemble.py`: "Not a stage: …", omitting `Reads:` because it consults neither), so absence of the three lines is a claim about the module rather than an oversight. Provenance: §5 of the 2026-07-11 conventions spec, recorded here 2026-08-16. - **A claim about WHICH STAGE or WHICH LAYER does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. That single claim then shipped SIX times across three correction rounds, which is the part worth internalizing: **when a mechanism claim turns out wrong, sweep for where else you wrote it, and sweep again at the END of the change over the words the change itself just added.** Prose density here means one idea lives in a docstring, a case note, a release-log entry and this file at once; the implementer working against a wrong mechanism is the person most likely to restate it; and rewriting a mechanism claim is writing one, so the correction earns the same one-command check as the original — two of the six instances were fresh errors introduced by the sentence fixing the previous one. **Adjacency is the trap.** The claim that feels already-known is the one about the neighbouring stage or the neighbouring layer: one comment block in `tests/v2/test_facade_cases.py` got the exception type, the raising layer, the skip mechanism, the count of skipped rows, and which row was blocked all wrong at once — every one a claim about `_config_shim` versus real 1.4.0, written from reasoning, in the file whose whole job is translating between them. What finally held was not better prose but moving the claim into a test (`_CORE_ONLY_IDS`), which cannot be wrong the way a sentence can. @@ -386,7 +386,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Expected-failure tests use `@pytest.mark.xfail`** — the conftest parametrized fixture breaks `@unittest.expectedFailure`; always use `@pytest.mark.xfail` instead. -**`lc()` strips leading and trailing periods** — `'M.D.'` → `'m.d'`, not `'md'` (interior periods are preserved). Exception keys in `capitalization_exceptions` are dot-free, so lookups must also try `.replace('.', '')`. +**`lc()` strips leading and trailing periods** — `'M.D.'` → `'m.d'`, not `'md'` (interior periods are preserved). Shipped keys in `capitalization_exceptions` are dot-free, so lookups must also try `.replace('.', '')`; whatever key matches, its value is a case mask applied to the word as written, never substituted for it (#459). **`is_suffix()`'s period-stripping is asymmetric** — it does `lc(piece).replace('.', '')` (strips *all* periods) before checking `suffix_acronyms`, but only `lc(piece)` (leading/trailing only) before checking `suffix_not_acronyms`. Code that reimplements this check instead of calling `is_suffix()` must mirror both branches or it will misclassify acronym suffixes with internal-only periods (e.g. `"M.D"` with no trailing dot) — bit `parse_nicknames()`'s `handle_match()` in PR #189. diff --git a/docs/customize.rst b/docs/customize.rst index 3f4a18e7..1f0e60f9 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -161,10 +161,14 @@ Fixing the case of a particular word ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``capitalization_exceptions`` is the one pair-valued field — each entry -maps a lowercase key to its exact-cased replacement (``"phd"`` → -``"Ph.D."``), so it isn't a fit for ``add()``/``remove()``. Change it with -``dataclasses.replace()`` instead, and pass the result to -``capitalized()``: +maps a lowercase key to a *case mask*: the key's own letters and +digits, each in the case it should take (``"phd"`` → ``"PhD"``). Case +repair lays the mask over the word as it was written and keeps every +other character where it stood, so it recases a word and never +re-spells it. A value that spells anything else raises ``ValueError`` +when the lexicon is built. Being pair-valued, the field isn't a fit for +``add()``/``remove()``. Change it with ``dataclasses.replace()`` +instead, and pass the result to ``capitalized()``: .. doctest:: @@ -179,15 +183,28 @@ maps a lowercase key to its exact-cased replacement (``"phd"`` → ... + (("dphil", "DPhil"),)) >>> str(parse("jane smith dphil").capitalized(lex)) 'Jane Smith DPhil' + >>> str(parse("JANE SMITH D.PHIL.").capitalized(lex)) + 'Jane Smith D.Phil.' Note the ``tuple(...) + ...``: assigning a bare ``(("dphil", "DPhil"),)`` would *replace* the default exceptions rather than extend them, so -``"phd"`` and the rest would stop being fixed. +``phd``, ``bsc`` and ``msc`` would lose their masks and fall back to +the all-capitals acronym repair: ``john smith phd`` would give +``John Smith PHD`` rather than ``John Smith PhD``. The key is matched against the token with punctuation normalized away, not against the raw text, so one ``"phd"`` entry covers ``"phd"``, ``"Phd"``, and ``"Ph.D."`` alike — you don't need a separate key for -each way a source might punctuate it. +each way a source might punctuate it, and each keeps its own +punctuation: ``Ph.D.`` repairs to ``Ph.D.``, not ``PhD``. Punctuation +in the *value* is never written into the word; it only marks which of +the mask's letters are joined. That matters for one case: a single +letter the writer split off beside a full stop is an initial, and is +capitalized where the mask keeps that letter inside a longer run, so +``p.h.d.`` repairs to ``P.H.D.`` under ``"PhD"``. A credential +acronym conventionally written all in capitals and a roman numeral +need no entry at all: case repair writes a suffix of either kind in +capitals by itself. Words that are also ordinary names ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -858,9 +875,11 @@ signatures: first letters, with its own ``delimiter``/``separator`` arguments. * :meth:`~nameparser.ParsedName.capitalized` returns a new, case-fixed :class:`~nameparser.ParsedName` instead of a string. It only touches - input that's already single-case (all lower, all upper) unless you - pass ``force=True`` — mixed case is left alone by default on the - assumption that someone already capitalized it on purpose. + a name whose words are written in a single case (all lower, all + upper) unless you pass ``force=True`` — mixed case is left alone by + default on the assumption that someone already capitalized it on + purpose. The suffixes don't count toward that: ``III`` or ``PhD`` + written the usual way says nothing about how the name was cased. .. doctest:: @@ -876,6 +895,8 @@ signatures: 'JuAn DE LA vEGA' >>> str(parse("JuAn DE LA vEGA").capitalized(force=True)) 'Juan de la Vega' + >>> str(parse("juan garcia III").capitalized()) + 'Juan Garcia III' Looking for v1's ``string_format``? It's the ``render(spec)`` argument now — pass your own format string per call instead of setting it once diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 19489dbb..bd20ae56 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -285,7 +285,7 @@ Recomputed 2026-09-07 with that recipe, after this section's own #342 decision l Open: [#348](https://github.com/derek73/python-nameparser/issues/348) applying C-i to the 711 title entries, then titles_ambiguous plus a TITLE_OR_GIVEN kind. Blocked on data, not on judgement — the census needs a given-name frequency corpus this repo does not have, which is why the criterion is recorded here and the census is not attempted. -### v1-xfail-triage — the eight inherited aspirations, four retired and four kept (two of those eight since fixed, 2026-09-08) +### v1-xfail-triage — the eight inherited aspirations, four retired and four kept (three of those eight since fixed, 2026-09-08 and 2026-09-23) The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xfail` tests inherited from v1, each an aspiration nobody had judged since. `xfail_strict = true`, so all eight were live claims that 2.0 still does not do the thing — but a bare marker says only that, never whether the thing is WANTED, and that is the gap this triage closes. After it, every surviving marker cites an issue and every retired one is a deliberate pin of current behavior. The parses quoted below are this entry's evidence and its own recompute: @@ -297,11 +297,12 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf Two halves, and only one was decided. DECIDED: `king` stays in TITLES, which is unchanged and is what still puts `King` inside the title run in the first place. RECORDED, not endorsed: what becomes of the leftover `Jr`. rules.md#S2 predicts suffix 'Jr' with an empty family — its Accepted clause consumes an unambiguous suffix even when nothing is left to be the family (`Smith Jr.` → family "") — but once the title chain had taken `Dr King`, H1 claimed the one remaining word and it read family 'Jr', suffix ''. `Dr Smith Jr` isolated the cause: family 'Smith', suffix 'Jr', exactly as S2 states. S2 carried a descriptive note saying so. This entry then said that a future change moving `Dr King Jr` toward S2's prediction is an IMPROVEMENT and updates the pin — that change is #H3's peel floor and it landed 2026-09-08. The reading is now title 'Dr', family 'King', suffix 'Jr': the run gives back its last word where everything behind it is post-nominal and that word is not itself suffix vocabulary, so `Dr King Jr` reads like `Dr Smith Jr` rather than against it, and S2's note is replaced by the argument it asked for. The pin in `tests/test_suffixes.py::test_king` moved with it, by decision rather than by drift, and `king` being title vocabulary now buys the row a `title-or-name` report (#H4) rather than a different reading. - `Ahmad ben Husain` — v1 wanted family "ben Husain"; it reads given Ahmad, middle ben, family Husain. Already decided in v0.2.5, when `ben` came out of the prefixes, and for the reason that still holds: `ben` collides with the given name Ben, in the position the particle claim would act on — `Ahmad Ben Husain` reads middle 'Ben' today, which is exactly the token a forward-joining particle claim would take. That is C-i's position test, and it keeps `ben` out. Recorded a second time as a standing keep-out in this file's Excluded block for the particle set, because that is where a wordlist sweep meets it: a keep-out that lives only in a triage entry is one the next Arabic/Hebrew patronymic-particle sweep never reads. Worth naming as a failure mode of its own — the marker was an aspiration that outlived its own resolution, and nothing in a bare xfail says which of the eight were like that. - `The Right Hon. the President of the Queen's Bench Division` — v1 wanted the whole string as one title; it reads title "The Right Hon. the President of the Queen's Bench", family 'Division'. This is a name parser, not a title parser: handed an input that is all titles it assumes the last title-word is the name. Accepted as convention rather than defended as correct — and since 2026-09-07 the guess is no longer silent: rules.md#H4 states the convention and the parse reports `title-or-name`, which is what [#491](https://github.com/derek73/python-nameparser/issues/491) asked for and is a report rather than a change of reading. See decisions.md#H4. -- 2026-09-01 — FIX CANDIDATES, four. Each carries its issue, so `pytest -rx` names the work instead of listing anonymous aspirations. THREE of the four still carry a marker; #489 was fixed 2026-09-08 and its marker is gone, which is why this line no longer says the marker stays: +- 2026-09-01 — FIX CANDIDATES, four. Each carries its issue, so `pytest -rx` names the work instead of listing anonymous aspirations. Two markers remain, both #490's (re-counted 2026-09-23, #492); #489 was fixed 2026-09-08 and #492 on 2026-09-23 and their markers are gone, which is why this line no longer says the marker stays: - [#489](https://github.com/derek73/python-nameparser/issues/489) — **FIXED 2026-09-08 (the #316/#489 bundle).** `Her Majesty Queen Elizabeth` should address by given name (`tests/test_conjunctions.py::test_conjunction_in_an_address_with_a_first_name_title`), and it does: a title RUN addresses as its LAST title does, `queen` being a given-name title (#H1). The xfail marker is removed and the test is an ordinary passing pin. - [#490](https://github.com/derek73/python-nameparser/issues/490) — `E.T. Smith` (`tests/test_conjunctions.py::test_two_initials_conflict_with_conjunction`) and `U.S. District Judge Marc Thomas Treadwell` (`tests/test_titles.py::test_chained_title_first_name_title_is_initials`). One issue for two tests deliberately: each test's own comment names the other's shape as what blocks a fix — dotted initials against dotted title and credential vocabulary — so they are one question, and fixing either alone is what has failed before. - [#492](https://github.com/derek73/python-nameparser/issues/492) — `capitalize()` leaves `juan garcia III` lowercase (`tests/test_capitalization.py::test_capitalization_exception_for_already_capitalized_III_KNOWN_FAILURE`). The `_KNOWN_FAILURE` suffix is kept: it is still true, and beside the annotated reason it reads as redundant rather than misleading. -- [#485](https://github.com/derek73/python-nameparser/issues/485) is superseded. It proposed prefixing corpus labels with `xfail:` so a radar diff on a name like `Dr King Jr` reads as a known-bad parse improving rather than a regression. The premise was that a v1 xfail marker is a usable triage signal; after this triage it is not, because the marker no longer tracks the disposition — `Dr King Jr` is now a PIN, so a radar diff on it means a decided reading moved, which is precisely the signal the prefix would have suppressed. The four that remain carry issue numbers, which is the same information in a place that cannot go stale against the pinned historical ref. The 2026-09-08 fix STRENGTHENS this rather than unsettling it, and the argument is worth stating because the fix looks at first like the counterexample: `Dr King Jr` moved, so a diff on it is exactly what #485's prefix predicted — but it moved by DECISION, with a ledger rule and a rewritten pin, which is a classified change and not a known-bad parse quietly improving. Had the prefix shipped, that diff would have read as the aspiration finally coming true and nobody would have been asked to classify it. Three markers remain rather than four, and they still carry issue numbers. +- 2026-09-23 [#492](https://github.com/derek73/python-nameparser/issues/492) — FIXED (the #459/#492/#478 bundle): `juan garcia III` repairs to `Juan Garcia III`, R5's gate leaving the suffixes out (decisions.md#R5, 2026-09-23). The marker is gone and the test is renamed `tests/test_capitalization.py::test_capitalization_exception_for_already_capitalized_III`, the `_KNOWN_FAILURE` suffix the bullet above kept being no longer true. Two markers remain, both #490's, and the marker counts above — this entry's heading, the FIX CANDIDATES line and the #485 bullet — were amended in place on this date, following the #489 fix's precedent. `tools/differential/corpus.jsonl` still carries the old test name as a label: that file is v1's own suite harvested at a pinned ref (tools/differential/README.md), so the label names where the row came from and is left as it is. +- [#485](https://github.com/derek73/python-nameparser/issues/485) is superseded. It proposed prefixing corpus labels with `xfail:` so a radar diff on a name like `Dr King Jr` reads as a known-bad parse improving rather than a regression. The premise was that a v1 xfail marker is a usable triage signal; after this triage it is not, because the marker no longer tracks the disposition — `Dr King Jr` is now a PIN, so a radar diff on it means a decided reading moved, which is precisely the signal the prefix would have suppressed. The four that remain carry issue numbers, which is the same information in a place that cannot go stale against the pinned historical ref. The 2026-09-08 fix STRENGTHENS this rather than unsettling it, and the argument is worth stating because the fix looks at first like the counterexample: `Dr King Jr` moved, so a diff on it is exactly what #485's prefix predicted — but it moved by DECISION, with a ledger rule and a rewritten pin, which is a classified change and not a known-bad parse quietly improving. Had the prefix shipped, that diff would have read as the aspiration finally coming true and nobody would have been asked to classify it. Two markers remain, both #490's (re-counted 2026-09-23, #492), and they still carry issue numbers. ### suffix-field-composition — three kinds of thing in one field @@ -357,6 +358,7 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf TWO OTHER CAUSES SHARE THE `_initials` FIELD IN THE LEDGERS AND NEITHER IS THIS PR'S, recorded because a reader meeting them under these names will reach for this entry. The Arabic `محمد و علي` diffs on `_initials` at 1.4.0 only (`م. و. ع.` → `م. ع.`), measured byte-identical either side of the fork: it is a pre-existing #269 consequence — a recognized non-Latin connective contributes no initial — surfaced by the row entering the contract corpus, and `expected_since_1.4.0.toml` ledgers it as `feat(#269)`. `JUAN Y GARCIA` diffs on `_initials` at 2.0.0 through 2.2.0 for a DIFFERENT reason than it does at 2.3.0: at those three baselines `fix(#462)` already admits the name (the facade moved `J. G.` → `J. Y. G.` there), and this PR's core move `J. Y. G.` → `J. G.` gets its own rule at 2.3.0 alone. Two causes, one field, documented in a dated paragraph on `fix(#462)` in those three ledgers rather than as a competing rule. Out of scope, each its own issue: #492 (whether a cased suffix token counts as case evidence — `is_one_case` is written so R5 and #492 can share it later, but render does not import it here); #478 (hyphenated connective repair — its spaced-form claim now depends on "y" staying OUT of the subset); #461 (R3's clause); #289 and #516 (the same case-class fact read at the suffix and post-comma slots); the render-side conjunction fallback for spliced raw text, which stays vocabulary-keyed as rules.md#R4 states. - 2026-09-15 #289/#516 — AMENDS the out-of-scope list above: #289 and #516 are no longer out of scope, having shipped in 2.4 on the fact this entry's own fork computes. The fact is now a `ParseState` field rather than a classify local, taken over the SAME own-words span this entry defines, and `_pieces.own_words` is where that span lives for both readers. #492 stays out of scope and stays deferred. See #S2. + - 2026-09-23 #492/#478 — AMENDS the out-of-scope list above again: both have landed, in RENDER only. #492: R5's gate leaves the suffixes out (decisions.md#R5, 2026-09-23); this entry's fork is untouched and still counts a cased suffix, which is the R5/parser split that entry records, and render still does not import `is_one_case`. #478: a connective part inside a hyphenated word keeps its lowercase (decisions.md#R4, 2026-09-23); the spaced-form claim it rested on is unchanged, and still depends on `y` staying out of the subset. - 2026-09-13 #383/#479 — `conjunctions_ambiguous` is deliberately NOT registered in `_SUBSET_FIELDS`, so an orphan marker entry raises nothing. An orphan there is INERT rather than harmful: the classify fork tests membership in `conjunctions` before it reads the subset, and the emitter tests it again, so a marker entry whose base word is gone is never consulted by either. AGENTS.md's invariants rule guards harm, not no-ops, and the precedent is this file's #given-name-titles Declined entry, where two attempts at a check each cost a working configuration to forbid a condition that costs nothing. What holds the SHIPPED constant to the subset relation is an import-time assert in `nameparser/config/conjunctions.py` (which also holds it to cased single letters, the only entries the fork can read), and the v1 shim's intersection, which is a provable no-op. So `remove(conjunctions={"e"})` simply works, leaving a stale marker entry behind that does nothing. Pinned by `test_removing_a_conjunction_leaves_its_ambiguous_marker_alone` in tests/v2/test_lexicon.py and by the orphan behavioral pin in tests/v2/pipeline/test_classify.py, both of which cite this entry. @@ -394,6 +396,10 @@ Excluded (Lexicon.conjunctions_ambiguous, the marked half of nameparser/config/c - `و` and any other caseless letter — inert rather than harmful, and excluded so nobody adds one believing it does something: the fork requires a token whose `upper()` and `lower()` differ, so a caseless member is never consulted (2026-09-13). - Multi-letter entries (`and`, `та`, `και`) and `&` — the fork tests a single CHARACTER, so an entry longer than one letter is silently inert. `nameparser/config/conjunctions.py` asserts against it at import (2026-09-13). +Excluded (Lexicon.conjunctions — nameparser/config/conjunctions.py): + +- `a` — the Māori link in `Te Awanui-a-Rangi` is the one hyphen-interior connective the #478 rule (rules.md#R4) leaves capitalized under force, `Te Awanui-A-Rangi Black`, and it must not be fixed by adding `a` here: a spaced `a` is an initial (`John A Smith`) or a name word elsewhere, and a connective entry would join or lowercase it in every one of those names. The non-reach is recorded under R4 with that name as its witness (2026-09-23, #478). + Open (Lexicon.conjunctions — contested membership; the issue is canonical): none. The block held one entry, [#397](https://github.com/derek73/python-nameparser/issues/397) — whether Catalan `i` belongs in the conjunctions at all — answered 2026-09-20 by the bullet of that date above: it does, and it joins `conjunctions_ambiguous` in the same change, which was never the contested half. The heading stays with nothing under it on purpose, the same reason an empty ledger roster section is a statement and a missing one is nobody having looked. ### given-name-titles — deliberately unvalidated @@ -651,6 +657,7 @@ for n in ('Smith, John','Smith, XYZ'): print(n, calls_for(off.parse, n), calls_f - 2026-09-18 (Derek), #531 — CAPITALS DECIDE FOR `do`, AND THE PARTICLE RULE KEEPS EVERY OTHER SPELLING. `do` is the one member of the ambiguous credential class (`ba`, `do`, `ed`, `jd`, `ma` — recompute with `sorted(Lexicon.default().suffix_acronyms_ambiguous)`) that is also particle vocabulary, so the trailing slot of the given part and P6's attachment want the same word. Where the writing carries a positive credential lean — an all-caps member in a name written in more than one case — the credential reading wins and P6 stands down: `Doe, John DO` reads suffix `DO` and reports `suffix-or-name`. Every other spelling attaches exactly as it did before, with P6's own `particle-or-given` and no second report: `Doe, John do`, `Doe, John Do`, `DOE, JOHN DO`, `doe, john do` all read family `do Doe` in their own case. THE PAIRING IS THE ARGUMENT, and the accepted cost and the reason are the two halves of it. In ONE CASE the rule cannot tell `NASCIMENTO, EDSON ARANTES DO` from `SMITH, JOHN DO` and reads both as the particle — right about the Brazilian record, wrong about the osteopath, whose `DO` 1.4.0 read as a suffix; both report `particle-or-given`, which is how a caller finds the second. In MIXED CASE it can, and does: `Nascimento, Edson Arantes do` keeps family `do Nascimento` while `Doe, John DO` gives suffix `DO`. Accepted rather than repaired: the one-case reading is the commoner of the two collisions, and the Nascimento record is the name a wrong answer would damage. The other four members are not particles and are unaffected. This is the exception rules.md#P6's "this attachment outranks the suffix reading (S2)" clause now names. Implementing it takes TWO carve-outs rather than one, which is worth recording because the second is in the REPORT and not the reading: the reading carve-out takes a particle-tagged member only on a POSITIVE credential lean, since `Doe, John do` leans nothing and the positional reading would otherwise take it — the wrong ANSWER, not merely a stray report; and the emitter beside it skips a particle-tagged member it did not itself take, since an emitter gated on tag membership alone reported both kinds on every `do` row. Where P6 owns the fork, P6 reports it, which is mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE read strictly. ONE SHAPE THE PAIRING DOES NOT REACH — a shape, not a spelling, and the difference is the review finding: ANY particle or particle chain standing in front of the caps `DO` takes it out of this slot, because grouping's prefix chain merges the two into ONE particle piece before the slot exists and there is then no lone member for the capitals to lean on. It is a fact about this WORD being particle vocabulary itself, not about particles generally — the caps `MA` in `Doe, John van MA` is not swallowed, reads as the credential, and leaves `van` to attach behind it. `Doe, John van DO` reads family `van DO Doe`, and so do the `de`, `von`, `del`, `la` and `van der` spellings of it (measured 2026-09-19 — parse them and read `family`). Pre-existing and unmoved by #531; `Doe, John van DO` is the case row and rules.md#P6's example line, and P6's statement carries the carve-out. The mirror shape is silent rather than merely unreached and is recorded on `AmbiguityKind.SUFFIX_OR_NAME`'s fourth boundary: a particle BEHIND the member ends the given part where this walk reads it, so `Doe, John MA do` keeps middle `MA` beside family `do Doe` though the capitals would otherwise have taken it, the attachment that moves `do` running a stage later. - 2026-09-18 (Derek), #531 — THE COMMA REPORT'S REACH NOW INCLUDES THE GIVEN SEGMENT'S TRAILING SLOT, AND THE OPEN FOLLOW-UP IS CLOSED. The 2026-09-18 bullet above headed THE COMMA REPORT'S REACH IS THE FIRST POST-COMMA PIECE recorded this slot as an open maintainer decision and named the two questions it turned on — whether a middle initial's neighbourhood should start reporting (a noise judgement, rules.md#A1) and whether the silence was a 1.4 parity gap. Both are answered here, and that bullet stands as it landed. The slot now reads and reports: `Doe, John MA` gives suffix `MA` and `Doe, John Ma` keeps middle `Ma`, each saying which way it went. Derek chose to restore the ROLE and report both ways — one rule for both spellings — over a report-only change and over a capitals-only one, because the issue exists in the first place because two spellings of one name disagree. The noise question was settled by MEASURING THE DISAGREEMENT rather than by argument: over a generated sweep of 78 pairs — five listed members and two by-shape tokens in three cased spellings each, plus five controls in one spelling apiece, so 26 words against three name shapes — 48 pairs disagreed about whether the word was a credential or a name, and every one of the 48 disagreed in the same direction, the comma form declining what the comma-less form took. After this change 3 disagree and all three are the lower-case `do` rows P6 owns. The sweep and its allowlist are `tests/v2/test_properties.py`; a slot that answers differently from the same name written without a comma is not a quiet slot, it is an inconsistent one. Recorded under `3-0-reevaluations`' standing rule because v1 parity is LOAD-BEARING for one half of this and explicitly NOT for the other: `Doe, John MA` reads suffix `MA` on the 1.4.0 wheel, so the bare-acronym half RESTORES v1's role and the report is all that is new there — its 1.4.0 ledger rule neither retired nor narrowed to `_ambiguities` (nothing below baseline 2.0 can diff on that pseudo-field) but was RE-POINTED to the four names where the writing declines the credential, which keep the 2.0-era middle name against v1. `Doe, John X.Y.Z.` reads middle `X.Y.Z.` at 1.4.0 too, so the dotted half LEAVES v1 and carries a 1.4.0 ledger rule of its own; it moves to match the comma-less `John Doe X.Y.Z.` and rules.md#S3's shape rule, not to restore anything. BLAST RADIUS, stated the way this log's own rule asks: twenty corpus names move, and only TWO of them were in any corpus before this branch (`Doe, John MA` and `Doe, John X.Y.Z.`, both admitted by #530's own arc) — the other eighteen are this change's own case rows, so what the differential measures on pre-existing data is two names, and the population the rule reaches is a SHAPE (every family-comma listing whose given part ends in a member of this class) that the corpora barely sample. ACCEPTED COSTS, all measured on the differential corpora: `SMITH, JOHN DO` keeps family `DO SMITH` where 1.4.0 read suffix `DO`, paired with the Nascimento record in the bullet above; one-case and caseless names take the credential with no case evidence at all, so `DOE, MARY JO MA`, `doe, john ma`, `田中, 太郎 MA` and `김, 민준 MA` all read a suffix, which is what 1.4.0 read for each of the four — "caseless is inert" holds for the LEAN and not for the outcome, since `is_one_case` answers True for a script with no case and the positional reading then decides; `Doe, John van MA` loses its middle to the family, reading family `van Doe`, suffix `MA` with two reports where it read middle `van MA` in silence, which is `Berg, Jan van Jr.`'s reading arriving through a shape it could not reach before (Derek accepted it 2026-09-18 as P6 working correctly rather than as a cascade to carve out); `Doe, John Prof. MA` gains a TITLE role `Prof.` never had, H5's transparency reaching it once `MA` leaves the walk, landing it on the same answer as `Doe, John MA Prof.`; `Smith, LEED AP` moves under the default-off caps switch alone, to given `LEED`, family `Smith`, suffix `AP` with two reports, so no default reading is at stake; and a genuine middle name that is also a class member is now a credential wherever it ends the given part and is not Title-cased — `DOE, JOHN ED` reads suffix `ED` — which is the same cost the comma-less form has carried since 2.0. NOT REPAIRED HERE and left open: the maiden walk claims `Smith MA` whole in `Doe, Jane nee Smith MA` before this slot exists, so the rule cannot reach it and the name stays silent with maiden `Smith MA` — the same gap #530's close-out saw from the other side with `John Smith nee Jones R.A.I.`, and it is out of scope for #531. - 2026-09-19 (Derek), #533 — THE CLASS'S LAST SILENT TRAILING POSITION IS CLOSED, AND #531'S READING NOW LIVES IN ONE PLACE. The slot list in rules.md#S2 gains the trailing slot of a maiden marker's clause, and #S3's enumeration gains it with the by-shape spellings; the rule that governs what the clause does with those words is M2's, and the entry under `### M2` above is where its reasoning lives. The gap the bullet above left open as out of scope for #531 is the one this closes, from both sides: `Doe, Jane nee Smith MA` now gives maiden 'Smith' with suffix 'MA' and reports, and `John Smith nee Jones R.A.I.` gives suffix 'R.A.I.' again — a RESTORATION rather than a change, since 2.3.0 read it that way (measured on the wheel) and this unreleased cycle moved it into the maiden name when #516 took dotted tokens out of the certain-suffix class, where no corpus file held the name and no gate could see it. Two things belong here rather than under M2. FIRST, `credential_at_the_given_slot` in `_pipeline/_pieces.py` is now where #531's reading of a class member ending the given part is written, with two callers — assign's walk over that part, and the maiden walk's second check over the name a take would leave. Spelling it twice is precisely the "condition written to match it" mechanisms.md#ONE-PREDICATE-PER-QUESTION names, and the drift would have been silent, since each site's own tests would have gone on passing. It is a text-and-tags question, which is what puts it in `_pieces` rather than beside either caller — the destination follows the LAYER, not the topic. It costs one frame PER MEMBER asked at that slot: measured 2026-09-19 against 2f57ff21 per `Parser.parse`, `Doe, John MA` goes 310 → 311 and `Doe, John MA Ma MA`, which asks four times, 439 → 443, while a name with no member there never reaches it and pays nothing. The reference band does not move (412/449 on `uv run python tools/perf/call_count.py`), no test pins 310, and Derek took the trade rather than keep two conditions in step across two stages with no test that asks them both. The maiden path got one frame CHEAPER in the same pass, the numeral reading calling the peel pair it had wrapped rather than the wrapper: `Jane Doe nee Smith` 249 → 248. SECOND, THE ONE-CASE HEAD IS AN ACCEPTED EXCEPTION AND IT IS THE M2 INSTANCE OF #492'S DEFERRED QUESTION about whether a cased suffix token counts as case evidence. `DOE, JANE nee Smith Ma` reads suffix 'Ma' where the clause-less `DOE, JANE Ma` keeps middle 'Ma', because the own-words span stops at the marker — rules.md#P3 puts "a maiden marker's run and every word after it" outside the name's own words from the moment the marker is tagged — so the member's own Title-casing is not in the span `one_case` is computed over and the clause HIDES the contrast. That is THE PREDICATE KEEPS THE JUDGED TOKEN IN THE SPAN, the 2026-09-14 #289/#516 entry at the head of this section, failing structurally rather than by oversight: at this slot the judged token is never in the span, and that entry's own answer — include it — cannot be had here. Widening the span for this one question would change `one_case` for the whole name, and three sites read it, so the exception is accepted instead. Measured 2026-09-19: 114 of 2016 generated pairs — every listed member and both by-shape spellings, in three cased spellings, against sixteen heads and six clause bodies of NAME WORDS ONLY — against 186 allowlisted and 984 disagreeing outside the class before the change, and 0 outside it after. The PR review widened that grid to three markers (`nee`, `née`, `geb.`) and three policies (the default and each 2.4 switch), and the class is INDIFFERENT to both: 1,026 of 18,144, exactly 9x the original in both columns, and still 0 disagreeing outside it. `tests/v2/test_properties.py` carries the sweep with the class defined STRUCTURALLY (`one_case` true of the clause form and false of the clause-less one) rather than as a name list. The recorded control beside it is now the SET and not only its size, as a digest over the members: a count cannot notice a swap, one pair leaving and another arriving, which is the same blindness the count was added to close one level up. +- 2026-09-23 #492 — THE DEFERRED QUESTION the 2026-09-14 predicate paragraph and the 2026-09-19 #533 bullet above name — whether a cased suffix token counts as case evidence — IS ANSWERED FOR RENDER ONLY. R5's gate now leaves the suffixes out (decisions.md#R5, 2026-09-23); the parse-time fact this section rests on keeps counting the judged token exactly as written above, so `Jack MA` still leans credential and `DOE, JANE nee Smith Ma` still reads suffix `Ma`. The split is decided and recorded under R5, with the shapes it makes visible there (`jack MA` at this slot, `john e jones III` at P3's). ### indic-honorifics — the renunciate class and the Indic honorific vocabulary (2026-09-06, #346/#344/#343) @@ -1341,6 +1348,36 @@ Accepted costs, deferred to the rescoped #459 rather than relitigated here: the AND THEN THE REPAIR ITSELF WAS TOO WIDE, found by the second review round (2026-09-20). The guard shipped as the suffix ROLE alone, and the role says only where a word LANDED. A plain connective can land in the suffix field without being generational vocabulary at all — the third part of a comma form is read as the suffix run whatever its words are — so `parse("Smith, John, and").capitalized(force=True)` gave 'John Smith And' where 1.4.0, 2.0.0 through 2.3.0 and the parent 46651750 all give 'John Smith and', and the same for `, y`, `, e`, `, und`, `, of` and `Doe, Jane, and Jr.`. A field SPLICED in went the same way and against R4's own Accepted paragraph, which says a field nobody read asks the vocabulary: `HumanName(first="John", last="Smith", suffix="de y")` repaired forced to 'John Smith De, Y' against 'John Smith De, y' on both wheels. THE GUARD IS NOW THE PAIR — the suffix role AND the `vocab:suffix` tag — because the clause turns on the GENERATION and the role alone cannot see it; classify's tag is the vocabulary half of the decision and reading both re-derives neither. Invisible to everything already watching: no name of the off-switch grid carries the shape (its oracle is a letter of the class, and none of these words is one) and no corpus name carries it either, which is why INV7b was written over a grid of its own. ACCEPTED, and the one reading this moves rather than restores: a field spliced to `suffix="i"` now repairs to 'John Smith i' where the parent gave 'John Smith I'. `i` is connective vocabulary on this branch and was not at the parent, so the spliced field follows `y` now — which is R4's Accepted paragraph working as written, the PARSED name being unaffected because it has a reading and the reading is the generation. MEASURED, and the numbers are the fix's own control. Over the same eight configurations the forced-only set is ten names before the fix and two after; the invariant `tests/v2/test_properties.py::test_a_letter_that_did_not_join_repairs_as_the_off_switch_does` — a letter that became no connective of this name repairs as the off-switch parse repairs it, plain and forced, on both surfaces — fails on 7,860 repairs at `e540d4c5` (6,843 core, 1,017 v1) and on 0 here, and removing the generation conjunct outright fails it on 8,054. (11,341 / 10,424 / 917 was the same control over the full cross product before the grid was trimmed by shape and then grown by the second review; it is a dated snapshot and the test's docstring is the live figure.) The differential still cannot see any of it, case being no compared surface (the 2026-08-29 bullets above settle why), which is why this needed an invariant rather than a ledger rule. +- 2026-09-23 #459 — DECIDED (Derek, 2026-09-23): the exceptions map holds case MASKS, and nothing else. A value is its key's own letters and digits, each in the case it takes, and case repair lays it over the word as the writer punctuated it, keeping every other character where the writer put it — so one `phd` entry repairs `phd` to `PhD`, `ph.d.` to `Ph.D.` and `PH.D.` to `Ph.D.`, and the map can hold a conventionally mixed-case acronym (`BSc`, `MSc`) without a dotted spelling per source. A replacement string is substitution, not capitalization: it is how `md` became `M.D.`, how `iii.` lost its period, and why the map never grew past the five entries it was imported with in 2011. The shipped map is `phd` → `PhD`, `bsc` → `BSc`, `msc` → `MSc`; `md`, `ii`, `iii` and `iv` left it, a suffix `md` now repairing by the acronym clause and a suffix numeral by the numeral clause (both below). Lookup is unchanged — the edge-stripped key, then the period-free one — and the map stays ROLE-FREE: an entry is the caller saying how a word is written wherever it stands (`phd smith` gives `PhD Smith`). rules.md#R4 now states the invariant every clause serves — repair changes case and nothing else — and `tests/v2/test_properties.py::test_case_repair_changes_case_and_nothing_else` holds it over every corpus name and case-table text, plain and forced, on both surfaces, under the shipped lexicon and one with the map emptied, comparing under `str.casefold()` (the Unicode-boundary bullet below says why that comparator). `md` → `MD` and `phd` → `PhD` are DELIBERATE DEVIATIONS from every release, 1.4.0 through 2.3.0, all of which gave the dotted forms (measured 2026-09-23 on all five wheels). There is no contributor intent behind the period insertion to preserve: the map, its `md` entry and `capitalize()` itself arrived together in the 2011 svn import (45a15394), authored by Derek. + +- 2026-09-23 #459 — THE CHECK, and two places it departs from the approved spec, both by measurement. A value that does not spell its key's letters and digits is a `ValueError` at `Lexicon` construction (`_lexicon._normpairs`). Both sides are folded through `_normalize` and compared as ALPHANUMERICS (`_lexicon._alnum`), so `('2nd', '2ND')` is accepted and `('2nd', '3ND')` raises — a comparison of letters alone would have let a digit key take any other digit. The message names the entry and offers the key at hand recased — `capitalization_exceptions=(('jr', 'JR'),)` for a `jr` key — a tuple of pairs, because the field is annotated with what it stores and a dict literal there would be an arg-type error in the pasted code; where `.upper()` changes the alphanumeric count (`straße` → `STRASSE`) it offers the key unchanged instead, an identity mask, so the offered code always constructs. Every entry point reaches it: construction, `dataclasses.replace()`, `|`, unpickling (`__setstate__` re-runs `__post_init__`), and the v1 shim's snapshot; `tests/v2/test_lexicon.py` sweeps all five. FIRST departure: the spec compared the value against the KEY as "already normalized, period-free and lowercase" — it is not period-free, `_normalize` stripping only EDGE full stops, so `Ph.D.` is stored `ph.d` and, compared raw, a key written with an interior period could never validate. SECOND: the spec's "the two letter sequences are always equal in length" is false. The key is NFC-composed and the word is not, so a word written in decomposed hangul (`씨` as two jamo) finds a one-letter key. The applier, `_render._apply_mask`, declines there (it returns None) and `_cap_word` falls through to its next clause — a caseless script has no case to recase, so nothing is lost — pinned by `tests/v2/test_render.py::test_the_mask_keeps_every_non_letter_and_declines_a_miscount`. Without the decline the naive applier raised `IndexError` inside `capitalized()` (measured in review on 2026-09-23, on the prototype and again by the spec review). + +- 2026-09-23 #459 — A DECIDED EXCEPTION to the shim's never-raise rule (AGENTS.md: "The shim TRANSLATES; it never raises on a config v1 accepted"). A v1 `Constants` whose `capitalization_exceptions` carries a value that does not spell its key — `{'jr': 'Junior'}` — raises the `ValueError` above at `Constants._snapshot()`, i.e. at the first parse. Taken rather than translated because there is nothing to translate TO: 1.4.0 substituted such a value for the word, 2.4's repair only recases, and so the only v1-faithful reading is the one this change removes. No such value was found in the repository, the docs, the issue tracker or any test when the design was approved; the hypothesis fuzzer drew arbitrary pairs and now draws recasings of its key. A value that merely carries PUNCTUATION is not one of these: it is a legal mask (the joins bullet below), so a v1 `Constants` unpickled from 1.4.0, still carrying `md` → `M.D.` and `phd` → `Ph.D.`, loads and parses — measured 2026-09-23, `john smith md` giving `John Smith MD` under it. Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse`; recorded in AGENTS.md's shim bullet and cross-referenced from #3-0-reevaluations. + +- 2026-09-23 #459 (review) — A MASK'S PUNCTUATION MARKS WHERE ITS LETTERS ARE JOINED, and is never written into the word. Letters written side by side in a value are one RUN, and any character between them that is not a letter — a full stop, a space, a digit — ends the run. That matters for exactly one case, the LONE INITIAL: a single letter the writer split off beside a full stop (a `FULL_STOPS` member, not the ASCII period alone, #322) is written in capitals whatever the mask says there, but only where the mask writes that letter inside a run of two or more — the writer split a chunk the mask keeps together, so each split piece is an initial. So `p.h.d.` under `PhD` gives `P.H.D.` (the mask is ONE three-letter run), `b.sc.` under `BSc` gives `B.Sc.`, `ph.d.` gives `Ph.D.`, and `d.phil.` under a caller's `DPhil` gives `D.Phil.`; while `h.c.` under a caller's own `h.c` mask, which spells each letter alone, stays `h.c.`, `a.bc` under `abc` gives `A.bc`, `ab.c` gives `ab.C`, `2b.` under `2b` stays `2b.`, and `x2b.c` under `x2bc` gives `x2B.C`. Everywhere else a value's punctuation changes nothing: `('md', 'M.D.')` and `('md', 'MD')` repair every word alike (measured on `md`, `m.d.`, `md smith` and `m.d`). The rule took three drafts, and the two wrong ones are the record worth keeping. The FIRST laid the mask letter by letter with no initial rule and REGRESSED what substitution had got right: `b.s.c.` repaired to `B.S.c.` and `p.h.d.` to `P.h.D.`, where the parent 4d0680e6 gave `B.S.C.` and `Ph.D.`. The SECOND capitalized every lone letter beside a stop, which overrode a caller's own mask — measured in review on 2026-09-23, `_apply_mask('h.c.', 'h.c')` gave `'H.C.'` and `dr. h.c. hans meier` forced gave `Dr. H.C. Hans Meier`, where the caller had written the case they wanted. The third reads the mask's runs, as above, and is rules.md#R4's `p.h.d.` line. CASING goes through the whole word — `word.lower()`/`word.upper()` — wherever both keep the word's length, so context-sensitive letters come out as the language writes them (`ΚΟΣ` under `Κος` gives `Κος`, the final sigma intact), and per character otherwise (`straße` under `STRAßE` gives `STRAßE`). KNOWN LIMIT: a mask letter reads as upper when it is not lower, so a titlecase letter (`Dž`, Unicode category Lt) reads as upper and `dž` under `Dž` gives `DŽ`; no per-character titlecase mapping is attempted. Pinned in tests/v2/test_render.py by `test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined`, `test_a_masks_punctuation_marks_its_joins_and_is_never_written` and `test_a_mask_cases_through_the_whole_word_for_context_sensitive_letters`. + +- 2026-09-23 #459 (review) — DECLINED, after being written: a construction-time `UserWarning` for a punctuated value, and a shim translation of the two pre-2.4 shipped pairs. Both were implemented during review and then dropped, because both rested on a premise the code falsified — that a value's punctuation is IGNORED. The warning fired on a value whose punctuation differed from its key's (`'M.D.'` for `md`, `'h.c'` for `hc`) and told the caller to write the value without it (`Write the value as 'hc'`); but `_render._letter_run_ge2` READS a mask's punctuation, so following the advice changed the output — `('hc', 'h.c')` keeps `dr. h.c. hans meier` at `h.c.` under force, where `('hc', 'hc')` gives `H.C.`. The translation rewrote `md` → `M.D.` and `phd` → `Ph.D.`, byte-equal, to their unpunctuated masks in `Constants._snapshot()`; it changed no output (those pairs repair every word alike, above) and existed only so a v1 `Constants` still carrying them — one unpickled from 1.4.0, for instance — would not trip the warning, so with the warning gone it had nothing left to do. A punctuated value is simply a legal mask, accepted silently at every entry point (`tests/v2/test_lexicon.py::test_a_value_may_carry_punctuation_that_marks_its_joins`). The lesson generalizes: "this input is silently ignored" is a MECHANISM claim about what the code reads, and it earns the same one-command check as any other before a diagnostic is built on it. + +- 2026-09-23 #459 (review) — THE INVARIANT'S UNICODE BOUNDARY, and why rules.md#R4 words it as it does. The approved spec stated the invariant as "no character is added, removed or reordered"; rules.md#R4 says "the repaired word is the word as written, recased", because two inherited cases break the stronger reading and neither is this change's to fix. Python's casing tables uppercase `ß` to `SS`, so a clause that uppercases can lengthen a word: `john smith ß.x.` reads a by-shape credential and repairs to `John Smith SS.X.`, casefold-equal and one character longer (the parent 4d0680e6 gave `Ss.x.`, longer too). And the roman shape matches under `re.I`, which admits the dotless `ı`, so `john smith ıı` reads suffix and repairs to `John Smith II` — not even casefold-equal to its input, `'ıı'.casefold()` being `ıı` (the parent gave `Iı`). The first comes from the casing tables and the second from the parser's own pattern; neither is fixed. `tests/v2/test_properties.py::test_case_repair_changes_case_and_nothing_else` compares under `str.casefold()` because of the first, and that comparator has a boundary on each side: it CANNOT see a repair that trades `ss` for `ß` or back, the two folding alike — a recorded blind spot — and it WOULD report the second, which it does not today only because no corpus name or case-table text spells a numeral with `ı`. A length check beside it was considered and declined: it would flag the `ß` boundary this bullet records as accepted. + +- 2026-09-23 #459 item 4 — DECIDED (Derek, 2026-09-23): a word the parse put in a NAME role repairs as a name word, whatever vocabulary it is in. `qc mp` gives `Qc MP`, `ed smith` gives `Ed Smith`, `john smith ed` gives `John Smith ED`: the parse decides the role and reports the fork (`suffix-or-name`), and repair follows the role (mechanisms.md#RENDER-HONORS-THE-PARSE). Closes the item with no code. What the decision also reaches, found while planning and NOT in the approved spec, which said `iv smith` would keep `IV Smith` "through the numeral rule": the numeral clause is suffix-gated — gating it is what keeps the given name `Vi` from reading `VI` — so the four words that left the map lose, in any role but the suffix, the role-free capitals the map gave them. Measured 2026-09-23: `iv smith` gives `Iv Smith` and `john iii smith` gives `John Iii Smith`, where every release from 1.4.0 through 2.3.0 gave `IV Smith` and `John III Smith`; `II Van Johnson` gives `Ii van Johnson` under force; and `md` read as a TITLE — the parse's reading of a leading `MD` — repairs as ordinary title text, so `MD DDS` gives `Md Dds` and `Md Abdul Karim`, the usual abbreviation of Mohammed and a shape-tagged corpus row, gives `Md Abdul Karim` under force where every release gave `M.D. Abdul Karim`. DECIDED (Derek, 2026-09-23, when the plan surfaced it): a numeral in a name role repairs as a name word, the same rule read to its end. DECLINED, the alternative put to him with it: keeping `ii`, `iii` and `iv` in the map as masks, which would have restored the three numerals in a name role while leaving a title `md` at `Md`. + +- 2026-09-23 — PERIOD INSERTION IS OUT of `capitalized()` for good, and PARKED rather than declined: if anyone ever asks for a canonical dotted spelling (`md` → `M.D.`), it belongs in a normalization view — Derek's working name for it is `repair()` — which may re-spell a word, not in a case repair, which may not. Not a commitment; no issue filed. + +- 2026-09-23 #459 — NUMERALS: a SUFFIX-roled word whose text, edge periods stripped, fullmatches the roman shape repairs to capitals. It reaches `i` through `x`, which is why `ii`/`iii`/`iv` left the map: `vi` through `ix` read as suffix by that shape, carry no vocabulary tag, and title-cased to `Vi`, `Vii`, `Viii`, `Ix` (measured on 4d0680e6, and `Vi`, `Vii` on every release wheel 1.4.0 through 2.3.0); and `john smith iii.` now keeps its period, `III.`, where every release gave `III`. The shape is `_vocab._ROMAN`, v1's `roman_numeral`, and `_render` holds a HAND COPY pinned against config by `tests/v2/test_regex_sync.py`: the approved spec said the clause would import the predicate, but tests/v2/test_layering.py's ALLOWED table gives `_render.py` `_types` and `_lexicon` and nothing else, and `_render._INITIAL` is the precedent for exactly this. The copy re-derives no parse decision — the role is the parse's, read as recorded; the pattern answers only how a numeral is written. THE PARSE LIMIT beside it, recorded and not fixed here: the roman shape stops at `x`, so `john smith xi` reads family `xi` and repairs to `John Smith Xi`, and after a family comma `john smith, vi` reads given `vi` and repairs to `Vi John Smith` — both rules.md#R4 boundary lines, and both for #459's close-out. The #397 generation guard is unchanged and still decides first: `Carod i` forced gives `Carod I`, the numeral clause now being what writes the capital. + +- 2026-09-23 #459 — A SUFFIX READ BY ITS DOTTED SHAPE is an acronym to repair: the acronym clause fires on a SUFFIX-roled token whose period-free text is listed OR which carries `SHAPE_ACRONYM_TAG` (#516's by-shape reading, rules.md#S3), so `john smith x.y.z.` repairs to `John Smith X.Y.Z.` where the parent gave `X.y.z.`. The constant moved from `_pipeline/_state.py` to `_types.py`, which `_render` may import, and `_state` imports it back — still one spelling, which is what the constant exists for. The role gate holds: `Jack X.Y.Z.` keeps its surname and repairs as a name word, `Jack X.y.z.` under force. ACCEPTED: an unlisted dotted credential written in mixed case reads all-caps wherever repair acts, since a by-shape member has no listed casing to honor — `John Smith B.Tech.` gives `John Smith B.TECH.` under force, where every release gave `B.tech.`, no better. And since the suffix-free gate (decisions.md#R5, 2026-09-23) this reaches the DEFAULT path too wherever the name words are one-case: `john smith B.Tech.` gives `John Smith B.TECH.` where every release returned it untouched, while the corpus spelling `John Smith B.Tech.` stays untouched there, its name words being mixed. A caller who writes such a credential adds a mask: `('btech', 'BTech')` gives `B.Tech.` on either spelling (measured 2026-09-23). + +- 2026-09-23 #478 — DECIDED: inside `_cap_text`, a part of a HYPHENATED token that is connective vocabulary keeps its lowercase when a WORDED part stands on each side of it within the token (split on `-`; a part counts only if it holds a word, and an empty part does not count and is skipped, so a trailing hyphen supplies no neighbour while a doubled hyphen changes nothing: `juan garcia--y-lopez` gives `Juan Garcia--y-Lopez`). An edge part repairs as ordinary name text. `jose ortega-y-gasset` and `JOSE ORTEGA-Y-GASSET` both give `Jose Ortega-y-Gasset`, and `maria silva-e-sousa` gives `Maria Silva-e-Sousa`. It reads POSITION and VOCABULARY, as the particle arm already does per sub-word (`maria da-silva` → `Maria da-Silva`, unchanged) — a question about a sub-word the parse never asked, the parse having read the hyphenated word as one untagged token, which is the same standing mechanisms.md#RENDER-HONORS-THE-PARSE records for the particle arm — and it does not re-derive the initial-versus-conjunction class from a word's case, which is what #458 removed. The edges are pinned on both sides: `juan e-f smith` keeps `Juan E-F Smith` and `juan y-garcia` gives `Juan Y-Garcia`, and in a three-part word `juan y-garcia-lopez` gives `Juan Y-Garcia-Lopez` and `juan garcia-lopez-y` gives `Juan Garcia-Lopez-Y`. A SINGLE LETTER MARKED WITH A PERIOD is an initial there, as the parse reads it in spaced text, never the connective (the guard is `not re.fullmatch(r"\w\.", part)`): the first draft lowered the `e.` in `j.-e.-p. dupont`, giving `J.-e.-P. Dupont` where every release from 1.4.0 through 2.3.0 gave `J.-E.-P. Dupont`, a regression found in review and pinned since as a rules.md#R4 boundary line. A MULTI-letter word marked with a period is not an initial and stays reachable, as its spaced reading is (`hans smith und. jones` tags `und.` a conjunction), so `hans smith-und.-jones` gives `Hans Smith-und.-Jones`. This reverses, for the interior position only, what #458's tag read did to the shape: 2.2.0 and 2.3.0 gave `Ortega-Y-Gasset` for both spellings — the side effect #478 reported, which the 2026-08-29 #458 bullet above does not record — and #458's edge answer stands. Parity, measured on the released wheels: 1.4.0, 2.0.0 and 2.1.0 re-decided per word and gave `Jose Ortega-y-Gasset` lowered but `Jose Ortega-Y-Gasset` upper, so the all-caps half is a parity break with them on #479's one-case precedent; they also gave `Juan y-Garcia` at the edge, which neither 2.2.0, 2.3.0 nor this tree does. KNOWN NON-REACH, with its witness: the Māori corpus name `Te Awanui-a-Rangi Black` still repairs to `Te Awanui-A-Rangi Black` under force, because `a` is not connective vocabulary and must not become one (a spaced `a` is an initial or a name elsewhere; P3's Excluded block for the connectives says so). ACCEPTED: `mary-e-smith` reads `Mary-e-Smith` — no evidence the shape exists, and the hyphen is the writer joining the parts. Every role, since a hyphenated given name is the same text shape; a middle-role row is pinned beside the family ones (`smith, jose ortega-y-gasset` forced gives `Jose Ortega-y-Gasset Smith`). + +- 2026-09-23 #459/#478 — MEASURED, with the recipe; none of these numbers is a live count. Population: the deduped `tools/differential/corpus*.jsonl` glob, both line shapes (an object with `name`, a bare string), as this change leaves it — 1371 distinct names, the new rules.md example lines among them. Comparator: the parent 4d0680e6 — `git archive 4d0680e6` into a scratch directory, both trees imported BY PATH in separate processes over the SAME corpus directory, each asserting `nameparser.__file__` — comparing every token's `capitalized()` and `capitalized(force=True)` text on the core, and the rendered string on the facade, and attributing each moved token PER SUB-WORD: the token is split into its `_WORD` sub-words (`(\w|\.)+`), and the FIRST sub-word whose repair changed decides the bucket, tried in this order — its period-free key is a mask key (`phd`, `bsc`, `msc`) or one that left the map (`md`, `ii`, `iii`, `iv`); else the token is a suffix carrying `SHAPE_ACRONYM_TAG` or the sub-word fits the roman shape; else the sub-word is an interior connective part of a hyphenated token; else, on the default path only, the token's repair used to equal its input, which is R5's admission — with a bucket for anything else, asserted empty. The per-sub-word split is load-bearing: keyed on the token as written, the corpus name `John Doe, MD-PhD-` (key `md-phd`) falls to the hyphen bucket instead, giving 15 and 82, 3 and 5 here and 11 and 77, 0 and 1 over the parent's corpus (measured in review on 2026-09-23). Mask, or a word that left the map: 15 names on the default path and 83 under force. Dotted shape and numerals: 3 and 16. Hyphenated link: 3 and 4, every one of them a rules.md#R4 example line — no corpus name written before this change carries a connective part in a hyphenated word, at the interior or an edge (0 of 1340 at 4d0680e6). R5's gate is R5's bullet. The same comparison over the parent's own corpus directory, 1340 names and none of this change's example lines, gives 11 and 78, 1 and 14, and 0 and 0: the difference is this change's own rules.md lines, and those are the counts to read as reach into names written for other reasons. The core and the facade move on the same number of names in both views. No role field moves on any name, which is also all the differential gate can say: its compared surface is the seven role fields, `_ambiguities` and `_initials`, never case (the 2026-08-29 bullet above). + +- 2026-09-23 — mechanisms.md#RENDER-HONORS-THE-PARSE CORRECTED, found in this change's re-read of that entry: it still said case repair carries the connective carve-out "ON R3's authority", quoting R4's pre-#461 text ("the carve-out R3 states for initials"), and that `.initials()` gives `j. a. j. s.` for `john and jane smith`. Both had been stale since #461 (2026-09-20), which cut R4's cross-reference to R3 in favor of R4's own reason and stopped a connective with a name word beside it from initialing. The entry now states R4's own reason, dates the cut, and names where the two views part today (`Juan de y`: initials `J. y.`, forced repair `Juan de y`), re-measured 2026-09-23. + +Excluded (CAPITALIZATION_EXCEPTIONS — md, ii, iii, iv, removed 2026-09-23, #459): + +- `md`, `ii`, `iii` and `iv` are OUT of the exceptions map and must not be put back by a sweep that finds one of them title-cased somewhere. As a SUFFIX each is written in capitals with no entry — `md` by the acronym clause, the numerals by the numeral clause — and in any other role each repairs as the parse read it by decision (the item-4 bullet above), so an entry would only bring back the role-free capitals that bullet retired: `iv smith` → `IV Smith`, `Md Abdul Karim` → `MD Abdul Karim`. A caller who wants them back adds them to a private lexicon, as masks. + ### R5 — the case-repair gate - 2026-08-29 (#407 arc) — EXTRACTION, not a decision: the parser is untouched. The two halves of R5 have separate provenance, and conflating them is easy enough that the first draft of this entry did. The REFUSAL — repair skips any name already carrying more than one case — is older than the git history: `git log -S "name == name.upper() or name == name.lower()" --reverse` bottoms out at 45a1539 (2011-02-03), the initial import from svn, where `capitalize()` already opens with that guard and a bare `return`. (A path-filtered search answers 280895b instead, the same-day commit that moved the module into `nameparser/`; the code did not change there.) The OVERRIDE is bf1e0a5, 2016-06-02, which did not add the refusal but wrapped it — `if not (name == name.upper() or ...)` became `if not force and not (...)` — and shipped in 0.4.0 (June 2, 2016; its own release-log line is docs/release_log.rst under that heading). So R5's statement as a whole holds from 0.4.0 on. rules.md had never said any of it, though bf1e0a5's diff shows the API docstring already did: "It will not adjust the case of names entered in mixed case" was there before that commit edited around it. The gap was rules.md's alone. Evidence, measured on the released 1.4.0 wheel — the last v1 release and one of the differential baselines, so a natural thing to measure against and not a release that introduced anything here — and re-measured on this branch today, facade and core agreeing: `HumanName('Shirley Maclaine').capitalize()` leaves `'Shirley Maclaine'` — mixed case, wrong, and kept — while the same name under `force=True` gives `'Shirley MacLaine'`; `HumanName('Juan McDonald').capitalize()` leaves `'Juan McDonald'`. rules.md's own preamble classifies behavior in this position as "pinned-but-undocumented — an extraction gap to close, not a specification", which is why the fix lands in the document rather than in `_render.py`. @@ -1355,6 +1392,14 @@ Accepted costs, deferred to the rescoped #459 rather than relitigated here: the THE DIRECTION REVERSED, which is the part worth carrying: uppercase was the worse direction and is now the better one. Recipe, one comparator, both sides — over the deduped `tools/differential/corpus*.jsonl` glob, compare `parse(n).capitalized(force=True)` against `parse(n.upper()).capitalized()` and against `parse(n.lower()).capitalized()`, rendering all seven role fields, and count the names that differ in each direction; run it once on this tree and once against a released wheel installed in a throwaway environment (the tree shadows the wheel otherwise). Measured 2026-09-13 over 1174 names: the 2.3.0 wheel gives 69 upper and 23 lower, this tree gives 11 upper and 18 lower. On the facade surface — `HumanName(n).capitalize(force=True)` read back with `str()`, against the same call on the re-cased spellings — 2.3.0 gives 68 and 20, this tree 10 and 15; the surface still matters for the reasons the 2026-08-29 bullet names (the default render spec omits the maiden name, and `str()` concatenates adjacent roles). The uppercase count collapsed; the lowercase one did not, its remaining members being what the carve-out never explained. The 63/18 and 62/16 figures in the bullet above are a dated snapshot over a 1094-name corpus and are not reproducible today in either direction — do not treat them as this recipe's before-column. rules.md#R5 needs no matching amendment: it never carried the claim. The 2026-08-29 bullet that follows the override bullet above says so in its own words ("and therefore NOT stated in rules.md"), and R5's Rationale, statement and example lines were re-read on this branch to confirm it. What DID move in the suite is the mirrored pin in tests/test_capitalization.py, where `JUAN Y GARCIA` had been pinned as inherited 1.4.0 behavior ("Juan Y Garcia") and now agrees with its lowercase spelling ("Juan y Garcia"), with a mixed-case control beside it. +- 2026-09-23 #492 — DECIDED (Derek, 2026-09-23): the gate reads every token but the SUFFIX-roled ones. R5 says mixed case is evidence the writer cased the NAME deliberately, and a suffix written the way a suffix is written — `III`, `PhD`, `Jr.` — says nothing about how the writer cased the name words. So `juan garcia III` repairs to `Juan Garcia III` where every release, 1.4.0 through 2.3.0, returned it untouched, and `juan garcia PhD`, `JUAN GARCIA Jr.` and `dr. juan garcia III` repair the same way. v1's test for it, `test_capitalization_exception_for_already_capitalized_III`, was a known failure since 2012 — added 2012-10-07 (e2df6937) as a plain failing test against the Google Code tracker's issue 22, marked xfail 2026-09-01 (d02dc41e) — and this change retires the marker and the `_KNOWN_FAILURE` suffix the test carried. Every token not roled SUFFIX stays in the gate text, NICKNAME and MAIDEN included: `jane doe nee SMITH III` and `juan garcia (Bob)` are left alone (pinned in `tests/v2/test_render.py::test_the_gate_leaves_the_suffixes_out`, where a mutant filtering those roles too survived the suite otherwise). A name whose non-suffix tokens are caseless is one-case exactly as the whole name was, so only the suffix repairs (`김민준 phd` → `민준 김 PhD`); a name with NO non-suffix token is one the parse never produces (rules.md#H4 — measured, `Jr.` is a title, `PhD` a given name, `MD, PhD` family and suffix), so only a synthetic `ParsedName` reaches the empty text, which is one-case, and every suffix repairs to its own fixpoint. Force is unchanged. WHY "USE FORCE" IS NOT THE ANSWER: `force=True` is lossy on correctly cased names — `Danny DeVito` → `Danny Devito`, `Shia LaBeouf` → `Shia Labeouf`, `Anne-Marie DuPont` → `Anne-Marie Dupont`, `Bill deBlasio` → `Bill Deblasio` (re-measured 2026-09-23) — so a batch caller cannot force the lowercase rows without damaging the cased ones beside them. + +- 2026-09-23 #492 — THE R5 / PARSER SPLIT, decided and recorded rather than left for a reader to discover. The parser's own one-case readings — rules.md#P3's single-letter connective and rules.md#S2's case signal at the suffix slot — still count a cased suffix, as Derek's 2026-09-19 comment on #492 and decisions.md#S2's 2026-09-14 bullet require (the judged token stays in the span: `Jack MA` is mixed because of `MA`); R5's gate does not. So one name can be mixed to the parser and one-case to repair. `john e jones III` is mixed to P3, `III` included, so its `e` reads as a connective and it repairs to `John e Jones III`, while `john e jones iii` is one-case, reads `e` as an initial and repairs to `John E Jones III`; the corpus name `john e jones, III` moves the first way on the default path, where every release left it untouched. The gate follows the parse's suffix call wherever the parse makes one, the ambiguous credential class included, so two shapes outside the corpus move on the default path as well: `jack MA` gives `Jack MA` — S2's lean read `MA` as the credential, and the gate then leaves it out — and `MD, PhD` gives `Md PhD`, an input that is nothing but vocabulary (rules.md#H4's shape) read as family `MD` and suffix `PhD`, where 2.2.0 and 2.3.0 left `MD PhD` untouched; the family `MD` repairs as a name word, by the item-4 decision under R4. All four strings are pinned by `tests/v2/test_render.py::test_the_gate_and_the_parser_read_a_cased_suffix_differently`. + +- 2026-09-23 #492 — TITLES STAY EVIDENCE, and the reason is a limit of title REPAIR rather than a claim about titles. Title repair is still plain `str.capitalize()`, and widening the gate to titles would surface its defects on the default path. Measured under `force=True`, where they already show (re-measured 2026-09-23): `Lt.Gov. juan e garcia` → `Lt.gov. Juan e Garcia`, `The Lord of the Universe` → `the Lord of the Universe`, `Dr. med. univ. Margit Popp` → `Dr. Med. Univ. Margit Popp`. So `Dr. juan garcia` is left alone (rules.md#R5's boundary line). This may move when title repair is trusted. + +- 2026-09-23 #492 — MEASURED, population first. Recipe: for each name in the deduped corpus glob, the gate as shipped before #492 (the joined text of EVERY token) declines it AND the joined text of its non-suffix tokens is one-case; those are exactly the names the new gate admits, and the default path returns their forced repair. The movers are the admitted names whose default-path string changes, counted by the R4 MEASURED bullet's comparator. At 4d0680e6, over its 1340 names, the population was 17 and 12 of them changed their default-path string — `juan garcia III`, `dr. ben alex johnson III`, `john e jones, III`, `john smith MA` and eight `pennie von bergen wessels` spellings; the other five (`威廉·莎士比亚, PhD`, `田中さん, PhD`, `田中さん, Ph. D.`, `김민준씨 Jr.`, `1 & 2, 3 4 5, Mr.`) are admitted and repair to themselves. The approved spec said "17 move": 17 is the population, 12 the movers. Over the corpus this change leaves (1371 names, R5's new example lines among them) the population is 20 and the movers 15, the three new ones being those lines. No forced repair moves by this rule, as none can: the gate is never consulted under force. + ### parse-cost — what a parse is allowed to cost Every number below is a py3.11 measurement of 2026-08-31, recomputable with `uv run python tools/perf/call_count.py` (add `--modules`, `--stages`, or `--against `). That harness is in the tree BECAUSE of this entry: its first draft took figures across several sessions with throwaway scripts on whichever interpreter was to hand, and published a module table that spliced one interpreter's before-column onto another's after-column. Four of its eight module numbers were wrong, its four stage numbers were 2x (2000-parse totals labelled per 1000, which its own arithmetic contradicted — four stages summing past the whole parse), and it blamed the wrong PR. Quote nothing here without the interpreter beside it. @@ -1442,3 +1487,4 @@ Promoted 2026-08-15 from session memory (Derek's 2026-07-30 ask; promotion appro - (A) 2026-09-13 — SHOULD A LOCALE PACK BE ABLE TO REMOVE BASE VOCABULARY? `Locale.lexicon` is UNIONED onto the base and never removes — the field's own comment says so — which makes `conjunctions_ambiguous` (decisions.md#P3, 2026-09-13) expressible by a pack in ONE DIRECTION ONLY: an `nl` pack can add "y", and a `pt` pack cannot remove "e", so a Portuguese caller has to write `Parser(lexicon=Lexicon.default().remove(conjunctions_ambiguous={"e"}))` by hand. Accepted for 2.x and parked here rather than filed, because 3.0 is when a wider set of locale packs gets designed and this is part of that work (Derek, 2026-09-13). It is the 2026-08-17 PARTICLE VOCABULARY AS LOCALE PACKS bullet's "overrides rather than additions" requirement meeting a second tenant, which is what makes it a shape question rather than one pack's inconvenience. - (A) 2026-09-13 — THE DIFFERENTIAL HARNESS CANNOT SEE A LOCALE PACK'S EFFECT AT ALL. `compare.py` builds one `Parser` per name ORDER and nothing else, and a `Case.locale` row projects into the corpora as a bare name string — no corpus row carries a locale key, verified over every `tools/differential/corpus*.jsonl` file on 2026-09-13 — so every corpus name is parsed under the DEFAULT `Lexicon` on both sides. A pack's effect on a name can therefore never diff, at any baseline. That is fine while packs only ADD vocabulary the default already carries most of, and stops being fine the moment the bullet above is answered yes. The shape of the fix is a `locale` carried per corpus ENTRY the way `shape` already carries an order, resolved at the baselines that ship the pack and skipped below them. Parked with the bullet above rather than filed, for the same reason: the two are one piece of work (Derek, 2026-09-13). - (A) 2026-09-20 #397 — A SECOND INSTANCE OF THE SAME PER-LOCALE QUESTION, cross-referenced rather than parked again. Catalan/Polish `i` joins `conjunctions_ambiguous` beside `e`, so the marked subset now carries two letters and the 2026-08-18 bullet above — which counted six single-letter conjunctions and named the per-word-policy requirement — describes a set of seven, `i` included. Nothing in that bullet's argument moves: the subset IS the per-word policy the bullet asked for, made configurable without packs, and the pack-side half is still what the 2026-09-13 (A) bullet above asks, since a `ca` or `pl` caller still has to write `Lexicon.default().remove(conjunctions_ambiguous={"i"})` by hand. See `decisions.md#P3`'s 2026-09-20 bullet. +- (B) 2026-09-23 #459 — THE SHIM RAISES on a v1 config it once accepted: a `capitalization_exceptions` value that does not spell its key's letters and digits raises `ValueError` at the first parse (decisions.md#R4, 2026-09-23). Cited against the shim's never-raise rule and taken anyway, so it is recorded here: the raise stands on its own — a case mask has no reading for a value that re-spells its word, and the core `Lexicon` raises the same way with no shim involved — and only where it LANDS for a v1 caller, at the first parse rather than at construction, is shim-era. 3.0 has nothing left to decide. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index ef077e24..2d728513 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -59,7 +59,7 @@ Problem shape. Two stages need the same answer about the same input, and the one ## RENDER-HONORS-THE-PARSE — the parse decides it, the views honor it -Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a connective the parse placed among the NAME words with nothing to join, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). Narrowed twice, the second time by a defect the record itself turned up (2026-09-20): where the parse read such a word as the GENERATION it also spells, repair was reading classify's `conjunction` tag without asking the role and lower-casing a suffix, so `parse("John Quincy Smith i")` forced gave 'John Quincy Smith i'. A view honors what the parse DECIDED, and the role is that decision where the tag is only what the word could have been -- the same distinction this entry draws between re-deriving an answer and reading the record, one level up. The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. +Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words and, since #461, on its own reason rather than on R3's authority: a connective that initials because it joins nothing is still not written the way a name is written. Until 2026-09-20 R4's text read "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reached R4's TEXT; #461 cut that cross-reference, and nothing now ties the two rules' wording. They were never tied in BEHAVIOR either, though an earlier wording of this sentence asserted it: over the corpus names carrying a conjunction in the GIVEN group, `parse("john and jane smith").capitalized()` kept `and` lowercase while `.initials()` gave `j. a. j. s.` before #461 (decisions.md#R2 carries that population). Since #461 a connective with a name word beside it contributes no initial, so that name initials `j. j. s.` and the two views agree on it, while a connective joining nothing still initials and still repairs lowercase — `parse("Juan de y")` initials `J. y.` and repairs to `Juan de y` under force — which is where they part today, by decision (rules.md#R3 and #R4 each say so in their own words); only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a connective the parse placed among the NAME words with nothing to join, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). Narrowed twice, the second time by a defect the record itself turned up (2026-09-20): where the parse read such a word as the GENERATION it also spells, repair was reading classify's `conjunction` tag without asking the role and lower-casing a suffix, so `parse("John Quincy Smith i")` forced gave 'John Quincy Smith i'. A view honors what the parse DECIDED, and the role is that decision where the tag is only what the word could have been -- the same distinction this entry draws between re-deriving an answer and reading the record, one level up. The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. ## CLAUSE-CONTENT-OVERRULES-DELIMITER — content wins diff --git a/docs/migrate.rst b/docs/migrate.rst index 0df2d49f..13aacb29 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -220,7 +220,13 @@ fields: * - ``capitalization_exceptions`` - ``capitalization_exceptions`` - Pair-valued; set it via ``dataclasses.replace(lexicon, - capitalization_exceptions={...})``, not ``add()``/``remove()`` + capitalization_exceptions={...})``, not ``add()``/``remove()``. + Since 2.4 a value is a case mask — the key's own letters and + digits recased (``{"phd": "PhD"}``) — laid over the word as + written, so ``{"md": "M.D."}`` repairs ``md`` to ``MD``, not + ``M.D.``; a value that spells anything else raises + ``ValueError`` on both APIs, at the first parse for a v1 + ``Constants`` The vocabulary that feeds both columns lives in ``nameparser.config``, and in 2.2 its module and constant names moved to the vocabulary the @@ -315,13 +321,13 @@ could hold two disagreeing defaults with nothing to say so. ``CAPITALIZATION_EXCEPTIONS`` is the constant the freeze left out. It is a mapping rather than a set, and it is still a plain mutable -``dict`` — ``CAPITALIZATION_EXCEPTIONS["phd"] = "PhD"`` runs on 2.2 and +``dict`` — ``CAPITALIZATION_EXCEPTIONS["dphil"] = "DPhil"`` runs on 2.2 and raises nothing. Everything just said about split defaults still applies to it, unchanged and measured on 2.2: an edit after the first parse reaches a freshly built ``Constants``, and neither ``Lexicon.default()`` nor the shared ``CONSTANTS``. The advice below is the same advice — configure the object, with -``constants.capitalization_exceptions["phd"] = "PhD"`` on a private +``constants.capitalization_exceptions["dphil"] = "DPhil"`` on a private ``Constants``, or ``dataclasses.replace(lexicon, capitalization_exceptions={...})`` for the 2.0 API. diff --git a/docs/release_log.rst b/docs/release_log.rst index 433f873a..785d4ad4 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -26,15 +26,23 @@ Release Log - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) + - **Change case repair's exceptions map from replacement spellings to case masks, so md repairs to MD and phd to PhD.** ``HumanName("john smith phd").capitalize()`` gives ``John Smith PhD`` and ``john smith md`` gives ``John Smith MD``, where every release from 1.4.0 through 2.3.0 gave ``John Smith Ph.D.`` and ``John Smith M.D.``. A ``capitalization_exceptions`` value is now the key's own letters and digits in the case each should take, laid over the word as it was written, so the one ``phd`` entry repairs ``ph.d.`` to ``Ph.D.`` and ``JOHN SMITH PH.D.`` to ``John Smith Ph.D.``, and repair never adds or drops a character: ``john smith iii.`` gives ``John Smith III.`` where every release dropped the period. Punctuation in a value only marks which of its letters are joined and is never written into the word, which matters for a lone initial: ``john smith p.h.d.`` gives ``John Smith P.H.D.``, each letter the writer split off from the mask's one run ``PhD`` being an initial, where every release gave ``John Smith Ph.D.``. The shipped map is ``phd`` → ``PhD``, ``bsc`` → ``BSc`` and ``msc`` → ``MSc``, so ``john smith bsc`` gives ``John Smith BSc`` where every release gave ``John Smith Bsc``; ``md``, ``ii``, ``iii`` and ``iv`` left it, a suffix ``md`` now repairing by the acronym repair listed under Additions and a suffix numeral by the numeral repair below. A mask still applies wherever its word stands (``phd smith`` gives ``PhD Smith``), but a word that left the map and was parsed as anything but a suffix repairs as that reading: ``iv smith`` gives ``Iv Smith`` where every release gave ``IV Smith``, and ``Md Abdul Karim`` stays ``Md`` under ``force=True`` where every release gave ``M.D.``. A value that does not spell its key's letters and digits -- ``{"jr": "Junior"}`` -- now raises ``ValueError`` when the ``Lexicon`` is built, and at the first parse for a v1 ``Constants``; a value may still carry its own punctuation (``{"md": "M.D."}`` is accepted, and repairs ``md`` to ``MD``). Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 11 move on the default ``capitalize()`` path and 78 under ``force=True``; no role field moves. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #459) + + - **Fix case repair treating a suffix written in capitals as evidence that the whole name was cased on purpose.** ``HumanName("juan garcia III").capitalize()`` gives ``Juan Garcia III``, where every release from 1.4.0 through 2.3.0 returned it untouched -- v1's test for it had been a known failure since 2012 (the Google Code tracker's issue 22) -- and ``juan garcia PhD`` and ``JUAN GARCIA Jr.`` repair the same way. Repair still acts only on a name written wholly in one case, but the suffixes are left out of that test now: a credential or a generation written the way one is written says nothing about how the writer cased the name. A title still counts, so ``Dr. juan garcia`` is left alone, and so does every other word, nicknames and maiden names included (``Juan garcia III`` and ``jane doe nee SMITH III`` are left alone too). The parser's own reading of a name's case is unchanged and still counts the suffix, so the two can differ: ``john e jones III`` gives ``John e Jones III``, the capitals making the ``e`` a connective to the parser, where ``john e jones iii`` gives ``John E Jones III``. Because the test follows the parser's suffix reading, ``jack MA`` gives ``Jack MA`` and ``MD, PhD`` gives ``Md PhD``, where 2.3.0 left both untouched. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), the new test admits 17 and 12 of them move on the default path; ``force=True`` is unchanged. The recipe is the ``R5`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #492) + + - **Fix case repair capitalizing the connective inside a hyphenated compound surname.** ``HumanName("jose ortega-y-gasset").capitalize()`` gives ``Jose Ortega-y-Gasset`` and ``maria silva-e-sousa`` gives ``Maria Silva-e-Sousa``, the lowercase connective 1.4.0, 2.0.0 and 2.1.0 gave, where 2.2.0 and 2.3.0 gave ``Jose Ortega-Y-Gasset`` and ``Maria Silva-E-Sousa`` -- so this restores 1.4.0's answer after a 2.2 regression. ``JOSE ORTEGA-Y-GASSET`` gives the same ``Jose Ortega-y-Gasset``, where every release gave ``Jose Ortega-Y-Gasset``. A connective with a part on each side of it inside one hyphenated word keeps its lowercase, as the spaced spelling always has, in every role; at either end of the word it is ordinary name text, so ``juan e-f smith`` still gives ``Juan E-F Smith`` and ``juan y-garcia`` gives ``Juan Y-Garcia``. A single letter marked with a period is an initial there, never the connective, so ``j.-e.-p. dupont`` keeps ``J.-E.-P. Dupont``. Only connective vocabulary is read, so the Māori ``Te Awanui-a-Rangi Black`` still repairs to ``Te Awanui-A-Rangi Black`` under ``force=True``. The shape is rare: the only differential-corpus names carrying it are the rules document's own examples (three move on the default path, four under ``force=True``). See the ``R4`` entry of ``docs/design/decisions.md`` (closes #478) + + - **Change case repair to write an unlisted dotted credential and a roman numeral past iv in capitals.** ``HumanName("john smith x.y.z.").capitalize()`` gives ``John Smith X.Y.Z.`` where every release gave ``John Smith X.y.z.``, the dotted word being a suffix now (the ``unlisted_dotted_suffixes`` change above) and repaired as a listed acronym is; and ``john smith vi`` gives ``John Smith VI`` where every release gave ``John Smith Vi``, with ``vii``, ``viii`` and ``ix`` alike. Both are keyed on the suffix role: ``Jack X.Y.Z.``, which keeps its surname, still repairs as a name word (``Jack X.y.z.`` under ``force=True``), and ``john smith xi`` still gives ``John Smith Xi``, the parser reading ``xi`` as the surname. An unlisted dotted credential written in mixed case reads all capitals once repair acts on it -- ``john smith B.Tech.`` gives ``John Smith B.TECH.`` -- which a ``capitalization_exceptions`` mask such as ``{"btech": "BTech"}`` undoes. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 1 moves on the default path and 14 under ``force=True``. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (#459) + **Additions** - **Add Lexicon.conjunctions_ambiguous, the one-letter connectives that read as initials.** A subset of ``conjunctions`` holding ``e`` and ``i`` by default; it is the knob for the change above rather than a switch. Portuguese data, where ``e`` links surnames the way ``y`` does in Spanish, takes it out: ``Lexicon.default().remove(conjunctions_ambiguous={"e"})`` restores the joining reading. Dutch data, where a bare single letter is an initial and never a connective, adds the other one: ``Lexicon.default().add(conjunctions_ambiguous={"y"})``. A v1 ``Constants`` has no manager of its own for it -- deleting the word from ``conjunctions`` is what turns the marking off, the same rule the glued-honorific tails follow. See ``docs/customize.rst`` (#383, #479) - **Add AmbiguityKind.CONJUNCTION_OR_INITIAL, reported when a one-letter connective in a name written wholly in one case is read as an initial:** ``parse("jose e maria santos").ambiguities`` and ``parse("JOSE E MARIA SANTOS").ambiguities`` both name it, and ``detail`` names the letter. That is the call the behavior change above had to make. A letter outside the marked set reports nothing, its reading not being in doubt, so ``JUAN GARCIA Y LOPEZ`` is silent; so is every mixed-case name, where the writing decided it. See the ``P3`` entry of ``docs/design/decisions.md`` (#383, #479) - - **Repair a credential acronym the case-repair exceptions map does not carry to all-caps instead of title-casing it.** ``HumanName("JOHN SMITH MBA").capitalize()`` gives ``John Smith MBA`` where every release since 1.4.0 gave ``John Smith Mba``; ``john smith jd`` gives ``John Smith JD``. The repair is keyed on the word having parsed in the suffix role from the acronym vocabulary, so a word that is an ordinary name merely sharing a spelling with an acronym is untouched, and the exceptions map still wins first -- ``john smith md`` gives ``John Smith M.D.`` and ``john smith phd`` gives ``John Smith Ph.D.`` as before, and the generational ``jr`` is unaffected (``john smith jr`` gives ``John Smith Jr``). The given-name half of a mixed run is unchanged, so ``QC MP`` gives ``Qc MP`` with the ``QC`` (given role) still title-cased and only the ``MP`` (suffix role) repaired. Twenty-two names move in the differential corpora on the default ``capitalize()`` path and 119 under ``force=True``, every one a single-case name with an acronym suffix the map does not carry; no role field moves. See the ``R4`` entry of ``docs/design/decisions.md`` (#459) + - **Repair a credential acronym the case-repair exceptions map does not carry to all-caps instead of title-casing it.** ``HumanName("JOHN SMITH MBA").capitalize()`` gives ``John Smith MBA`` where every release since 1.4.0 gave ``John Smith Mba``; ``john smith jd`` gives ``John Smith JD``. The repair is keyed on the word having parsed in the suffix role from the acronym vocabulary, so a word that is an ordinary name merely sharing a spelling with an acronym is untouched, and the exceptions map is still asked first -- since the case-mask change above it holds masks, so ``john smith bsc`` gives ``John Smith BSc`` rather than ``BSC``, and ``md``, which left the map, is one of the acronyms this repair reaches (``john smith md`` gives ``John Smith MD``) -- while the generational ``jr`` is unaffected (``john smith jr`` gives ``John Smith Jr``). The given-name half of a mixed run is unchanged, so ``QC MP`` gives ``Qc MP`` with the ``QC`` (given role) still title-cased and only the ``MP`` (suffix role) repaired. When this repair landed (PR #521, before the case-mask change), twenty-two names moved in the differential corpora on the default ``capitalize()`` path and 119 under ``force=True``, every one a single-case name with an acronym suffix the map did not carry; no role field moves. See the ``R4`` entry of ``docs/design/decisions.md`` (#459) - - **Remove ph from the default post-nominal acronyms.** The fragment existed only so the merged ``Ph. D.`` token could pass the acronym test on its first piece, and the repair above would have read ``john smith ph. d.`` as ``John Smith PH. D.``; the parser merges the split spelling by its own rule, so ``HumanName("John Smith Ph. D.")`` still gives suffix ``Ph. D.``, ``john smith ph. d.`` capitalizes to ``John Smith Ph. D.``, and ``phd``/``Ph.D.`` are unchanged. The cost is a bare ``ph`` with no ``D.`` behind it, dotted or not, alone or inside a credential run: ``HumanName("John Smith Ph.")`` gives middle ``Smith``, last ``Ph.``, where every release since 1.4.0 gave suffix ``Ph.``, and ``John Smith MD Ph.`` gives middle ``Smith MD``, last ``Ph.``, the ``MD`` leaving the suffix with it. A caller who needs that back adds it: ``Lexicon.default().add(suffix_acronyms={"ph"})``. See the ``Excluded (SUFFIX_ACRONYMS -- ph)`` entry of ``docs/design/decisions.md`` (#459) + - **Remove ph from the default post-nominal acronyms.** The fragment existed only so the merged ``Ph. D.`` token could pass the acronym test on its first piece, and the repair above would have read ``john smith ph. d.`` as ``John Smith PH. D.``; the parser merges the split spelling by its own rule, so ``HumanName("John Smith Ph. D.")`` still gives suffix ``Ph. D.``, ``john smith ph. d.`` capitalizes to ``John Smith Ph. D.``, and ``phd``/``Ph.D.`` parse as they did. The cost is a bare ``ph`` with no ``D.`` behind it, dotted or not, alone or inside a credential run: ``HumanName("John Smith Ph.")`` gives middle ``Smith``, last ``Ph.``, where every release since 1.4.0 gave suffix ``Ph.``, and ``John Smith MD Ph.`` gives middle ``Smith MD``, last ``Ph.``, the ``MD`` leaving the suffix with it. A caller who needs that back adds it: ``Lexicon.default().add(suffix_acronyms={"ph"})``. See the ``Excluded (SUFFIX_ACRONYMS -- ph)`` entry of ``docs/design/decisions.md`` (#459) * 2.3.0 - September 12, 2026 diff --git a/docs/usage.rst b/docs/usage.rst index 3f9a8ffc..42bd99bd 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -931,8 +931,16 @@ empties and ``family_base`` takes the whole field. A token the parse never saw carries no decision to honor, so a view that is *handed* a vocabulary can fall back to it — of the parsed name's own views, :meth:`~nameparser.ParsedName.capitalized` is the -one that is, and it falls back for one question only: whether a word -is a conjunction or an initial, which a word answers on its own. +one that is. It asks the vocabulary whether a word is a conjunction +or an initial, which a word answers on its own. Its other special +cases never needed a reading in the first place: an exceptions-map +mask applies wherever its word stands, and a credential acronym the +vocabulary lists or a roman numeral is written in capitals wherever +the field is the suffix, so a spliced word repairs as a parsed one +does — ``replace(suffix="mba")`` repairs to ``MBA`` and +``suffix="vi"`` to ``VI``. A credential recognised only by its dotted +shape is the exception, since the shape is something the parse +records. (The v1 :class:`~nameparser.parser.HumanName` facade's ``initials()`` is the other view that is handed one, and takes the same fallback for spliced text; it is not a method of the parsed name and is not what diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 390483a9..7c4c1d66 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -1097,7 +1097,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # NOT translated: the one DECIDED exception to this # method's never-raise rule (#459, decisions.md#R4 and # #3-0-reevaluations) is a value that does not spell its - # key's letters, which raises ValueError from Lexicon + # key's letters and digits, which raises ValueError from Lexicon # here, at the first parse -- v1 substituted such a value # for the word, 2.4 repair only recases, so there is no # v1 behavior left to reproduce, and no such value was diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 07ea3d4b..add38e35 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -385,9 +385,10 @@ def _normpairs( ) -> tuple[tuple[str, str], ...]: """Canonicalize capitalization_exceptions input: _normset's sibling for the one pair-valued field. Dedupes on the NORMALIZED key so the - tuple and the derived map always agree ("Ph.D." and "phd" collide - after normalization); last occurrence wins, matching dict semantics - and the right-bias rule used elsewhere.""" + tuple and the derived map always agree ("PHD" and "phd" collide + after normalization; "Ph.D." does not, the fold keeping interior + periods, so it is a key of its own, "ph.d"); last occurrence wins, + matching dict semantics and the right-bias rule used elsewhere.""" if isinstance(raw, str): raise TypeError( "capitalization_exceptions must be a mapping or an " @@ -457,8 +458,9 @@ def _normpairs( normalized_key) else normalized_key raise ValueError( f"capitalization_exceptions value {v!r} for key {k!r} " - f"does not spell the key's letters: a value is a case " - f"mask, the key's own letters recased -- e.g. " + f"does not spell the key's letters and digits: a value " + f"is a case mask, the key's own letters and digits " + f"recased -- e.g. " f"capitalization_exceptions=(({normalized_key!r}, " f"{offered!r}),)") # capitalized() looks words up one at a time (the _WORD regex @@ -607,7 +609,7 @@ class Lexicon: #: where its letters are joined into one run versus split apart, #: and is never written into the word -- repair keeps the writer's #: own punctuation. A value that does not spell the key's letters - #: raises ValueError. Pair-valued: change it with + #: and digits raises ValueError. Pair-valued: change it with #: dataclasses.replace(), not add()/remove(); read it as a mapping #: via capitalization_exceptions_map. Full default mapping: #: :data:`~nameparser.config.capitalization.CAPITALIZATION_EXCEPTIONS`. diff --git a/nameparser/_render.py b/nameparser/_render.py index 1a12a29b..1c9e84ec 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -498,8 +498,10 @@ def cap(match: re.Match[str]) -> str: # initial class from a word's CASE, the thing #458 removed: an # EDGE part has a part on one side only and stays ordinary name # text, so 'juan e-f smith' keeps 'E-F'. A 'part' is one holding - # a word, so a doubled or trailing hyphen ('md-phd-') supplies no - # neighbour, and the two-part compound ('mcnabb-smith') never + # a word; an empty part does not count and is skipped, so a + # trailing hyphen ('md-phd-') supplies no neighbour while a + # doubled one changes nothing ('garcia--y-lopez' keeps its 'y' + # lowercase), and the two-part compound ('mcnabb-smith') never # gets past the count above. The period is the one mark classify # itself reads as an initial: a single letter marked with a # period is read as an initial there, as the parse reads it, diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index cc841f76..b40d142d 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -2539,11 +2539,14 @@ def test_case_repair_changes_case_and_nothing_else() -> None: under the shipped lexicon and under one whose exceptions map is empty. casefold(), not a character-for-character check, is the comparator because the rule was never stated against Python's own - casing tables: 'ss'/'SS' both fold to 'ss' though 'ß'.upper() is - 'SS' (two characters for one), and the roman-numeral shape under - re.I admits the dotless 'ı', which lower()s to itself but - upper()s to 'I' -- both are casing this walk accepts, not length - changes rules.md#R4 promises against. + casing tables: 'ß'.upper() is 'SS', two characters for one, and + 'ß' and 'ss' fold alike, so a lengthened word still passes -- and + so would a repair trading 'ss' for 'ß', the comparator's recorded + blind spot. It is not blind the other way: the roman-numeral + shape under re.I admits the dotless 'ı', which upper()s to 'I', + and 'I' does NOT fold back to 'ı', so a name spelling a numeral + with it would fail this walk; none in the walk does. Both cases + are decisions.md#R4's Unicode boundary. Recorded negative control, measured 2026-09-23 over this walk: at the parent 4d0680e6, where the exceptions map SUBSTITUTED its diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index ca2f6c55..be815f6d 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -591,6 +591,27 @@ def test_the_gate_leaves_the_suffixes_out() -> None: == ["PhD", "MD"] +def test_the_gate_and_the_parser_read_a_cased_suffix_differently() -> None: + """decisions.md#R5's split: the parser's own one-case readings + (rules.md#P3, #S2) still count a cased suffix, while R5's gate + leaves it out. So 'john e jones III' is MIXED to the parser, which + reads its 'e' as a connective, and one-case to the gate, which + repairs it -- keeping the connective lowercase -- where the one-case + 'john e jones iii' reads 'e' as an initial. The gate follows the + parse's suffix call, the ambiguous credential class included, so + 'jack MA' (S2's lean read MA as the credential) and 'MD, PhD' + (family MD, suffix PhD) repair on the default path too.""" + mixed = parse("john e jones III") + one_case = parse("john e jones iii") + assert "conjunction" in mixed.tokens[1].tags + assert "initial" in one_case.tokens[1].tags + for name, repaired in ((mixed, "John e Jones III"), + (one_case, "John E Jones III"), + (parse("jack MA"), "Jack MA"), + (parse("MD, PhD"), "Md PhD")): + assert str(name.capitalized()) == repaired, name.original + + def test_capitalized_is_idempotent() -> None: once = _lowercase_mac().capitalized() assert once.capitalized() == once From 7fe67c8ebac54f667c37fe45113c886df5a20c1f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 24 Sep 2026 01:05:59 -0700 Subject: [PATCH 6/9] fix(#459/#492/#478): the branch review round Three reviewers read the whole branch (a silent-failure hunt, a 74-mutant test analysis and a comment analysis); this is what they found. The v1 raise for a capitalization_exceptions value that does not spell its key now carries a hint spelled for the surface the caller is standing on -- constants.capitalization_exceptions['jr'] = 'JR' -- re-raised from the shim's snapshot with the Lexicon error chained. _normpairs raises a private ValueError subclass carrying the key and the offered spelling, so the shim catches that one shape by type and every other ValueError passes through untouched; the offer itself is one helper, _offered_mask, used by both messages. The hyphen clause's initial pattern is a module constant, _DOTTED_INITIAL, registered in test_regex_sync as _INITIAL's period alternative rather than an inline literal the sync test could not see. Mask values are stored NFC-composed, so a decomposed value can no longer change the split-off-initial reading the composed spelling gives. Eight test rows kill mutants that survived the branch's own tests: a mask outranks the numeral clause; the gate leaves out every suffix token, not the trailing one, and an untagged one; non-link parts of a three-part hyphenated token get the full per-word repair; the dotless-i boundary is an assertion; two facade assigned-field rows; a leading empty part; three comma shapes of the gate. Comments that referenced a helper that never shipped, claimed the mask clause reads no vocabulary, or restated rules.md#R4 in fresh words where an excerpt exists are corrected, and decisions.md#R4 records the round. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 +- docs/customize.rst | 9 +- docs/design/decisions.md | 2 + docs/release_log.rst | 2 +- nameparser/_config_shim.py | 234 ++++++++++++++++------------ nameparser/_lexicon.py | 70 +++++++-- nameparser/_render.py | 64 +++++--- nameparser/config/capitalization.py | 14 +- tests/test_capitalization.py | 32 +++- tests/v2/test_config_shim.py | 31 +++- tests/v2/test_lexicon.py | 63 ++++++-- tests/v2/test_regex_sync.py | 13 ++ tests/v2/test_render.py | 114 +++++++++++++- 13 files changed, 482 insertions(+), 168 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7ddee687..6b8606fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -319,7 +319,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, the R1 entry pass `suffix_entries` re-run over the forced state so a suffix value's entries follow its own commas, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. **Between raise and silence sits the construction-time `UserWarning`**, for a gap that is statically decidable, harmless to SOME deliberate caller, and indistinguishable-from-working for everyone else: the segmenterless activation (#337 — `parser_for(locales.JA)` without a segmenter behaved exactly like a working parser minus the feature) warns rather than raises because the inert JA registration is itself a pinned property, and a warning is filterable by the caller who wants exactly that. The message must carry every applicable remedy and no inapplicable one (the `ja_segmenter` hint fires only when a Japanese script is among the dead ones). Test fuzzers that legitimately construct such configs suppress the warning by MESSAGE, never by category — a blanket `UserWarning` ignore would mask the next construction diagnostic (`_quiet_parser` in `tests/v2/test_properties.py` is the pattern). -- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Seven exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), `conjunctions_ambiguous = CONJUNCTIONS_AMBIGUOUS ∩ conjunctions` (#383/#479 behavior with no v1 manager of its own either, so the one v1 knob that reaches it is deleting the conjunction — which turns the marking off, `test_snapshot_removing_a_conjunction_turns_the_marker_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. A `capitalization_exceptions` value that does not spell its key's letters and digits raises `ValueError` at snapshot rather than being translated (#459), because v1 substituted such a value for the word and 2.4 repair only recases, so there is no v1 behavior to reproduce; pinned by `test_a_mismatched_exception_value_raises_at_the_first_parse`. +- **The shim TRANSLATES; it never raises on a config v1 accepted (one decided exception, #459, at the end of this bullet), and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Seven exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), `conjunctions_ambiguous = CONJUNCTIONS_AMBIGUOUS ∩ conjunctions` (#383/#479 behavior with no v1 manager of its own either, so the one v1 knob that reaches it is deleting the conjunction — which turns the marking off, `test_snapshot_removing_a_conjunction_turns_the_marker_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. A `capitalization_exceptions` value that does not spell its key's letters and digits raises `ValueError` at snapshot rather than being translated (#459), because v1 substituted such a value for the word and 2.4 repair only recases, so there is no v1 behavior to reproduce; pinned by `test_a_mismatched_exception_value_raises_at_the_first_parse`. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Every pipeline stage is one module holding one public function of the same name** (`_tokenize.py`/`tokenize`), with the exceptions named where they stand — `_group.py`'s `marker_run_length`, a shared predicate (mechanisms.md#ONE-PREDICATE-PER-QUESTION), and since #511 `_post_rules.py`'s `suffix_entries`, the R1 entry pass as a function, because `Parser.revise` runs that one pass over a forced-role sub-parse and a stage's tail cannot be called on its own — **and its module docstring declares the contract in three labelled lines**: `Consumes:` what it takes from `ParseState`, `Produces:` what it hands back, `Reads:` which `Policy`/`Lexicon` fields it consults. `Reads:` is the load-bearing one — it makes "which stage do I touch for this feature?" a grep rather than a read-through. The authority for the stage set is `_pipeline/__init__.py`'s `STAGES` (eight, and not public API) with the field-ownership map in `ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`; NOT rules.md, which is implementation-free by its own preamble and whose `implemented:` names modules honoring a rule rather than stages. A `_pipeline/` module that is not a stage says so in its first line instead (`_assemble.py`: "Not a stage: …", omitting `Reads:` because it consults neither), so absence of the three lines is a claim about the module rather than an oversight. Provenance: §5 of the 2026-07-11 conventions spec, recorded here 2026-08-16. - **A claim about WHICH STAGE or WHICH LAYER does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. That single claim then shipped SIX times across three correction rounds, which is the part worth internalizing: **when a mechanism claim turns out wrong, sweep for where else you wrote it, and sweep again at the END of the change over the words the change itself just added.** Prose density here means one idea lives in a docstring, a case note, a release-log entry and this file at once; the implementer working against a wrong mechanism is the person most likely to restate it; and rewriting a mechanism claim is writing one, so the correction earns the same one-command check as the original — two of the six instances were fresh errors introduced by the sentence fixing the previous one. **Adjacency is the trap.** The claim that feels already-known is the one about the neighbouring stage or the neighbouring layer: one comment block in `tests/v2/test_facade_cases.py` got the exception type, the raising layer, the skip mechanism, the count of skipped rows, and which row was blocked all wrong at once — every one a claim about `_config_shim` versus real 1.4.0, written from reasoning, in the file whose whole job is translating between them. What finally held was not better prose but moving the claim into a test (`_CORE_ONLY_IDS`), which cannot be wrong the way a sentence can. diff --git a/docs/customize.rst b/docs/customize.rst index 1f0e60f9..6e422738 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -201,10 +201,13 @@ in the *value* is never written into the word; it only marks which of the mask's letters are joined. That matters for one case: a single letter the writer split off beside a full stop is an initial, and is capitalized where the mask keeps that letter inside a longer run, so -``p.h.d.`` repairs to ``P.H.D.`` under ``"PhD"``. A credential -acronym conventionally written all in capitals and a roman numeral +``p.h.d.`` repairs to ``P.H.D.`` under ``"PhD"``. An acronym already +listed in ``suffix_acronyms`` — plain or dotted — and a roman numeral need no entry at all: case repair writes a suffix of either kind in -capitals by itself. +capitals by itself. A caller's own acronym, one ``suffix_acronyms`` +doesn't already list, isn't covered by that and parses as an ordinary +name word instead — give it a ``suffix_acronyms`` entry (or a mask of +its own) rather than relying on this fallback. Words that are also ordinary names ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/design/decisions.md b/docs/design/decisions.md index bd20ae56..f9de4f81 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -1374,6 +1374,8 @@ Accepted costs, deferred to the rescoped #459 rather than relitigated here: the - 2026-09-23 — mechanisms.md#RENDER-HONORS-THE-PARSE CORRECTED, found in this change's re-read of that entry: it still said case repair carries the connective carve-out "ON R3's authority", quoting R4's pre-#461 text ("the carve-out R3 states for initials"), and that `.initials()` gives `j. a. j. s.` for `john and jane smith`. Both had been stale since #461 (2026-09-20), which cut R4's cross-reference to R3 in favor of R4's own reason and stopped a connective with a name word beside it from initialing. The entry now states R4's own reason, dates the cut, and names where the two views part today (`Juan de y`: initials `J. y.`, forced repair `Juan de y`), re-measured 2026-09-23. +- 2026-09-24 #459/#492/#478 (branch review) — FOUR FINDINGS FROM THE REVIEW ROUND, recorded together because each is a small correction to a bullet already above rather than a new decision. (a) THE UNICODE BOUNDARY IS NOT SCOPED TO THE TWO SHAPE CLAUSES, AND NOT TO ONE MASK BRANCH EITHER. The 2026-09-23 bullet above states the `ß` and dotless-`ı` boundary against `_cap_word`'s acronym/numeral clauses, but the MASK path can lengthen a word through `ß` exactly the same way, and both of its two branches can do it independently. `_apply_mask`'s split-off-initial force falls back to `c.upper()` when the whole-word uppered form does not keep the word's length, and `'ß'.upper()` is `'SS'`: an identity mask validates fine (`('a.ß', 'aß')` spells the key's own letters unchanged), so `john a.ß smith` forced gives middle `A.SS` -- casefold-equal to its input and one character longer, the same trade the shape clauses make. The ORDINARY mask branch -- the plain `else` arm, no full stop or split letter involved -- has the identical `c.upper()` fallback for the same reason (mask letter not lower, word not same-length under `.upper()`): `('straße', 'STRAẞE')`, the value spelling `straße`'s own letters unchanged with the actual German capital `ẞ` (not the digraph `'STRASSE'`, which the validator refuses as a VALUE for failing to spell `ß` at all), validates and `john straße` forced gives family `STRASSE`. The validator's refusal of `('straße', 'STRASSE')` is a different question -- whether the VALUE spells the KEY -- and says nothing about what either APPLIER branch can produce from a mask that does validate. Pinned by `tests/v2/test_render.py::test_a_masks_upper_fallback_can_lengthen_a_word_through_ss`. (b) MASK VALUES ARE STORED NFC-COMPOSED. `_normpairs` compared a value against its key NFC-composed but stored the value AS WRITTEN, so a value written in decomposed form spelled one composed letter as a base letter plus a non-alpha combining mark -- changing which letters `_letter_run_ge2` reads as split off beside a full stop from what the composed spelling of the identical value would read. `john smith pé.x.` forced gave `Pé.X.` under a composed mask and `Pé.x.` under the decomposed spelling of the SAME value. Fixed by NFC-composing the value at the same point the key is composed; pinned by `tests/v2/test_lexicon.py::test_a_decomposed_value_is_compared_and_stored_nfc_composed` and `tests/v2/test_render.py::test_a_decomposed_mask_value_reads_the_same_split_as_composed`. (c) THE HYPHEN CLAUSE'S INITIAL TEST IS NOW A PINNED MODULE CONSTANT. `_cap_text`'s inline `re.fullmatch(r"\w\.", part)` was a third hand copy of the initial shape's period half, invisible to `tests/v2/test_regex_sync.py`'s hand-copy sweep because it was never hoisted to a module-level pattern. Hoisted to `_render._DOTTED_INITIAL` and registered as `_INITIAL`'s period alternative; pinned by `tests/v2/test_regex_sync.py::test_dotted_initial_is_the_period_alternative_of_initial`. (d) THE V1 RAISE NOW CARRIES A V1-SPELLED HINT, VIA A TYPED EXCEPTION. The 2026-09-23 bullet above records that a mismatched-mask value raises `ValueError` at a v1 `Constants`' first parse; the message it raised was `Lexicon`'s own, offering a `Lexicon(capitalization_exceptions=...)` constructor call to a caller looking at a `TupleManager` assignment through `Constants`. `_lexicon._normpairs` now raises a dedicated `_MaskValueError(ValueError)` for exactly that one shape, carrying `key` (the raw key as given) and `offered` (from `_offered_mask`) as attributes -- picklable and copyable via its own `__reduce__`, since the inherited default (`type(self)(*self.args)`, `self.args` holding only the message) cannot reconstruct an instance that needs `key`/`offered` too and raises `TypeError` trying; a worker in a `ProcessPoolExecutor` pickles an exception it raises to deliver it to the caller, so an unpicklable one there surfaces as `BrokenProcessPool` instead. `_config_shim._build_snapshot` wraps ONLY the `Lexicon(...)` construction in `try`/`except _MaskValueError` and re-raises with `constants.capitalization_exceptions[key] = offered`. Every OTHER `ValueError` `Lexicon(...)` can raise passes through unchanged, by TYPE rather than by a message-substring test. Of those, most never reach `Lexicon(...)` from this shim at all: an orphaned `non_first_name_prefixes` entry cannot put an entry in `particles_ambiguous` that is outside `particles` (both terms of the union that builds it are already subsets), a bound/particle contradiction cannot arise (`bound & particles` is unconditionally folded into that same union), and the `suffix_acronyms_ambiguous`/`suffix_words` gate bypass cannot arise (`suffix_words` above already subtracts the ambiguous acronyms) -- all three measured by constructing, through a v1 `Constants`, the configuration that would trigger Lexicon's corresponding check, and observing no raise. What DOES reach `Lexicon(...)` here unchanged is an entry normalizing to empty, in `capitalization_exceptions` or in ANY set field -- measured: `c.titles.add("...")` then a parse raises Lexicon's own `titles entry '' normalizes to empty`, unaltered -- since nothing above translates that away. Declined: a first draft pre-validated `capitalization_exceptions` in isolation, a second call to `_normpairs` ahead of `Lexicon(...)` gated on a message-substring test; measured to make a multi-word key's dead-entry warning fire twice on the shim path, and to rest on a filter that was untested (deleting it passed the whole suite) and wrong (an unrelated ValueError then got the v1 hint appended regardless). Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse` (the hint), `tests/v2/test_config_shim.py::test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint` (the pass-through), `tests/v2/test_config_shim.py::test_a_multiword_capitalization_exceptions_key_warns_exactly_once` (the single warning), and `tests/v2/test_lexicon.py::test_mask_value_error_survives_pickle_and_copy` (the pickle/copy round-trip). + Excluded (CAPITALIZATION_EXCEPTIONS — md, ii, iii, iv, removed 2026-09-23, #459): - `md`, `ii`, `iii` and `iv` are OUT of the exceptions map and must not be put back by a sweep that finds one of them title-cased somewhere. As a SUFFIX each is written in capitals with no entry — `md` by the acronym clause, the numerals by the numeral clause — and in any other role each repairs as the parse read it by decision (the item-4 bullet above), so an entry would only bring back the role-free capitals that bullet retired: `iv smith` → `IV Smith`, `Md Abdul Karim` → `MD Abdul Karim`. A caller who wants them back adds them to a private lexicon, as masks. diff --git a/docs/release_log.rst b/docs/release_log.rst index 785d4ad4..d16d8dab 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -26,7 +26,7 @@ Release Log - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) - - **Change case repair's exceptions map from replacement spellings to case masks, so md repairs to MD and phd to PhD.** ``HumanName("john smith phd").capitalize()`` gives ``John Smith PhD`` and ``john smith md`` gives ``John Smith MD``, where every release from 1.4.0 through 2.3.0 gave ``John Smith Ph.D.`` and ``John Smith M.D.``. A ``capitalization_exceptions`` value is now the key's own letters and digits in the case each should take, laid over the word as it was written, so the one ``phd`` entry repairs ``ph.d.`` to ``Ph.D.`` and ``JOHN SMITH PH.D.`` to ``John Smith Ph.D.``, and repair never adds or drops a character: ``john smith iii.`` gives ``John Smith III.`` where every release dropped the period. Punctuation in a value only marks which of its letters are joined and is never written into the word, which matters for a lone initial: ``john smith p.h.d.`` gives ``John Smith P.H.D.``, each letter the writer split off from the mask's one run ``PhD`` being an initial, where every release gave ``John Smith Ph.D.``. The shipped map is ``phd`` → ``PhD``, ``bsc`` → ``BSc`` and ``msc`` → ``MSc``, so ``john smith bsc`` gives ``John Smith BSc`` where every release gave ``John Smith Bsc``; ``md``, ``ii``, ``iii`` and ``iv`` left it, a suffix ``md`` now repairing by the acronym repair listed under Additions and a suffix numeral by the numeral repair below. A mask still applies wherever its word stands (``phd smith`` gives ``PhD Smith``), but a word that left the map and was parsed as anything but a suffix repairs as that reading: ``iv smith`` gives ``Iv Smith`` where every release gave ``IV Smith``, and ``Md Abdul Karim`` stays ``Md`` under ``force=True`` where every release gave ``M.D.``. A value that does not spell its key's letters and digits -- ``{"jr": "Junior"}`` -- now raises ``ValueError`` when the ``Lexicon`` is built, and at the first parse for a v1 ``Constants``; a value may still carry its own punctuation (``{"md": "M.D."}`` is accepted, and repairs ``md`` to ``MD``). Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 11 move on the default ``capitalize()`` path and 78 under ``force=True``; no role field moves. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #459) + - **Change case repair's exceptions map from replacement spellings to case masks, so md repairs to MD and phd to PhD.** ``HumanName("john smith phd").capitalize()`` gives ``John Smith PhD`` and ``john smith md`` gives ``John Smith MD``, where every release from 1.4.0 through 2.3.0 gave ``John Smith Ph.D.`` and ``John Smith M.D.``. A ``capitalization_exceptions`` value is now the key's own letters and digits in the case each should take, laid over the word as it was written, so the one ``phd`` entry repairs ``ph.d.`` to ``Ph.D.`` and ``JOHN SMITH PH.D.`` to ``John Smith Ph.D.``, and repair never adds or drops a character: ``john smith iii.`` gives ``John Smith III.`` where every release dropped the period. Punctuation in a value only marks which of its letters are joined and is never written into the word, which matters for a lone initial: ``john smith p.h.d.`` gives ``John Smith P.H.D.``, each letter the writer split off from the mask's one run ``PhD`` being an initial, where every release gave ``John Smith Ph.D.``. The shipped map is ``phd`` → ``PhD``, ``bsc`` → ``BSc`` and ``msc`` → ``MSc``, so ``john smith bsc`` gives ``John Smith BSc`` where every release gave ``John Smith Bsc``; ``md``, ``ii``, ``iii`` and ``iv`` left it, a suffix ``md`` now repairing by the acronym repair listed under Additions and a suffix numeral by the numeral repair below. A mask still applies wherever its word stands (``phd smith`` gives ``PhD Smith``), but a word that left the map and was parsed as anything but a suffix repairs as that reading: ``iv smith`` gives ``Iv Smith`` where every release gave ``IV Smith``, and ``Md Abdul Karim`` stays ``Md`` under ``force=True`` where every release gave ``M.D.``. A value that does not spell its key's letters and digits -- ``{"jr": "Junior"}`` -- now raises ``ValueError`` when the ``Lexicon`` is built, and at the first parse for a v1 ``Constants``, where the raise names the fix in v1's own spelling (``constants.capitalization_exceptions['jr'] = 'JR'``) rather than the ``Lexicon()`` constructor call the same check offers a 2.0 caller; a value may still carry its own punctuation (``{"md": "M.D."}`` is accepted, and repairs ``md`` to ``MD``). Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 11 move on the default ``capitalize()`` path and 78 under ``force=True``; no role field moves. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #459) - **Fix case repair treating a suffix written in capitals as evidence that the whole name was cased on purpose.** ``HumanName("juan garcia III").capitalize()`` gives ``Juan Garcia III``, where every release from 1.4.0 through 2.3.0 returned it untouched -- v1's test for it had been a known failure since 2012 (the Google Code tracker's issue 22) -- and ``juan garcia PhD`` and ``JUAN GARCIA Jr.`` repair the same way. Repair still acts only on a name written wholly in one case, but the suffixes are left out of that test now: a credential or a generation written the way one is written says nothing about how the writer cased the name. A title still counts, so ``Dr. juan garcia`` is left alone, and so does every other word, nicknames and maiden names included (``Juan garcia III`` and ``jane doe nee SMITH III`` are left alone too). The parser's own reading of a name's case is unchanged and still counts the suffix, so the two can differ: ``john e jones III`` gives ``John e Jones III``, the capitals making the ``e`` a connective to the parser, where ``john e jones iii`` gives ``John E Jones III``. Because the test follows the parser's suffix reading, ``jack MA`` gives ``Jack MA`` and ``MD, PhD`` gives ``Md PhD``, where 2.3.0 left both untouched. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), the new test admits 17 and 12 of them move on the default path; ``force=True`` is unchanged. The recipe is the ``R5`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #492) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 7c4c1d66..e5aa921d 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -28,7 +28,7 @@ ) from typing import NamedTuple, Self -from nameparser._lexicon import Lexicon, _title_key +from nameparser._lexicon import Lexicon, _MaskValueError, _title_key from nameparser._parser import Parser from nameparser._policy import PatronymicRule, Policy from nameparser.util import lc @@ -1011,104 +1011,140 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: suffix_words = frozenset(self.suffix_not_acronyms) - ambiguous_acronyms # keep in sync with _lexicon._default_lexicon() (pinned by # tests/v2/test_config_shim.py::test_snapshot_field_translation) - lexicon = Lexicon( - titles=frozenset(self.titles), - # TRANSLATE, do not filter. The two versions build the same - # lookup key differently: v1 joins the raw title run and - # then applies lc(), which strips only the whole string's - # edge periods, so an interior word keeps its own ("lt. - # col"). v2 normalizes each token and then joins ("lt col"). - # Re-folding per word converts a v1 entry into the v2 - # spelling; filtering instead dropped every multi-word - # honorific containing an abbreviation or a conjunction and - # silently swapped given and family. - # Only entries v1 could actually match: its key is the - # joined title run, so always single-spaced and never - # empty. An entry holding a whitespace run was inert there - # (translating it would start matching), and one that folds - # away entirely would trip _normset's empty-entry check on - # a config v1 simply ignored. - given_name_titles=frozenset( - t for t in ( - _title_key(e.split()) - for e in self.first_name_titles - if e == " ".join(e.split()) - ) if t), - suffix_acronyms=acronyms, - suffix_words=suffix_words, - # Intersect with acronyms: Lexicon enforces ambiguous <= - # acronyms; v1 behaves the same when an acronym is deleted - # but its ambiguous entry lingers (the entry stops - # mattering). - suffix_acronyms_ambiguous=ambiguous_acronyms, - particles=particles, - # complement translation: v1 marks the never-given subset; - # v2 marks the may-be-given subset. The trailing union keeps - # a config v1 accepted: particles.py asserts its own data has - # no word in both NON_GIVEN_NAME_PARTICLES and - # BOUND_GIVEN_NAMES, but nothing stops a caller adding one at - # runtime, and v1 then lets the bound rule win (leading "dos - # Santos Silva" parses first="dos Santos"). Treating such a - # word as may-be-given reproduces that rather than raising. - # - # KNOWN DEVIATION, pinned by - # test_bound_never_given_prefix_deviates_on_two_pieces: v1's - # join has a reserve_last guard, so with only two pieces it - # does NOT fire and the word stays never-given ("dos Santos" - # -> last="dos Santos"). Promotion here is unconditional, so - # that case reads given="dos", family="Santos". v1's rule is - # piece-count dependent and a static vocabulary set cannot - # express it; the alternative is raising on a config v1 - # accepted, which is worse. Only reachable via a runtime - # config the shipped data forbids. - particles_ambiguous=( - particles - frozenset(self.non_first_name_prefixes)) - | (bound & particles), - conjunctions=conjunctions, - # no v1 manager of its own: the ambiguous-connective - # subset is 2.4 behavior (#383/#479), so it rides in the - # snapshot only. Lexicon does NOT check this pair -- unlike - # honorific_tails against suffix_words below -- so the - # intersection is a provable no-op, kept only for - # `_snapshot() == Lexicon.default()` legibility; the v1 - # knob (deleting the conjunction) turns the marking off - # through the fork's own base-vocabulary test rather than - # through this intersection. - conjunctions_ambiguous=CONJUNCTIONS_AMBIGUOUS & conjunctions, - bound_given_names=bound, - # v1 Constants has no manager for these (#274 is 2.0 - # behavior); the data module is the only source - maiden_markers=MAIDEN_MARKERS, - # likewise no v1 manager: the unspaced-name segmentation - # vocabulary is 2.0 behavior (#271), so it rides in the - # snapshot only -- v1's Constants surface stays frozen. - surnames=KOREAN_SURNAMES, - # likewise no v1 manager: the glued-honorific tail set is - # 2.1 behavior (#308), so it rides in the snapshot only. - # Intersect with the word set: Lexicon enforces tails <= - # suffix_words, and v1 semantics are that deleting a suffix - # word turns the behavior off -- a lingering tail simply - # stops mattering, the same rule ambiguous_acronyms gets - # against suffix_acronyms above. - honorific_tails=GLUED_HONORIFICS & suffix_words, - # TupleManager is dict[str, object] (v1 parity: values were - # never statically str-typed); every real entry is a str, - # same assumption _DelimiterManager's sentinel lookup makes. - # NOT translated: the one DECIDED exception to this - # method's never-raise rule (#459, decisions.md#R4 and - # #3-0-reevaluations) is a value that does not spell its - # key's letters and digits, which raises ValueError from Lexicon - # here, at the first parse -- v1 substituted such a value - # for the word, 2.4 repair only recases, so there is no - # v1 behavior left to reproduce, and no such value was - # found in the tracker, the docs or any test. A value that - # merely carries punctuation is NOT such an exception -- - # it is a legal mask (the punctuation marks which letters - # are joined and is never written into the word), so it - # passes through untranslated and unwarned like any other. - capitalization_exceptions=tuple( - sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] - ) + # The one shape the v1-spelled hint fits: _normpairs raises + # _MaskValueError, carrying the raw key and the offered fix, + # for exactly one shape -- a capitalization_exceptions value + # that does not spell its key. Caught by TYPE, not by a + # message-substring filter, so every OTHER ValueError Lexicon + # can raise passes through unchanged automatically -- no + # enumeration needed to keep them that way. Most of those never + # reach Lexicon at all: they are translated away by the + # computations above. An orphaned non_first_name_prefixes entry + # cannot put an entry in particles_ambiguous that is outside + # particles (it is built as two subtractions/intersections of + # particles, always inside it); a bound/particle contradiction + # cannot arise either (bound & particles is unconditionally + # folded into particles_ambiguous above, so the gate Lexicon + # checks is satisfied by construction); and the + # suffix_acronyms_ambiguous/suffix_words gate bypass cannot + # arise since suffix_words above already subtracts + # ambiguous_acronyms (measured: constructing each of the three + # configurations that would trigger Lexicon's corresponding + # check raises nothing through a v1 Constants). What DOES + # reach Lexicon here unchanged is an entry normalizing to + # empty -- in capitalization_exceptions or in any SET field + # (titles, particles, ...; measured: c.titles.add("...") then + # a parse raises Lexicon's own "entry '' normalizes to empty", + # unchanged) -- since nothing above translates that away, so + # it propagates by type, past this except clause, exactly as + # v1 never validated an entry's content either way. A + # wrong-TYPE value (the comment above: TupleManager never + # statically typed its values) does not need special handling + # for the same reason -- v1 never accepted one silently + # either, it raised at capitalize() trying to use the value as + # a string, so the TypeError _normpairs raises for it here + # reproduces a raise v1 already had, at an earlier point in + # the same call chain rather than a new one. + try: + lexicon = Lexicon( + titles=frozenset(self.titles), + # TRANSLATE, do not filter. The two versions build the same + # lookup key differently: v1 joins the raw title run and + # then applies lc(), which strips only the whole string's + # edge periods, so an interior word keeps its own ("lt. + # col"). v2 normalizes each token and then joins ("lt col"). + # Re-folding per word converts a v1 entry into the v2 + # spelling; filtering instead dropped every multi-word + # honorific containing an abbreviation or a conjunction and + # silently swapped given and family. + # Only entries v1 could actually match: its key is the + # joined title run, so always single-spaced and never + # empty. An entry holding a whitespace run was inert there + # (translating it would start matching), and one that folds + # away entirely would trip _normset's empty-entry check on + # a config v1 simply ignored. + given_name_titles=frozenset( + t for t in ( + _title_key(e.split()) + for e in self.first_name_titles + if e == " ".join(e.split()) + ) if t), + suffix_acronyms=acronyms, + suffix_words=suffix_words, + # Intersect with acronyms: Lexicon enforces ambiguous <= + # acronyms; v1 behaves the same when an acronym is deleted + # but its ambiguous entry lingers (the entry stops + # mattering). + suffix_acronyms_ambiguous=ambiguous_acronyms, + particles=particles, + # complement translation: v1 marks the never-given subset; + # v2 marks the may-be-given subset. The trailing union keeps + # a config v1 accepted: particles.py asserts its own data has + # no word in both NON_GIVEN_NAME_PARTICLES and + # BOUND_GIVEN_NAMES, but nothing stops a caller adding one at + # runtime, and v1 then lets the bound rule win (leading "dos + # Santos Silva" parses first="dos Santos"). Treating such a + # word as may-be-given reproduces that rather than raising. + # + # KNOWN DEVIATION, pinned by + # test_bound_never_given_prefix_deviates_on_two_pieces: v1's + # join has a reserve_last guard, so with only two pieces it + # does NOT fire and the word stays never-given ("dos Santos" + # -> last="dos Santos"). Promotion here is unconditional, so + # that case reads given="dos", family="Santos". v1's rule is + # piece-count dependent and a static vocabulary set cannot + # express it; the alternative is raising on a config v1 + # accepted, which is worse. Only reachable via a runtime + # config the shipped data forbids. + particles_ambiguous=( + particles - frozenset(self.non_first_name_prefixes)) + | (bound & particles), + conjunctions=conjunctions, + # no v1 manager of its own: the ambiguous-connective + # subset is 2.4 behavior (#383/#479), so it rides in the + # snapshot only. Lexicon does NOT check this pair -- unlike + # honorific_tails against suffix_words below -- so the + # intersection is a provable no-op, kept only for + # `_snapshot() == Lexicon.default()` legibility; the v1 + # knob (deleting the conjunction) turns the marking off + # through the fork's own base-vocabulary test rather than + # through this intersection. + conjunctions_ambiguous=CONJUNCTIONS_AMBIGUOUS & conjunctions, + bound_given_names=bound, + # v1 Constants has no manager for these (#274 is 2.0 + # behavior); the data module is the only source + maiden_markers=MAIDEN_MARKERS, + # likewise no v1 manager: the unspaced-name segmentation + # vocabulary is 2.0 behavior (#271), so it rides in the + # snapshot only -- v1's Constants surface stays frozen. + surnames=KOREAN_SURNAMES, + # likewise no v1 manager: the glued-honorific tail set is + # 2.1 behavior (#308), so it rides in the snapshot only. + # Intersect with the word set: Lexicon enforces tails <= + # suffix_words, and v1 semantics are that deleting a suffix + # word turns the behavior off -- a lingering tail simply + # stops mattering, the same rule ambiguous_acronyms gets + # against suffix_acronyms above. + honorific_tails=GLUED_HONORIFICS & suffix_words, + # TupleManager is dict[str, object] (v1 parity: values were + # never statically str-typed); every real entry is a str, + # same assumption _DelimiterManager's sentinel lookup makes. + # NOT translated: the one DECIDED exception to this + # method's never-raise rule (#459, decisions.md#R4 and + # #3-0-reevaluations) -- a value that does not spell its + # key's letters and digits -- and the key-normalizes-to- + # empty case beside it are both explained in the comment + # above the try: block, which is where the catching + # happens now. + capitalization_exceptions=tuple( + sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] + ) + except _MaskValueError as e: + raise ValueError( + f"{e} -- on a v1 Constants, write " + f"constants.capitalization_exceptions[{e.key!r}] = " + f"{e.offered!r}" + ) from e rules = frozenset({PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) \ if self.patronymic_name_order else frozenset() policy = Policy( diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index add38e35..e9e97626 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -380,6 +380,50 @@ def _alnum(text: str) -> str: return "".join(c for c in text if c.isalnum()) +def _offered_mask(normalized_key: str) -> str: + """The spelling _normpairs' mismatched-value error offers as a + working replacement, and what the shim's re-raise (_config_shim's + _build_snapshot) offers a v1 caller too -- factored so the two + messages can never name different fixes for the same key. + + .upper() can change a letter's COUNT ('straße' -> 'STRASSE', ß + growing to two letters), which would fail the mask check itself + if pasted back in. Falling back to the key unchanged -- an + identity mask, always legal -- keeps the offer actionable in + every case rather than only the common one.""" + upper = normalized_key.upper() + return upper if _alnum(_normalize(upper)) == _alnum( + normalized_key) else normalized_key + + +class _MaskValueError(ValueError): + """Raised by _normpairs for a value that does not spell its key; + carries the raw key and the offered fix (`key`, `offered`) so a + caller reached through another surface -- the v1 shim's + _build_snapshot -- can re-spell the same fix for that surface + rather than re-deriving it from the message text. + + __reduce__ pickles/copies as (message, key, offered) rather than + the default (args,) -- ValueError's own __reduce__ would call + type(self)(*self.args), and self.args holds only the message + (super().__init__ below is passed the message alone), so the + default would call this __init__ with two required arguments + missing. This is an exception, not one of _types.py's frozen + dataclasses, so the _guarded_getstate/slots convention there does + not apply -- a plain __reduce__ is the right-sized fix. A worker + in a ProcessPoolExecutor pickles an exception raised in the + worker to deliver it to the caller, so an unpicklable one there + surfaces as BrokenProcessPool instead.""" + + def __init__(self, message: str, key: str, offered: str) -> None: + super().__init__(message) + self.key = key + self.offered = offered + + def __reduce__(self) -> tuple[type[_MaskValueError], tuple[str, str, str]]: + return (type(self), (self.args[0], self.key, self.offered)) + + def _normpairs( raw: Mapping[str, str] | Iterable[tuple[str, str]], ) -> tuple[tuple[str, str], ...]: @@ -446,23 +490,27 @@ def _normpairs( # used to be SUBSTITUTED for the word, and rules.md#R4: # "Repair changes case and nothing else", so there is no # reading of one that repair can honor. + # Stored NFC-composed (case-preserving -- unicodedata.normalize, + # not _normalize, which also lowercases). _apply_mask walks the + # mask's alphanumeric characters one at a time + # (_letter_run_ge2), so a decomposed value spells one composed + # letter as two characters -- a base letter plus a combining + # mark that is not alpha -- and the base reads as a + # SPLIT-OFF letter next to a non-letter, changing which + # letters _apply_mask reads as an initial beside a full stop. + # The check just above already compares composed to composed + # (_normalize NFC-composes both sides), so a decomposed value + # passes it and would otherwise be stored exactly as written. + v = unicodedata.normalize("NFC", v) if _alnum(_normalize(v)) != _alnum(normalized_key): - upper = normalized_key.upper() - # The offered spelling must itself construct: .upper() can - # change a letter's COUNT ('straße' -> 'STRASSE', ß -> SS), - # which would fail the very check above if pasted. Falling - # back to the key unchanged -- an identity mask, always - # legal -- keeps the offer actionable in every case rather - # than only the common one. - offered = upper if _alnum(_normalize(upper)) == _alnum( - normalized_key) else normalized_key - raise ValueError( + offered = _offered_mask(normalized_key) + raise _MaskValueError( f"capitalization_exceptions value {v!r} for key {k!r} " f"does not spell the key's letters and digits: a value " f"is a case mask, the key's own letters and digits " f"recased -- e.g. " f"capitalization_exceptions=(({normalized_key!r}, " - f"{offered!r}),)") + f"{offered!r}),)", key=k, offered=offered) # capitalized() looks words up one at a time (the _WORD regex # never yields spaces), so a multi-word key is unreachable. # interior whitespace test; split() covers all Unicode whitespace diff --git a/nameparser/_render.py b/nameparser/_render.py index 1c9e84ec..75894634 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -88,6 +88,19 @@ # still not worth the import layering forbids. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") +#: The hyphen clause's own initial test (_cap_text, #478): _INITIAL's +#: period alternative alone. The bare-capital half is deliberately not +#: used here -- admitting it would exempt a capital connective from +#: the clause on its CASE alone, reading it as an initial and leaving +#: it uppercase. The dotted test runs before the vocabulary test, and +#: a capital DOES lower there: 'Y' in 'JOSE ORTEGA-Y-GASSET' reaches +#: the vocabulary test and lowers to 'Ortega-y-Gasset'. Exempting it +#: on case is exactly the reading #458 removed from case repair +#: generally. Registered in tests/v2/test_regex_sync.py as the period +#: alternative of _INITIAL's pattern, so the two cannot drift apart +#: unnoticed. +_DOTTED_INITIAL = re.compile(r"^\w\.$") + # v1 regexes.py "roman_numeral" -- the pipeline's _vocab._ROMAN, # copied by hand because layering forbids this module the import # (the reason _INITIAL above is a copy too) and pinned against config @@ -244,6 +257,9 @@ def _letter_run_ge2(text: str) -> list[bool]: return flags +# rules.md#R4: "the writer split a chunk the mask keeps together, so +# the split-off letter is an initial" -- the split-initial exception +# the docstring below states in one sentence and defers to. def _apply_mask(word: str, mask: str) -> str | None: """rules.md#R4's mask: `word` with each letter or digit recased to the case of the mask's alphanumeric in the same position, and @@ -256,18 +272,16 @@ def _apply_mask(word: str, mask: str) -> str | None: rather than guess. One exception the mask does not decide: a SINGLE LETTER split off - alone beside a full stop (FULL_STOPS, not the ASCII period alone - -- #322) is written in capitals whatever the mask says there -- - but only where the MASK ALSO writes that letter inside a run of - two or more letters: the writer split a chunk the mask keeps - together, so the split-off letter is an initial. 'p.h.d.' under - 'PhD' is 'P.H.D.': p, h and d all sit inside the mask's ONE run, - 'PhD', so each, written alone beside a stop, is an initial and is - capitalized. Where the mask spells the letter alone too -- a - caller's own 'h.c' mask on 'h.c.' -- there is no split to repair, - and the mask's own case stands unchanged. A run of two or more - letters beside a full stop ('sc' in 'b.sc.') is never an initial - either way and always takes the mask's case letter by letter. + alone beside a full stop is written in capitals whatever the mask + says there, but only where the mask itself writes that letter + inside a run of two or more letters (the rule stated above this + function). What the code adds past that statement: a run is of + LETTERS only, so a DIGIT ends one exactly as a full stop does; the + full stop tested is any of FULL_STOPS, not the ASCII period alone + (#322) -- though through capitalized() only the ASCII period is + ever reachable, since _WORD splits a token at any other stop and + this function never sees the rest of the word; the fullwidth row + in test_render pins the direct call, which is reachable. Casing goes through the WHOLE word (word.lower()/word.upper()) rather than per character, when both have the same length as @@ -454,10 +468,10 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], normalized.replace(".", "") in lex.suffix_acronyms or SHAPE_ACRONYM_TAG in tags): return word.upper() - # A roman numeral the parse put in the suffix role is written in - # capitals whether or not the vocabulary lists it: 'vi' through - # 'x' carry no vocabulary tag and title-cased to 'Vi'/'Ix' until - # #459, while 'ii'/'iii'/'iv' rode the exceptions map, which is + # rules.md#R4: "A roman numeral the parse put in the suffix role is + # written in capitals" whether or not the vocabulary lists it: 'vi' + # through 'x' carry no vocabulary tag and title-cased to 'Vi'/'Ix' + # until #459, while 'ii'/'iii'/'iv' rode the exceptions map, which is # why they left it. Suffix-gated for the acronym clause's reason # -- 'Vi' is a given name -- and it is also what writes a # generational 'i' the connective arm's `generation` guard let @@ -502,15 +516,16 @@ def cap(match: re.Match[str]) -> str: # trailing hyphen ('md-phd-') supplies no neighbour while a # doubled one changes nothing ('garcia--y-lopez' keeps its 'y' # lowercase), and the two-part compound ('mcnabb-smith') never - # gets past the count above. The period is the one mark classify - # itself reads as an initial: a single letter marked with a - # period is read as an initial there, as the parse reads it, - # never as the connective, so 'j.-e.-p. dupont' keeps 'E.' while - # the multi-letter 'und.' in 'hans smith-und.-jones' still lowers. + # gets past the count above. The period is the one punctuation + # mark classify itself reads as an initial: a single letter marked + # with a period is read as an initial there, as the parse reads + # it, never as the connective, so 'j.-e.-p. dupont' keeps 'E.' + # while the multi-letter 'und.' in 'hans smith-und.-jones' still + # lowers. first, last = named[0], named[-1] return "-".join( part.lower() - if (first < at < last and not re.fullmatch(r"\w\.", part) + if (first < at < last and not _DOTTED_INITIAL.fullmatch(part) and _normalize(part) in lex.conjunctions) else _WORD.sub(cap, part) for at, part in enumerate(parts)) @@ -526,8 +541,9 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, the joined texts of every token not roled SUFFIX (#492) -- not render() output, so it stays decoupled from spec formatting and the #254 collapse. - Repair changes case and nothing else: an exceptions-map value is - a mask recasing the word as written (#459), never a replacement. + Repair changes case and nothing else (see the rules.md#R4 citation + in _cap_word): an exceptions-map value is a mask recasing the word + as written (#459), never a replacement. The repair reads token TAGS as well as texts: a part whose every word is particle vocabulary is repaired as ordinary name words, and the mark saying so comes from the pipeline, as does the diff --git a/nameparser/config/capitalization.py b/nameparser/config/capitalization.py index 4e431918..3c08cf2a 100644 --- a/nameparser/config/capitalization.py +++ b/nameparser/config/capitalization.py @@ -14,10 +14,16 @@ ``Ph.D.``. The mask's own punctuation marks where its letters are joined into one run versus split apart, and is never written into the word -- repair keeps the writer's own punctuation. A value that -does not spell its key's letters is a ``ValueError`` when a -``Lexicon`` is built from it. A credential acronym written all in -capitals and a roman numeral need no entry: case repair writes a -suffix of either kind in capitals by itself. +does not spell its key's letters and digits is a ``ValueError`` when +a ``Lexicon`` is built from it. An acronym the suffix vocabulary +already lists, written plainly or in dotted form, and a roman +numeral need no entry: case repair writes a suffix of either kind in +capitals by itself. A caller's OWN acronym -- one absent from +``suffix_acronyms`` -- is not covered by that and parses as an +ordinary name word instead; it needs either a ``suffix_acronyms`` +entry (``Lexicon.add(suffix_acronyms={...})`` in the 2.0 API, +``constants.suffix_acronyms.add(...)`` in the v1 one) or a mask of +its own. """ diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index 0ad05e0d..630bcf4c 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -166,19 +166,44 @@ def test_capitalize_exceptions_still_win_over_acronyms(self) -> None: # Lexicon. A DECIDED exception to the shim's never-raise rule # (decisions.md#R4, and #3-0-reevaluations): 1.4.0 substituted # such a value for the word, repair now only recases, and no such - # value was found in the tracker, the docs or any test. + # value was found in the tracker, the docs or any test. The raise + # carries a v1-spelled hint: the 2.0 message's own offer + # ("capitalization_exceptions=(('jr', 'JR'),)") is a Lexicon() + # constructor call, and a v1 caller is looking at a TupleManager + # assignment through Constants, not that constructor. def test_a_mismatched_exception_value_raises_at_the_first_parse( self, ) -> None: c = Constants(capitalization_exceptions={'jr': 'Junior'}) with pytest.raises(ValueError, - match="does not spell the key's letters"): + match="does not spell the key's letters") \ + as caught: HumanName('john smith jr', constants=c) + assert "constants.capitalization_exceptions['jr'] = 'JR'" \ + in str(caught.value) + # pasted onto a fresh Constants, the offered fix works + fixed = Constants() + fixed.capitalization_exceptions['jr'] = 'JR' + hn = HumanName('john smith jr', constants=fixed) + hn.capitalize() + self.m(str(hn), 'John Smith JR', hn) ok = Constants(capitalization_exceptions={'jr': 'JR'}) hn = HumanName('john smith jr', constants=ok) hn.capitalize() self.m(str(hn), 'John Smith JR', hn) + # rules.md#R5 (#492): a suffix ASSIGNED to the facade after + # construction carries UNCLASSIFIED_TAG (no `vocab:suffix` tag, no + # roman-numeral shape read from a parse) and still excluded from + # the one-case gate -- the gate reads the ROLE, not the tag. + def test_capitalize_leaves_an_assigned_suffix_out_of_the_gate( + self, + ) -> None: + hn = HumanName('juan garcia') + hn.suffix = 'III' + hn.capitalize() + self.m(str(hn), 'Juan Garcia III', hn) + # A word in the acronym vocabulary that parses as a family name # still repairs as an ordinary name word, not an acronym (#459). def test_capitalize_family_name_in_acronym_vocab_stays_title_case(self) -> None: @@ -420,7 +445,8 @@ def test_an_assigned_field_keeps_v1_conjunction_repair(self) -> None: # part is ordinary name text (pinned below), and the # difference is that one carries a reading. An # interior link is lowered on both paths since #478. - ('last', 'smith-y', 'John Smith-y')): + ('last', 'smith-y', 'John Smith-y'), + ('last', 'smith-y-jones', 'John Smith-y-Jones')): hn = HumanName('john smith') setattr(hn, field, value) hn.capitalize(force=True) diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 585f60e3..b43480f5 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -375,6 +375,21 @@ def test_v14_pickle_restores_and_parses_warning_free() -> None: HumanName("Jane Roe", constants=c) +def test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint( +) -> None: + """The v1-spelled hint is added ONLY for _MaskValueError -- a value + that does not spell its key -- caught by TYPE in _build_snapshot's + try/except. A capitalization_exceptions key normalizing to empty + is a plain ValueError (_normpairs raises the base class for it, + not _MaskValueError), so it passes through the except clause + unchanged: no v1-spelled hint is appended to an error about a + shape the hint does not fit.""" + c = Constants(capitalization_exceptions={'...': 'x'}) + with pytest.raises(ValueError, match="normalizes to empty") as caught: + HumanName("john smith", constants=c) + assert "on a v1 Constants" not in str(caught.value) + + def test_a_non_str_exception_value_raises_typeerror_at_the_first_parse( ) -> None: """The shim does not translate or otherwise intercept a @@ -857,7 +872,21 @@ def test_multiword_warning_through_shim_points_at_caller() -> None: warnings.simplefilter("always") HumanName("John Smith", constants=c) # snapshot builds lazily here multi = [x for x in w if "matched one word at a time" in str(x.message)] - assert multi and all(x.filename == __file__ for x in multi) + assert len(multi) == 1 + assert multi[0].filename == __file__ + + +def test_a_multiword_capitalization_exceptions_key_warns_exactly_once( +) -> None: + """_build_snapshot's Lexicon(...) construction is the ONLY place + capitalization_exceptions is validated, so a multi-word key must + warn once, not twice.""" + c = Constants(capitalization_exceptions={'zqx zqy': 'ZqXZqY'}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + HumanName("John Smith", constants=c) # snapshot builds lazily here + multi = [x for x in w if "matched one word at a time" in str(x.message)] + assert len(multi) == 1 def test_2x_pickle_roundtrip_keeps_a_readded_dead_entry() -> None: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 293d11cb..e8c9793a 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1,3 +1,4 @@ +import copy import dataclasses import pickle import unicodedata @@ -8,8 +9,8 @@ from nameparser import Parser from nameparser._lexicon import ( - Lexicon, _PHRASE_FIELDS, _VOCAB_FIELDS, _default_lexicon, _normalize, - _title_key, + Lexicon, _MaskValueError, _PHRASE_FIELDS, _VOCAB_FIELDS, _default_lexicon, + _normalize, _title_key, ) from nameparser._policy import Script, _SCRIPT_RANGES from nameparser.config import Constants @@ -166,24 +167,34 @@ def test_an_exception_value_that_does_not_spell_its_key_raises( def test_a_mask_may_recase_a_digit_key_unchanged() -> None: - """#459 review: the mask compares ALPHANUMERICS, not letters alone - (_alnum, not the old letters-only _letters), so a digit is part of - what a value must spell -- '2nd' accepts '2ND' ('2' unchanged, - letters recased) and rejects '3ND' above.""" + """#459 review: the mask compares ALPHANUMERICS (_lexicon._alnum), + not letters alone, so a digit is part of what a value must spell + -- '2nd' accepts '2ND' ('2' unchanged, letters recased) and + rejects '3ND' above.""" lex = Lexicon(capitalization_exceptions=(("2nd", "2ND"),)) assert lex.capitalization_exceptions_map["2nd"] == "2ND" -def test_a_decomposed_value_is_compared_nfc_composed() -> None: +def test_a_decomposed_value_is_compared_and_stored_nfc_composed() -> None: """Both the key and the value fold through _normalize, which NFC- composes non-ASCII text (#459's decomposed-hangul note applies - here too): a value written in decomposed form is accepted against - a key written composed, since both sides read the same letters - once composed.""" + here too), so a value written in decomposed form is accepted + against a key written composed: both sides read the same letters + once composed. The value is also STORED NFC-composed, not as + written -- _apply_mask walks the mask's alphanumeric characters + one at a time, and a decomposed value spells one composed letter + as a base letter plus a non-alpha combining mark, which changes + which letters read as split off beside a full stop. Both + spellings of the same value therefore construct to the identical + stored value.""" + composed = unicodedata.normalize("NFC", "CAFÉ") decomposed = unicodedata.normalize("NFD", "CAFÉ") - assert decomposed != "CAFÉ" # the draw actually decomposed something + assert decomposed != composed # the draw actually decomposed something lex = Lexicon(capitalization_exceptions=(("café", decomposed),)) - assert lex.capitalization_exceptions_map["café"] == decomposed + assert lex.capitalization_exceptions_map["café"] == composed + same = Lexicon(capitalization_exceptions=(("café", composed),)) + assert (lex.capitalization_exceptions_map["café"] + == same.capitalization_exceptions_map["café"]) @pytest.mark.parametrize("key, value", [ @@ -195,8 +206,8 @@ def test_a_value_may_carry_punctuation_that_marks_its_joins( """#459 review: a value's punctuation is not ignored -- it marks which of the key's letters are JOINED (one run) versus split, and _apply_mask's initial rule reads that structure -- but it is never - WRITTEN into the word, so there is nothing here for a warning to - catch. Every one of these constructs silently.""" + WRITTEN into the word. Every one of these is warning-free at + construction, where a construction diagnostic could be raised.""" with warnings.catch_warnings(): warnings.simplefilter("error") lex = Lexicon(capitalization_exceptions=((key, value),)) @@ -239,6 +250,30 @@ def test_the_mask_error_falls_back_when_upper_would_not_itself_validate( .capitalization_exceptions_map == {"straße": "straße"} +def test_mask_value_error_survives_pickle_and_copy() -> None: + """_MaskValueError's default __reduce__ (inherited from + ValueError -> BaseException, which calls type(self)(*self.args)) + would call __init__ with only the message -- self.args holds just + that, key/offered being stored as separate attributes -- and + __init__ requires all three positionally. A worker process in a + ProcessPoolExecutor pickles an exception raised in the worker to + deliver it to the caller, so an unpicklable one surfaces as + BrokenProcessPool there instead of the ValueError HEAD delivered.""" + try: + Lexicon(capitalization_exceptions=(("jr", "Junior"),)) + except _MaskValueError as caught: + original = caught + else: + raise AssertionError("expected _MaskValueError") + for restored in (pickle.loads(pickle.dumps(original)), + copy.copy(original)): + assert isinstance(restored, ValueError) + assert isinstance(restored, _MaskValueError) + assert restored.key == original.key == "jr" + assert restored.offered == original.offered + assert str(restored) == str(original) + + def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 86f9f077..bef90910 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -99,6 +99,18 @@ def test_initial_copies_agree_with_each_other_and_config() -> None: assert source.flags == _vocab._INITIAL.flags +def test_dotted_initial_is_the_period_alternative_of_initial() -> None: + # _render._DOTTED_INITIAL (#478's hyphen clause) is _INITIAL's + # period alternative alone -- the bare-capital half is deliberately + # not used there (#458). _INITIAL is "^(\\w\\.|[A-Z])$"; splitting + # its alternation and re-wrapping the first half must reproduce + # _DOTTED_INITIAL exactly, so the two cannot drift apart unnoticed. + inner = _render._INITIAL.pattern[2:-2] # strip "^(" and ")$" + period_alt = inner.split("|")[0] + assert _render._DOTTED_INITIAL.pattern == f"^{period_alt}$" + assert _render._DOTTED_INITIAL.flags == _render._INITIAL.flags + + # The roster above grew one test at a time, and four hand-copies were # never added to it -- _render's _SPACES, _SPACE_BEFORE_COMMA, _MAC and # _WORD. They had not diverged, but nothing would have said so, which @@ -137,6 +149,7 @@ def test_initial_copies_agree_with_each_other_and_config() -> None: # above, which assert the documented RELATIONSHIP instead: ("_render", "_INITIAL"): None, # config's pattern minus one "?" ("_vocab", "_INITIAL"): None, # same + ("_render", "_DOTTED_INITIAL"): None, # _INITIAL's period alternative ("_tokenize", "_BIDI"): None, # re_bidi, not a REGEXES key # Mirrors _pipeline._state.COMMA_CHARS, not nameparser.config ("_render", "_COMMA_CHAR"): None, diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index be815f6d..159f64b3 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -564,8 +564,25 @@ def test_the_gate_leaves_the_suffixes_out() -> None: ("juan garcia PhD", "Juan Garcia PhD"), ("JUAN GARCIA Jr.", "Juan Garcia Jr."), ("dr. juan garcia III", "Dr. Juan Garcia III"), - ("JUAN GARCIA iii", "Juan Garcia III")): + ("JUAN GARCIA iii", "Juan Garcia III"), + # a cased suffix that is NOT the trailing + # token: the gate excludes every SUFFIX-roled + # token, not only the last one + ("juan garcia PhD MD", "Juan Garcia PhD MD"), + ("juan garcia, PhD, MD", "Juan Garcia PhD, MD"), + # an untagged suffix: 'VI' reads suffix by + # shape (roman numeral) and carries no + # vocab:* tag, so a tag-driven gate would + # wrongly keep it in the one-case test + ("juan garcia VI", "Juan Garcia VI"), + # the gate over each comma shape: SUFFIX_COMMA, + # FAMILY_COMMA, and a FAMILY_COMMA plus a + # trailing suffix segment + ("juan garcia, III", "Juan Garcia III"), + ("garcia, juan III", "Juan Garcia III"), + ("GARCIA, JUAN, Jr.", "Juan Garcia Jr.")): assert str(parse(text).capitalized()) == repaired, text + assert parse("juan garcia VI").tokens[-1].tags == frozenset() for untouched in ("Dr. juan garcia", "DR. juan garcia III", "Juan garcia III", "Juan Garcia iii", # NICKNAME and MAIDEN tokens stay IN the gate @@ -645,6 +662,13 @@ def test_a_mask_recases_the_word_as_the_writer_punctuated_it() -> None: # forced: a mixed-case corpus name R5 would otherwise hold back assert str(parse("Dr. med. univ. Margit Popp, MSc").capitalized( force=True)) == "Dr. Med. Univ. Margit Popp MSc" + # the mask is also asked BEFORE the numeral clause, which would + # give 'III' -- an identity mask ('iii' stays lowercase) proves + # the mask decided rather than merely agreeing with it + lower_iii = dataclasses.replace( + Lexicon.default(), capitalization_exceptions=(("iii", "iii"),)) + p = Parser(lexicon=lower_iii) + assert p.capitalized(p.parse("john smith iii")).suffix == "iii" def test_a_mask_applies_whatever_role_the_word_took() -> None: @@ -735,6 +759,12 @@ def test_a_suffix_numeral_repairs_to_capitals_by_its_shape() -> None: Token("smith", Span(5, 10), Role.FAMILY), Token("vi", Span(11, 13), Role.SUFFIX), ]).capitalized(Lexicon.empty()).suffix == "VI" + # the documented Unicode boundary of the case-only invariant: _ROMAN + # matches under re.I, which admits the dotless Turkish 'ı' (casefold- + # unequal to 'i') as a roman-numeral suffix, and the numeral clause + # writes it in capitals same as any other + assert "ı".casefold() != "i".casefold() + assert parse("john smith ıv").capitalized().suffix == "IV" def test_the_mask_keeps_every_non_letter_and_declines_a_miscount() -> None: @@ -833,6 +863,57 @@ def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( force=True)) == "Dr. h.c. Hans Meier" +def test_a_decomposed_mask_value_reads_the_same_split_as_composed() -> None: + """A mask value is stored NFC-composed (#459 review): the split-off- + initial rule (_letter_run_ge2) walks the mask's alphanumeric + characters one at a time, so a decomposed value spells one + composed letter as a base letter plus a non-alpha combining mark + -- reading the base as split off beside a non-letter rather than + inside a run. Both spellings of the same value must therefore + repair the split word identically.""" + composed = unicodedata.normalize("NFC", "Péx") + decomposed = unicodedata.normalize("NFD", "Péx") + assert decomposed != composed + for value in (composed, decomposed): + lex = dataclasses.replace( + Lexicon.default(), + capitalization_exceptions=(("pé.x.", value),)) + p = Parser(lexicon=lex) + assert p.capitalized(p.parse("john smith pé.x."), + force=True).suffix == "Pé.X." + + +def test_a_masks_upper_fallback_can_lengthen_a_word_through_ss() -> None: + """decisions.md#R4 (2026-09-24 review): _apply_mask's `c.upper()` + fallback, taken whenever the whole-word uppered form does not + keep the word's length, can lengthen a word through ss exactly as + the acronym/numeral clauses' plain `word.upper()` can + (decisions.md#R4's 2026-09-23 Unicode-boundary bullet): the + boundary applies to every clause that upper-cases, not only those + two. And it is not only the split-off-initial force that reaches + it -- the ORDINARY mask branch (the plain `else` arm, no full + stop involved) falls back to `c.upper()` the same way whenever the + mask's own letter at that position is not lower and the word is + not same-length under `.upper()`. An identity mask validates fine + ('aß' spells 'a.ß's own letters unchanged; 'STRAẞE' -- the actual + German capital ẞ, not 'STRASSE' -- spells 'straße's own letters + unchanged too), so both are reachable with no separate defect in + the validator: the validator's refusal of ('straße', 'STRASSE') is + about the VALUE failing to spell the KEY, a different question + from what the applier's output can be once a mask DOES validate.""" + split = dataclasses.replace( + Lexicon.default(), capitalization_exceptions=(("a.ß", "aß"),)) + p = Parser(lexicon=split) + assert p.capitalized(p.parse("john a.ß smith"), + force=True).middle == "A.SS" + ordinary = dataclasses.replace( + Lexicon.default(), + capitalization_exceptions=(("straße", "STRAẞE"),)) + p = Parser(lexicon=ordinary) + assert p.capitalized(p.parse("john straße"), + force=True).family == "STRASSE" + + def test_a_masks_punctuation_marks_its_joins_and_is_never_written() -> None: """#459 review: a value's punctuation is never written into the word -- 'md' repairs to 'MD' and 'm.d.' to 'M.D.' under the @@ -854,8 +935,9 @@ def test_a_masks_punctuation_marks_its_joins_and_is_never_written() -> None: assert str(Parser(lexicon=lex).capitalized( Parser(lexicon=lex).parse("john smith m.d."))) \ == "John Smith M.D." - # the facade twin: warning-free at the first parse (the shim's - # Lexicon snapshot is built lazily, so this is where it would fire) + # the facade twin: warning-free at the first parse, where the + # shim's lazily built Lexicon snapshot is the one place a + # construction diagnostic could be raised c = Constants(capitalization_exceptions={'md': 'M.D.'}) with warnings.catch_warnings(): warnings.simplefilter("error") @@ -879,9 +961,12 @@ def test_a_mask_recases_a_digit_key_unchanged() -> None: """#459 review: the mask walks ALPHANUMERICS, not letters alone, so a digit in the word is carried through unchanged (it has no case) while the surrounding letters still take the mask's case. - The clause reads no vocabulary (like the roman-numeral test - above), so a synthetic token proves it without depending on how - '2nd' happens to parse on its own.""" + The mask clause reads no ROLE or TAGS, unlike the acronym and + numeral clauses that follow it -- an entry applies wherever its + word stands. A synthetic GIVEN-roled token proves that: a role + the acronym/numeral clauses would never reach still finds the + mask, without depending on how '2nd' happens to parse on its + own.""" lex = dataclasses.replace(Lexicon.default(), capitalization_exceptions=(("2nd", "2ND"),)) assert _pn("2nd", [ @@ -997,7 +1082,22 @@ def test_a_link_inside_a_hyphenated_word_keeps_its_lowercase() -> None: # three worded parts on the ambiguous side of `first < at # < last`, so a mutant dropping that bound cannot pass ("juan y-garcia-lopez", "Juan Y-Garcia-Lopez"), - ("juan garcia-lopez-y", "Juan Garcia-Lopez-Y")): + ("juan garcia-lopez-y", "Juan Garcia-Lopez-Y"), + # non-link parts inside a THREE-part hyphenated token still + # get the full per-word repair -- the particle arm + # ("de"/"la") and the Mac rule ("mcnabb") each apply per + # part, not only to a two-part compound + ("juan garcia-de-la-vega", "Juan Garcia-de-la-Vega"), + ("donovan mcnabb-y-smith", "Donovan McNabb-y-Smith"), + # a leading EMPTY part shifts which named index is "first": + # 'y' sits at the edge of the NAMED parts (index 0 of + # ['y', 'garcia', 'lopez']) though it is not part 0 of the + # split, so it stays ordinary name text -- a mutant using + # the raw part index (0 < at < len(parts)-1) instead of the + # named-relative bound would wrongly read it as interior + # and lower it + ("jose -y-garcia-lopez", "Jose -Y-Garcia-Lopez"), + ("jose garcia-lopez-y-", "Jose Garcia-Lopez-Y-")): assert str(parse(text).capitalized()) == repaired, text # mixed case is R5's: untouched unless forced mixed = parse("Jose Ortega-Y-Gasset") From c35aec24e07de5fbefbb11dad0bd41addd59d539 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 24 Sep 2026 01:18:26 -0700 Subject: [PATCH 7/9] refactor(#459/#492/#478): the simplify pass One walk decides a mask letter's case: _letter_run_ge2 is a list comprehension and _apply_mask reads it for the word as well as the mask, so the split-off-initial test is one expression and the two casing branches are one step. A local names the undotted key the mask lookup and the acronym clause both read. Comments and docstrings state each mechanism once and cite rules.md#R4 for the rule; the shim's comment keeps the measured facts about which errors are translated away and drops the rest. The tests share one custom-lexicon helper. Byte-identical against an oracle of 141,120 recorded repairs (every corpus name, case row, rules.md example input and new test input, plain and forced, core and facade, under the default lexicon and sixteen custom maps), the exact text of every new ValueError, the shim's warning count, and 400,000 random word/mask pairs through the applier. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 2 +- nameparser/_config_shim.py | 57 +++------- nameparser/_lexicon.py | 78 +++++-------- nameparser/_render.py | 168 ++++++++++------------------ tests/test_capitalization.py | 2 +- tests/v2/test_lexicon.py | 30 ++--- tests/v2/test_render.py | 205 ++++++++++++----------------------- 7 files changed, 181 insertions(+), 361 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index f9de4f81..21fe1eed 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -1374,7 +1374,7 @@ Accepted costs, deferred to the rescoped #459 rather than relitigated here: the - 2026-09-23 — mechanisms.md#RENDER-HONORS-THE-PARSE CORRECTED, found in this change's re-read of that entry: it still said case repair carries the connective carve-out "ON R3's authority", quoting R4's pre-#461 text ("the carve-out R3 states for initials"), and that `.initials()` gives `j. a. j. s.` for `john and jane smith`. Both had been stale since #461 (2026-09-20), which cut R4's cross-reference to R3 in favor of R4's own reason and stopped a connective with a name word beside it from initialing. The entry now states R4's own reason, dates the cut, and names where the two views part today (`Juan de y`: initials `J. y.`, forced repair `Juan de y`), re-measured 2026-09-23. -- 2026-09-24 #459/#492/#478 (branch review) — FOUR FINDINGS FROM THE REVIEW ROUND, recorded together because each is a small correction to a bullet already above rather than a new decision. (a) THE UNICODE BOUNDARY IS NOT SCOPED TO THE TWO SHAPE CLAUSES, AND NOT TO ONE MASK BRANCH EITHER. The 2026-09-23 bullet above states the `ß` and dotless-`ı` boundary against `_cap_word`'s acronym/numeral clauses, but the MASK path can lengthen a word through `ß` exactly the same way, and both of its two branches can do it independently. `_apply_mask`'s split-off-initial force falls back to `c.upper()` when the whole-word uppered form does not keep the word's length, and `'ß'.upper()` is `'SS'`: an identity mask validates fine (`('a.ß', 'aß')` spells the key's own letters unchanged), so `john a.ß smith` forced gives middle `A.SS` -- casefold-equal to its input and one character longer, the same trade the shape clauses make. The ORDINARY mask branch -- the plain `else` arm, no full stop or split letter involved -- has the identical `c.upper()` fallback for the same reason (mask letter not lower, word not same-length under `.upper()`): `('straße', 'STRAẞE')`, the value spelling `straße`'s own letters unchanged with the actual German capital `ẞ` (not the digraph `'STRASSE'`, which the validator refuses as a VALUE for failing to spell `ß` at all), validates and `john straße` forced gives family `STRASSE`. The validator's refusal of `('straße', 'STRASSE')` is a different question -- whether the VALUE spells the KEY -- and says nothing about what either APPLIER branch can produce from a mask that does validate. Pinned by `tests/v2/test_render.py::test_a_masks_upper_fallback_can_lengthen_a_word_through_ss`. (b) MASK VALUES ARE STORED NFC-COMPOSED. `_normpairs` compared a value against its key NFC-composed but stored the value AS WRITTEN, so a value written in decomposed form spelled one composed letter as a base letter plus a non-alpha combining mark -- changing which letters `_letter_run_ge2` reads as split off beside a full stop from what the composed spelling of the identical value would read. `john smith pé.x.` forced gave `Pé.X.` under a composed mask and `Pé.x.` under the decomposed spelling of the SAME value. Fixed by NFC-composing the value at the same point the key is composed; pinned by `tests/v2/test_lexicon.py::test_a_decomposed_value_is_compared_and_stored_nfc_composed` and `tests/v2/test_render.py::test_a_decomposed_mask_value_reads_the_same_split_as_composed`. (c) THE HYPHEN CLAUSE'S INITIAL TEST IS NOW A PINNED MODULE CONSTANT. `_cap_text`'s inline `re.fullmatch(r"\w\.", part)` was a third hand copy of the initial shape's period half, invisible to `tests/v2/test_regex_sync.py`'s hand-copy sweep because it was never hoisted to a module-level pattern. Hoisted to `_render._DOTTED_INITIAL` and registered as `_INITIAL`'s period alternative; pinned by `tests/v2/test_regex_sync.py::test_dotted_initial_is_the_period_alternative_of_initial`. (d) THE V1 RAISE NOW CARRIES A V1-SPELLED HINT, VIA A TYPED EXCEPTION. The 2026-09-23 bullet above records that a mismatched-mask value raises `ValueError` at a v1 `Constants`' first parse; the message it raised was `Lexicon`'s own, offering a `Lexicon(capitalization_exceptions=...)` constructor call to a caller looking at a `TupleManager` assignment through `Constants`. `_lexicon._normpairs` now raises a dedicated `_MaskValueError(ValueError)` for exactly that one shape, carrying `key` (the raw key as given) and `offered` (from `_offered_mask`) as attributes -- picklable and copyable via its own `__reduce__`, since the inherited default (`type(self)(*self.args)`, `self.args` holding only the message) cannot reconstruct an instance that needs `key`/`offered` too and raises `TypeError` trying; a worker in a `ProcessPoolExecutor` pickles an exception it raises to deliver it to the caller, so an unpicklable one there surfaces as `BrokenProcessPool` instead. `_config_shim._build_snapshot` wraps ONLY the `Lexicon(...)` construction in `try`/`except _MaskValueError` and re-raises with `constants.capitalization_exceptions[key] = offered`. Every OTHER `ValueError` `Lexicon(...)` can raise passes through unchanged, by TYPE rather than by a message-substring test. Of those, most never reach `Lexicon(...)` from this shim at all: an orphaned `non_first_name_prefixes` entry cannot put an entry in `particles_ambiguous` that is outside `particles` (both terms of the union that builds it are already subsets), a bound/particle contradiction cannot arise (`bound & particles` is unconditionally folded into that same union), and the `suffix_acronyms_ambiguous`/`suffix_words` gate bypass cannot arise (`suffix_words` above already subtracts the ambiguous acronyms) -- all three measured by constructing, through a v1 `Constants`, the configuration that would trigger Lexicon's corresponding check, and observing no raise. What DOES reach `Lexicon(...)` here unchanged is an entry normalizing to empty, in `capitalization_exceptions` or in ANY set field -- measured: `c.titles.add("...")` then a parse raises Lexicon's own `titles entry '' normalizes to empty`, unaltered -- since nothing above translates that away. Declined: a first draft pre-validated `capitalization_exceptions` in isolation, a second call to `_normpairs` ahead of `Lexicon(...)` gated on a message-substring test; measured to make a multi-word key's dead-entry warning fire twice on the shim path, and to rest on a filter that was untested (deleting it passed the whole suite) and wrong (an unrelated ValueError then got the v1 hint appended regardless). Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse` (the hint), `tests/v2/test_config_shim.py::test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint` (the pass-through), `tests/v2/test_config_shim.py::test_a_multiword_capitalization_exceptions_key_warns_exactly_once` (the single warning), and `tests/v2/test_lexicon.py::test_mask_value_error_survives_pickle_and_copy` (the pickle/copy round-trip). +- 2026-09-24 #459/#492/#478 (branch review) — FOUR FINDINGS FROM THE REVIEW ROUND, recorded together because each is a small correction to a bullet already above rather than a new decision. (a) THE UNICODE BOUNDARY IS NOT SCOPED TO THE TWO SHAPE CLAUSES, AND NOT TO ONE WAY A MASK LETTER COMES OUT UPPER EITHER. The 2026-09-23 bullet above states the `ß` and dotless-`ı` boundary against `_cap_word`'s acronym/numeral clauses, but the MASK path can lengthen a word through `ß` exactly the same way. `_apply_mask` decides per letter whether it is written upper -- by the mask's own case, or by the split-off-initial force -- and then takes one casing step: the letter at that position in the whole-word `word.upper()`/`word.lower()` when both keep the word's length, otherwise a per-character `c.upper()`/`c.lower()`. `'ß'.upper()` is `'SS'`, so wherever the word holds a `ß` the whole-word form changes the length and every letter goes through `c.upper()`, which lengthens a `ß` written upper to `SS` whichever of the two reasons made it upper. Through the force: an identity mask validates fine (`('a.ß', 'aß')` spells the key's own letters unchanged), so `john a.ß smith` forced gives middle `A.SS` -- casefold-equal to its input and one character longer, the same trade the shape clauses make. Through the mask's own case, no full stop or split letter involved: `('straße', 'STRAẞE')`, the value spelling `straße`'s own letters unchanged with the actual German capital `ẞ` (not the digraph `'STRASSE'`, which the validator refuses as a VALUE for failing to spell `ß` at all), validates and `john straße` forced gives family `STRASSE`. The validator's refusal of `('straße', 'STRASSE')` is a different question -- whether the VALUE spells the KEY -- and says nothing about what the applier can produce from a mask that does validate. Pinned by `tests/v2/test_render.py::test_a_masks_upper_fallback_can_lengthen_a_word_through_ss`. (b) MASK VALUES ARE STORED NFC-COMPOSED. `_normpairs` compared a value against its key NFC-composed but stored the value AS WRITTEN, so a value written in decomposed form spelled one composed letter as a base letter plus a non-alpha combining mark -- changing which letters `_letter_run_ge2` reads as split off beside a full stop from what the composed spelling of the identical value would read. `john smith pé.x.` forced gave `Pé.X.` under a composed mask and `Pé.x.` under the decomposed spelling of the SAME value. Fixed by NFC-composing the value at the same point the key is composed; pinned by `tests/v2/test_lexicon.py::test_a_decomposed_value_is_compared_and_stored_nfc_composed` and `tests/v2/test_render.py::test_a_decomposed_mask_value_reads_the_same_split_as_composed`. (c) THE HYPHEN CLAUSE'S INITIAL TEST IS NOW A PINNED MODULE CONSTANT. `_cap_text`'s inline `re.fullmatch(r"\w\.", part)` was a third hand copy of the initial shape's period half, invisible to `tests/v2/test_regex_sync.py`'s hand-copy sweep because it was never hoisted to a module-level pattern. Hoisted to `_render._DOTTED_INITIAL` and registered as `_INITIAL`'s period alternative; pinned by `tests/v2/test_regex_sync.py::test_dotted_initial_is_the_period_alternative_of_initial`. (d) THE V1 RAISE NOW CARRIES A V1-SPELLED HINT, VIA A TYPED EXCEPTION. The 2026-09-23 bullet above records that a mismatched-mask value raises `ValueError` at a v1 `Constants`' first parse; the message it raised was `Lexicon`'s own, offering a `Lexicon(capitalization_exceptions=...)` constructor call to a caller looking at a `TupleManager` assignment through `Constants`. `_lexicon._normpairs` now raises a dedicated `_MaskValueError(ValueError)` for exactly that one shape, carrying `key` (the raw key as given) and `offered` (from `_offered_mask`) as attributes -- picklable and copyable via its own `__reduce__`, since the inherited default (`type(self)(*self.args)`, `self.args` holding only the message) cannot reconstruct an instance that needs `key`/`offered` too and raises `TypeError` trying; a worker in a `ProcessPoolExecutor` pickles an exception it raises to deliver it to the caller, so an unpicklable one there surfaces as `BrokenProcessPool` instead. `_config_shim._build_snapshot` wraps ONLY the `Lexicon(...)` construction in `try`/`except _MaskValueError` and re-raises with `constants.capitalization_exceptions[key] = offered`. Every OTHER `ValueError` `Lexicon(...)` can raise passes through unchanged, by TYPE rather than by a message-substring test. Of those, most never reach `Lexicon(...)` from this shim at all: an orphaned `non_first_name_prefixes` entry cannot put an entry in `particles_ambiguous` that is outside `particles` (both terms of the union that builds it are already subsets), a bound/particle contradiction cannot arise (`bound & particles` is unconditionally folded into that same union), and the `suffix_acronyms_ambiguous`/`suffix_words` gate bypass cannot arise (`suffix_words` above already subtracts the ambiguous acronyms) -- all three measured by constructing, through a v1 `Constants`, the configuration that would trigger Lexicon's corresponding check, and observing no raise. What DOES reach `Lexicon(...)` here unchanged is an entry normalizing to empty, in `capitalization_exceptions` or in ANY set field -- measured: `c.titles.add("...")` then a parse raises Lexicon's own `titles entry '' normalizes to empty`, unaltered -- since nothing above translates that away. Declined: a first draft pre-validated `capitalization_exceptions` in isolation, a second call to `_normpairs` ahead of `Lexicon(...)` gated on a message-substring test; measured to make a multi-word key's dead-entry warning fire twice on the shim path, and to rest on a filter that was untested (deleting it passed the whole suite) and wrong (an unrelated ValueError then got the v1 hint appended regardless). Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse` (the hint), `tests/v2/test_config_shim.py::test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint` (the pass-through), `tests/v2/test_config_shim.py::test_a_multiword_capitalization_exceptions_key_warns_exactly_once` (the single warning), and `tests/v2/test_lexicon.py::test_mask_value_error_survives_pickle_and_copy` (the pickle/copy round-trip). Excluded (CAPITALIZATION_EXCEPTIONS — md, ii, iii, iv, removed 2026-09-23, #459): diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index e5aa921d..f1a3757c 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -1011,40 +1011,21 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: suffix_words = frozenset(self.suffix_not_acronyms) - ambiguous_acronyms # keep in sync with _lexicon._default_lexicon() (pinned by # tests/v2/test_config_shim.py::test_snapshot_field_translation) - # The one shape the v1-spelled hint fits: _normpairs raises - # _MaskValueError, carrying the raw key and the offered fix, - # for exactly one shape -- a capitalization_exceptions value - # that does not spell its key. Caught by TYPE, not by a - # message-substring filter, so every OTHER ValueError Lexicon - # can raise passes through unchanged automatically -- no - # enumeration needed to keep them that way. Most of those never - # reach Lexicon at all: they are translated away by the - # computations above. An orphaned non_first_name_prefixes entry - # cannot put an entry in particles_ambiguous that is outside - # particles (it is built as two subtractions/intersections of - # particles, always inside it); a bound/particle contradiction - # cannot arise either (bound & particles is unconditionally - # folded into particles_ambiguous above, so the gate Lexicon - # checks is satisfied by construction); and the - # suffix_acronyms_ambiguous/suffix_words gate bypass cannot - # arise since suffix_words above already subtracts - # ambiguous_acronyms (measured: constructing each of the three - # configurations that would trigger Lexicon's corresponding - # check raises nothing through a v1 Constants). What DOES - # reach Lexicon here unchanged is an entry normalizing to - # empty -- in capitalization_exceptions or in any SET field - # (titles, particles, ...; measured: c.titles.add("...") then - # a parse raises Lexicon's own "entry '' normalizes to empty", - # unchanged) -- since nothing above translates that away, so - # it propagates by type, past this except clause, exactly as - # v1 never validated an entry's content either way. A - # wrong-TYPE value (the comment above: TupleManager never - # statically typed its values) does not need special handling - # for the same reason -- v1 never accepted one silently - # either, it raised at capitalize() trying to use the value as - # a string, so the TypeError _normpairs raises for it here - # reproduces a raise v1 already had, at an earlier point in - # the same call chain rather than a new one. + # A capitalization_exceptions value that does not spell its key + # is the one DECIDED exception to this method's never-raise + # rule (#459, decisions.md#R4 and #3-0-reevaluations). + # _normpairs raises _MaskValueError for exactly that shape, and + # the except clause re-spells its offered fix for a v1 caller. + # Caught by TYPE, so every other Lexicon error passes through + # unchanged. Measured: three of Lexicon's checks cannot fire + # from here, this method's translations satisfying them by + # construction (particles_ambiguous inside particles, the + # bound/never-given contradiction, and an ambiguous acronym in + # suffix_words); what does reach it is an entry normalizing to + # empty, in capitalization_exceptions or any set field + # (c.titles.add("...") raises Lexicon's own "normalizes to + # empty"), and a non-str value's TypeError -- a raise v1 also + # had, later, at capitalize(). try: lexicon = Lexicon( titles=frozenset(self.titles), @@ -1129,13 +1110,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # TupleManager is dict[str, object] (v1 parity: values were # never statically str-typed); every real entry is a str, # same assumption _DelimiterManager's sentinel lookup makes. - # NOT translated: the one DECIDED exception to this - # method's never-raise rule (#459, decisions.md#R4 and - # #3-0-reevaluations) -- a value that does not spell its - # key's letters and digits -- and the key-normalizes-to- - # empty case beside it are both explained in the comment - # above the try: block, which is where the catching - # happens now. + # NOT translated: see the comment above the try: block. capitalization_exceptions=tuple( sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] ) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index e9e97626..a01ef8f1 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -381,39 +381,27 @@ def _alnum(text: str) -> str: def _offered_mask(normalized_key: str) -> str: - """The spelling _normpairs' mismatched-value error offers as a - working replacement, and what the shim's re-raise (_config_shim's - _build_snapshot) offers a v1 caller too -- factored so the two - messages can never name different fixes for the same key. - - .upper() can change a letter's COUNT ('straße' -> 'STRASSE', ß - growing to two letters), which would fail the mask check itself - if pasted back in. Falling back to the key unchanged -- an - identity mask, always legal -- keeps the offer actionable in - every case rather than only the common one.""" + """The working value a mismatched-mask error offers (carried to the + v1 shim's re-raise on _MaskValueError, so both name one fix): the + key upper-cased, or the key unchanged -- an identity mask, always + legal -- where .upper() changes the letter count ('straße' -> + 'STRASSE') and would fail the check if pasted back in.""" upper = normalized_key.upper() return upper if _alnum(_normalize(upper)) == _alnum( normalized_key) else normalized_key class _MaskValueError(ValueError): - """Raised by _normpairs for a value that does not spell its key; - carries the raw key and the offered fix (`key`, `offered`) so a - caller reached through another surface -- the v1 shim's - _build_snapshot -- can re-spell the same fix for that surface - rather than re-deriving it from the message text. - - __reduce__ pickles/copies as (message, key, offered) rather than - the default (args,) -- ValueError's own __reduce__ would call - type(self)(*self.args), and self.args holds only the message - (super().__init__ below is passed the message alone), so the - default would call this __init__ with two required arguments - missing. This is an exception, not one of _types.py's frozen - dataclasses, so the _guarded_getstate/slots convention there does - not apply -- a plain __reduce__ is the right-sized fix. A worker - in a ProcessPoolExecutor pickles an exception raised in the - worker to deliver it to the caller, so an unpicklable one there - surfaces as BrokenProcessPool instead.""" + """_normpairs' error for a value that does not spell its key. It + carries the raw `key` and the `offered` fix so the v1 shim can + re-spell that fix for its own surface without parsing the message. + + __reduce__ is explicit because the inherited one re-calls + __init__ with self.args, which holds the message alone, so + pickling or copying would fail -- and a ProcessPoolExecutor + pickles a worker's exception to deliver it, surfacing an + unpicklable one as BrokenProcessPool. (An exception, so + _types.py's frozen-dataclass pickle guards do not apply.)""" def __init__(self, message: str, key: str, offered: str) -> None: super().__init__(message) @@ -477,30 +465,18 @@ def _normpairs( f"nothing)" ) # A value is a case MASK (#459): the key's own letters and - # digits recased. Its punctuation marks where its letters are - # JOINED -- letters written side by side are one run, and any - # character that is not a letter between them (a full stop, a - # space, a digit) ends the run, so the mask's own case stands - # for a letter it spells alone ('h.c' on 'h.c.' stays 'h.c.'). - # It is never written into the word: case repair keeps the - # WRITER's own punctuation, not the mask's. Compared through - # the same fold as the key, which keeps interior periods - # ('Ph.D.' is stored 'ph.d') and composes NFC, so both sides - # are read alike. A raise, not a warning: a mismatched value - # used to be SUBSTITUTED for the word, and rules.md#R4: - # "Repair changes case and nothing else", so there is no - # reading of one that repair can honor. - # Stored NFC-composed (case-preserving -- unicodedata.normalize, - # not _normalize, which also lowercases). _apply_mask walks the - # mask's alphanumeric characters one at a time - # (_letter_run_ge2), so a decomposed value spells one composed - # letter as two characters -- a base letter plus a combining - # mark that is not alpha -- and the base reads as a - # SPLIT-OFF letter next to a non-letter, changing which - # letters _apply_mask reads as an initial beside a full stop. - # The check just above already compares composed to composed - # (_normalize NFC-composes both sides), so a decomposed value - # passes it and would otherwise be stored exactly as written. + # digits recased, compared through the key's own fold. Its + # punctuation marks which letters are joined into one run + # (_render._apply_mask reads that) and is never written into + # the word. A raise, not a warning: a mismatched value used to + # be SUBSTITUTED for the word, and rules.md#R4: "Repair + # changes case and nothing else", so there is no reading of + # one that repair can honor. + # Stored NFC-composed, case kept (unicodedata, not _normalize): + # _apply_mask reads the mask one character at a time, and a + # decomposed letter would read as a base letter split off + # beside a non-alpha combining mark. The check below passes + # either spelling, _normalize composing both of its sides. v = unicodedata.normalize("NFC", v) if _alnum(_normalize(v)) != _alnum(normalized_key): offered = _offered_mask(normalized_key) diff --git a/nameparser/_render.py b/nameparser/_render.py index 75894634..f5682774 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -88,27 +88,18 @@ # still not worth the import layering forbids. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") -#: The hyphen clause's own initial test (_cap_text, #478): _INITIAL's -#: period alternative alone. The bare-capital half is deliberately not -#: used here -- admitting it would exempt a capital connective from -#: the clause on its CASE alone, reading it as an initial and leaving -#: it uppercase. The dotted test runs before the vocabulary test, and -#: a capital DOES lower there: 'Y' in 'JOSE ORTEGA-Y-GASSET' reaches -#: the vocabulary test and lowers to 'Ortega-y-Gasset'. Exempting it -#: on case is exactly the reading #458 removed from case repair -#: generally. Registered in tests/v2/test_regex_sync.py as the period -#: alternative of _INITIAL's pattern, so the two cannot drift apart -#: unnoticed. +#: The hyphen clause's initial test (_cap_text, #478): _INITIAL's +#: period alternative alone. The bare-capital half would exempt a +#: capital connective on its CASE -- 'Y' in 'JOSE ORTEGA-Y-GASSET' +#: must lower -- the reading #458 removed. Pinned as that alternative +#: by tests/v2/test_regex_sync.py. _DOTTED_INITIAL = re.compile(r"^\w\.$") -# v1 regexes.py "roman_numeral" -- the pipeline's _vocab._ROMAN, -# copied by hand because layering forbids this module the import -# (the reason _INITIAL above is a copy too) and pinned against config -# by tests/v2/test_regex_sync.py. Its one reader is _cap_word's -# numeral clause, which asks a RENDERING question of a word the parse -# already put in the suffix role -- how a numeral is written -- and -# not the parse's own question of whether the word is a suffix, so -# the role is honored rather than re-derived +# v1 regexes.py "roman_numeral", the pipeline's _vocab._ROMAN copied +# by hand (layering, as for _INITIAL above) and pinned by +# tests/v2/test_regex_sync.py. Read only by _cap_word's numeral +# clause, which asks how a word the parse already put in the suffix +# role is WRITTEN, never whether it is a suffix # (mechanisms.md#RENDER-HONORS-THE-PARSE). _ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I) @@ -236,92 +227,61 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str def _letter_run_ge2(text: str) -> list[bool]: - """One flag per alphanumeric character of `text`, in the order - _apply_mask below walks them: True where that character is a - LETTER with a letter immediately before or after it WITHIN - `text` -- a full stop or a digit breaks a run, exactly as either - breaks one in the word _apply_mask is casing. A digit's own flag - is always False; the initial-beside-a-stop exception only ever - concerns letters.""" - n = len(text) - flags: list[bool] = [] - for i, c in enumerate(text): - if not c.isalnum(): - continue - if not c.isalpha(): - flags.append(False) - continue - prev_letter = i > 0 and text[i - 1].isalpha() - next_letter = i + 1 < n and text[i + 1].isalpha() - flags.append(prev_letter or next_letter) - return flags + """One flag per alphanumeric character of `text`, in order: True + where it is a LETTER with a letter immediately before or after + it. Any other character -- a full stop, a space, a digit -- ends + a run, and a digit's own flag is always False.""" + last = len(text) - 1 + return [c.isalpha() and ((i > 0 and text[i - 1].isalpha()) + or (i < last and text[i + 1].isalpha())) + for i, c in enumerate(text) if c.isalnum()] # rules.md#R4: "the writer split a chunk the mask keeps together, so -# the split-off letter is an initial" -- the split-initial exception -# the docstring below states in one sentence and defers to. +# the split-off letter is an initial" -- the one override below. def _apply_mask(word: str, mask: str) -> str | None: """rules.md#R4's mask: `word` with each letter or digit recased to - the case of the mask's alphanumeric in the same position, and - every other character kept where the writer put it -- 'ph.d.' - under 'PhD' is 'Ph.D.'. None where the two alphanumeric counts - differ. The lookup key makes that rare and not impossible: the - key is the word NFC-composed, so a word written in decomposed - hangul spells one syllable in two letters and still finds a - one-letter key. The caller falls through to its next clause - rather than guess. - - One exception the mask does not decide: a SINGLE LETTER split off - alone beside a full stop is written in capitals whatever the mask - says there, but only where the mask itself writes that letter - inside a run of two or more letters (the rule stated above this - function). What the code adds past that statement: a run is of - LETTERS only, so a DIGIT ends one exactly as a full stop does; the - full stop tested is any of FULL_STOPS, not the ASCII period alone - (#322) -- though through capitalized() only the ASCII period is - ever reachable, since _WORD splits a token at any other stop and - this function never sees the rest of the word; the fullwidth row - in test_render pins the direct call, which is reachable. - - Casing goes through the WHOLE word (word.lower()/word.upper()) - rather than per character, when both have the same length as - `word`: a per-character str.lower()/str.upper() call is - context-free and gets some letters wrong that the whole-word form - gets right -- a medial sigma where Greek wants a final one, for - one. A mask letter is read as upper when it is not lower - (`not letter.islower()`), so a titlecase letter (Unicode category - Lt) reads as upper rather than lower -- a known limit: no - per-character titlecase mapping is attempted.""" + the case of the mask's alphanumeric in the same position, every + other character kept where the writer put it ('ph.d.' under 'PhD' + is 'Ph.D.'). None where the two alphanumeric counts differ -- + rare, since the key is the word folded, but a word in decomposed + hangul spells one syllable in two letters and still finds its + one-letter key -- and the caller falls through rather than guess. + + The override cited above: a letter standing alone beside a full + stop (any of FULL_STOPS, though through capitalized() only the + ASCII period reaches here, _WORD splitting a token at any other) + is written upper wherever the mask writes it inside a run of two + or more letters (_letter_run_ge2). + + Casing goes through the whole word (word.lower()/word.upper()) + when both keep its length, since per-character casing is + context-free (a Greek final sigma needs its neighbours). A mask + letter that is not lowercase reads as upper, so a titlecase + letter (Lt) is written upper -- a known limit.""" mask_chars = [c for c in mask if c.isalnum()] if len(mask_chars) != sum(1 for c in word if c.isalnum()): return None mask_run = _letter_run_ge2(mask) - lowered = word.lower() - uppered = word.upper() - same_length = len(lowered) == len(word) and len(uppered) == len(word) - n = len(word) + word_run = _letter_run_ge2(word) + lowered, uppered = word.lower(), word.upper() + same_length = len(lowered) == len(uppered) == len(word) + last = len(word) - 1 out: list[str] = [] at = 0 for i, c in enumerate(word): if not c.isalnum(): out.append(c) continue - mask_char = mask_chars[at] - in_mask_run = mask_run[at] + upper = not mask_chars[at].islower() or ( + mask_run[at] and c.isalpha() and not word_run[at] + and ((i > 0 and word[i - 1] in FULL_STOPS) + or (i < last and word[i + 1] in FULL_STOPS))) at += 1 - if c.isalpha(): - prev_letter = i > 0 and word[i - 1].isalpha() - next_letter = i + 1 < n and word[i + 1].isalpha() - beside_stop = (i > 0 and word[i - 1] in FULL_STOPS) or ( - i + 1 < n and word[i + 1] in FULL_STOPS) - if (not prev_letter and not next_letter and beside_stop - and in_mask_run): - out.append(uppered[i] if same_length else c.upper()) - continue - if mask_char.islower(): - out.append(lowered[i] if same_length else c.lower()) + if same_length: + out.append(uppered[i] if upper else lowered[i]) else: - out.append(uppered[i] if same_length else c.upper()) + out.append(c.upper() if upper else c.lower()) return "".join(out) @@ -445,7 +405,8 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # rules.md#R4: "Repair changes case and nothing else". Role-free, # as the map always was: an entry is the caller saying how a word # is written wherever it stands. - for key in (normalized, normalized.replace(".", "")): + undotted = normalized.replace(".", "") + for key in (normalized, undotted): mask = lex.capitalization_exceptions_map.get(key) if mask is not None: masked = _apply_mask(word, mask) @@ -465,7 +426,7 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # name word -- #459's given-role half, decided: repair follows the # role the parse chose ('qc mp' -> 'Qc MP'). if role is Role.SUFFIX and ( - normalized.replace(".", "") in lex.suffix_acronyms + undotted in lex.suffix_acronyms or SHAPE_ACRONYM_TAG in tags): return word.upper() # rules.md#R4: "A roman numeral the parse put in the suffix role is @@ -504,24 +465,15 @@ def cap(match: re.Match[str]) -> str: return _WORD.sub(cap, text) # rules.md#R4: "Inside a hyphenated word, a part that is # connective vocabulary with a worded part on each side of it - # keeps its lowercase" -- the hyphens are the writer joining the - # name around it, as the spaced connective would (#478). Position - # and vocabulary both come from the whole token, which is why - # this sits here and not in _cap_word, whose word has lost its - # neighbours. It does NOT re-derive the conjunction-versus- - # initial class from a word's CASE, the thing #458 removed: an - # EDGE part has a part on one side only and stays ordinary name - # text, so 'juan e-f smith' keeps 'E-F'. A 'part' is one holding - # a word; an empty part does not count and is skipped, so a - # trailing hyphen ('md-phd-') supplies no neighbour while a - # doubled one changes nothing ('garcia--y-lopez' keeps its 'y' - # lowercase), and the two-part compound ('mcnabb-smith') never - # gets past the count above. The period is the one punctuation - # mark classify itself reads as an initial: a single letter marked - # with a period is read as an initial there, as the parse reads - # it, never as the connective, so 'j.-e.-p. dupont' keeps 'E.' - # while the multi-letter 'und.' in 'hans smith-und.-jones' still - # lowers. + # keeps its lowercase" (#478). Decided here, not in _cap_word, + # whose word has lost its neighbours. Only a part holding a word + # is a neighbour: an empty one is skipped ('garcia--y-lopez' keeps + # its 'y') and supplies none ('md-phd-'). An EDGE part stays + # ordinary name text ('juan e-f smith' keeps 'E-F'), so no word is + # re-read as connective or initial by its case (#458). A single + # letter with a period is an initial, as classify reads it, never + # the connective ('j.-e.-p. dupont' keeps 'E.'); the multi-letter + # 'und.' in 'hans smith-und.-jones' still lowers. first, last = named[0], named[-1] return "-".join( part.lower() diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index 630bcf4c..a504666a 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -194,7 +194,7 @@ def test_a_mismatched_exception_value_raises_at_the_first_parse( # rules.md#R5 (#492): a suffix ASSIGNED to the facade after # construction carries UNCLASSIFIED_TAG (no `vocab:suffix` tag, no - # roman-numeral shape read from a parse) and still excluded from + # roman-numeral shape read from a parse) and is still excluded from # the one-case gate -- the gate reads the ROLE, not the tag. def test_capitalize_leaves_an_assigned_suffix_out_of_the_gate( self, diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index e8c9793a..ddf41569 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -176,25 +176,15 @@ def test_a_mask_may_recase_a_digit_key_unchanged() -> None: def test_a_decomposed_value_is_compared_and_stored_nfc_composed() -> None: - """Both the key and the value fold through _normalize, which NFC- - composes non-ASCII text (#459's decomposed-hangul note applies - here too), so a value written in decomposed form is accepted - against a key written composed: both sides read the same letters - once composed. The value is also STORED NFC-composed, not as - written -- _apply_mask walks the mask's alphanumeric characters - one at a time, and a decomposed value spells one composed letter - as a base letter plus a non-alpha combining mark, which changes - which letters read as split off beside a full stop. Both - spellings of the same value therefore construct to the identical - stored value.""" + """A decomposed value is accepted against a composed key, the + check folding both sides through _normalize, and is STORED + composed, so both spellings of one value construct alike + (test_render's decomposed-mask test shows why that matters).""" composed = unicodedata.normalize("NFC", "CAFÉ") decomposed = unicodedata.normalize("NFD", "CAFÉ") assert decomposed != composed # the draw actually decomposed something lex = Lexicon(capitalization_exceptions=(("café", decomposed),)) assert lex.capitalization_exceptions_map["café"] == composed - same = Lexicon(capitalization_exceptions=(("café", composed),)) - assert (lex.capitalization_exceptions_map["café"] - == same.capitalization_exceptions_map["café"]) @pytest.mark.parametrize("key, value", [ @@ -251,14 +241,10 @@ def test_the_mask_error_falls_back_when_upper_would_not_itself_validate( def test_mask_value_error_survives_pickle_and_copy() -> None: - """_MaskValueError's default __reduce__ (inherited from - ValueError -> BaseException, which calls type(self)(*self.args)) - would call __init__ with only the message -- self.args holds just - that, key/offered being stored as separate attributes -- and - __init__ requires all three positionally. A worker process in a - ProcessPoolExecutor pickles an exception raised in the worker to - deliver it to the caller, so an unpicklable one surfaces as - BrokenProcessPool there instead of the ValueError HEAD delivered.""" + """The inherited __reduce__ would re-call __init__ with self.args, + the message alone, and __init__ needs `key` and `offered` too -- + so without the override a ProcessPoolExecutor worker raising it + would surface as BrokenProcessPool rather than this ValueError.""" try: Lexicon(capitalization_exceptions=(("jr", "Junior"),)) except _MaskValueError as caught: diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 159f64b3..eec36308 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -7,7 +7,7 @@ from nameparser import FAMILY_FIRST, HumanName, Parser, Policy, parse from nameparser._lexicon import Lexicon from nameparser.config import Constants -from nameparser._render import _collapse, render +from nameparser._render import _apply_mask, _collapse, render from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, UNJOINED_TAG, Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token) @@ -642,6 +642,15 @@ def test_capitalized_with_explicit_lexicon() -> None: assert out.suffix == "Phd" +def _repaired_under(pairs: tuple[tuple[str, str], ...], text: str, *, + force: bool = True) -> ParsedName: + """`text` parsed and repaired under the default lexicon with its + exceptions map replaced by `pairs`.""" + p = Parser(lexicon=dataclasses.replace( + Lexicon.default(), capitalization_exceptions=pairs)) + return p.capitalized(p.parse(text), force=force) + + def test_a_mask_recases_the_word_as_the_writer_punctuated_it() -> None: """rules.md#R4's mask (#459): an exceptions-map value is its key's letters in the case each takes, laid over the word as written -- @@ -665,10 +674,8 @@ def test_a_mask_recases_the_word_as_the_writer_punctuated_it() -> None: # the mask is also asked BEFORE the numeral clause, which would # give 'III' -- an identity mask ('iii' stays lowercase) proves # the mask decided rather than merely agreeing with it - lower_iii = dataclasses.replace( - Lexicon.default(), capitalization_exceptions=(("iii", "iii"),)) - p = Parser(lexicon=lower_iii) - assert p.capitalized(p.parse("john smith iii")).suffix == "iii" + assert _repaired_under((("iii", "iii"),), "john smith iii", + force=False).suffix == "iii" def test_a_mask_applies_whatever_role_the_word_took() -> None: @@ -768,7 +775,6 @@ def test_a_suffix_numeral_repairs_to_capitals_by_its_shape() -> None: def test_the_mask_keeps_every_non_letter_and_declines_a_miscount() -> None: - from nameparser._render import _apply_mask assert _apply_mask("ph.d.", "PhD") == "Ph.D." assert _apply_mask("PHD", "PhD") == "PhD" assert _apply_mask("bsc", "BSc") == "BSc" @@ -779,25 +785,18 @@ def test_the_mask_keeps_every_non_letter_and_declines_a_miscount() -> None: decomposed = unicodedata.normalize("NFD", "씨") assert len(decomposed) == 2 assert _apply_mask(decomposed, "씨") is None - lex = dataclasses.replace(Lexicon.default(), - capitalization_exceptions=(("씨", "씨"),)) - p = Parser(lexicon=lex) - name = p.parse(unicodedata.normalize("NFD", "John Smith 씨")) - assert p.capitalized(name, force=True).suffix == decomposed + assert _repaired_under((("씨", "씨"),), unicodedata.normalize( + "NFD", "John Smith 씨")).suffix == decomposed def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( ) -> None: """rules.md#R4 (#459 review, narrowed): a letter written alone beside a full stop is an initial ONLY where the MASK writes that - same letter inside a run of two or more letters -- the writer - split a chunk the mask keeps together, so each split piece is an - initial. Where the mask spells the letter alone too, there is no - split to repair, and the mask's own case stands unchanged -- a - caller's own 'h.c' mask on 'h.c.' stays 'h.c.'. A run of two or - more letters beside a full stop ('sc' in 'b.sc.') is never an - initial either way.""" - from nameparser._render import _apply_mask + same letter inside a run of two or more letters. Where the mask + spells it alone too, the mask's own case stands ('h.c' on 'h.c.' + stays 'h.c.'), and a run of two or more letters beside a full + stop ('sc' in 'b.sc.') is never an initial.""" for word, mask, expected in ( ("p.h.d.", "PhD", "P.H.D."), ("b.sc.", "BSc", "B.Sc."), @@ -806,39 +805,18 @@ def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( (".a", "a", ".a"), # single-letter mask never overrides ("a.", "a", "a."), ("2b.", "2b", "2b."), # 'b' has no LETTER neighbor in '2b' - # a full stop on one side only, under a multi-letter mask: - # 'a' splits from the mask's 'Abc' run and is forced - # upper; 'b' and 'c' sit next to each other in the WORD - # (not lone) and take the mask's own lowercase case. Weak - # as a mutation pin -- the mask's own 'A' is ALREADY - # upper, so mask-driven casing alone (override skipped) - # gives the same answer; the next two rows use a mask - # with no letter already capitalized, so only the - # override, not the mask's own case, can produce the 'A'. + # the mask's own 'A' already agrees, so this row alone + # cannot tell the override from plain masking; the two + # all-lowercase masks after it can ("a.bc", "Abc", "A.bc"), - # stop on the NEXT side, at the mask's run START: 'a' - # splits from the mask's one run 'abc' and is forced - # upper; 'b'/'c' are not lone in the word and take the - # mask's own (here lowercase) case - ("a.bc", "abc", "A.bc"), - # stop on the PREVIOUS side, at the mask's run END: 'c' - # splits from the same one-run mask and is forced upper; - # 'a'/'b' are not lone and stay lowercase - ("ab.c", "abc", "ab.C"), - # a digit breaks a letter run on BOTH sides: 'x2bc' is two - # runs, {x} and {bc} (the digit itself gets no run, and - # is not a letter, so it never triggers the override - # either). 'x' is adjacent to the DIGIT '2' -- not a full - # stop -- so it is never even beside-a-stop and keeps the - # mask's lowercase; 'b' and 'c' are each lone in the word - # AND inside the mask's {bc} run, so both are forced upper + ("a.bc", "abc", "A.bc"), # stop after the run's first letter + ("ab.c", "abc", "ab.C"), # stop before the run's last letter + # a digit ends a run: the mask's runs are {x} and {bc}, so + # 'x' keeps its lowercase (and has no stop beside it) while + # the split 'b' and 'c' are forced upper ("x2b.c", "x2bc", "x2B.C"), - # the fullwidth stop U+FF0E is a FULL_STOPS member, so the - # word-side adjacency test reads it like an ASCII period - # -- but _render._WORD ("(\\w|\\.)+") only ever yields the - # ASCII period inside a token's text, so a word carrying - # this stop reaches _apply_mask only through a direct - # call, never through _cap_text's normal tokenizing + # a fullwidth stop is a full stop too -- reachable by a + # direct call only, since _WORD splits a token at it ("a．bc", "abc", "A．bc"), ): assert _apply_mask(word, mask) == expected, (word, mask) @@ -856,84 +834,53 @@ def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( # end to end: a caller's own mask that ALSO spells each letter # alone (honoris causa -- "Dr. h.c.") leaves the split-looking # word unchanged rather than forcing capitals nobody asked for - lex = dataclasses.replace(Lexicon.default(), - capitalization_exceptions=(("h.c", "h.c"),)) - p = Parser(lexicon=lex) - assert str(p.capitalized(p.parse("dr. h.c. hans meier"), - force=True)) == "Dr. h.c. Hans Meier" + assert str(_repaired_under((("h.c", "h.c"),), "dr. h.c. hans meier")) \ + == "Dr. h.c. Hans Meier" def test_a_decomposed_mask_value_reads_the_same_split_as_composed() -> None: - """A mask value is stored NFC-composed (#459 review): the split-off- - initial rule (_letter_run_ge2) walks the mask's alphanumeric - characters one at a time, so a decomposed value spells one - composed letter as a base letter plus a non-alpha combining mark - -- reading the base as split off beside a non-letter rather than - inside a run. Both spellings of the same value must therefore - repair the split word identically.""" + """A mask value is stored NFC-composed (#459 review): decomposed, + 'é' would be a base letter plus a non-alpha combining mark, and + the split-off-initial rule would read the base as split off. Both + spellings of one value must repair the split word identically.""" composed = unicodedata.normalize("NFC", "Péx") decomposed = unicodedata.normalize("NFD", "Péx") assert decomposed != composed for value in (composed, decomposed): - lex = dataclasses.replace( - Lexicon.default(), - capitalization_exceptions=(("pé.x.", value),)) - p = Parser(lexicon=lex) - assert p.capitalized(p.parse("john smith pé.x."), - force=True).suffix == "Pé.X." + assert _repaired_under((("pé.x.", value),), + "john smith pé.x.").suffix == "Pé.X." def test_a_masks_upper_fallback_can_lengthen_a_word_through_ss() -> None: - """decisions.md#R4 (2026-09-24 review): _apply_mask's `c.upper()` - fallback, taken whenever the whole-word uppered form does not - keep the word's length, can lengthen a word through ss exactly as - the acronym/numeral clauses' plain `word.upper()` can - (decisions.md#R4's 2026-09-23 Unicode-boundary bullet): the - boundary applies to every clause that upper-cases, not only those - two. And it is not only the split-off-initial force that reaches - it -- the ORDINARY mask branch (the plain `else` arm, no full - stop involved) falls back to `c.upper()` the same way whenever the - mask's own letter at that position is not lower and the word is - not same-length under `.upper()`. An identity mask validates fine - ('aß' spells 'a.ß's own letters unchanged; 'STRAẞE' -- the actual - German capital ẞ, not 'STRASSE' -- spells 'straße's own letters - unchanged too), so both are reachable with no separate defect in - the validator: the validator's refusal of ('straße', 'STRASSE') is - about the VALUE failing to spell the KEY, a different question - from what the applier's output can be once a mask DOES validate.""" - split = dataclasses.replace( - Lexicon.default(), capitalization_exceptions=(("a.ß", "aß"),)) - p = Parser(lexicon=split) - assert p.capitalized(p.parse("john a.ß smith"), - force=True).middle == "A.SS" - ordinary = dataclasses.replace( - Lexicon.default(), - capitalization_exceptions=(("straße", "STRAẞE"),)) - p = Parser(lexicon=ordinary) - assert p.capitalized(p.parse("john straße"), - force=True).family == "STRASSE" + """decisions.md#R4 (2026-09-24 review): where the whole-word + upper-casing does not keep the word's length, _apply_mask falls + back to a per-character `c.upper()`, which lengthens 'ß' to 'SS' + as the acronym/numeral clauses' `word.upper()` does -- whether + the letter is upper by the split-off-initial override ('a.ß') or + by the mask's own case ('STRAẞE', with the capital ẞ). Both masks + validate, spelling their keys' own letters; the validator's + refusal of ('straße', 'STRASSE') is a different question.""" + assert _repaired_under((("a.ß", "aß"),), + "john a.ß smith").middle == "A.SS" + assert _repaired_under((("straße", "STRAẞE"),), + "john straße").family == "STRASSE" def test_a_masks_punctuation_marks_its_joins_and_is_never_written() -> None: """#459 review: a value's punctuation is never written into the - word -- 'md' repairs to 'MD' and 'm.d.' to 'M.D.' under the - ('md','M.D.') mask, identically to what the plain 'MD' mask - gives -- but it is not IGNORED either: it marks which of the - mask's letters are one run versus split apart, and the - lone-initial clause reads that structure. Two masks that differ - ONLY in punctuation, 'h.c' and 'hc', therefore give DIFFERENT - output on the same split-looking word: the writer wrote 'h.c.', - and whether that reads as one abbreviation or two initials - depends on how the mask itself joins the letters.""" + word -- under ('md', 'M.D.') 'md' repairs to 'MD' and 'm.d.' to + 'M.D.', as under a plain 'MD' -- but it is not IGNORED: it marks + which of the mask's letters are one run, and the split-off-initial + rule reads that. So 'h.c' and 'hc', differing ONLY in punctuation, + repair the writer's 'h.c.' differently.""" with warnings.catch_warnings(): warnings.simplefilter("error") lex = dataclasses.replace( Lexicon.default(), capitalization_exceptions=(("md", "M.D."),)) - assert str(Parser(lexicon=lex).capitalized( - Parser(lexicon=lex).parse("john smith md"))) == "John Smith MD" - assert str(Parser(lexicon=lex).capitalized( - Parser(lexicon=lex).parse("john smith m.d."))) \ + p = Parser(lexicon=lex) + assert str(p.capitalized(p.parse("john smith md"))) == "John Smith MD" + assert str(p.capitalized(p.parse("john smith m.d."))) \ == "John Smith M.D." # the facade twin: warning-free at the first parse, where the # shim's lazily built Lexicon snapshot is the one place a @@ -958,15 +905,10 @@ def test_a_masks_punctuation_marks_its_joins_and_is_never_written() -> None: def test_a_mask_recases_a_digit_key_unchanged() -> None: - """#459 review: the mask walks ALPHANUMERICS, not letters alone, - so a digit in the word is carried through unchanged (it has no - case) while the surrounding letters still take the mask's case. - The mask clause reads no ROLE or TAGS, unlike the acronym and - numeral clauses that follow it -- an entry applies wherever its - word stands. A synthetic GIVEN-roled token proves that: a role - the acronym/numeral clauses would never reach still finds the - mask, without depending on how '2nd' happens to parse on its - own.""" + """#459 review: the mask walks ALPHANUMERICS, so a digit is carried + through (it has no case) while the letters take the mask's case. + A synthetic GIVEN-roled token keeps the row off the role-gated + acronym/numeral clauses and off how '2nd' happens to parse.""" lex = dataclasses.replace(Lexicon.default(), capitalization_exceptions=(("2nd", "2ND"),)) assert _pn("2nd", [ @@ -976,25 +918,14 @@ def test_a_mask_recases_a_digit_key_unchanged() -> None: def test_a_mask_cases_through_the_whole_word_for_context_sensitive_letters( ) -> None: - """#459 review: casing goes through word.lower()/word.upper() - when both keep the word's length, not a per-character - str.lower()/str.upper() call, which is context-free and gets some - letters wrong that the whole-word form gets right -- a Greek - medial sigma where a FINAL one belongs, here. A titlecase mask - letter reads as upper rather than lower (a known limit: no - per-character titlecase mapping is attempted, so a digraph's - OWN titlecase spelling is not reproduced -- only the Greek case - is pinned here, since it recovers the correct written form).""" - from nameparser._render import _apply_mask - assert _apply_mask("ΚΟΣ", "Κος") \ - == "Κος" # 'ΚΟΣ' under 'Κος' -> 'Κος' (final sigma) - # the FALLBACK side of the same branch: 'straße'.upper() is - # 'STRASSE' (ß grows to two letters), longer than the 6-char - # word, so same_length is False and this exercises the - # per-character c.upper()/c.lower() path rather than the - # whole-word indexed lookup -- ß keeps its own lowercase form - # under the mask's lowercase letter there (c.lower() on 'ß' is - # 'ß', not the two-letter 'ss' casefold() would give) + """#459 review: casing goes through word.lower()/word.upper() when + both keep the word's length, since a per-character call is + context-free and writes a Greek medial sigma where a FINAL one + belongs.""" + assert _apply_mask("ΚΟΣ", "Κος") == "Κος" # final sigma + # the per-character fallback: 'straße'.upper() is 'STRASSE', + # longer than the word, and there ß keeps its own lowercase under + # the mask's lowercase letter assert _apply_mask("straße", "STRAßE") == "STRAßE" # end to end, through a custom Lexicon: a per-character # c.lower()/c.upper() walk gave 'Κοσ' here (medial sigma), wrong From cd10010d4ad5f8920ad2e7e9f6ec66b0dfc02617 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 24 Sep 2026 12:08:56 -0700 Subject: [PATCH 8/9] fix(#459/#492/#478): the toolkit review round Five reviewers read the pull request on its final tree. _MaskValueError now takes its structure and builds its own message (key, normalized key, value in; offered and text out), so an instance cannot disagree with itself, and its __reduce__ carries __dict__ so notes survive a process boundary; the shim's re-raise is a v1-only message built from those fields, one remedy for the surface the caller is on. A dead multi-word exceptions key warns as before and is no longer checked as a mask, since it can produce nothing; the error echoes the caller's own spelling of the value. The hyphen clause counts a neighbour only where a part holds an alphanumeric, not a period or underscore. Ten more mask rows kill the mutants a fresh sweep left standing (both sides of the digit-ends-a-run rule, both index guards, the per-character lowering path, each equality of the whole-word test, the titlecase limit), and the non-fixpoints the Unicode boundary allows are pinned and recorded: a lengthening mask, a letter whose uppercase is longer, a combining mark that splits the word. Comments and guides are corrected where they overclaimed (a dotted unlisted acronym is a suffix by shape; the shim's subset checks are the ones construction satisfies, and two others are reachable through an entry with edge whitespace, pre-existing); the tag constant's rationale lives in _types beside the constant. Co-Authored-By: Claude Fable 5.1 --- docs/customize.rst | 10 ++- docs/design/decisions.md | 2 +- nameparser/_config_shim.py | 49 ++++++++++--- nameparser/_lexicon.py | 103 ++++++++++++++++++---------- nameparser/_pipeline/_state.py | 19 ++--- nameparser/_render.py | 48 ++++++++++--- nameparser/_types.py | 20 ++++-- nameparser/config/capitalization.py | 14 ++-- tests/test_capitalization.py | 7 ++ tests/v2/test_config_shim.py | 14 ++++ tests/v2/test_lexicon.py | 45 +++++++++++- tests/v2/test_render.py | 63 ++++++++++++++++- 12 files changed, 303 insertions(+), 91 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 6e422738..d480df22 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -205,9 +205,13 @@ capitalized where the mask keeps that letter inside a longer run, so listed in ``suffix_acronyms`` — plain or dotted — and a roman numeral need no entry at all: case repair writes a suffix of either kind in capitals by itself. A caller's own acronym, one ``suffix_acronyms`` -doesn't already list, isn't covered by that and parses as an ordinary -name word instead — give it a ``suffix_acronyms`` entry (or a mask of -its own) rather than relying on this fallback. +doesn't already list, depends on how it is written: plain (``dphil``) +it parses as an ordinary name word and repairs as one (``Dphil``, +above); dotted (``d.phil.``) it is a suffix by shape alone, with no +vocabulary entry needed to read it as one, and repairs in all +capitals the same as a listed acronym does (``D.PHIL.``). Either +way, give it a ``suffix_acronyms`` entry (or a mask of its own) +rather than relying on this fallback. Words that are also ordinary names ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 21fe1eed..9a03d9aa 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -1374,7 +1374,7 @@ Accepted costs, deferred to the rescoped #459 rather than relitigated here: the - 2026-09-23 — mechanisms.md#RENDER-HONORS-THE-PARSE CORRECTED, found in this change's re-read of that entry: it still said case repair carries the connective carve-out "ON R3's authority", quoting R4's pre-#461 text ("the carve-out R3 states for initials"), and that `.initials()` gives `j. a. j. s.` for `john and jane smith`. Both had been stale since #461 (2026-09-20), which cut R4's cross-reference to R3 in favor of R4's own reason and stopped a connective with a name word beside it from initialing. The entry now states R4's own reason, dates the cut, and names where the two views part today (`Juan de y`: initials `J. y.`, forced repair `Juan de y`), re-measured 2026-09-23. -- 2026-09-24 #459/#492/#478 (branch review) — FOUR FINDINGS FROM THE REVIEW ROUND, recorded together because each is a small correction to a bullet already above rather than a new decision. (a) THE UNICODE BOUNDARY IS NOT SCOPED TO THE TWO SHAPE CLAUSES, AND NOT TO ONE WAY A MASK LETTER COMES OUT UPPER EITHER. The 2026-09-23 bullet above states the `ß` and dotless-`ı` boundary against `_cap_word`'s acronym/numeral clauses, but the MASK path can lengthen a word through `ß` exactly the same way. `_apply_mask` decides per letter whether it is written upper -- by the mask's own case, or by the split-off-initial force -- and then takes one casing step: the letter at that position in the whole-word `word.upper()`/`word.lower()` when both keep the word's length, otherwise a per-character `c.upper()`/`c.lower()`. `'ß'.upper()` is `'SS'`, so wherever the word holds a `ß` the whole-word form changes the length and every letter goes through `c.upper()`, which lengthens a `ß` written upper to `SS` whichever of the two reasons made it upper. Through the force: an identity mask validates fine (`('a.ß', 'aß')` spells the key's own letters unchanged), so `john a.ß smith` forced gives middle `A.SS` -- casefold-equal to its input and one character longer, the same trade the shape clauses make. Through the mask's own case, no full stop or split letter involved: `('straße', 'STRAẞE')`, the value spelling `straße`'s own letters unchanged with the actual German capital `ẞ` (not the digraph `'STRASSE'`, which the validator refuses as a VALUE for failing to spell `ß` at all), validates and `john straße` forced gives family `STRASSE`. The validator's refusal of `('straße', 'STRASSE')` is a different question -- whether the VALUE spells the KEY -- and says nothing about what the applier can produce from a mask that does validate. Pinned by `tests/v2/test_render.py::test_a_masks_upper_fallback_can_lengthen_a_word_through_ss`. (b) MASK VALUES ARE STORED NFC-COMPOSED. `_normpairs` compared a value against its key NFC-composed but stored the value AS WRITTEN, so a value written in decomposed form spelled one composed letter as a base letter plus a non-alpha combining mark -- changing which letters `_letter_run_ge2` reads as split off beside a full stop from what the composed spelling of the identical value would read. `john smith pé.x.` forced gave `Pé.X.` under a composed mask and `Pé.x.` under the decomposed spelling of the SAME value. Fixed by NFC-composing the value at the same point the key is composed; pinned by `tests/v2/test_lexicon.py::test_a_decomposed_value_is_compared_and_stored_nfc_composed` and `tests/v2/test_render.py::test_a_decomposed_mask_value_reads_the_same_split_as_composed`. (c) THE HYPHEN CLAUSE'S INITIAL TEST IS NOW A PINNED MODULE CONSTANT. `_cap_text`'s inline `re.fullmatch(r"\w\.", part)` was a third hand copy of the initial shape's period half, invisible to `tests/v2/test_regex_sync.py`'s hand-copy sweep because it was never hoisted to a module-level pattern. Hoisted to `_render._DOTTED_INITIAL` and registered as `_INITIAL`'s period alternative; pinned by `tests/v2/test_regex_sync.py::test_dotted_initial_is_the_period_alternative_of_initial`. (d) THE V1 RAISE NOW CARRIES A V1-SPELLED HINT, VIA A TYPED EXCEPTION. The 2026-09-23 bullet above records that a mismatched-mask value raises `ValueError` at a v1 `Constants`' first parse; the message it raised was `Lexicon`'s own, offering a `Lexicon(capitalization_exceptions=...)` constructor call to a caller looking at a `TupleManager` assignment through `Constants`. `_lexicon._normpairs` now raises a dedicated `_MaskValueError(ValueError)` for exactly that one shape, carrying `key` (the raw key as given) and `offered` (from `_offered_mask`) as attributes -- picklable and copyable via its own `__reduce__`, since the inherited default (`type(self)(*self.args)`, `self.args` holding only the message) cannot reconstruct an instance that needs `key`/`offered` too and raises `TypeError` trying; a worker in a `ProcessPoolExecutor` pickles an exception it raises to deliver it to the caller, so an unpicklable one there surfaces as `BrokenProcessPool` instead. `_config_shim._build_snapshot` wraps ONLY the `Lexicon(...)` construction in `try`/`except _MaskValueError` and re-raises with `constants.capitalization_exceptions[key] = offered`. Every OTHER `ValueError` `Lexicon(...)` can raise passes through unchanged, by TYPE rather than by a message-substring test. Of those, most never reach `Lexicon(...)` from this shim at all: an orphaned `non_first_name_prefixes` entry cannot put an entry in `particles_ambiguous` that is outside `particles` (both terms of the union that builds it are already subsets), a bound/particle contradiction cannot arise (`bound & particles` is unconditionally folded into that same union), and the `suffix_acronyms_ambiguous`/`suffix_words` gate bypass cannot arise (`suffix_words` above already subtracts the ambiguous acronyms) -- all three measured by constructing, through a v1 `Constants`, the configuration that would trigger Lexicon's corresponding check, and observing no raise. What DOES reach `Lexicon(...)` here unchanged is an entry normalizing to empty, in `capitalization_exceptions` or in ANY set field -- measured: `c.titles.add("...")` then a parse raises Lexicon's own `titles entry '' normalizes to empty`, unaltered -- since nothing above translates that away. Declined: a first draft pre-validated `capitalization_exceptions` in isolation, a second call to `_normpairs` ahead of `Lexicon(...)` gated on a message-substring test; measured to make a multi-word key's dead-entry warning fire twice on the shim path, and to rest on a filter that was untested (deleting it passed the whole suite) and wrong (an unrelated ValueError then got the v1 hint appended regardless). Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse` (the hint), `tests/v2/test_config_shim.py::test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint` (the pass-through), `tests/v2/test_config_shim.py::test_a_multiword_capitalization_exceptions_key_warns_exactly_once` (the single warning), and `tests/v2/test_lexicon.py::test_mask_value_error_survives_pickle_and_copy` (the pickle/copy round-trip). +- 2026-09-24 #459/#492/#478 (branch review) — FOUR FINDINGS FROM THE REVIEW ROUND, recorded together because each is a small correction to a bullet already above rather than a new decision. (a) THE UNICODE BOUNDARY IS NOT SCOPED TO THE TWO SHAPE CLAUSES, AND NOT TO ONE WAY A MASK LETTER COMES OUT UPPER EITHER. The 2026-09-23 bullet above states the `ß` and dotless-`ı` boundary against `_cap_word`'s acronym/numeral clauses, but the MASK path can lengthen a word through `ß` exactly the same way. `_apply_mask` decides per letter whether it is written upper -- by the mask's own case, or by the split-off-initial force -- and then takes one casing step: the letter at that position in the whole-word `word.upper()`/`word.lower()` when both keep the word's length, otherwise a per-character `c.upper()`/`c.lower()`. `'ß'.upper()` is `'SS'`, so wherever the word holds a `ß` the whole-word form changes the length and every letter goes through `c.upper()`, which lengthens a `ß` written upper to `SS` whichever of the two reasons made it upper. Through the force: an identity mask validates fine (`('a.ß', 'aß')` spells the key's own letters unchanged), so `john a.ß smith` forced gives middle `A.SS` -- casefold-equal to its input and one character longer, the same trade the shape clauses make. Through the mask's own case, no full stop or split letter involved: `('straße', 'STRAẞE')`, the value spelling `straße`'s own letters unchanged with the actual German capital `ẞ` (not the digraph `'STRASSE'`, which the validator refuses as a VALUE for failing to spell `ß` at all), validates and `john straße` forced gives family `STRASSE`. The validator's refusal of `('straße', 'STRASSE')` is a different question -- whether the VALUE spells the KEY -- and says nothing about what the applier can produce from a mask that does validate. This same boundary is where `capitalized()`'s own "Idempotent" docstring claim narrows: forcing `A.SS` (that first pass's own output) through the SAME lexicon a second time gives `A.ss`, not `A.SS` back, because the second pass folds the word to `a.ss` and the exceptions map's key is still `a.ß` -- a different letter sequence, not merely a different case of one -- so the lookup that found the entry on the first pass misses on the second and the word falls through to plain title-casing. Pinned by `tests/v2/test_render.py::test_a_masks_upper_fallback_can_lengthen_a_word_through_ss` and `tests/v2/test_render.py::test_a_lengthening_mask_is_not_a_fixpoint_under_a_second_forced_pass`. The same boundary reaches plain title-casing with no mask involved, by two distinct mechanisms: LENGTHENING, where a letter's own upper() is a different letter SEQUENCE that stays \w -- 'ʼn' (one letter) upper-cases to the two cased characters 'ʼN', so `parse("ʼna smith").capitalized(force=True)` gives given `ʼNa`, and forcing THAT output a second time gives `ʼna` ('N' is no longer the word's first character, so `str.capitalize()` lower-cases it on the second pass) -- and SPLITTING, where a letter's own upper() is a base letter plus a COMBINING MARK that `_WORD` does not match at all -- 'ǰ' upper-cases to 'J' + a combining caron, so `parse("ǰo smith").capitalized(force=True)` gives given `J̌o`, and forcing THAT output a second time gives `J̌O` (`_cap_text` reads the output as two separate words, 'J' and 'o', the combining mark matching neither, and 'o' capitalized alone is 'O'). (b) MASK VALUES ARE STORED NFC-COMPOSED. `_normpairs` compared a value against its key NFC-composed but stored the value AS WRITTEN, so a value written in decomposed form spelled one composed letter as a base letter plus a non-alpha combining mark -- changing which letters `_letter_run_ge2` reads as split off beside a full stop from what the composed spelling of the identical value would read. `john smith pé.x.` forced gave `Pé.X.` under a composed mask and `Pé.x.` under the decomposed spelling of the SAME value. Fixed by NFC-composing the value at the same point the key is composed; pinned by `tests/v2/test_lexicon.py::test_a_decomposed_value_is_compared_and_stored_nfc_composed` and `tests/v2/test_render.py::test_a_decomposed_mask_value_reads_the_same_split_as_composed`. (c) THE HYPHEN CLAUSE'S INITIAL TEST IS NOW A PINNED MODULE CONSTANT. `_cap_text`'s inline `re.fullmatch(r"\w\.", part)` was a third hand copy of the initial shape's period half, invisible to `tests/v2/test_regex_sync.py`'s hand-copy sweep because it was never hoisted to a module-level pattern. Hoisted to `_render._DOTTED_INITIAL` and registered as `_INITIAL`'s period alternative; pinned by `tests/v2/test_regex_sync.py::test_dotted_initial_is_the_period_alternative_of_initial`. (d) THE V1 RAISE NOW CARRIES A V1-SPELLED HINT, VIA A TYPED EXCEPTION. The 2026-09-23 bullet above records that a mismatched-mask value raises `ValueError` at a v1 `Constants`' first parse; the message it raised was `Lexicon`'s own, offering a `Lexicon(capitalization_exceptions=...)` constructor call to a caller looking at a `TupleManager` assignment through `Constants`. `_lexicon._normpairs` now raises a dedicated `_MaskValueError(ValueError)` for exactly that one shape, built in the `UnicodeDecodeError` idiom: `__init__(self, key, normalized_key, value)` stores the three offending inputs and computes `self.offered` (from `_offered_mask`) and the message from them, rather than taking a pre-built message. `__reduce__` returns `(type(self), (self.key, self.normalized_key, self.value), self.__dict__)` -- reconstructing from the three fields and then restoring `self.__dict__` (so an attribute added after construction, e.g. `add_note()`, survives too) -- since the inherited default (`type(self)(*self.args)`, `self.args` holding only the message) cannot reconstruct an instance that needs three fields, not one, and raises `TypeError` trying; a worker in a `ProcessPoolExecutor` pickles an exception it raises to deliver it to the caller, so an unpicklable one there surfaces as `BrokenProcessPool` instead. `_config_shim._build_snapshot` wraps ONLY the `Lexicon(...)` construction in `try`/`except _MaskValueError` and re-raises a message built fresh from `e.key`/`e.value`/`e.offered` -- not from `str(e)`, which is the 2.0 exception's own message offering the 2.0 `capitalization_exceptions=((...),)` constructor spelling no v1 caller can paste -- with `constants.capitalization_exceptions[e.key] = e.offered`. Every OTHER `ValueError` `Lexicon(...)` can raise passes through unchanged, by TYPE rather than by a message-substring test. Of those, the three SUBSET checks (`particles_ambiguous` inside `particles`, `suffix_acronyms_ambiguous` inside `suffix_acronyms`, `honorific_tails` inside `suffix_words`) cannot fire from here: each is built as an intersection or union with its OWN base, so the normalized image of that relation is still a subset whatever the raw spelling. THE CONTRADICTION CHECK AND THE GATE-BYPASS CHECK ARE A DIFFERENT SHAPE, AND DO REACH `Lexicon(...)` -- this bullet claimed otherwise ("cannot arise ... observing no raise") until the confirmation round measured the opposite. Both compare two sets built by SEPARATE set arithmetic in `_build_snapshot`, on the RAW SetManager entries, before Lexicon's own per-field normalization ever runs: `SetManager` strips edge periods but not edge whitespace, while `Lexicon`'s normalization strips both, so an entry carrying edge whitespace can slip past the shim's own subtraction or union and still collide once Lexicon normalizes it independently. Measured: `c.suffix_not_acronyms.add('ba ')` then a parse raises Lexicon's own gate-bypass check (`an ambiguous suffix acronym must not also be a suffix word; in both: ba`), and `c.bound_first_names.add("'t ")` then a parse raises the contradiction check -- both pass through the `except _MaskValueError` clause above unchanged, being a different exception type. Pre-existing at HEAD and at master alike, not introduced by this session and not fixed here -- left for the orchestrator to file as its own issue. What DOES reach `Lexicon(...)` here unchanged, always, is an entry normalizing to empty, in `capitalization_exceptions` or in ANY set field -- measured: `c.titles.add("...")` then a parse raises Lexicon's own `titles entry '' normalizes to empty`, unaltered -- since nothing above translates that away, and a non-str value's `TypeError`, a raise v1 also had, later, at `capitalize()`. Declined: a first draft pre-validated `capitalization_exceptions` in isolation, a second call to `_normpairs` ahead of `Lexicon(...)` gated on a message-substring test; measured to make a multi-word key's dead-entry warning fire twice on the shim path, and to rest on a filter that was untested (deleting it passed the whole suite) and wrong (an unrelated ValueError then got the v1 hint appended regardless). Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse` (the hint, and that it carries no 2.0 constructor spelling), `tests/v2/test_config_shim.py::test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint` (the pass-through), `tests/v2/test_config_shim.py::test_a_multiword_capitalization_exceptions_key_warns_exactly_once` (the single warning), and `tests/v2/test_lexicon.py::test_mask_value_error_survives_pickle_and_copy` (the pickle/copy round-trip). Excluded (CAPITALIZATION_EXCEPTIONS — md, ii, iii, iv, removed 2026-09-23, #459): diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index f1a3757c..678cc39d 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -1017,15 +1017,37 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # _normpairs raises _MaskValueError for exactly that shape, and # the except clause re-spells its offered fix for a v1 caller. # Caught by TYPE, so every other Lexicon error passes through - # unchanged. Measured: three of Lexicon's checks cannot fire - # from here, this method's translations satisfying them by - # construction (particles_ambiguous inside particles, the - # bound/never-given contradiction, and an ambiguous acronym in - # suffix_words); what does reach it is an entry normalizing to - # empty, in capitalization_exceptions or any set field - # (c.titles.add("...") raises Lexicon's own "normalizes to - # empty"), and a non-str value's TypeError -- a raise v1 also - # had, later, at capitalize(). + # unchanged. Only the three SUBSET checks (particles_ambiguous + # inside particles, suffix_acronyms_ambiguous inside + # suffix_acronyms, honorific_tails inside suffix_words) are + # satisfied BY CONSTRUCTION: each is built below as an + # intersection or union with its OWN base -- particles_ambiguous + # from `particles` itself, honorific_tails from `suffix_words` + # itself -- so the image of a subset relation under any + # element-wise normalization is still a subset, whatever the + # raw spelling. The CONTRADICTION check (bound_given_names & + # particles inside particles_ambiguous) and the GATE-BYPASS + # check (suffix_acronyms_ambiguous disjoint from suffix_words) + # do NOT have that property -- each compares two sets built by + # SEPARATE set arithmetic (particles_ambiguous's own union + # includes `bound & particles`, computed before Lexicon + # normalizes anything; suffix_words above subtracts + # ambiguous_acronyms the same way) -- and are reachable through + # an entry carrying EDGE WHITESPACE: SetManager strips edge + # periods but not edge whitespace, while Lexicon's own + # normalization strips both, so `c.suffix_not_acronyms.add('ba + # ')` (raw 'ba ' survives the shim's subtraction against + # 'ba', then Lexicon normalizes both to 'ba' and the + # gate-bypass check collides) and `c.bound_first_names.add("'t + # ")` (analogous miss against the contradiction check) both + # reach Lexicon(...) and raise -- measured, and true of this + # method unchanged since before this session; pre-existing, + # not fixed here, and left for the orchestrator to file. What + # else reaches it is an entry normalizing to empty, in + # capitalization_exceptions or any set field (c.titles.add("...") + # raises Lexicon's own "normalizes to empty"), and a non-str + # value's TypeError -- a raise v1 also had, later, at + # capitalize(). try: lexicon = Lexicon( titles=frozenset(self.titles), @@ -1115,8 +1137,15 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] ) except _MaskValueError as e: + # A v1-ONLY message built from the fields, not `{e}` -- + # the 2.0 exception's own message offers the 2.0 + # `capitalization_exceptions=((...),)` constructor spelling, + # which no v1 Constants caller can paste. raise ValueError( - f"{e} -- on a v1 Constants, write " + f"capitalization_exceptions value {e.value!r} for key " + f"{e.key!r} does not spell the key's letters and " + f"digits: a value is a case mask, the key's own " + f"letters and digits recased -- write " f"constants.capitalization_exceptions[{e.key!r}] = " f"{e.offered!r}" ) from e diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index a01ef8f1..0d6ace01 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -392,24 +392,41 @@ def _offered_mask(normalized_key: str) -> str: class _MaskValueError(ValueError): - """_normpairs' error for a value that does not spell its key. It - carries the raw `key` and the `offered` fix so the v1 shim can - re-spell that fix for its own surface without parsing the message. + """_normpairs' error for a value that does not spell its key. + Built in the UnicodeDecodeError idiom -- structure in, message + out: `key`, `normalized_key` and `value` are the offending inputs; + `offered` (the v1 shim's re-spelled fix, and the v2 one quoted in + the message below) is computed from them, not carried separately. __reduce__ is explicit because the inherited one re-calls - __init__ with self.args, which holds the message alone, so - pickling or copying would fail -- and a ProcessPoolExecutor - pickles a worker's exception to deliver it, surfacing an - unpicklable one as BrokenProcessPool. (An exception, so - _types.py's frozen-dataclass pickle guards do not apply.)""" - - def __init__(self, message: str, key: str, offered: str) -> None: - super().__init__(message) + __init__ with self.args, which holds the message alone, not the + three fields __init__ needs -- and a ProcessPoolExecutor pickles a + worker's exception to deliver it, surfacing a construction failure + as BrokenProcessPool. Passing self.__dict__ as reduce's third + element (restored via a plain __dict__.update, there being no + __setstate__ here) carries along anything added after construction + -- e.g. add_note() -- that reconstructing from the three fields + alone would not. (An exception, so _types.py's frozen-dataclass + pickle guards do not apply.)""" + + def __init__(self, key: str, normalized_key: str, value: str) -> None: self.key = key - self.offered = offered - - def __reduce__(self) -> tuple[type[_MaskValueError], tuple[str, str, str]]: - return (type(self), (self.args[0], self.key, self.offered)) + self.normalized_key = normalized_key + self.value = value + self.offered = _offered_mask(normalized_key) + super().__init__( + f"capitalization_exceptions value {value!r} for key {key!r} " + f"does not spell the key's letters and digits: a value " + f"is a case mask, the key's own letters and digits " + f"recased -- e.g. " + f"capitalization_exceptions=(({normalized_key!r}, " + f"{self.offered!r}),)") + + def __reduce__( + self, + ) -> tuple[type[_MaskValueError], tuple[str, str, str], dict[str, object]]: + return (type(self), (self.key, self.normalized_key, self.value), + self.__dict__) def _normpairs( @@ -464,37 +481,53 @@ def _normpairs( f"empty (lowercase + strip full stops/whitespace leaves " f"nothing)" ) - # A value is a case MASK (#459): the key's own letters and - # digits recased, compared through the key's own fold. Its - # punctuation marks which letters are joined into one run - # (_render._apply_mask reads that) and is never written into - # the word. A raise, not a warning: a mismatched value used to - # be SUBSTITUTED for the word, and rules.md#R4: "Repair - # changes case and nothing else", so there is no reading of - # one that repair can honor. # Stored NFC-composed, case kept (unicodedata, not _normalize): # _apply_mask reads the mask one character at a time, and a # decomposed letter would read as a base letter split off - # beside a non-alpha combining mark. The check below passes - # either spelling, _normalize composing both of its sides. + # beside a non-alpha combining mark. The mask check below + # passes either spelling, _normalize composing both of its + # sides. `written` keeps the caller's own spelling -- composed + # or not -- for the mismatch error below: a decomposed value + # (e.g. 'e' + a combining acute) and its NFC form print + # identically to a reader's eye but are different str objects, + # so echoing the composed spelling back would show the caller + # a value they did not write. + written = v v = unicodedata.normalize("NFC", v) - if _alnum(_normalize(v)) != _alnum(normalized_key): - offered = _offered_mask(normalized_key) - raise _MaskValueError( - f"capitalization_exceptions value {v!r} for key {k!r} " - f"does not spell the key's letters and digits: a value " - f"is a case mask, the key's own letters and digits " - f"recased -- e.g. " - f"capitalization_exceptions=(({normalized_key!r}, " - f"{offered!r}),)", key=k, offered=offered) # capitalized() looks words up one at a time (the _WORD regex - # never yields spaces), so a multi-word key is unreachable. + # never yields spaces), so a multi-word key is unreachable -- + # checked FIRST, ahead of the mask check below, because an + # invariant guards harm and a value that can never be read + # does no harm no matter what it says (AGENTS.md: "a check + # guards harm, not no-ops"). Raising the mask error for such a + # key -- as a naive mask-first order does -- would refuse a + # config that was always going to be a no-op. # interior whitespace test; split() covers all Unicode whitespace if normalized_key != "".join(normalized_key.split()): _warn_dead_entry( f"capitalization_exceptions keys are matched one word " f"at a time; multi-word key {k!r} can never match. " f"Split it into per-word entries") + deduped[normalized_key] = v + continue + # A value is a case MASK (#459): the key's own letters and + # digits recased, compared through the key's own fold. Its + # punctuation marks which letters are joined into one run + # (_render._apply_mask reads that) and is never written into + # the word. A raise, not a warning: a mismatched value used to + # be SUBSTITUTED for the word, and rules.md#R4: "Repair + # changes case and nothing else", so there is no reading of + # one that repair can honor. + if _alnum(_normalize(v)) != _alnum(normalized_key): + # key is the RAW key, not normalized_key: the v1 shim's + # hint must overwrite the offending dict entry, which is + # stored under the raw spelling -- writing the normalized + # spelling would add a second entry and still raise + # (measured with {"PHD": "Junior"}: the shim's suggested + # constants.capitalization_exceptions['PHD'] = ... has to + # match the key already there, not 'phd'). `written`, not + # `v`: the caller's own spelling, not the NFC-composed one. + raise _MaskValueError(k, normalized_key, written) deduped[normalized_key] = v return tuple(sorted(deduped.items())) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 968d9844..95e70b7f 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -65,21 +65,10 @@ class WorkToken: #: cannot reach the other way). _NEVER_FLIPPED = frozenset({"vocab:bound-given", "initial"}) -# SHAPE_ACRONYM_TAG, imported from _types above -- where it lives -# since #459, because case repair reads it too and _render may not -# import the pipeline -- is the by-shape half of #289/#516's -# ambiguous credential class: a token classify admits to -# `vocab:suffix-ambiguous`'s READING by SHAPE rather than by the -# listed vocabulary (`Policy.unlisted_dotted_suffixes` is the first -# emitter; `Policy.unlisted_caps_suffixes` is the second, and -# classify writes the tag from both branches). One constant, not a -# string literal at each site, because the readers that must tell a -# by-shape member apart from a listed one -- `_pieces.peel_trailing` -# and `_pieces.listed_lean`, which `segment_suffix_reading` asks -# through, so the reading it decides is second-hand -- cannot afford -# to spell it several ways and have one of them typo silently past -# the others. The two sites that want EITHER half read -# `_AMBIGUOUS_CREDENTIAL_TAGS` below rather than this constant. +# SHAPE_ACRONYM_TAG is defined and explained in _types (its #: block +# above the constant, not restated here -- defined there so _render +# can read it too); re-exported here beside the membership half below +# because the stages read both halves from one module. #: The MEMBERSHIP half of the same class: classify's tag for a token #: the ambiguous credential vocabulary claims, by listing diff --git a/nameparser/_render.py b/nameparser/_render.py index f5682774..6ce0d9f5 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -422,7 +422,7 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # first, which is what keeps a conventionally mixed-case acronym as # it is written (bsc -> BSc) though it is listed here too. Gated # on the SUFFIX role so a word that is a family name only happens - # to be in the vocabulary (anh van DO) still repairs as an ordinary + # to be in the vocabulary (anh van do) still repairs as an ordinary # name word -- #459's given-role half, decided: repair follows the # role the parse chose ('qc mp' -> 'Qc MP'). if role is Role.SUFFIX and ( @@ -460,20 +460,27 @@ def cap(match: re.Match[str]) -> str: if "-" not in text: return _WORD.sub(cap, text) parts = text.split("-") - named = [at for at, part in enumerate(parts) if _WORD.search(part)] + # A "named" part needs an alphanumeric, not just a _WORD match: + # _WORD also matches a run of bare periods (or underscores), so a + # part holding only punctuation -- the family TOKEN of 'jose + # .-y-garcia' is '.-y-garcia', which splits to ['.', 'y', 'garcia'] + # -- is not a worded neighbour and must not count as one. + named = [at for at, part in enumerate(parts) + if any(c.isalnum() for c in part)] if len(named) < 3: return _WORD.sub(cap, text) # rules.md#R4: "Inside a hyphenated word, a part that is # connective vocabulary with a worded part on each side of it # keeps its lowercase" (#478). Decided here, not in _cap_word, - # whose word has lost its neighbours. Only a part holding a word - # is a neighbour: an empty one is skipped ('garcia--y-lopez' keeps - # its 'y') and supplies none ('md-phd-'). An EDGE part stays - # ordinary name text ('juan e-f smith' keeps 'E-F'), so no word is - # re-read as connective or initial by its case (#458). A single - # letter with a period is an initial, as classify reads it, never - # the connective ('j.-e.-p. dupont' keeps 'E.'); the multi-letter - # 'und.' in 'hans smith-und.-jones' still lowers. + # whose word has lost its neighbours. Only a part holding an + # alphanumeric is a neighbour: an empty one is skipped + # ('garcia--y-lopez' still lowers its 'y') and supplies none + # ('md-phd-'). An EDGE part stays ordinary name text ('juan e-f + # smith' keeps 'E-F'), so no word is re-read as connective or + # initial by its case (#458). A single letter with a period is an + # initial, as classify reads it, never the connective + # ('j.-e.-p. dupont' keeps 'E.'); the multi-letter 'und.' in + # 'hans smith-und.-jones' still lowers. first, last = named[0], named[-1] return "-".join( part.lower() @@ -514,7 +521,26 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, _cap_text, is a fixpoint on its own output, so a repaired name comes back unchanged whether or not the gate admits it again (a name whose non-suffix words are caseless, 'Kim Minjun' in - hangul with a 'phd', is admitted every time).""" + hangul with a 'phd', is admitted every time) -- except where a + LETTER'S OWN CASE MAPPING changes its length or splits the word + (decisions.md#R4's Unicode boundary; 'ß' recasing to 'SS' through + a mask is one example, not the only one). Lengthening: a mask's + own per-character casing fallback can turn one letter into a + different LETTER SEQUENCE ('ß' upper is 'SS', not one recased + letter), so the repaired word's own folded spelling ('a.ss') no + longer matches the exceptions map's key ('a.ß'), and a second + forced pass over that output cannot find the entry the first + pass did. The same lengthening reaches plain title-casing with no + mask involved: 'ʼn' (a single letter) upper-cases to the two + CASED characters 'ʼN', so str.capitalize() on 'ʼna' gives 'ʼNa' + but on THAT output gives 'ʼna' -- the 'N' is no longer the + word's first character, so the second pass lower-cases it. + Splitting: some letters upper-case to a base letter plus a + COMBINING MARK, which _WORD does not match -- 'ǰ' upper-cases to + 'J' + a combining caron, so 'ǰo' capitalizes to 'J̌o', but + _cap_text reads THAT text as two separate words ('J', then 'o', + the combining mark between them matching neither), and 'o' + capitalized alone is 'O'.""" if lexicon is not None and not isinstance(lexicon, Lexicon): # eager, before the gate: a garbage argument must not become a # silent no-op on mixed-case input or a deep AttributeError diff --git a/nameparser/_types.py b/nameparser/_types.py index f331de31..80e248d0 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -153,12 +153,20 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: The by-shape half of #289/#516's ambiguous credential class: the #: tag classify writes on a token it admits to the credential reading -#: by its SHAPE rather than by listed vocabulary. Defined here, at -#: the bottom of the graph, because a render view reads it as well -#: as the pipeline: case repair writes such a suffix in capitals as -#: it does a listed acronym (#459), and _render may not import -#: _pipeline. _pipeline/_state.py re-exports it beside the -#: membership half and says why the stages need ONE constant. +#: by its SHAPE rather than by listed vocabulary (`Policy. +#: unlisted_dotted_suffixes` is the first emitter, `Policy. +#: unlisted_caps_suffixes` the second, classify writing the tag from +#: both branches). Defined here, at the bottom of the graph, because +#: a render view reads it as well as the pipeline: case repair writes +#: such a suffix in capitals as it does a listed acronym (#459), and +#: _render may not import _pipeline. One constant, not a string +#: literal at each site, because the readers that must tell a +#: by-shape member apart from a listed one -- `_pieces.peel_trailing` +#: and `_pieces.listed_lean`, which `segment_suffix_reading` asks +#: through, so the reading it decides is second-hand -- cannot afford +#: to spell it several ways and have one of them typo silently past +#: the others. `_pipeline/_state.py` re-exports it beside the +#: membership half, since the stages read both halves from one module. SHAPE_ACRONYM_TAG = "shape:acronym" #: The one-element tag set its two producers stamp, built once. diff --git a/nameparser/config/capitalization.py b/nameparser/config/capitalization.py index 3c08cf2a..fe63021e 100644 --- a/nameparser/config/capitalization.py +++ b/nameparser/config/capitalization.py @@ -19,11 +19,15 @@ already lists, written plainly or in dotted form, and a roman numeral need no entry: case repair writes a suffix of either kind in capitals by itself. A caller's OWN acronym -- one absent from -``suffix_acronyms`` -- is not covered by that and parses as an -ordinary name word instead; it needs either a ``suffix_acronyms`` -entry (``Lexicon.add(suffix_acronyms={...})`` in the 2.0 API, -``constants.suffix_acronyms.add(...)`` in the v1 one) or a mask of -its own. +``suffix_acronyms`` -- depends on how it is written. Written +PLAINLY (``dphil``) it parses as an ordinary name word and repairs +as one (``Dphil``). Written DOTTED (``d.phil.``) it is a suffix by +shape alone, with no vocabulary entry needed to read it as one, and +repairs in all capitals the same as a listed acronym does +(``D.PHIL.``). Either way, a caller who wants a specific spelling +needs a ``suffix_acronyms`` entry (``Lexicon.add(suffix_acronyms= +{...})`` in the 2.0 API, ``constants.suffix_acronyms.add(...)`` in +the v1 one) or a mask of its own. """ diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index a504666a..a5965e90 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -174,6 +174,7 @@ def test_capitalize_exceptions_still_win_over_acronyms(self) -> None: def test_a_mismatched_exception_value_raises_at_the_first_parse( self, ) -> None: + from nameparser._lexicon import _MaskValueError c = Constants(capitalization_exceptions={'jr': 'Junior'}) with pytest.raises(ValueError, match="does not spell the key's letters") \ @@ -181,6 +182,12 @@ def test_a_mismatched_exception_value_raises_at_the_first_parse( HumanName('john smith jr', constants=c) assert "constants.capitalization_exceptions['jr'] = 'JR'" \ in str(caught.value) + # The v1 message is built fresh from the fields, not from the + # wrapped 2.0 exception's own text -- so it never carries the + # 2.0 constructor spelling a v1 caller cannot paste. + assert "Lexicon(" not in str(caught.value) + assert "capitalization_exceptions=((" not in str(caught.value) + assert isinstance(caught.value.__cause__, _MaskValueError) # pasted onto a fresh Constants, the offered fix works fixed = Constants() fixed.capitalization_exceptions['jr'] = 'JR' diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index b43480f5..b117873d 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -889,6 +889,20 @@ def test_a_multiword_capitalization_exceptions_key_warns_exactly_once( assert len(multi) == 1 +def test_a_multiword_capitalization_exceptions_key_with_a_bad_mask_only_warns( +) -> None: + """#459 review, shim half of test_a_multiword_key_skips_the_mask_ + check_entirely in tests/v2/test_lexicon.py: a v1 caller's + multi-word key is unreachable through capitalized() regardless of + its value, so a value that does not spell it must not raise here + either.""" + c = Constants(capitalization_exceptions={'ph d': 'Doctor'}) + with pytest.warns(UserWarning, match="matched one word at a time"): + HumanName("John Smith", constants=c) + lexicon, _policy, _render = c._snapshot() + assert lexicon.capitalization_exceptions_map == {"ph d": "Doctor"} + + def test_2x_pickle_roundtrip_keeps_a_readded_dead_entry() -> None: # the all-eight gate: a 2.0 user's deliberate re-add of ONE legacy # string survives a round-trip (only a full pre-2.0 blob, which diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ddf41569..2dff0825 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -240,17 +240,37 @@ def test_the_mask_error_falls_back_when_upper_would_not_itself_validate( .capitalization_exceptions_map == {"straße": "straße"} +def test_the_mask_error_echoes_the_callers_own_spelling_not_the_composed_one( +) -> None: + """#459 review: the value used to be captured AFTER NFC + normalization, so the message (and _MaskValueError.value) showed + the composed spelling even when the caller wrote a decomposed one. + A decomposed 'e' + a combining acute and its NFC-composed 'é' look + identical printed, but are different str objects -- repr() shows + it (different escaping), and here it also changes len().""" + decomposed = "PhéD" # 'PhéD', 'e' + U+0301 COMBINING ACUTE + composed = unicodedata.normalize("NFC", decomposed) + assert repr(decomposed) != repr(composed) + with pytest.raises(_MaskValueError) as caught: + Lexicon(capitalization_exceptions=(("phed", decomposed),)) + assert caught.value.value == decomposed + assert repr(decomposed) in str(caught.value) + assert repr(composed) not in str(caught.value) + + def test_mask_value_error_survives_pickle_and_copy() -> None: """The inherited __reduce__ would re-call __init__ with self.args, - the message alone, and __init__ needs `key` and `offered` too -- - so without the override a ProcessPoolExecutor worker raising it - would surface as BrokenProcessPool rather than this ValueError.""" + the message alone, and __init__ needs `key`, `normalized_key` and + `value` too -- so without the override a ProcessPoolExecutor + worker raising it would surface as BrokenProcessPool rather than + this ValueError.""" try: Lexicon(capitalization_exceptions=(("jr", "Junior"),)) except _MaskValueError as caught: original = caught else: raise AssertionError("expected _MaskValueError") + original.add_note("seen in worker 3") for restored in (pickle.loads(pickle.dumps(original)), copy.copy(original)): assert isinstance(restored, ValueError) @@ -258,6 +278,10 @@ def test_mask_value_error_survives_pickle_and_copy() -> None: assert restored.key == original.key == "jr" assert restored.offered == original.offered assert str(restored) == str(original) + # __reduce__'s third element (self.__dict__) is what carries a + # note added after construction -- reconstructing from the + # three __init__ fields alone would drop it. + assert restored.__notes__ == ["seen in worker 3"] def test_add_and_remove_return_new_lexicons() -> None: @@ -808,6 +832,21 @@ def test_multiword_capitalization_key_warns() -> None: capitalization_exceptions=(("zqx zqy", "ZqXZqY"),)) +def test_a_multiword_key_skips_the_mask_check_entirely() -> None: + """#459 review: the mask check ran before the multi-word-key + warning, so a key that can never match -- 'ph d' has no word in + capitalized()'s per-word lookup -- still raised over a value that + does not spell it (AGENTS.md: an invariant guards harm, and an + unreachable entry does none). Reordered so a multi-word key only + warns, storing its value verbatim regardless of the mask + question.""" + with pytest.warns(UserWarning, match="matched one word at a time"): + lex = dataclasses.replace( + Lexicon.empty(), + capitalization_exceptions=(("ph d", "Doctor"),)) + assert lex.capitalization_exceptions_map == {"ph d": "Doctor"} + + def test_default_lexicon_builds_warning_free() -> None: # Would have caught the eight dead entries the previous commit # repaired. diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index eec36308..cbacbd03 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -662,7 +662,8 @@ def test_a_mask_recases_the_word_as_the_writer_punctuated_it() -> None: ("john smith bsc", "BSc"), ("JOHN SMITH MSC", "MSc")): assert parse(text).capitalized().suffix == suffix, text - # two tokens, and 'ph.' is no key: nothing to mask, nothing moves + # two tokens, and 'ph.' is no key: no mask applies, and each word + # title-cases on its own assert str(parse("john smith ph. d.").capitalized()) \ == "John Smith Ph. D." # the mask is asked BEFORE the acronym clause, which would give @@ -805,6 +806,10 @@ def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( (".a", "a", ".a"), # single-letter mask never overrides ("a.", "a", "a."), ("2b.", "2b", "2b."), # 'b' has no LETTER neighbor in '2b' + # a digit ends a run on the NEXT side too, not only the + # previous one the row above covers + ("b.2", "b2", "b.2"), + ("2.b.", "2b", "2.b."), # the mask's own 'A' already agrees, so this row alone # cannot tell the override from plain masking; the two # all-lowercase masks after it can @@ -818,8 +823,23 @@ def test_a_split_initial_is_capitalized_only_where_the_mask_keeps_it_joined( # a fullwidth stop is a full stop too -- reachable by a # direct call only, since _WORD splits a token at it ("a．bc", "abc", "A．bc"), + ("ab．c", "abc", "ab．C"), # fullwidth stop on the previous side + # a TITLECASE mask letter (DŽ, the digraph DŽ's title form) + # reads as upper -- `not mask_chars[at].islower()` is true + # for it same as for a plain uppercase letter -- a + # documented limit (_apply_mask's own docstring) + ("dž", "Dž", "DŽ"), ): assert _apply_mask(word, mask) == expected, (word, mask) + # The override's two index guards ('i > 0' before reading + # word[i - 1], 'i < last' before reading word[i + 1]) matter only + # for a split letter at an actual EDGE of the word -- reachable + # here only by calling _apply_mask directly, since through + # _cap_text a hyphen is never handed to it: _WORD splits a token + # at a hyphen first (and the hyphen clause in _cap_text handles + # that text separately), so only a full stop reaches this far. + assert _apply_mask("ab-c", "abc") == "ab-c" + assert _apply_mask("a-bc.", "abc") == "a-bc." for text, suffix in (("john smith b.s.c.", "B.S.C."), ("JOHN SMITH B.S.C.", "B.S.C."), ("john smith m.s.c.", "M.S.C."), @@ -866,6 +886,27 @@ def test_a_masks_upper_fallback_can_lengthen_a_word_through_ss() -> None: "john straße").family == "STRASSE" +def test_a_lengthening_mask_is_not_a_fixpoint_under_a_second_forced_pass( +) -> None: + """decisions.md#R4 (2026-09-24 review, sub-clause (a)): capitalized()'s + own docstring claims every clause is a fixpoint, so a repaired name + comes back unchanged if repaired again -- true everywhere except + this one boundary. The FIRST forced pass over 'a.ß' under the + ('a.ß', 'aß') mask gives 'A.SS' (the previous test). Forcing that + OUTPUT through the same lexicon a second time folds it to 'a.ss', + which is a different letter sequence from the stored key 'a.ß' -- + not merely a different case of the same one -- so the exceptions + map lookup that found the entry on the first pass misses on the + second, and the word falls through to plain title-casing.""" + lex = dataclasses.replace(Lexicon.default(), + capitalization_exceptions=(("a.ß", "aß"),)) + first = parse("john a.ß smith").capitalized(lex, force=True) + assert first.middle == "A.SS" + second = first.capitalized(lex, force=True) + assert second.middle == "A.ss" + assert second.middle != first.middle + + def test_a_masks_punctuation_marks_its_joins_and_is_never_written() -> None: """#459 review: a value's punctuation is never written into the word -- under ('md', 'M.D.') 'md' repairs to 'MD' and 'm.d.' to @@ -927,6 +968,19 @@ def test_a_mask_cases_through_the_whole_word_for_context_sensitive_letters( # longer than the word, and there ß keeps its own lowercase under # the mask's lowercase letter assert _apply_mask("straße", "STRAßE") == "STRAßE" + # same fallback (the word's own upper/lower length still + # disagrees regardless of the mask's case), with a LOWERCASE mask + # this time: the per-character path lowers every letter, + # including ß's own, to 'straße' + assert _apply_mask("STRAßE", "straße") == "straße" + # `same_length` chains three lengths (lowered, uppered, word); the + # ß rows above both fail it through the UPPERED side ('ß' grows + # under .upper()). This row fails it through the LOWERED side + # instead -- 'İ' (capital dotted I) grows under .lower() to 'i̇' + # (dotless i + combining dot above) -- and the last row fails it + # through BOTH at once, mixing ß and İ in one word. + assert _apply_mask("İx", "ix") == "İx".lower() + assert _apply_mask("ßİ", "ßi") == "".join(c.lower() for c in "ßİ") # end to end, through a custom Lexicon: a per-character # c.lower()/c.upper() walk gave 'Κοσ' here (medial sigma), wrong lex = dataclasses.replace(Lexicon.default(), @@ -1028,7 +1082,12 @@ def test_a_link_inside_a_hyphenated_word_keeps_its_lowercase() -> None: # named-relative bound would wrongly read it as interior # and lower it ("jose -y-garcia-lopez", "Jose -Y-Garcia-Lopez"), - ("jose garcia-lopez-y-", "Jose Garcia-Lopez-Y-")): + ("jose garcia-lopez-y-", "Jose Garcia-Lopez-Y-"), + # a part holding only punctuation ('.') is not a worded + # neighbour -- the split is ['.', 'y', 'garcia'], so 'y' + # has only ONE worded neighbour and stays ordinary name + # text rather than reading as the connective + ("jose .-y-garcia", "Jose .-Y-Garcia")): assert str(parse(text).capitalized()) == repaired, text # mixed case is R5's: untouched unless forced mixed = parse("Jose Ortega-Y-Gasset") From 3fd6b82098e7ffb5f68f298f848d233fdeaa23b0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 24 Sep 2026 20:33:39 -0700 Subject: [PATCH 9/9] fix(#492/#459/#478): a mixed-case suffix is kept as written, and the mixed-case acronyms carry masks Two decisions from the pull request's review (Derek, 2026-09-24). A suffix the gate no longer reads as case evidence is no longer repaired either: on the default path a suffix token written in more than one case is kept as the writer wrote it, so 'john smith EdD' gives 'John Smith EdD' where the suffix-free gate had admitted the name and the acronym clause had written 'EDD', and 'B.Tech.' stays 'B.Tech.' rather than the 'B.TECH.' an earlier bullet accepted. 'juan garcia Iii' keeps 'Iii' on that path, the cost of "as written"; force=True repairs every token as before. The gate and the per-suffix check share one helper. Fifteen listed acronyms whose usual spelling is mixed case ship as masks (Bt, ChFC, CPhT, DiplAc, DMin, DrPH, DSc, Kt, MDiv, PharmD, PhC, PsyD, RPh, ThD, ThM), so 'john smith psyd' gives 'PsyD' from a single-case name too. Four are excluded and recorded: a mask is role-free, and 'Meng Li', 'Edd Smith', 'Lac' and 'Ded' are names. The hyphen clause keeps its reading: a hyphen is the writer joining the surname, so the interior word is a connective even in a name written in one case, where the spaced form reads a marked letter as an initial. 'J-E-P DUPONT' -> 'J-e-P Dupont' is the recorded, pinned boundary, and Lexicon.conjunctions_ambiguous does not reach a hyphenated word. rules.md#R4/#R5, decisions.md, the guides and the release log carry both decisions; the corpus counts move only by the new example lines. Co-Authored-By: Claude Fable 5.1 --- docs/customize.rst | 20 ++++- docs/design/decisions.md | 20 ++++- docs/design/rules.md | 26 +++++- docs/release_log.rst | 8 +- nameparser/_pipeline/_vocab.py | 35 +++++--- nameparser/_render.py | 27 +++++- nameparser/_types.py | 6 +- nameparser/config/capitalization.py | 50 ++++++++--- tests/test_capitalization.py | 4 + tests/v2/test_render.py | 123 +++++++++++++++++++++++++- tools/differential/corpus_rules.jsonl | 4 + 11 files changed, 280 insertions(+), 43 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index d480df22..c72342db 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -188,9 +188,10 @@ instead, and pass the result to ``capitalized()``: Note the ``tuple(...) + ...``: assigning a bare ``(("dphil", "DPhil"),)`` would *replace* the default exceptions rather than extend them, so -``phd``, ``bsc`` and ``msc`` would lose their masks and fall back to -the all-capitals acronym repair: ``john smith phd`` would give -``John Smith PHD`` rather than ``John Smith PhD``. +the shipped masks (``phd``, ``bsc``, ``psyd`` and the rest) would be +lost and those words fall back to the all-capitals acronym repair: +``john smith phd`` would give ``John Smith PHD`` rather than +``John Smith PhD``. The key is matched against the token with punctuation normalized away, not against the raw text, so one ``"phd"`` entry covers ``"phd"``, @@ -204,7 +205,15 @@ capitalized where the mask keeps that letter inside a longer run, so ``p.h.d.`` repairs to ``P.H.D.`` under ``"PhD"``. An acronym already listed in ``suffix_acronyms`` — plain or dotted — and a roman numeral need no entry at all: case repair writes a suffix of either kind in -capitals by itself. A caller's own acronym, one ``suffix_acronyms`` +capitals by itself. Most of the listed acronyms whose usual spelling +is not all capitals already carry a shipped mask (``DSc``, ``PsyD``, +``PharmD``, ``MDiv`` and others), so ``john smith psyd`` gives +``John Smith PsyD``. The exception is an acronym that is also a name +word, such as ``meng`` or ``edd``: a mask applies wherever its word +stands, so it would re-spell a person called Meng or Edd, and these +get none — ``john smith edd`` gives ``John Smith EDD`` (the reasoning +is the Excluded block for ``CAPITALIZATION_EXCEPTIONS`` under ``R4`` +in ``docs/design/decisions.md``). A caller's own acronym, one ``suffix_acronyms`` doesn't already list, depends on how it is written: plain (``dphil``) it parses as an ordinary name word and repairs as one (``Dphil``, above); dotted (``d.phil.``) it is a suffix by shape alone, with no @@ -887,6 +896,9 @@ signatures: default on the assumption that someone already capitalized it on purpose. The suffixes don't count toward that: ``III`` or ``PhD`` written the usual way says nothing about how the name was cased. + A suffix written in more than one case is the writer's spelling + and is kept as written (``john smith EdD`` gives ``John Smith + EdD``) unless you pass ``force=True``. .. doctest:: diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 9a03d9aa..348bf3f2 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -387,6 +387,8 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf THE FIGURES, AND THE INTERPRETER, because the pair above spliced two and this one does not. Re-measured 2026-09-21 on py3.11 through `tests/v2/test_benchmark.py`'s own `_frames_for` shape, `b9ed1429` → `9fd84463` → this tree: `Josep Carod i Rovira` 311 → 314 → 313, `Josep Lluis Carod i III` 377 → 381 → 380, `Jane Doe nee Puig i Soler` 315 → 320 → 319, `John Quincy Adams i MA Prof.` 438 → 443 → 442, and the clause guard's own pair 1,125 → 875 → 859 at sixteen links and 6,741 → 2,651 → 2,587 at sixty-four (3.01x, re-pinned in the test). `tools/perf/call_count.py` is unmoved at parse=406.00 facade=443.00 — its reference name carries no link — and so are `John Smith` 172, `Smith, John` 203, `Juan Garcia y Lopez` 291, `John and Jane Smith` 293, `Jane Doe nee Smith` 245 and `Jane Doe nee Smith PhD` 340, none of which reaches the predicate. WHICH OF THE EARLIER FIGURES REPRODUCE, stated exactly where this sentence first said only that one half "very nearly" did: re-measured 2026-09-22 on py3.11, the short-name pairs (304 → 307, 307 → 312) do not reproduce at all, and of the run-of-64 pair (6,741 → 2,652) the LEFT half does and the right does not — `b9ed1429` reads 6,741 on the nose, while `6048eb5d`, the tree its 2,652 was taken on, reads 2,651 here, the same figure `9fd84463` reads. So the splice runs through a single arrow rather than between two of them, which is exactly the spliced table `tools/perf/call_count.py`'s docstring was written about. Timings are unmoved: `"Josep " + "i " * n + "Rovira"` and `"Jane Doe nee Puig " + "i " * n + "Soler"` both read 1.96-2.02x per doubling from n=200 to n=1,600, 11.36ms and 9.39ms at the top end against 11.58ms and 9.58ms at `9fd84463`. AND THE FOLD BOUGHT A TEST, which is the part worth keeping: while the predicate was called once per side, a mutation of the suffix or the title test hit BOTH sides at once and the left-hand rows killed it, so the right-hand halves were never separately covered — their own rows (`Josep Lluis Carod i Jr.`, `i Mr.`) stand at the END of the name, where `hi` refuses them before either piece test is asked. Folded, the two sides mutate independently and both right-hand tests survived the whole suite. `test_a_credential_or_honorific_mid_name_on_the_right_too` is the pair that kills them, and every arm of the folded predicate now dies by a named test (recorded in its docstring). +- 2026-09-24 #478 — CROSS-REFERENCE, no change to this rule: case repair's hyphen clause is the one place a marked letter in a name written wholly in one case is read as the connective rather than an initial — `maria silva-e-sousa` repairs to `Maria Silva-e-Sousa` beside the spaced `Maria Silva E Sousa`, and `conjunctions_ambiguous` does not reach the hyphenated form. Decided there, with its cost (`J-E-P DUPONT` → `J-e-P Dupont`): decisions.md#R4's 2026-09-24 #478 bullet. + - Provenance: the single-letter-connective guard is v1's fix for Google Code issue 11 ("john e smith", 2013, commit 33676c9) — the "#11" citations that circulated pointed at a GitHub accident, not the real source. Recorded so the archaeology stays done. Excluded (Lexicon.conjunctions_ambiguous, the marked half of nameparser/config/conjunctions.py — an entry here reads as an INITIAL in a name written wholly in one case): @@ -1370,12 +1372,24 @@ Accepted costs, deferred to the rescoped #459 rather than relitigated here: the - 2026-09-23 #478 — DECIDED: inside `_cap_text`, a part of a HYPHENATED token that is connective vocabulary keeps its lowercase when a WORDED part stands on each side of it within the token (split on `-`; a part counts only if it holds a word, and an empty part does not count and is skipped, so a trailing hyphen supplies no neighbour while a doubled hyphen changes nothing: `juan garcia--y-lopez` gives `Juan Garcia--y-Lopez`). An edge part repairs as ordinary name text. `jose ortega-y-gasset` and `JOSE ORTEGA-Y-GASSET` both give `Jose Ortega-y-Gasset`, and `maria silva-e-sousa` gives `Maria Silva-e-Sousa`. It reads POSITION and VOCABULARY, as the particle arm already does per sub-word (`maria da-silva` → `Maria da-Silva`, unchanged) — a question about a sub-word the parse never asked, the parse having read the hyphenated word as one untagged token, which is the same standing mechanisms.md#RENDER-HONORS-THE-PARSE records for the particle arm — and it does not re-derive the initial-versus-conjunction class from a word's case, which is what #458 removed. The edges are pinned on both sides: `juan e-f smith` keeps `Juan E-F Smith` and `juan y-garcia` gives `Juan Y-Garcia`, and in a three-part word `juan y-garcia-lopez` gives `Juan Y-Garcia-Lopez` and `juan garcia-lopez-y` gives `Juan Garcia-Lopez-Y`. A SINGLE LETTER MARKED WITH A PERIOD is an initial there, as the parse reads it in spaced text, never the connective (the guard is `not re.fullmatch(r"\w\.", part)`): the first draft lowered the `e.` in `j.-e.-p. dupont`, giving `J.-e.-P. Dupont` where every release from 1.4.0 through 2.3.0 gave `J.-E.-P. Dupont`, a regression found in review and pinned since as a rules.md#R4 boundary line. A MULTI-letter word marked with a period is not an initial and stays reachable, as its spaced reading is (`hans smith und. jones` tags `und.` a conjunction), so `hans smith-und.-jones` gives `Hans Smith-und.-Jones`. This reverses, for the interior position only, what #458's tag read did to the shape: 2.2.0 and 2.3.0 gave `Ortega-Y-Gasset` for both spellings — the side effect #478 reported, which the 2026-08-29 #458 bullet above does not record — and #458's edge answer stands. Parity, measured on the released wheels: 1.4.0, 2.0.0 and 2.1.0 re-decided per word and gave `Jose Ortega-y-Gasset` lowered but `Jose Ortega-Y-Gasset` upper, so the all-caps half is a parity break with them on #479's one-case precedent; they also gave `Juan y-Garcia` at the edge, which neither 2.2.0, 2.3.0 nor this tree does. KNOWN NON-REACH, with its witness: the Māori corpus name `Te Awanui-a-Rangi Black` still repairs to `Te Awanui-A-Rangi Black` under force, because `a` is not connective vocabulary and must not become one (a spaced `a` is an initial or a name elsewhere; P3's Excluded block for the connectives says so). ACCEPTED: `mary-e-smith` reads `Mary-e-Smith` — no evidence the shape exists, and the hyphen is the writer joining the parts. Every role, since a hyphenated given name is the same text shape; a middle-role row is pinned beside the family ones (`smith, jose ortega-y-gasset` forced gives `Jose Ortega-y-Gasset Smith`). -- 2026-09-23 #459/#478 — MEASURED, with the recipe; none of these numbers is a live count. Population: the deduped `tools/differential/corpus*.jsonl` glob, both line shapes (an object with `name`, a bare string), as this change leaves it — 1371 distinct names, the new rules.md example lines among them. Comparator: the parent 4d0680e6 — `git archive 4d0680e6` into a scratch directory, both trees imported BY PATH in separate processes over the SAME corpus directory, each asserting `nameparser.__file__` — comparing every token's `capitalized()` and `capitalized(force=True)` text on the core, and the rendered string on the facade, and attributing each moved token PER SUB-WORD: the token is split into its `_WORD` sub-words (`(\w|\.)+`), and the FIRST sub-word whose repair changed decides the bucket, tried in this order — its period-free key is a mask key (`phd`, `bsc`, `msc`) or one that left the map (`md`, `ii`, `iii`, `iv`); else the token is a suffix carrying `SHAPE_ACRONYM_TAG` or the sub-word fits the roman shape; else the sub-word is an interior connective part of a hyphenated token; else, on the default path only, the token's repair used to equal its input, which is R5's admission — with a bucket for anything else, asserted empty. The per-sub-word split is load-bearing: keyed on the token as written, the corpus name `John Doe, MD-PhD-` (key `md-phd`) falls to the hyphen bucket instead, giving 15 and 82, 3 and 5 here and 11 and 77, 0 and 1 over the parent's corpus (measured in review on 2026-09-23). Mask, or a word that left the map: 15 names on the default path and 83 under force. Dotted shape and numerals: 3 and 16. Hyphenated link: 3 and 4, every one of them a rules.md#R4 example line — no corpus name written before this change carries a connective part in a hyphenated word, at the interior or an edge (0 of 1340 at 4d0680e6). R5's gate is R5's bullet. The same comparison over the parent's own corpus directory, 1340 names and none of this change's example lines, gives 11 and 78, 1 and 14, and 0 and 0: the difference is this change's own rules.md lines, and those are the counts to read as reach into names written for other reasons. The core and the facade move on the same number of names in both views. No role field moves on any name, which is also all the differential gate can say: its compared surface is the seven role fields, `_ambiguities` and `_initials`, never case (the 2026-08-29 bullet above). +- 2026-09-23 #459/#478 — MEASURED, with the recipe; none of these numbers is a live count. Population: the deduped `tools/differential/corpus*.jsonl` glob, both line shapes (an object with `name`, a bare string), as this change leaves it — 1375 distinct names, the new rules.md example lines among them (1371 before the two 2026-09-24 decisions below added four; re-measured 2026-09-24). Comparator: the parent 4d0680e6 — `git archive 4d0680e6` into a scratch directory, both trees imported BY PATH in separate processes over the SAME corpus directory, each asserting `nameparser.__file__` — comparing every token's `capitalized()` and `capitalized(force=True)` text on the core, and the rendered string on the facade, and attributing each moved token PER SUB-WORD: the token is split into its `_WORD` sub-words (`(\w|\.)+`), and the FIRST sub-word whose repair changed decides the bucket, tried in this order — its period-free key is a shipped mask key (`phd`, `bsc`, `msc`, and since 2026-09-24 the fifteen below) or one that left the map (`md`, `ii`, `iii`, `iv`); else the token is a suffix carrying `SHAPE_ACRONYM_TAG` or the sub-word fits the roman shape; else the sub-word is an interior connective part of a hyphenated token; else, on the default path only, the token's repair used to equal its input, which is R5's admission — with a bucket for anything else, asserted empty. The per-sub-word split is load-bearing: keyed on the token as written, the corpus name `John Doe, MD-PhD-` (key `md-phd`) falls to the hyphen bucket instead, giving 15 and 82, 3 and 5 over that day's 1371 names and 11 and 77, 0 and 1 over the parent's corpus (measured in review on 2026-09-23). Mask, or a word that left the map: 16 names on the default path and 84 under force over the 1375 (15 and 83 over the 1371 names before the 2026-09-24 decisions). Dotted shape and numerals: 3 and 16 over both. Hyphenated link: 4 and 5 over the 1375 (3 and 4 over the 1371), every one of them a rules.md#R4 example line — no corpus name written before this change carries a connective part in a hyphenated word, at the interior or an edge (0 of 1340 at 4d0680e6). R5's gate is R5's bullet. The same comparison over the parent's own corpus directory, 1340 names and none of this change's example lines, gives 11 and 78, 1 and 14, and 0 and 0 — the same before and after the two 2026-09-24 decisions, which move no name written before them: the difference is this change's own rules.md lines, and those are the counts to read as reach into names written for other reasons. The core and the facade move on the same number of names in both views. No role field moves on any name, which is also all the differential gate can say: its compared surface is the seven role fields, `_ambiguities` and `_initials`, never case (the 2026-08-29 bullet above). - 2026-09-23 — mechanisms.md#RENDER-HONORS-THE-PARSE CORRECTED, found in this change's re-read of that entry: it still said case repair carries the connective carve-out "ON R3's authority", quoting R4's pre-#461 text ("the carve-out R3 states for initials"), and that `.initials()` gives `j. a. j. s.` for `john and jane smith`. Both had been stale since #461 (2026-09-20), which cut R4's cross-reference to R3 in favor of R4's own reason and stopped a connective with a name word beside it from initialing. The entry now states R4's own reason, dates the cut, and names where the two views part today (`Juan de y`: initials `J. y.`, forced repair `Juan de y`), re-measured 2026-09-23. - 2026-09-24 #459/#492/#478 (branch review) — FOUR FINDINGS FROM THE REVIEW ROUND, recorded together because each is a small correction to a bullet already above rather than a new decision. (a) THE UNICODE BOUNDARY IS NOT SCOPED TO THE TWO SHAPE CLAUSES, AND NOT TO ONE WAY A MASK LETTER COMES OUT UPPER EITHER. The 2026-09-23 bullet above states the `ß` and dotless-`ı` boundary against `_cap_word`'s acronym/numeral clauses, but the MASK path can lengthen a word through `ß` exactly the same way. `_apply_mask` decides per letter whether it is written upper -- by the mask's own case, or by the split-off-initial force -- and then takes one casing step: the letter at that position in the whole-word `word.upper()`/`word.lower()` when both keep the word's length, otherwise a per-character `c.upper()`/`c.lower()`. `'ß'.upper()` is `'SS'`, so wherever the word holds a `ß` the whole-word form changes the length and every letter goes through `c.upper()`, which lengthens a `ß` written upper to `SS` whichever of the two reasons made it upper. Through the force: an identity mask validates fine (`('a.ß', 'aß')` spells the key's own letters unchanged), so `john a.ß smith` forced gives middle `A.SS` -- casefold-equal to its input and one character longer, the same trade the shape clauses make. Through the mask's own case, no full stop or split letter involved: `('straße', 'STRAẞE')`, the value spelling `straße`'s own letters unchanged with the actual German capital `ẞ` (not the digraph `'STRASSE'`, which the validator refuses as a VALUE for failing to spell `ß` at all), validates and `john straße` forced gives family `STRASSE`. The validator's refusal of `('straße', 'STRASSE')` is a different question -- whether the VALUE spells the KEY -- and says nothing about what the applier can produce from a mask that does validate. This same boundary is where `capitalized()`'s own "Idempotent" docstring claim narrows: forcing `A.SS` (that first pass's own output) through the SAME lexicon a second time gives `A.ss`, not `A.SS` back, because the second pass folds the word to `a.ss` and the exceptions map's key is still `a.ß` -- a different letter sequence, not merely a different case of one -- so the lookup that found the entry on the first pass misses on the second and the word falls through to plain title-casing. Pinned by `tests/v2/test_render.py::test_a_masks_upper_fallback_can_lengthen_a_word_through_ss` and `tests/v2/test_render.py::test_a_lengthening_mask_is_not_a_fixpoint_under_a_second_forced_pass`. The same boundary reaches plain title-casing with no mask involved, by two distinct mechanisms: LENGTHENING, where a letter's own upper() is a different letter SEQUENCE that stays \w -- 'ʼn' (one letter) upper-cases to the two cased characters 'ʼN', so `parse("ʼna smith").capitalized(force=True)` gives given `ʼNa`, and forcing THAT output a second time gives `ʼna` ('N' is no longer the word's first character, so `str.capitalize()` lower-cases it on the second pass) -- and SPLITTING, where a letter's own upper() is a base letter plus a COMBINING MARK that `_WORD` does not match at all -- 'ǰ' upper-cases to 'J' + a combining caron, so `parse("ǰo smith").capitalized(force=True)` gives given `J̌o`, and forcing THAT output a second time gives `J̌O` (`_cap_text` reads the output as two separate words, 'J' and 'o', the combining mark matching neither, and 'o' capitalized alone is 'O'). (b) MASK VALUES ARE STORED NFC-COMPOSED. `_normpairs` compared a value against its key NFC-composed but stored the value AS WRITTEN, so a value written in decomposed form spelled one composed letter as a base letter plus a non-alpha combining mark -- changing which letters `_letter_run_ge2` reads as split off beside a full stop from what the composed spelling of the identical value would read. `john smith pé.x.` forced gave `Pé.X.` under a composed mask and `Pé.x.` under the decomposed spelling of the SAME value. Fixed by NFC-composing the value at the same point the key is composed; pinned by `tests/v2/test_lexicon.py::test_a_decomposed_value_is_compared_and_stored_nfc_composed` and `tests/v2/test_render.py::test_a_decomposed_mask_value_reads_the_same_split_as_composed`. (c) THE HYPHEN CLAUSE'S INITIAL TEST IS NOW A PINNED MODULE CONSTANT. `_cap_text`'s inline `re.fullmatch(r"\w\.", part)` was a third hand copy of the initial shape's period half, invisible to `tests/v2/test_regex_sync.py`'s hand-copy sweep because it was never hoisted to a module-level pattern. Hoisted to `_render._DOTTED_INITIAL` and registered as `_INITIAL`'s period alternative; pinned by `tests/v2/test_regex_sync.py::test_dotted_initial_is_the_period_alternative_of_initial`. (d) THE V1 RAISE NOW CARRIES A V1-SPELLED HINT, VIA A TYPED EXCEPTION. The 2026-09-23 bullet above records that a mismatched-mask value raises `ValueError` at a v1 `Constants`' first parse; the message it raised was `Lexicon`'s own, offering a `Lexicon(capitalization_exceptions=...)` constructor call to a caller looking at a `TupleManager` assignment through `Constants`. `_lexicon._normpairs` now raises a dedicated `_MaskValueError(ValueError)` for exactly that one shape, built in the `UnicodeDecodeError` idiom: `__init__(self, key, normalized_key, value)` stores the three offending inputs and computes `self.offered` (from `_offered_mask`) and the message from them, rather than taking a pre-built message. `__reduce__` returns `(type(self), (self.key, self.normalized_key, self.value), self.__dict__)` -- reconstructing from the three fields and then restoring `self.__dict__` (so an attribute added after construction, e.g. `add_note()`, survives too) -- since the inherited default (`type(self)(*self.args)`, `self.args` holding only the message) cannot reconstruct an instance that needs three fields, not one, and raises `TypeError` trying; a worker in a `ProcessPoolExecutor` pickles an exception it raises to deliver it to the caller, so an unpicklable one there surfaces as `BrokenProcessPool` instead. `_config_shim._build_snapshot` wraps ONLY the `Lexicon(...)` construction in `try`/`except _MaskValueError` and re-raises a message built fresh from `e.key`/`e.value`/`e.offered` -- not from `str(e)`, which is the 2.0 exception's own message offering the 2.0 `capitalization_exceptions=((...),)` constructor spelling no v1 caller can paste -- with `constants.capitalization_exceptions[e.key] = e.offered`. Every OTHER `ValueError` `Lexicon(...)` can raise passes through unchanged, by TYPE rather than by a message-substring test. Of those, the three SUBSET checks (`particles_ambiguous` inside `particles`, `suffix_acronyms_ambiguous` inside `suffix_acronyms`, `honorific_tails` inside `suffix_words`) cannot fire from here: each is built as an intersection or union with its OWN base, so the normalized image of that relation is still a subset whatever the raw spelling. THE CONTRADICTION CHECK AND THE GATE-BYPASS CHECK ARE A DIFFERENT SHAPE, AND DO REACH `Lexicon(...)` -- this bullet claimed otherwise ("cannot arise ... observing no raise") until the confirmation round measured the opposite. Both compare two sets built by SEPARATE set arithmetic in `_build_snapshot`, on the RAW SetManager entries, before Lexicon's own per-field normalization ever runs: `SetManager` strips edge periods but not edge whitespace, while `Lexicon`'s normalization strips both, so an entry carrying edge whitespace can slip past the shim's own subtraction or union and still collide once Lexicon normalizes it independently. Measured: `c.suffix_not_acronyms.add('ba ')` then a parse raises Lexicon's own gate-bypass check (`an ambiguous suffix acronym must not also be a suffix word; in both: ba`), and `c.bound_first_names.add("'t ")` then a parse raises the contradiction check -- both pass through the `except _MaskValueError` clause above unchanged, being a different exception type. Pre-existing at HEAD and at master alike, not introduced by this session and not fixed here -- left for the orchestrator to file as its own issue. What DOES reach `Lexicon(...)` here unchanged, always, is an entry normalizing to empty, in `capitalization_exceptions` or in ANY set field -- measured: `c.titles.add("...")` then a parse raises Lexicon's own `titles entry '' normalizes to empty`, unaltered -- since nothing above translates that away, and a non-str value's `TypeError`, a raise v1 also had, later, at `capitalize()`. Declined: a first draft pre-validated `capitalization_exceptions` in isolation, a second call to `_normpairs` ahead of `Lexicon(...)` gated on a message-substring test; measured to make a multi-word key's dead-entry warning fire twice on the shim path, and to rest on a filter that was untested (deleting it passed the whole suite) and wrong (an unrelated ValueError then got the v1 hint appended regardless). Pinned by `tests/test_capitalization.py::test_a_mismatched_exception_value_raises_at_the_first_parse` (the hint, and that it carries no 2.0 constructor spelling), `tests/v2/test_config_shim.py::test_an_unrelated_capitalization_exceptions_valueerror_has_no_v1_hint` (the pass-through), `tests/v2/test_config_shim.py::test_a_multiword_capitalization_exceptions_key_warns_exactly_once` (the single warning), and `tests/v2/test_lexicon.py::test_mask_value_error_survives_pickle_and_copy` (the pickle/copy round-trip). +- 2026-09-24 #459 — DECIDED (Derek, 2026-09-24): the shipped map gains a mask for each listed acronym whose conventional spelling is mixed case, taken together with decisions.md#R5's keep-as-written bullet of the same day, which is the half that surfaced the need. The reviewer who measured the R5 regression named nine keys — `edd`, `dsc`, `psyd`, `meng`, `dmin`, `thd`, `drph`, `pharmd`, `mdiv` — and a sweep of `SUFFIX_ACRONYMS` for further entries whose usual spelling is unambiguously mixed found ten more: `bt`, `chfc`, `cpht`, `diplac`, `kt`, `phc`, `rph` and `thm`, and `lac` and `ded`. Of those nineteen, four are left out (below), so SHIPPED is fifteen — seven of the reviewer's nine and eight of the sweep's ten — every key a `suffix_acronyms` member: `bt` → `Bt` (Baronet), `chfc` → `ChFC` (Chartered Financial Consultant), `cpht` → `CPhT` (Certified Pharmacy Technician), `diplac` → `DiplAc` (Diplomate in Acupuncture, written `Dipl. Ac.`), `dmin` → `DMin` (Doctor of Ministry), `drph` → `DrPH` (Doctor of Public Health), `dsc` → `DSc` (Doctor of Science), `kt` → `Kt` (Knight Bachelor), `mdiv` → `MDiv` (Master of Divinity), `pharmd` → `PharmD` (Doctor of Pharmacy), `phc` → `PhC` (Pharmaceutical Chemist), `psyd` → `PsyD` (Doctor of Psychology), `rph` → `RPh` (Registered Pharmacist), `thd` → `ThD` (Doctor of Theology), `thm` → `ThM` (Master of Theology). The four LEFT OUT are two of the reviewer's nine, `meng` and `edd`, and two of the sweep's ten, `lac` and `ded` (the Excluded block below): a mask applies wherever its word stands (the 2026-09-23 masks bullet above), so a key borne as a name re-spells a person — the #vocabulary-collisions C-i test read with the mask's reach, which is every position. Candidates considered and not taken for want of one settled spelling: `djur` (`D.Jur.` beside the German `Dr. jur.`), and `litk`, `litl`, `litp`, `mscmsm`, whose credentials could not be identified. WHY BOTH HALVES: each serves a population the other cannot. Keeping a suffix as written serves the writer who spelled it (`john smith EdD`), but a name written wholly in one case carries no spelling to keep, its suffix being one-case too, so only a mask gives `john smith psyd` → `John Smith PsyD` (`PSYD` by the acronym clause without it; `Psyd` on every release 1.4.0 through 2.3.0, measured 2026-09-24); and a mask cannot reach an unlisted credential or a caller's own spelling, which the keep rule does. No corpus name written before this change carries any of the fifteen keys (measured 2026-09-24 over the parent's 1340 names and this tree's 1375), so the MEASURED bullet above moves only by the rules.md line `john smith psyd`. Recompute the list: `sorted(Lexicon.default().capitalization_exceptions)`. Pinned on both surfaces by `tests/v2/test_render.py::test_a_shipped_mask_spells_its_suffix_in_either_single_case`, which derives its rows from the shipped pairs and so pins the property rather than the list, and the exclusion's fork by `test_a_listed_acronym_that_is_a_name_word_gets_no_mask`, whose recorded negative control is the `meng`/`edd` masks added back. + +- 2026-09-24 — SUPERSEDED IN PART: the 2026-09-23 bullet above headed "A SUFFIX READ BY ITS DOTTED SHAPE" accepted that `john smith B.Tech.` gives `John Smith B.TECH.` on the DEFAULT path once the suffix-free gate admitted the name. That half no longer holds: decisions.md#R5's keep-as-written bullet (2026-09-24) keeps a suffix written in more than one case, so the default path gives `John Smith B.Tech.`, the credential as written — every release 1.4.0 through 2.3.0 returned the whole name untouched, `john smith B.Tech.` (measured 2026-09-24). The acceptance stands under force only — `John Smith B.TECH.` with `force=True`, where every release gave `B.tech.` — and the mask remedy it names is unchanged. The rules.md#R5 example line proposed for this spelling (`juan garcia B.Tech.`) was not added: the differential gate reported it UNEXPLAINED at every baseline, the by-shape suffix reading being a role change no ledger rule claims for that name, so it is pinned as a unit test instead (`tests/v2/test_render.py::test_a_suffix_written_in_more_than_one_case_is_kept_as_written`). + +- 2026-09-24 #478 — DECIDED (Derek, 2026-09-24): the hyphen clause keeps its reading in a name written wholly in one case, and the case where that disagrees with rules.md#P3 is a recorded boundary, not a change. The two collide on a marked letter. Spaced, a letter the vocabulary marks as reading both ways reads as an initial in a one-case name (P3), so `maria silva e sousa` repairs to `Maria Silva E Sousa`; hyphenated, the interior part reads as the connective whatever the name's case, so `maria silva-e-sousa` repairs to `Maria Silva-e-Sousa`. Derek's reasoning: hyphenating `Silva-e-Sousa` is the writer joining the surname on purpose, so the interior word is a connective by that act, whatever the one-case rule would say of the spaced form. THE COST, measured 2026-09-24: a one-case name whose hyphenated bare initials happen to spell a connective — `J-E-P DUPONT` gives `J-e-P Dupont`, where every release 1.4.0 through 2.3.0 and the parent 4d0680e6 gave `J-E-P Dupont`, and `JOHN A-Y-B SMITH` gives `John A-y-B Smith` (every release: `John A-Y-B Smith`). The period-marked spelling is unaffected (`j.-e.-p. dupont` keeps `J.-E.-P. Dupont`, the 2026-09-23 #478 bullet). `conjunctions_ambiguous`, P3's knob, does NOT reach a hyphenated word: under `Lexicon.default().add(conjunctions_ambiguous={"y"})` the spaced `JOSE ORTEGA Y GASSET` repairs to `Jose Ortega Y Gasset` (default lexicon: `Jose Ortega y Gasset`) while `JOSE ORTEGA-Y-GASSET` stays `Jose Ortega-y-Gasset` under both. rules.md#R4's hyphen sentence said the hyphens join the name "as the spaced connective would", which claimed an equality the one-case case breaks; it now says the hyphens are the writer's join and states the split, with an Accepted paragraph and the `J-E-P DUPONT` boundary line. Pinned by `tests/v2/test_render.py::test_the_hyphen_is_the_writers_join_even_in_a_one_case_name`. + +Excluded (CAPITALIZATION_EXCEPTIONS — meng, edd, lac, ded, left out of the 2026-09-24 masks, #459): + +- `meng` (MEng) and `edd` (EdD) are conventionally mixed-case credentials and listed acronyms, and must not get a mask while the map stays role-free: a mask applies in every role, and both are borne as given names or surnames. Measured 2026-09-24 with each mask added to a private lexicon: `MENG LI` gives `MEng Li`, `meng wang` gives `MEng Wang`, `edd smith` gives `EdD Smith` and `EDD KIMBER` gives `EdD Kimber`, where the shipped map and every release give `Meng Li`, `Meng Wang`, `Edd Smith` and `Edd Kimber`. The same holds for two sweep candidates, `lac` (LAc, Licensed Acupuncturist; `Lac` is a Vietnamese and French surname) and `ded` (DEd; `Ded` is an Albanian given name, `ded gjo luli` giving `DEd Gjo Luli` under the mask). A writer who spells one mixed-case keeps it (decisions.md#R5, 2026-09-24); a caller who wants the mask adds it to a private lexicon. A NOTE on `SUFFIX_ACRONYMS` itself, which this map does not decide. The two name-word collisions there come from two bulk additions, not one: `lac` (with `edd` and `ded`) from the Wikipedia post-nominals import, `af5bdab` (2019-12-11, #93), and `meng` from `3e14ea20` (2026-07-01, the German/Dutch title and degree batch, first shipped in 1.3.0). Both read as a suffix after a given name, and lose a family name to it. Measured 2026-09-24 on the released wheels, the parent 4d0680e6 and this tree: `wang meng`, `li meng` and `zhang meng` read family `meng` on 1.4.0 and suffix `meng`, with no family, on every release from 2.0.0; and since #521 on master the suffix renders `Wang MENG` (parent and tree alike), where 2.3.0 gave `Wang Meng`. `nguyen van lac` reads suffix `lac` with family `van` on every release from 1.4.0, and since #521 renders `Nguyen Van LAC` where 2.3.0 gave `Nguyen Van Lac`. This is a #vocabulary-collisions C-i question about the acronym set, not about this map. + +Open: a follow-up issue on `SUFFIX_ACRONYMS`'s `meng`/`lac` name-word collision is to be filed (the note above carries the measurements). + Excluded (CAPITALIZATION_EXCEPTIONS — md, ii, iii, iv, removed 2026-09-23, #459): - `md`, `ii`, `iii` and `iv` are OUT of the exceptions map and must not be put back by a sweep that finds one of them title-cased somewhere. As a SUFFIX each is written in capitals with no entry — `md` by the acronym clause, the numerals by the numeral clause — and in any other role each repairs as the parse read it by decision (the item-4 bullet above), so an entry would only bring back the role-free capitals that bullet retired: `iv smith` → `IV Smith`, `Md Abdul Karim` → `MD Abdul Karim`. A caller who wants them back adds them to a private lexicon, as masks. @@ -1400,7 +1414,9 @@ Excluded (CAPITALIZATION_EXCEPTIONS — md, ii, iii, iv, removed 2026-09-23, #45 - 2026-09-23 #492 — TITLES STAY EVIDENCE, and the reason is a limit of title REPAIR rather than a claim about titles. Title repair is still plain `str.capitalize()`, and widening the gate to titles would surface its defects on the default path. Measured under `force=True`, where they already show (re-measured 2026-09-23): `Lt.Gov. juan e garcia` → `Lt.gov. Juan e Garcia`, `The Lord of the Universe` → `the Lord of the Universe`, `Dr. med. univ. Margit Popp` → `Dr. Med. Univ. Margit Popp`. So `Dr. juan garcia` is left alone (rules.md#R5's boundary line). This may move when title repair is trusted. -- 2026-09-23 #492 — MEASURED, population first. Recipe: for each name in the deduped corpus glob, the gate as shipped before #492 (the joined text of EVERY token) declines it AND the joined text of its non-suffix tokens is one-case; those are exactly the names the new gate admits, and the default path returns their forced repair. The movers are the admitted names whose default-path string changes, counted by the R4 MEASURED bullet's comparator. At 4d0680e6, over its 1340 names, the population was 17 and 12 of them changed their default-path string — `juan garcia III`, `dr. ben alex johnson III`, `john e jones, III`, `john smith MA` and eight `pennie von bergen wessels` spellings; the other five (`威廉·莎士比亚, PhD`, `田中さん, PhD`, `田中さん, Ph. D.`, `김민준씨 Jr.`, `1 & 2, 3 4 5, Mr.`) are admitted and repair to themselves. The approved spec said "17 move": 17 is the population, 12 the movers. Over the corpus this change leaves (1371 names, R5's new example lines among them) the population is 20 and the movers 15, the three new ones being those lines. No forced repair moves by this rule, as none can: the gate is never consulted under force. +- 2026-09-23 #492 — MEASURED, population first. Recipe: for each name in the deduped corpus glob, the gate as shipped before #492 (the joined text of EVERY token) declines it AND the joined text of its non-suffix tokens is one-case; those are exactly the names the new gate admits, and the default path returns their forced repair (since 2026-09-24, less any suffix written in more than one case, which it keeps as written — the bullet below). The movers are the admitted names whose default-path string changes, counted by the R4 MEASURED bullet's comparator. At 4d0680e6, over its 1340 names, the population was 17 and 12 of them changed their default-path string — `juan garcia III`, `dr. ben alex johnson III`, `john e jones, III`, `john smith MA` and eight `pennie von bergen wessels` spellings; the other five (`威廉·莎士比亚, PhD`, `田中さん, PhD`, `田中さん, Ph. D.`, `김민준씨 Jr.`, `1 & 2, 3 4 5, Mr.`) are admitted and repair to themselves. The approved spec said "17 move": 17 is the population, 12 the movers. Over the corpus this change leaves (1375 names, R5's new example lines among them) the population is 22 and the movers 17, the five new ones being those lines (20 and 15 over 1371 names before the 2026-09-24 decision below added `john smith EdD` and `juan garcia Iii`; re-measured 2026-09-24, the parent-corpus 17 and 12 unchanged). No forced repair moves by this rule, as none can: the gate is never consulted under force. + +- 2026-09-24 #492 — DECIDED (Derek, 2026-09-24): where repair was not forced, a SUFFIX token written in more than one case is kept as written; one written in a single case is repaired like any other token, and force repairs both, unchanged. It is the gate's own one-case test asked of each suffix on its own (`_render._in_one_case`, one predicate for both), and it is the 2026-09-23 #492 reasoning read to its end: a suffix is left out of the gate because a cased one says nothing about how the writer cased the NAME, and for the same reason a mixed-case one is the writer's spelling of the credential, which repair has no evidence to overrule. THE REGRESSION THAT SURFACED IT, measured by the branch's code reviewer and re-measured 2026-09-24: once the gate stopped counting the suffix, a one-case name with a conventionally mixed-case listed acronym was admitted and the acronym clause wrote the acronym in capitals — `john smith EdD` → `John Smith EDD`, `JANE DOE, DSc` → `Jane Doe DSC`, `juan garcia PsyD` → `Juan Garcia PSYD` — where the parent 4d0680e6 and every release 1.4.0 through 2.3.0 returned those names untouched. They now give `John Smith EdD`, `Jane Doe DSc` and `Juan Garcia PsyD`. The accepted `B.Tech.` → `B.TECH.` on the default path (decisions.md#R4, 2026-09-23) was the same shape, and its default-path half is superseded (decisions.md#R4, 2026-09-24). THE COST, accepted: "as written" keeps a garbled spelling with the deliberate one, so `juan garcia Iii` gives `Juan Garcia Iii` on the default path (`Juan Garcia III` forced) and `john smith Mba` gives `John Smith Mba`, where the branch before this decision gave `III` and `MBA`; every release returned both names untouched. A rules.md#R5 boundary line pins `Iii`. Why not keep only a spelling the vocabulary does not know how to repair: that would re-derive, per word, a judgment about which mixed spellings are deliberate, which is the evidence R5 says repair cannot read. The masks added the same day (decisions.md#R4) are the other half: they serve the one-case names this rule cannot reach (`john smith psyd` → `PsyD`). No corpus name written before this decision moves by it (measured 2026-09-24 against the branch before it, over this tree's 1375 names: the only movers are the new rules.md lines). Pinned by `tests/v2/test_render.py::test_a_suffix_written_in_more_than_one_case_is_kept_as_written`, core and facade; a mutant dropping the per-token test fails it, `test_the_gate_leaves_the_suffixes_out` and two rules.md#R5 lines, and one dropping its `force` guard fails it and the forced `EdD` line. ### parse-cost — what a parse is allowed to cost diff --git a/docs/design/rules.md b/docs/design/rules.md index 31a71b8e..9ca3d42d 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2209,11 +2209,15 @@ R4. Rationale: case repair is a display concern, applied only on generational `jr`, `sr` -- keeps its title case. Inside a hyphenated word, a part that is connective vocabulary with a worded part on each side of it keeps its lowercase, the - hyphens joining the name around it as the spaced connective would + hyphens being the writer joining the name around it (Ortega-y-Gasset); at either end of the hyphenated word it is repaired as ordinary name text. A single letter marked with a period is read as an initial there, as the parse reads it, never - as the connective. + as the connective. A bare letter is not: the hyphen is the + writer's join, so the interior connective is read as one even in + a name written wholly in one case, where the same letter spaced + reads as an initial if the vocabulary marks it as reading both + ways (P3). "juan mcdonald" → capitalized="Juan McDonald" "Juan McDonald" → capitalized_forced="Juan McDonald" "ANH DO" → capitalized="Anh Do" @@ -2224,6 +2228,7 @@ R4. Rationale: case repair is a display concern, applied only on "john smith p.h.d." → capitalized="John Smith P.H.D." "john smith ph. d." → capitalized="John Smith Ph. D." "john smith bsc" → capitalized="John Smith BSc" + "john smith psyd" → capitalized="John Smith PsyD" "Dr. med. univ. Margit Popp, MSc" → capitalized_forced="Dr. Med. Univ. Margit Popp MSc" "MSc Dr. med. univ." → capitalized_forced="MSc Dr. Med. Univ." "Md Abdul Karim" → capitalized_forced="Md Abdul Karim" @@ -2260,6 +2265,14 @@ R4. Rationale: case repair is a display concern, applied only on "maria da-silva" → capitalized="Maria da-Silva" "juan y-garcia-lopez" → capitalized="Juan Y-Garcia-Lopez" · boundary "j.-e.-p. dupont" → capitalized_forced="J.-E.-P. Dupont" · boundary + "J-E-P DUPONT" → capitalized="J-e-P Dupont" · boundary + Accepted: a name written in one case whose hyphenated bare + initials spell a connective pays for the hyphen clause — its + interior letter is lowered as the connective though the writer + meant an initial, where the same letters spaced keep their + capitals. The hyphen is the writer's join, and reading the one + shape both ways would need the case evidence a one-case name does + not carry. Accepted: the all-particle clause reaches a part the parser read. A field spliced in as raw text after the parse carries no reading of its own, so a family set that way to "de la" stays lowercase @@ -2322,7 +2335,11 @@ R5. Rationale: mixed case is evidence that the writer cased the name out, and a cased title still holds repair back. A name whose words outside the suffix are written in more than one case is kept as it was written, and whether that casing is right does - not enter into it, unless repair was asked for anyway. + not enter into it, unless repair was asked for anyway. The same + test is then asked of each suffix on its own: a suffix written + in more than one case is the writer's spelling and is kept as + written where repair was not forced; one written in a single + case is repaired like any other. "juan mcdonald" → capitalized="Juan McDonald" "SHIRLEY MACLAINE" → capitalized="Shirley MacLaine" "Shirley Maclaine" → capitalized="Shirley Maclaine" @@ -2333,6 +2350,9 @@ R5. Rationale: mixed case is evidence that the writer cased the name "dr. juan garcia III" → capitalized="Dr. Juan Garcia III" "Dr. juan garcia" → capitalized="Dr. juan garcia" · boundary "Juan garcia III" → capitalized="Juan garcia III" · boundary + "john smith EdD" → capitalized="John Smith EdD" + "juan garcia Iii" → capitalized="Juan Garcia Iii" · boundary + "john smith EdD" → capitalized_forced="John Smith EDD" history: decisions.md#R5 · interacts: R4, P3, S2 · implemented: nameparser/_render.py ## Construction & configuration diagnostics (D) diff --git a/docs/release_log.rst b/docs/release_log.rst index d16d8dab..43a553cf 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -26,13 +26,13 @@ Release Log - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) - - **Change case repair's exceptions map from replacement spellings to case masks, so md repairs to MD and phd to PhD.** ``HumanName("john smith phd").capitalize()`` gives ``John Smith PhD`` and ``john smith md`` gives ``John Smith MD``, where every release from 1.4.0 through 2.3.0 gave ``John Smith Ph.D.`` and ``John Smith M.D.``. A ``capitalization_exceptions`` value is now the key's own letters and digits in the case each should take, laid over the word as it was written, so the one ``phd`` entry repairs ``ph.d.`` to ``Ph.D.`` and ``JOHN SMITH PH.D.`` to ``John Smith Ph.D.``, and repair never adds or drops a character: ``john smith iii.`` gives ``John Smith III.`` where every release dropped the period. Punctuation in a value only marks which of its letters are joined and is never written into the word, which matters for a lone initial: ``john smith p.h.d.`` gives ``John Smith P.H.D.``, each letter the writer split off from the mask's one run ``PhD`` being an initial, where every release gave ``John Smith Ph.D.``. The shipped map is ``phd`` → ``PhD``, ``bsc`` → ``BSc`` and ``msc`` → ``MSc``, so ``john smith bsc`` gives ``John Smith BSc`` where every release gave ``John Smith Bsc``; ``md``, ``ii``, ``iii`` and ``iv`` left it, a suffix ``md`` now repairing by the acronym repair listed under Additions and a suffix numeral by the numeral repair below. A mask still applies wherever its word stands (``phd smith`` gives ``PhD Smith``), but a word that left the map and was parsed as anything but a suffix repairs as that reading: ``iv smith`` gives ``Iv Smith`` where every release gave ``IV Smith``, and ``Md Abdul Karim`` stays ``Md`` under ``force=True`` where every release gave ``M.D.``. A value that does not spell its key's letters and digits -- ``{"jr": "Junior"}`` -- now raises ``ValueError`` when the ``Lexicon`` is built, and at the first parse for a v1 ``Constants``, where the raise names the fix in v1's own spelling (``constants.capitalization_exceptions['jr'] = 'JR'``) rather than the ``Lexicon()`` constructor call the same check offers a 2.0 caller; a value may still carry its own punctuation (``{"md": "M.D."}`` is accepted, and repairs ``md`` to ``MD``). Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 11 move on the default ``capitalize()`` path and 78 under ``force=True``; no role field moves. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #459) + - **Change case repair's exceptions map from replacement spellings to case masks, so md repairs to MD and phd to PhD.** ``HumanName("john smith phd").capitalize()`` gives ``John Smith PhD`` and ``john smith md`` gives ``John Smith MD``, where every release from 1.4.0 through 2.3.0 gave ``John Smith Ph.D.`` and ``John Smith M.D.``. A ``capitalization_exceptions`` value is now the key's own letters and digits in the case each should take, laid over the word as it was written, so the one ``phd`` entry repairs ``ph.d.`` to ``Ph.D.`` and ``JOHN SMITH PH.D.`` to ``John Smith Ph.D.``, and repair never adds or drops a character: ``john smith iii.`` gives ``John Smith III.`` where every release dropped the period. Punctuation in a value only marks which of its letters are joined and is never written into the word, which matters for a lone initial: ``john smith p.h.d.`` gives ``John Smith P.H.D.``, each letter the writer split off from the mask's one run ``PhD`` being an initial, where every release gave ``John Smith Ph.D.``. The shipped map holds ``phd`` → ``PhD``, ``bsc`` → ``BSc`` and ``msc`` → ``MSc`` and fifteen more of the listed post-nominals whose usual spelling is mixed case -- ``DSc``, ``PsyD``, ``PharmD``, ``MDiv``, ``ThD`` and ``Bt`` among them -- so ``john smith bsc`` gives ``John Smith BSc`` and ``john smith psyd`` gives ``John Smith PsyD``, where every release gave ``John Smith Bsc`` and ``John Smith Psyd``; ``meng`` and ``edd`` get no mask, since a mask applies wherever its word stands and both are also names (``Meng Li``, ``Edd Smith``); ``md``, ``ii``, ``iii`` and ``iv`` left it, a suffix ``md`` now repairing by the acronym repair listed under Additions and a suffix numeral by the numeral repair below. A mask still applies wherever its word stands (``phd smith`` gives ``PhD Smith``), but a word that left the map and was parsed as anything but a suffix repairs as that reading: ``iv smith`` gives ``Iv Smith`` where every release gave ``IV Smith``, and ``Md Abdul Karim`` stays ``Md`` under ``force=True`` where every release gave ``M.D.``. A value that does not spell its key's letters and digits -- ``{"jr": "Junior"}`` -- now raises ``ValueError`` when the ``Lexicon`` is built, and at the first parse for a v1 ``Constants``, where the raise names the fix in v1's own spelling (``constants.capitalization_exceptions['jr'] = 'JR'``) rather than the ``Lexicon()`` constructor call the same check offers a 2.0 caller; a value may still carry its own punctuation (``{"md": "M.D."}`` is accepted, and repairs ``md`` to ``MD``). Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 11 move on the default ``capitalize()`` path and 78 under ``force=True``; no role field moves. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #459) - - **Fix case repair treating a suffix written in capitals as evidence that the whole name was cased on purpose.** ``HumanName("juan garcia III").capitalize()`` gives ``Juan Garcia III``, where every release from 1.4.0 through 2.3.0 returned it untouched -- v1's test for it had been a known failure since 2012 (the Google Code tracker's issue 22) -- and ``juan garcia PhD`` and ``JUAN GARCIA Jr.`` repair the same way. Repair still acts only on a name written wholly in one case, but the suffixes are left out of that test now: a credential or a generation written the way one is written says nothing about how the writer cased the name. A title still counts, so ``Dr. juan garcia`` is left alone, and so does every other word, nicknames and maiden names included (``Juan garcia III`` and ``jane doe nee SMITH III`` are left alone too). The parser's own reading of a name's case is unchanged and still counts the suffix, so the two can differ: ``john e jones III`` gives ``John e Jones III``, the capitals making the ``e`` a connective to the parser, where ``john e jones iii`` gives ``John E Jones III``. Because the test follows the parser's suffix reading, ``jack MA`` gives ``Jack MA`` and ``MD, PhD`` gives ``Md PhD``, where 2.3.0 left both untouched. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), the new test admits 17 and 12 of them move on the default path; ``force=True`` is unchanged. The recipe is the ``R5`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #492) + - **Fix case repair treating a suffix written in capitals as evidence that the whole name was cased on purpose.** ``HumanName("juan garcia III").capitalize()`` gives ``Juan Garcia III``, where every release from 1.4.0 through 2.3.0 returned it untouched -- v1's test for it had been a known failure since 2012 (the Google Code tracker's issue 22) -- and ``juan garcia PhD`` and ``JUAN GARCIA Jr.`` repair the same way. Repair still acts only on a name written wholly in one case, but the suffixes are left out of that test now: a credential or a generation written the way one is written says nothing about how the writer cased the name. A title still counts, so ``Dr. juan garcia`` is left alone, and so does every other word, nicknames and maiden names included (``Juan garcia III`` and ``jane doe nee SMITH III`` are left alone too). A suffix written in more than one case is the writer's spelling and is kept as written: ``john smith EdD`` gives ``John Smith EdD`` and ``juan garcia PsyD`` gives ``Juan Garcia PsyD``, and so does a garbled one, ``juan garcia Iii`` giving ``Juan Garcia Iii``; ``force=True`` repairs it (``John Smith EDD``, ``Juan Garcia III``), and every release returned those names untouched. The parser's own reading of a name's case is unchanged and still counts the suffix, so the two can differ: ``john e jones III`` gives ``John e Jones III``, the capitals making the ``e`` a connective to the parser, where ``john e jones iii`` gives ``John E Jones III``. Because the test follows the parser's suffix reading, ``jack MA`` gives ``Jack MA`` and ``MD, PhD`` gives ``Md PhD``, where 2.3.0 left both untouched. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), the new test admits 17 and 12 of them move on the default path; ``force=True`` is unchanged. The recipe is the ``R5`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (closes #492) - - **Fix case repair capitalizing the connective inside a hyphenated compound surname.** ``HumanName("jose ortega-y-gasset").capitalize()`` gives ``Jose Ortega-y-Gasset`` and ``maria silva-e-sousa`` gives ``Maria Silva-e-Sousa``, the lowercase connective 1.4.0, 2.0.0 and 2.1.0 gave, where 2.2.0 and 2.3.0 gave ``Jose Ortega-Y-Gasset`` and ``Maria Silva-E-Sousa`` -- so this restores 1.4.0's answer after a 2.2 regression. ``JOSE ORTEGA-Y-GASSET`` gives the same ``Jose Ortega-y-Gasset``, where every release gave ``Jose Ortega-Y-Gasset``. A connective with a part on each side of it inside one hyphenated word keeps its lowercase, as the spaced spelling always has, in every role; at either end of the word it is ordinary name text, so ``juan e-f smith`` still gives ``Juan E-F Smith`` and ``juan y-garcia`` gives ``Juan Y-Garcia``. A single letter marked with a period is an initial there, never the connective, so ``j.-e.-p. dupont`` keeps ``J.-E.-P. Dupont``. Only connective vocabulary is read, so the Māori ``Te Awanui-a-Rangi Black`` still repairs to ``Te Awanui-A-Rangi Black`` under ``force=True``. The shape is rare: the only differential-corpus names carrying it are the rules document's own examples (three move on the default path, four under ``force=True``). See the ``R4`` entry of ``docs/design/decisions.md`` (closes #478) + - **Fix case repair capitalizing the connective inside a hyphenated compound surname.** ``HumanName("jose ortega-y-gasset").capitalize()`` gives ``Jose Ortega-y-Gasset`` and ``maria silva-e-sousa`` gives ``Maria Silva-e-Sousa``, the lowercase connective 1.4.0, 2.0.0 and 2.1.0 gave, where 2.2.0 and 2.3.0 gave ``Jose Ortega-Y-Gasset`` and ``Maria Silva-E-Sousa`` -- so this restores 1.4.0's answer after a 2.2 regression. ``JOSE ORTEGA-Y-GASSET`` gives the same ``Jose Ortega-y-Gasset``, where every release gave ``Jose Ortega-Y-Gasset``. A connective with a part on each side of it inside one hyphenated word keeps its lowercase, as the spaced spelling always has, in every role; at either end of the word it is ordinary name text, so ``juan e-f smith`` still gives ``Juan E-F Smith`` and ``juan y-garcia`` gives ``Juan Y-Garcia``. A single letter marked with a period is an initial there, never the connective, so ``j.-e.-p. dupont`` keeps ``J.-E.-P. Dupont``. The hyphen is read as the writer's join even in a name written wholly in one case, where the same letter spaced reads as an initial, so a bare hyphenated initial that spells a connective is lowered -- ``J-E-P DUPONT`` gives ``J-e-P Dupont`` where every release gave ``J-E-P Dupont``, a recorded boundary -- while the spaced ``maria silva e sousa`` gives ``Maria Silva E Sousa``. Only connective vocabulary is read, so the Māori ``Te Awanui-a-Rangi Black`` still repairs to ``Te Awanui-A-Rangi Black`` under ``force=True``. The shape is rare: none of the 1340 names in the differential corpora at the commit before this change (2026-09-23) carries it, the only movers being the rules document's own example lines. See the ``R4`` entry of ``docs/design/decisions.md`` (closes #478) - - **Change case repair to write an unlisted dotted credential and a roman numeral past iv in capitals.** ``HumanName("john smith x.y.z.").capitalize()`` gives ``John Smith X.Y.Z.`` where every release gave ``John Smith X.y.z.``, the dotted word being a suffix now (the ``unlisted_dotted_suffixes`` change above) and repaired as a listed acronym is; and ``john smith vi`` gives ``John Smith VI`` where every release gave ``John Smith Vi``, with ``vii``, ``viii`` and ``ix`` alike. Both are keyed on the suffix role: ``Jack X.Y.Z.``, which keeps its surname, still repairs as a name word (``Jack X.y.z.`` under ``force=True``), and ``john smith xi`` still gives ``John Smith Xi``, the parser reading ``xi`` as the surname. An unlisted dotted credential written in mixed case reads all capitals once repair acts on it -- ``john smith B.Tech.`` gives ``John Smith B.TECH.`` -- which a ``capitalization_exceptions`` mask such as ``{"btech": "BTech"}`` undoes. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 1 moves on the default path and 14 under ``force=True``. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (#459) + - **Change case repair to write an unlisted dotted credential and a roman numeral past iv in capitals.** ``HumanName("john smith x.y.z.").capitalize()`` gives ``John Smith X.Y.Z.`` where every release gave ``John Smith X.y.z.``, the dotted word being a suffix now (the ``unlisted_dotted_suffixes`` change above) and repaired as a listed acronym is; and ``john smith vi`` gives ``John Smith VI`` where every release gave ``John Smith Vi``, with ``vii``, ``viii`` and ``ix`` alike. Both are keyed on the suffix role: ``Jack X.Y.Z.``, which keeps its surname, still repairs as a name word (``Jack X.y.z.`` under ``force=True``), and ``john smith xi`` still gives ``John Smith Xi``, the parser reading ``xi`` as the surname. An unlisted dotted credential written in mixed case is kept as written on the default path, by the suffix change above (``john smith B.Tech.`` gives ``John Smith B.Tech.``), and reads all capitals under ``force=True`` (``John Smith B.TECH.``, where every release gave ``John Smith B.tech.``), which a ``capitalization_exceptions`` mask such as ``{"btech": "BTech"}`` undoes. Over the 1340 names in the differential corpora at the commit before this change (2026-09-23), 1 moves on the default path and 14 under ``force=True``. The recipe is the ``R4`` entry's 2026-09-23 MEASURED bullet in ``docs/design/decisions.md`` (#459) **Additions** diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index a1877501..ec0035a7 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -228,25 +228,34 @@ def is_one_case(texts: Sequence[str]) -> bool: reading of words that did not change. Mirrors the SHAPE of the comparison the R5 gate in - `_render.capitalized` makes, not its SPAN: R5 joins every token, - nickname and maiden content included, while classify's caller hands - in only the name's own words (rules.md#P3's own-words doctrine, see - above) -- so a clause-bearing name can be one-case to this function - and mixed to R5 (measured: `'JUAN GARCIA Y LOPEZ née Jones'` is one - case here, mixed there). Not shared by import today -- render is a - layer this module does not reach into, and #492 is where the two - spans are reconciled if they ever need to be. + `_render.capitalized` makes, not its SPAN: R5 joins every token but + the suffixes, nickname and maiden content included, while + classify's caller hands in only the name's own words (rules.md#P3's + own-words doctrine, see above) -- so a clause-bearing name can be + one-case to this function and mixed to R5 (measured: + `'JUAN GARCIA Y LOPEZ née Jones'` is one case here, mixed there), + and a name with a cased suffix mixed here and one-case there + (`'john e jones III'`). Not shared by import today -- render is a + layer this module does not reach into -- and #492 left the two + spans apart by decision (decisions.md#R5, "THE R5 / PARSER SPLIT"). `Sequence`, not `Iterable`: the caller passes a list it already built rather than a fresh generator, so `is_one_case` costs one profiler frame per parse rather than one per token (#475). A CASELESS script answers True, harmlessly: `'محمد و علي'.upper()` - is the string itself, so the comparison holds, and the only caller - also requires a token whose own `upper()` and `lower()` differ -- - which a caseless letter's never do. So a caseless name never - reaches the decision this gates, and "one case" is the honest - verdict for text that has only one. + is the string itself, so the comparison holds. There are two + callers, classify and -- where a comma form asks -- segment, and + both record the answer as `ParseState.one_case`; the decisions + that read it each also require the token they judge to be CASED: + classify's connective-or-initial fork asks for a token whose own + `upper()` and `lower()` differ, `ambiguous_lean` -- which the + suffix readers in `_pieces`, `_group` and `_assign` reach through + `listed_lean` -- returns no lean for a caseless token, and + `caps_shape_candidate` requires + `isupper()`. A caseless letter is none of those, so a caseless + name never reaches a decision this gates, and "one case" is the + honest verdict for text that has only one. """ joined = " ".join(texts) return joined in (joined.upper(), joined.lower()) diff --git a/nameparser/_render.py b/nameparser/_render.py index 6ce0d9f5..20c5cfd1 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -490,6 +490,12 @@ def cap(match: re.Match[str]) -> str: for at, part in enumerate(parts)) +def _in_one_case(text: str) -> bool: + """R5's test: `text` is written all upper or all lower (a caseless + text is both, and so passes).""" + return text in (text.upper(), text.lower()) + + # rules.md#R4: "case repair returns a repaired copy and never mutates # the parse" def capitalized(name: ParsedName, lexicon: Lexicon | None, *, @@ -499,7 +505,11 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, written in one case is touched unless force=True; the gate reads the joined texts of every token not roled SUFFIX (#492) -- not render() output, so it stays decoupled from spec formatting and - the #254 collapse. + the #254 collapse. The same one-case test is then asked of each + SUFFIX token on its own, and unless force=True one written in more + than one case is kept as written -- 'EdD', 'B.Tech.', and the + garbled 'Iii' alike -- while one written in a single case is + repaired like any other token. Repair changes case and nothing else (see the rules.md#R4 citation in _cap_word): an exceptions-map value is a mask recasing the word as written (#459), never a replacement. @@ -555,10 +565,21 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, # titles that showed why). gate = " ".join(t.text for t in name.tokens if t.role is not Role.SUFFIX) - if not force and gate not in (gate.upper(), gate.lower()): + if not force and not _in_one_case(gate): return name + # rules.md#R5: "a suffix written in more than one case is the + # writer's spelling and is kept as written where repair was not + # forced" -- the gate's own test, asked of each suffix token. The + # gate leaves the suffixes out because a cased one says nothing + # about the NAME; read to its end, that also means repair has no + # business re-spelling it ('EdD' stays 'EdD', where the acronym + # clause would write 'EDD'). The cost is the garbled spelling kept + # with the deliberate one ('Iii' stays 'Iii'); force repairs both. new_tokens = tuple( - Token(_cap_text(t.text, t.role, t.tags, lex), t.span, t.role, t.tags) + t if (not force and t.role is Role.SUFFIX + and not _in_one_case(t.text)) + else Token(_cap_text(t.text, t.role, t.tags, lex), + t.span, t.role, t.tags) for t in name.tokens) # equal tokens (possible only for synthetic span=None duplicates) # collapse to one mapping entry -- benign: the rebuilt ambiguity diff --git a/nameparser/_types.py b/nameparser/_types.py index 80e248d0..c1a468cf 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -1124,7 +1124,9 @@ def capitalized(self, lexicon: Lexicon | None = None, *, particle rules; None uses the DEFAULT lexicon -- if this name came from a custom Parser, pass its lexicon or use Parser.capitalized. force=False preserves a name whose words - outside the suffix are written in more than one case - (rules.md#R5). Idempotent.""" + outside the suffix are written in more than one case, and in a + name it does repair keeps a suffix written in more than one + case as written ('EdD'), repairing only one written in a + single case (rules.md#R5). Idempotent.""" import nameparser._render as _render return _render.capitalized(self, lexicon, force=force) diff --git a/nameparser/config/capitalization.py b/nameparser/config/capitalization.py index fe63021e..e63b38d0 100644 --- a/nameparser/config/capitalization.py +++ b/nameparser/config/capitalization.py @@ -4,6 +4,27 @@ 'bsc': 'BSc', 'msc': 'MSc', 'phd': 'PhD', + # Listed post-nominals whose conventional spelling is mixed case + # (#459): without a mask the acronym repair writes each in + # capitals. A mask applies wherever its word stands, so a word + # borne as a name must not get one -- meng and edd are listed + # acronyms that are, and stay out on purpose (decisions.md#R4's + # Excluded block). + 'bt': 'Bt', + 'chfc': 'ChFC', + 'cpht': 'CPhT', + 'diplac': 'DiplAc', + 'dmin': 'DMin', + 'drph': 'DrPH', + 'dsc': 'DSc', + 'kt': 'Kt', + 'mdiv': 'MDiv', + 'pharmd': 'PharmD', + 'phc': 'PhC', + 'psyd': 'PsyD', + 'rph': 'RPh', + 'thd': 'ThD', + 'thm': 'ThM', } """ Words whose case ``str.capitalize()`` gets wrong, each mapped to a @@ -18,16 +39,25 @@ a ``Lexicon`` is built from it. An acronym the suffix vocabulary already lists, written plainly or in dotted form, and a roman numeral need no entry: case repair writes a suffix of either kind in -capitals by itself. A caller's OWN acronym -- one absent from -``suffix_acronyms`` -- depends on how it is written. Written -PLAINLY (``dphil``) it parses as an ordinary name word and repairs -as one (``Dphil``). Written DOTTED (``d.phil.``) it is a suffix by -shape alone, with no vocabulary entry needed to read it as one, and -repairs in all capitals the same as a listed acronym does -(``D.PHIL.``). Either way, a caller who wants a specific spelling -needs a ``suffix_acronyms`` entry (``Lexicon.add(suffix_acronyms= -{...})`` in the 2.0 API, ``constants.suffix_acronyms.add(...)`` in -the v1 one) or a mask of its own. +capitals by itself. The map carries most of the listed acronyms +whose conventional spelling is NOT all capitals (``DSc``, ``PsyD``, +``PharmD``), so ``psyd`` repairs to ``PsyD`` rather than ``PSYD``. +The exception is an acronym that is also a name word (``meng``, +``edd``): a mask applies in every role, so it gets none, and +``john smith edd`` repairs to ``EDD`` (decisions.md#R4's Excluded +block for CAPITALIZATION_EXCEPTIONS -- meng, edd, lac, ded). A +suffix the writer spelled in more than one case (``EdD``) needs no +entry either: repair keeps it as written unless forced. A caller's +OWN acronym -- one absent from ``suffix_acronyms`` -- depends on how +it is written. Written PLAINLY (``dphil``) it parses as an ordinary +name word and repairs as one (``Dphil``). Written DOTTED +(``d.phil.``) it is a suffix by shape alone, with no vocabulary +entry needed to read it as one, and repairs in all capitals the same +as a listed acronym does (``D.PHIL.``). Either way, a caller who +wants a specific spelling needs a ``suffix_acronyms`` entry +(``Lexicon.add(suffix_acronyms={...})`` in the 2.0 API, +``constants.suffix_acronyms.add(...)`` in the v1 one) or a mask of +its own. """ diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index a5965e90..8ead3862 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -155,6 +155,10 @@ def test_capitalize_exceptions_still_win_over_acronyms(self) -> None: ('john smith ph.d.', 'John Smith Ph.D.'), ('john smith bsc', 'John Smith BSc'), ('JOHN SMITH MSC', 'John Smith MSc'), + # the conventionally mixed-case acronyms given masks on + # 2026-09-24; without one, the acronym clause gave 'PSYD' + ('john smith psyd', 'John Smith PsyD'), + ('JOHN SMITH PHARMD', 'John Smith PharmD'), ]: hn = HumanName(src) hn.capitalize() diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index cbacbd03..cb693ea6 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -596,8 +596,11 @@ def test_the_gate_leaves_the_suffixes_out() -> None: # a caseless name with a MIXED-CASE suffix: the old gate counted # the suffix, read the whole joined text as mixed case and # refused; the new gate excludes it, and the non-suffix text has - # no case at all (trivially one-case), so only the suffix repairs - assert str(parse("김민준 Phd").capitalized()) == "민준 김 PhD" + # no case at all (trivially one-case), so the name is admitted -- + # and then the suffix, written in more than one case, is kept as + # written (2026-09-24), while one written in one case repairs + assert str(parse("김민준 Phd").capitalized()) == "민준 김 Phd" + assert str(parse("김민준 PHD").capitalized()) == "민준 김 PhD" # no non-suffix token at all -- a synthetic name only, since every # parse names somebody (rules.md#H4): the empty text is one-case only_suffixes = _pn("phd md", [ @@ -608,6 +611,46 @@ def test_the_gate_leaves_the_suffixes_out() -> None: == ["PhD", "MD"] +def test_a_suffix_written_in_more_than_one_case_is_kept_as_written( +) -> None: + """rules.md#R5 (decided 2026-09-24): the gate leaves the suffixes + out because a cased suffix says nothing about the NAME, and read + to its end that also means repair has no business re-spelling + one. So where repair was not forced, a SUFFIX token written in + more than one case is kept as written -- 'EdD' stays 'EdD' where + the acronym clause would write 'EDD' -- while one written in a + single case is repaired as before. Force repairs both. The cost is + the garbled spelling kept with the deliberate one ('Iii'), pinned + here as a boundary. Both surfaces.""" + for text, plain, forced in ( + ("john smith EdD", "John Smith EdD", "John Smith EDD"), + ("JANE DOE, DSc", "Jane Doe DSc", "Jane Doe DSc"), + ("juan garcia PsyD", "Juan Garcia PsyD", "Juan Garcia PsyD"), + ("john smith B.Tech.", "John Smith B.Tech.", + "John Smith B.TECH."), + ("john smith, EdD, PhD", "John Smith EdD, PhD", + "John Smith EDD, PhD"), + # boundary: the garbled spelling is kept too + ("juan garcia Iii", "Juan Garcia Iii", "Juan Garcia III"), + ("john smith Mba", "John Smith Mba", "John Smith MBA"), + # a suffix written in ONE case is repaired as any token is + ("juan garcia III", "Juan Garcia III", "Juan Garcia III"), + ("JUAN GARCIA iii", "Juan Garcia III", "Juan Garcia III"), + ("john smith edd", "John Smith EDD", "John Smith EDD"), + # unchanged by the rule: already written as repair writes + ("juan garcia PhD", "Juan Garcia PhD", "Juan Garcia PhD"), + ("juan garcia Jr.", "Juan Garcia Jr.", "Juan Garcia Jr.")): + name = parse(text) + assert str(name.capitalized()) == plain, text + assert str(name.capitalized(force=True)) == forced, text + hn = HumanName(text) + hn.capitalize() + assert str(hn) == plain, text + hn = HumanName(text) + hn.capitalize(force=True) + assert str(hn) == forced, text + + def test_the_gate_and_the_parser_read_a_cased_suffix_differently() -> None: """decisions.md#R5's split: the parser's own one-case readings (rules.md#P3, #S2) still count a cased suffix, while R5's gate @@ -1115,6 +1158,82 @@ def test_a_link_inside_a_hyphenated_word_keeps_its_lowercase() -> None: force=True)) == "Jose Ortega-y-Gasset Smith" +def test_the_hyphen_is_the_writers_join_even_in_a_one_case_name() -> None: + """rules.md#R4's hyphen clause against rules.md#P3's one-case fork, + a split DECIDED 2026-09-24 (decisions.md#R4): spaced, a marked + letter in a name written in one case reads as an initial and + repairs to a capital; hyphenated, the writer joined the surname on + purpose, so the interior word is the connective whatever case the + name is in. The cost is a one-case name whose hyphenated bare + initials spell a connective -- 'J-E-P DUPONT' repairs to + 'J-e-P Dupont', where 1.4.0 and the parent gave 'J-E-P'. And + conjunctions_ambiguous, P3's knob, does not reach a hyphenated + word: marking 'y' moves the spaced spelling only.""" + for text, repaired in (("J-E-P DUPONT", "J-e-P Dupont"), + ("JOHN A-Y-B SMITH", "John A-y-B Smith"), + ("maria silva-e-sousa", "Maria Silva-e-Sousa"), + ("maria silva e sousa", "Maria Silva E Sousa")): + assert str(parse(text).capitalized()) == repaired, text + hn = HumanName(text) + hn.capitalize() + assert str(hn) == repaired, text + marked = Parser(lexicon=Lexicon.default().add( + conjunctions_ambiguous={"y"})) + for text, default, under_mark in ( + ("JOSE ORTEGA Y GASSET", "Jose Ortega y Gasset", + "Jose Ortega Y Gasset"), + ("JOSE ORTEGA-Y-GASSET", "Jose Ortega-y-Gasset", + "Jose Ortega-y-Gasset")): + assert str(parse(text).capitalized()) == default, text + assert str(marked.capitalized(marked.parse(text))) == under_mark, \ + text + + +def test_a_shipped_mask_spells_its_suffix_in_either_single_case() -> None: + """#459 (decisions.md#R4, 2026-09-24): the PROPERTY every shipped + mask pair serves, derived from the pairs rather than restating + them -- `john smith ` puts the key in the suffix role, and + repair spells it as the mask whether it was written all lower or + all upper, where the acronym clause alone would write it in + capitals. Both surfaces.""" + pairs = tuple(Lexicon.default().capitalization_exceptions) + assert pairs # an empty map would make the loop vacuous + for key, mask in pairs: + for text in (f"john smith {key}", f"JOHN SMITH {key.upper()}"): + name = parse(text) + assert name.suffix.lower() == key, text + assert name.capitalized().suffix == mask, text + hn = HumanName(text) + hn.capitalize() + assert hn.suffix == mask, text + + +def test_a_listed_acronym_that_is_a_name_word_gets_no_mask() -> None: + """decisions.md#R4's Excluded block for CAPITALIZATION_EXCEPTIONS + (meng, edd, lac, ded): a mask applies in every role, so an acronym + that is also a name word must not carry one. The fork it protects: + in a NAME role the word repairs as a title-cased name word, and in + the suffix role, with no mask, the acronym clause writes it in + capitals. The recorded negative control is the mask added back, + which re-spells the person.""" + for text, repaired in (("MENG LI", "Meng Li"), + ("edd smith", "Edd Smith"), + ("john smith meng", "John Smith MENG"), + ("john smith edd", "John Smith EDD")): + assert str(parse(text).capitalized()) == repaired, text + hn = HumanName(text) + hn.capitalize() + assert str(hn) == repaired, text + default = Lexicon.default() + masked = Parser(lexicon=dataclasses.replace( + default, capitalization_exceptions=tuple( + default.capitalization_exceptions) + + (("meng", "MEng"), ("edd", "EdD")))) + for text, respelled in (("MENG LI", "MEng Li"), + ("edd smith", "EdD Smith")): + assert str(masked.capitalized(masked.parse(text))) == respelled + + def test_case_repair_falls_back_for_text_the_parse_never_read() -> None: """A token carrying UNCLASSIFIED_TAG holds raw text no parse read, so there is no decision to honor and case repair -- which is handed diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index e85bfac6..7088c901 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -81,6 +81,7 @@ "Her Royal Highness Princess Anne" "His Excellency Lord Duncan" "II Van Johnson" +"J-E-P DUPONT" "J. Smith" "J. née Jones Smith V" "J.R. Smith" @@ -341,6 +342,7 @@ "j.-e.-p. dupont" "john doe q.c." "john e smith" +"john smith EdD" "john smith bsc" "john smith ed" "john smith iii" @@ -355,6 +357,7 @@ "john smith ph. d." "john smith ph.d." "john smith phd" +"john smith psyd" "john smith vi" "john smith x.y.z." "john smith xi" @@ -366,6 +369,7 @@ "juan de la vega" "juan e-f smith" "juan garcia III" +"juan garcia Iii" "juan garcia PhD" "juan garcia y lopez" "juan mcdonald"