Text Normalization & Sentence Segmentation

The Story That Explains Text Normalization

Text Normalization is the collection of preprocessing steps that convert raw, noisy human language into a standardized representation. It is not about losing meaning — it's about removing irrelevant variation so that identical concepts map to identical tokens. Without it, "Hello", "hello", "HELLO" look like three completely different words to a model.

The Full NLP Preprocessing Pipeline

Before diving into each technique, here is where text normalization and sentence segmentation sit in the broader NLP pipeline. Every stage feeds the next — order matters enormously.

Technique 1 — Lowercasing

The simplest and most universally applied normalization step. Every character is converted to its lowercase form. This ensures that "Python", "PYTHON", and "python" are treated as the same word.

import re

# ── 1. Basic lowercasing ─────────────────────────────
text = "The QUICK Brown FOX jumps over THE lazy Dog."
lowered = text.lower()
print(lowered)
# → "the quick brown fox jumps over the lazy dog."

# ── 2. Lowercasing with unicode awareness ────────────
# Python's .lower() handles accented chars correctly
text_fr = "École Nationale SUPÉRIEURE"
print(text_fr.lower())
# → "école nationale supérieure"

# ── 3. When NOT to lowercase ─────────────────────────
# Named Entity Recognition (NER) tasks: "apple" (fruit)
# vs "Apple" (company) — case is semantically meaningful.
# For such tasks, skip or apply case-sensitive models.

def safe_lowercase(text: str, preserve_ner: bool = False) -> str:
    """Lowercase text; optionally skip if NER task."""
    if preserve_ner:
        return text   # keep original case for NER
    return text.lower()

Technique 2 — Contraction Expansion

Contractions are shortened word combinations held together by apostrophes: "don't", "I'm", "they've". Models treat "don't" and "do not" as completely different strings unless you expand them first. Expansion ensures both forms map to the same concept.

import re

# ── Contraction dictionary (extend as needed) ─────────
CONTRACTIONS = {
    "won't": "will not",
    "can't": "cannot",
    "don't": "do not",
    "doesn't": "does not",
    "didn't": "did not",
    "isn't": "is not",
    "aren't": "are not",
    "wasn't": "was not",
    "weren't": "were not",
    "hasn't": "has not",
    "haven't": "have not",
    "hadn't": "had not",
    "i'm": "i am",
    "i've": "i have",
    "i'll": "i will",
    "i'd": "i would",
    "you're": "you are",
    "they've": "they have",
    "it's": "it is",
    "that's": "that is",
    "let's": "let us",
}

def expand_contractions(text: str) -> str:
    """Expand English contractions to their full forms."""
    # Build single regex from all keys for efficiency
    pattern = re.compile(
        '(' + '|'.join(re.escape(k) for k in CONTRACTIONS) + ')',
        re.IGNORECASE
    )
    return pattern.sub(
        lambda m: CONTRACTIONS[m.group(0).lower()],
        text
    )

# ── Example ───────────────────────────────────────────
sample = "I can't believe they've won't cooperate. It's ridiculous."
print(expand_contractions(sample))

Technique 3 — Special Character & Noise Removal

Raw text is littered with characters that carry no linguistic information for most NLP tasks: HTML tags, URLs, email addresses, hashtags, punctuation storms, emoji, and control characters. Removing them reduces noise and vocabulary size dramatically.

import re
import unicodedata

def remove_html_tags(text: str) -> str:
    """Strip all HTML/XML tags."""
    return re.sub(r'<[^>]+>', ' ', text)

def remove_urls(text: str, placeholder: str = '') -> str:
    """Remove URLs, optionally replace with placeholder."""
    url_pattern = re.compile(
        r'https?://\S+|www\.\S+', re.IGNORECASE
    )
    return url_pattern.sub(placeholder, text)

def remove_mentions_hashtags(text: str) -> str:
    """Remove @mentions; strip # but keep hashtag word."""
    text = re.sub(r'@\w+', '', text)       # remove @user
    text = re.sub(r'#(\w+)', r'\1', text)   # #NLP → NLP
    return text

def remove_special_characters(text: str,
                               keep_punct: bool = False) -> str:
    """Remove non-alphanumeric characters."""
    if keep_punct:
        return re.sub(r'[^a-zA-Z0-9\s.,!?\'"-]', '', text)
    return re.sub(r'[^a-zA-Z0-9\s]', '', text)

def normalize_whitespace(text: str) -> str:
    """Collapse multiple spaces, tabs, newlines."""
    return re.sub(r'\s+', ' ', text).strip()

def normalize_unicode(text: str) -> str:
    """Normalize unicode (NFD → NFC, remove diacritics optionally)."""
    return unicodedata.normalize('NFC', text)

# ── Full pipeline example ─────────────────────────────
raw = "<p>Check out https://example.com! @JohnDoe loves #NLP 😊  &&  AI.</p>"

clean = remove_html_tags(raw)
clean = remove_urls(clean)
clean = remove_mentions_hashtags(clean)
clean = remove_special_characters(clean)
clean = normalize_whitespace(clean)
clean = clean.lower()
print(clean)

The Normalization Decision Matrix

Not every normalization step applies to every task. The table below tells you exactly what to apply — and what to skip — for each common NLP use case.

The Full Normalization Pipeline in Code

Here is a production-ready text normalization class that chains all techniques together in the correct order, with full configurability per task.

import re
import unicodedata
from dataclasses import dataclass, field
from typing import List

CONTRACTIONS = {
    "won't": "will not", "can't": "cannot",
    "don't": "do not", "doesn't": "does not",
    "didn't": "did not", "isn't": "is not",
    "i'm": "i am", "i've": "i have",
    "i'll": "i will", "it's": "it is",
    "that's": "that is", "they've": "they have",
}

@dataclass
class NormalizationConfig:
    lowercase: bool         = True
    expand_contractions: bool = True
    remove_html: bool       = True
    remove_urls: bool       = True
    remove_mentions: bool   = True
    remove_punctuation: bool= True
    normalize_unicode: bool = True
    normalize_whitespace: bool = True

class TextNormalizer:
    def __init__(self, config: NormalizationConfig = None):
        self.cfg = config or NormalizationConfig()
        self._contraction_re = re.compile(
            '(' + '|'.join(re.escape(k) for k in CONTRACTIONS) + ')',
            re.IGNORECASE
        )

    def normalize(self, text: str) -> str:
        """Run the full normalization pipeline."""
        if self.cfg.normalize_unicode:
            text = unicodedata.normalize('NFC', text)
        if self.cfg.remove_html:
            text = re.sub(r'<[^>]+>', ' ', text)
        if self.cfg.remove_urls:
            text = re.sub(r'https?://\S+|www\.\S+', '', text, flags=re.I)
        if self.cfg.remove_mentions:
            text = re.sub(r'@\w+', '', text)
            text = re.sub(r'#(\w+)', r'\1', text)
        if self.cfg.lowercase:
            text = text.lower()
        if self.cfg.expand_contractions:
            text = self._contraction_re.sub(
                lambda m: CONTRACTIONS[m.group(0).lower()], text
            )
        if self.cfg.remove_punctuation:
            text = re.sub(r'[^a-z0-9\s]', '', text)
        if self.cfg.normalize_whitespace:
            text = re.sub(r'\s+', ' ', text).strip()
        return text

# ── Usage ─────────────────────────────────────────────
normalizer = TextNormalizer()

samples = [
    "<b>I can't BELIEVE it's this good!</b> Visit https://example.com",
    "@alice They've  won't stop   #MachineLearning",
    "Python isn't hard — it's just different.",
]
for s in samples:
    print(normalizer.normalize(s))

Sentence Segmentation — The Hidden Hard Problem

Sentence Segmentation (also called Sentence Boundary Detection) is the task of splitting a block of text into its constituent sentences. It is the prerequisite for tokenization, parsing, machine translation, and summarization — all of which operate at the sentence level.

The Three Approaches to Sentence Segmentation

Approach 1 — Rule-Based Regex Segmentation

import re

def simple_sent_tokenize(text: str) -> list:
    """
    Naïve rule-based sentence splitter.
    Works for clean text; fails on abbreviations.
    """
    # Split after . ! ? followed by whitespace + uppercase
    pattern = re.compile(r'(?<=[.!?])\s+(?=[A-Z])')
    sentences = pattern.split(text)
    return [s.strip() for s in sentences if s.strip()]

# ── Test it ───────────────────────────────────────────
text1 = "The sun rose over the mountains. Birds began to sing. It was a beautiful morning!"
text2 = "Dr. Smith earned a Ph.D. She now works in Washington, D.C. It is remarkable."

print("=== Clean Text ===")
for i, s in enumerate(simple_sent_tokenize(text1), 1):
    print(f"  [{i}] {s}")

print("\n=== Abbreviation Text (fails) ===")
for i, s in enumerate(simple_sent_tokenize(text2), 1):
    print(f"  [{i}] {s}")

Approach 2 — Statistical Segmentation with NLTK Punkt

NLTK's Punkt algorithm was published by Kiss & Strunk (2006). It uses unsupervised learning to build a list of abbreviations from the training corpus, then uses collocation statistics to decide whether a period ends a sentence. It is the industry standard for lightweight English NLP.

import nltk
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)

from nltk.tokenize import sent_tokenize

# ── Test cases ────────────────────────────────────────
texts = [
    "Dr. Smith earned her Ph.D. at M.I.T. She now works in Washington, D.C.",
    "The price is $3.99. It went up by 0.5%. Buy now!",
    "He said: 'Wait... really?' She nodded. 'Yes, really.'",
    "The U.S.A. is large. Canada is larger. Russia is largest.",
]

for text in texts:
    sentences = sent_tokenize(text)
    print(f"INPUT : {text}")
    print(f"SPLITS: {len(sentences)} sentence(s)")
    for i, s in enumerate(sentences, 1):
        print(f"  [{i}] {s}")
    print()

Approach 3 — Neural Segmentation with spaCy

spaCy's segmenter uses a dependency parser — it understands the grammatical structure of the sentence to find boundaries, not just punctuation patterns. This makes it the most accurate option, especially for complex, informal, or domain-specific text.

import spacy

# Load English model (run once: python -m spacy download en_core_web_sm)
nlp = spacy.load("en_core_web_sm")

# ── Test with complex text ────────────────────────────
complex_text = """
Prof. Johnson (B.Sc., M.D.) published new research today.
The study, conducted at Johns Hopkins Univ., found a 3.7% improvement.
"This is groundbreaking," she said. "We never expected these results."
The FDA (U.S. Food & Drug Admin.) will review findings by Q4 2025.
""".strip()

doc = nlp(complex_text)
sentences = list(doc.sents)

print(f"Total sentences detected: {len(sentences)}")
for i, sent in enumerate(sentences, 1):
    print(f"  [{i}] {sent.text.strip()}")

# ── Additional metadata spaCy provides ───────────────
print("\nSentence start tokens:")
for token in doc:
    if token.is_sent_start:
        print(f"  '{token.text}' (pos={token.pos_})")

Visual Diagram — How Sentence Boundaries Are Decided

The diagram below shows the decision tree a robust sentence segmenter uses when it encounters a period. Each node tests a specific property of the token to determine whether the period terminates a sentence.

Head-to-Head Comparison — All Three Methods

import re
import nltk
import spacy

nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)
from nltk.tokenize import sent_tokenize
nlp = spacy.load("en_core_web_sm")

# ── Benchmark text ────────────────────────────────────
text = ("Dr. Angela Yu (Ph.D., M.D.) presented findings at 9:30 a.m. "
        "The study cost approx. $2.5M. Results showed a 4.7% gain. "
        "She said: 'We didn't expect this.' The FDA, U.S. Dept. of Health, "
        "will review by Jan. 2026. 'Exciting times,' remarked Prof. Lee.")

# Method 1: Regex
regex_sents = re.split(r'(?<=[.!?])\s+(?=[A-Z])', text)

# Method 2: NLTK Punkt
nltk_sents  = sent_tokenize(text)

# Method 3: spaCy neural
spacy_sents = [s.text.strip() for s in nlp(text).sents]

for name, sents in [("REGEX", regex_sents),
                      ("NLTK",  nltk_sents),
                      ("SPACY", spacy_sents)]:
    print(f"── {name} → {len(sents)} segments ──")
    for i, s in enumerate(sents, 1):
        print(f"  [{i}] {s}")
    print()

Complete Production Pipeline — Normalization + Segmentation

Here is the fully integrated pipeline: normalize raw text first, then segment into sentences, ready for tokenization and downstream modelling.

import re
import unicodedata
import nltk
from nltk.tokenize import sent_tokenize
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)

CONTRACTIONS = {
    "won't": "will not", "can't": "cannot",
    "don't": "do not", "isn't": "is not",
    "i'm": "i am", "it's": "it is",
    "they've": "they have", "i'll": "i will",
}

_contraction_re = re.compile(
    '(' + '|'.join(re.escape(k) for k in CONTRACTIONS) + ')',
    re.IGNORECASE
)

def normalize(text: str,
              lowercase: bool = True,
              expand_contractions: bool = True,
              remove_html: bool = True,
              remove_urls: bool = True,
              remove_punct: bool = False) -> str:
    """
    Full normalization pipeline.
    NOTE: remove_punct=False by default so sentence
    segmentation can still use punctuation as signal.
    Always segment BEFORE removing punctuation.
    """
    text = unicodedata.normalize('NFC', text)
    if remove_html:
        text = re.sub(r'<[^>]+>', ' ', text)
    if remove_urls:
        text = re.sub(r'https?://\S+|www\.\S+', '', text, flags=re.I)
    text = re.sub(r'@\w+', '', text)
    text = re.sub(r'#(\w+)', r'\1', text)
    if lowercase:
        text = text.lower()
    if expand_contractions:
        text = _contraction_re.sub(
            lambda m: CONTRACTIONS[m.group(0).lower()], text
        )
    if remove_punct:
        text = re.sub(r'[^a-z0-9\s]', '', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text

def full_pipeline(raw_text: str) -> list[str]:
    """
    Step 1: Normalize (keep punctuation for segmentation)
    Step 2: Segment into sentences
    Step 3: Remove punctuation from each sentence
    Returns: list of clean, normalized sentence strings
    """
    # Step 1 — normalize (preserve punct for segmentation)
    normalized = normalize(raw_text, remove_punct=False)

    # Step 2 — sentence segmentation
    sentences = sent_tokenize(normalized)

    # Step 3 — remove punctuation from each sentence
    clean_sentences = []
    for sent in sentences:
        clean = re.sub(r'[^a-z0-9\s]', '', sent)
        clean = re.sub(r'\s+', ' ', clean).strip()
        if clean:
            clean_sentences.append(clean)
    return clean_sentences

# ── Run the pipeline ──────────────────────────────────
raw = """<p>@alice I can't believe it's true! Dr. Smith
won't confirm — but they've posted at https://example.com.
The result was 99.7% accuracy. Amazing!</p>"""

sentences = full_pipeline(raw)
print(f"Detected {len(sentences)} sentences:")
for i, s in enumerate(sentences, 1):
    print(f"  [{i}] {s}")

Golden Rules