OCR & Computer Vision

The Story That Explains OCR

OCR (Optical Character Recognition) is a branch of Computer Vision that converts images of typed, handwritten, or printed text into machine-readable text. It is one of the oldest and most commercially impactful AI fields — powering bank cheque readers, passport scanners, Google Books, invoice automation, and real-time translation apps used by billions of people.

The OCR Pipeline — How It Works End to End

Modern OCR is not a single algorithm — it is a pipeline of stages, each transforming the image progressively from raw pixels to clean, structured text.

OCR Approaches — Three Generations of Technology

Pre-processing — The Most Important Step

import cv2
import numpy as np
from PIL import Image

# ── Load image ────────────────────────────────────────────
img = cv2.imread('invoice.jpg')

# ── Step 1: Convert to grayscale ──────────────────────────
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# ── Step 2: Denoise (preserve edges with fastNlMeans) ─────
denoised = cv2.fastNlMeansDenoising(gray, h=10, templateWindowSize=7, searchWindowSize=21)

# ── Step 3: Adaptive binarisation (handles shadows) ───────
binary = cv2.adaptiveThreshold(
    denoised, 255,
    cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
    cv2.THRESH_BINARY,
    blockSize=11,
    C=2
)

# ── Step 4: Deskew using Hough transform ──────────────────
def deskew(image):
    coords = np.column_stack(np.where(image < 127))
    angle  = cv2.minAreaRect(coords)[-1]
    if angle < -45:
        angle = -(90 + angle)
    else:
        angle = -angle
    h, w  = image.shape
    centre = (w // 2, h // 2)
    M     = cv2.getRotationMatrix2D(centre, angle, 1.0)
    return cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)

deskewed = deskew(binary)

# ── Step 5: Morphological closing to fill stroke gaps ─────
kernel  = cv2.getStructuringElement(cv2.MORPH_RECT, (1, 1))
cleaned = cv2.morphologyEx(deskewed, cv2.MORPH_CLOSE, kernel)

cv2.imwrite('preprocessed.png', cleaned)
print("Pre-processing complete. Image ready for OCR.")

Tesseract OCR — The Industry Open-Source Standard

Tesseract is a free, open-source OCR engine originally developed by HP in the 1980s, then open-sourced by Google in 2006. Version 4+ replaced the old pipeline with an LSTM-based recognition engine and now rivals commercial products for clean printed text.

import pytesseract
import cv2
from PIL import Image
import pandas as pd

# ── Basic extraction ───────────────────────────────────────
img  = cv2.imread('preprocessed.png')
text = pytesseract.image_to_string(img, lang='eng', config='--psm 6')
print("Extracted Text:\n", text)

# ── Extract with bounding boxes (TSV) ─────────────────────
data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DATAFRAME)

# Filter confident words only (confidence > 60)
confident = data[data['conf'] > 60].copy()
print(confident[['text', 'conf', 'left', 'top', 'width', 'height']].head(10))

# ── Draw bounding boxes on image ──────────────────────────
import cv2
result = img.copy()
for _, row in confident.iterrows():
    x, y, w, h = int(row['left']), int(row['top']), int(row['width']), int(row['height'])
    cv2.rectangle(result, (x, y), (x+w, y+h), (0, 255, 0), 2)

cv2.imwrite('annotated.png', result)
print(f"Found {len(confident)} confident word detections.")

EasyOCR — Deep Learning Out of the Box

EasyOCR is a Python library built on PyTorch that bundles a CRAFT text detector and a CRNN recognition model. It supports 80+ languages, works on GPU/CPU, and handles natural scene text far better than Tesseract out of the box — with no configuration required.

import easyocr
import cv2
import numpy as np

# ── Initialise reader (downloads models on first run) ──────
# Pass ['en','hi'] for bilingual, GPU=True for CUDA
reader = easyocr.Reader(['en'], gpu=False)

# ── Read image ─────────────────────────────────────────────
results = reader.readtext('street_sign.jpg')

# results: list of (bbox, text, confidence)
for (bbox, text, confidence) in results:
    print(f"Text: {text:30s} | Confidence: {confidence:.2%}")

# ── Draw detections on image ───────────────────────────────
img = cv2.imread('street_sign.jpg')
for (bbox, text, conf) in results:
    pts = np.array(bbox, dtype=np.int32)
    cv2.polylines(img, [pts], isClosed=True, color=(0,255,0), thickness=2)
    origin = (int(bbox[0][0]), int(bbox[0][1]) - 5)
    cv2.putText(img, f"{text} ({conf:.0%})", origin,
               cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,200,255), 1)

cv2.imwrite('easyocr_result.jpg', img)

How Neural OCR Works — The CRNN Architecture

The dominant architecture for sequence-based OCR is CRNN — a Convolutional Recurrent Neural Network. It treats a word image as a sequence of visual features to be decoded left-to-right, mirroring how we read.

OCR on Real Documents — Invoice Extraction

import pytesseract
import cv2
import re
from dataclasses import dataclass
from typing import Optional

# ── Data model for extracted invoice fields ────────────────
@dataclass
class InvoiceData:
    invoice_number: Optional[str]
    date:           Optional[str]
    vendor_name:    Optional[str]
    total_amount:   Optional[str]

def extract_invoice(image_path: str) -> InvoiceData:
    img  = cv2.imread(image_path)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)

    # Extract full text with Tesseract
    text = pytesseract.image_to_string(binary, config='--psm 6')

    # Regex-based field extraction
    inv_match    = re.search(r'Invoice\s*[#No.:]+\s*(\w+)',   text, re.IGNORECASE)
    date_match   = re.search(r'\b(\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4})\b', text)
    total_match  = re.search(r'Total[^£$\d]*([\£\$€]?\s?\d[\d,\.]+)', text, re.IGNORECASE)
    vendor_match = re.search(r'From[:\s]+(.+)', text, re.IGNORECASE)

    return InvoiceData(
        invoice_number = inv_match.group(1)    if inv_match    else None,
        date           = date_match.group(1)   if date_match   else None,
        total_amount   = total_match.group(1)  if total_match  else None,
        vendor_name    = vendor_match.group(1).strip() if vendor_match else None
    )

result = extract_invoice('invoice_scan.jpg')
print(f"Invoice #:  {result.invoice_number}")
print(f"Date:       {result.date}")
print(f"Vendor:     {result.vendor_name}")
print(f"Total:      {result.total_amount}")

Handwriting Recognition — HTR

Handwriting recognition is dramatically harder than printed OCR. No two people write the same character the same way. Ligatures connect letters ambiguously. Baseline wanders. Word spacing is inconsistent. Modern HTR uses sequence-to-sequence Transformers trained on millions of handwritten samples.

Document Layout Analysis — Beyond Characters

Modern documents are not just text. They contain tables, figures, headings, footnotes, headers, sidebars, and multi-column layouts. Understanding structure is as important as reading characters.

OCR Accuracy — Key Metrics and Benchmarks

PaddleOCR — The State-of-the-Art Open Engine

PaddleOCR by Baidu is currently one of the most accurate open-source OCR engines available. It ships with a detection model (DBNet), a direction classifier, and a recognition model (SVTR or PP-OCRv4), all pre-trained and ready to use in three lines of Python.

from paddleocr import PaddleOCR, draw_ocr
from PIL import Image
import numpy as np

# ── Initialise (downloads ~15MB models on first run) ───────
ocr = PaddleOCR(
    use_angle_cls=True,   # auto-rotate rotated text (upsidedown etc)
    lang='en',            # or 'ch' for Chinese, 'hi' for Hindi
    use_gpu=False
)

# ── Run OCR ────────────────────────────────────────────────
result = ocr.ocr('document.jpg', cls=True)

# ── Print results ──────────────────────────────────────────
for line in result[0]:
    bbox, (text, score) = line
    print(f"[{score:.2%}] {text}")

# ── Visualise with draw_ocr ────────────────────────────────
boxes  = [line[0]      for line in result[0]]
txts   = [line[1][0]   for line in result[0]]
scores = [line[1][1]   for line in result[0]]

img    = Image.open('document.jpg').convert('RGB')
vis    = draw_ocr(img, boxes, txts, scores)
Image.fromarray(vis).save('paddle_result.jpg')
print(f"Total lines detected: {len(result[0])}")

TrOCR — Transformer-Based OCR for Handwriting

TrOCR by Microsoft Research (2021) is a pure Transformer architecture — an image encoder (ViT or BEiT) combined with a text decoder (RoBERTa/GPT-2 style) — trained end-to-end for OCR. It achieves new state-of-the-art results on handwritten text datasets (IAM, RIMES) with no need for explicit character segmentation.

from transformers import TrOCRProcessor, VisionEncoderDecoderModel
from PIL import Image
import torch

# ── Load model ─────────────────────────────────────────────
# Options: trocr-base-printed | trocr-large-printed
#          trocr-base-handwritten | trocr-large-handwritten
processor = TrOCRProcessor.from_pretrained('microsoft/trocr-large-handwritten')
model     = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-large-handwritten')

# ── Load and prepare image ─────────────────────────────────
# Crop to a single line of handwriting first
image       = Image.open('handwritten_line.png').convert('RGB')
pixel_values = processor(images=image, return_tensors='pt').pixel_values

# ── Generate text (beam search with 4 beams) ──────────────
with torch.no_grad():
    generated_ids = model.generate(
        pixel_values,
        num_beams=4,
        max_new_tokens=128
    )

generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(f"Recognised: {generated_text}")

Common OCR Failure Modes

OCR vs. Vision-Language Models — When to Use What

Fine-Tuning TrOCR on Custom Data

Out-of-the-box models struggle with specialist fonts, domain vocabulary (medical, legal), or unusual scripts. Fine-tuning TrOCR on as few as 500 labelled examples can cut error rates by 40–60% in domain-specific tasks.

from transformers import (TrOCRProcessor, VisionEncoderDecoderModel,
                            Seq2SeqTrainer, Seq2SeqTrainingArguments)
from torch.utils.data import Dataset
from PIL import Image
import torch, json

# ── Dataset: JSONL with {"image_path": ..., "text": ...} ──
class OCRDataset(Dataset):
    def __init__(self, jsonl_path, processor, max_length=128):
        self.data      = [json.loads(l) for l in open(jsonl_path)]
        self.processor = processor
        self.max_length = max_length

    def __len__(self):  return len(self.data)

    def __getitem__(self, idx):
        item  = self.data[idx]
        image = Image.open(item['image_path']).convert('RGB')
        enc   = self.processor(
            images=image, text=item['text'],
            return_tensors='pt', padding='max_length',
            max_length=self.max_length, truncation=True
        )
        labels = enc.labels.squeeze()
        labels[labels == self.processor.tokenizer.pad_token_id] = -100
        return {'pixel_values': enc.pixel_values.squeeze(), 'labels': labels}

# ── Load base model ────────────────────────────────────────
MODEL_ID  = 'microsoft/trocr-base-printed'
processor = TrOCRProcessor.from_pretrained(MODEL_ID)
model     = VisionEncoderDecoderModel.from_pretrained(MODEL_ID)

model.config.decoder_start_token_id = processor.tokenizer.cls_token_id
model.config.pad_token_id           = processor.tokenizer.pad_token_id

train_ds = OCRDataset('train.jsonl', processor)
eval_ds  = OCRDataset('eval.jsonl',  processor)

# ── Training arguments ─────────────────────────────────────
args = Seq2SeqTrainingArguments(
    output_dir='./trocr-finetuned',
    num_train_epochs=10,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    predict_with_generate=True,
    fp16=True,
    learning_rate=5e-5,
    evaluation_strategy='epoch',
    save_strategy='best',
    load_best_model_at_end=True,
)

trainer = Seq2SeqTrainer(model=model, args=args, train_dataset=train_ds, eval_dataset=eval_ds)
trainer.train()
print("Fine-tuning complete. Model saved to ./trocr-finetuned")

Golden Rules of Production OCR