Computer Vision — The Complete Deep Dive

The Story Behind Computer Vision

Computer vision (CV) is a field of artificial intelligence that trains computers to interpret and understand visual data — images, video frames, depth maps, point clouds — extracting structured information from them. It powers face unlock on your phone, quality inspection on factory floors, cancer detection in radiology, and self-driving cars navigating city streets at 60 km/h.

The Visual Cortex — How Humans vs. Machines See

This analogy is not coincidence — early deep learning researchers were inspired by neuroscience. Hubel and Wiesel's Nobel Prize-winning cat-cortex experiments (1981) directly informed Yann LeCun's 1998 convolutional network design. Biology and AI evolved in parallel.

The Image as Data — Pixels, Channels, and Tensors

Before any algorithm runs, you need to understand what an image actually is as data.

import cv2
import numpy as np
from PIL import Image

# Load an image (OpenCV loads as BGR by default)
img_bgr = cv2.imread('photo.jpg')
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)

print(f"Shape: {img_rgb.shape}")     # (H, W, 3)
print(f"Dtype: {img_rgb.dtype}")     # uint8
print(f"Min/Max: {img_rgb.min()}/{img_rgb.max()}")  # 0 / 255

# Normalise for neural network input
img_float = img_rgb.astype(np.float32) / 255.0

# Convert to PyTorch tensor (C, H, W)
import torch
tensor = torch.from_numpy(img_float).permute(2, 0, 1)
print(f"Tensor shape: {tensor.shape}")  # torch.Size([3, H, W])

Core Computer Vision Tasks — The Taxonomy

Computer vision is not a single problem — it is a family of related tasks. Understanding the taxonomy is essential before choosing an architecture or dataset.

The Convolutional Neural Network — Deep Dive

Building a CNN from Scratch in PyTorch

import torch
import torch.nn as nn
import torch.nn.functional as F

class SimpleCNN(nn.Module):
    '''
    A clean 3-block CNN for 32×32 images (e.g. CIFAR-10, 10 classes).
    Architecture: [Conv→BN→ReLU→Pool] × 3 → Flatten → FC → FC
    '''
    def __init__(self, num_classes=10):
        super().__init__()

        # Block 1: 3 → 32 channels, 32×32 → 16×16
        self.block1 = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2)            # 32×32 → 16×16
        )

        # Block 2: 32 → 64 channels, 16×16 → 8×8
        self.block2 = nn.Sequential(
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2)            # 16×16 → 8×8
        )

        # Block 3: 64 → 128 channels, 8×8 → 4×4
        self.block3 = nn.Sequential(
            nn.Conv2d(64, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2)            # 8×8 → 4×4
        )

        # Classifier head
        self.classifier = nn.Sequential(
            nn.Flatten(),              # 128×4×4 = 2048
            nn.Linear(2048, 512),
            nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(512, num_classes)
        )

    def forward(self, x):
        x = self.block1(x)
        x = self.block2(x)
        x = self.block3(x)
        return self.classifier(x)

# Sanity-check with a random batch
model = SimpleCNN(num_classes=10)
dummy = torch.randn(4, 3, 32, 32)   # batch of 4 CIFAR images
logits = model(dummy)
print(f"Output shape: {logits.shape}")   # (4, 10)

total_params = sum(p.numel() for p in model.parameters())
print(f"Total parameters: {total_params:,}")

Landmark CNN Architectures — The Evolution

Transfer Learning — The Greatest Productivity Hack in Vision

import torch
import torch.nn as nn
from torchvision import models, transforms
from torch.utils.data import DataLoader, ImageFolder

# ── 1. Load pretrained ResNet50 ────────────────────────────────
backbone = models.resnet50(weights='IMAGENET1K_V2')

# ── 2. Freeze ALL backbone parameters ─────────────────────────
for param in backbone.parameters():
    param.requires_grad = False

# ── 3. Replace the final FC layer for your task (5 classes) ───
in_feats = backbone.fc.in_features           # 2048
backbone.fc = nn.Sequential(
    nn.Dropout(0.4),
    nn.Linear(in_feats, 256),
    nn.ReLU(),
    nn.Linear(256, 5)                   # 5 custom classes
)

# Only the new head has gradients
trainable = sum(p.numel() for p in backbone.parameters() if p.requires_grad)
print(f"Trainable params: {trainable:,}")   # ~527,621

# ── 4. ImageNet normalisation for pretrained models ────────────
transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225]
    )
])

# ── 5. Training loop ──────────────────────────────────────────
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = backbone.to(device)
optimiser = torch.optim.Adam(model.parameters(), lr=3e-4)
criterion = nn.CrossEntropyLoss()

# Assume train_loader is a DataLoader over your image folder
model.train()
for epoch in range(10):
    for imgs, labels in train_loader:
        imgs, labels = imgs.to(device), labels.to(device)
        optimiser.zero_grad()
        loss = criterion(model(imgs), labels)
        loss.backward()
        optimiser.step()

Data Augmentation — Making One Image into Many

A model that only ever sees one orientation of a cat will fail on an upside-down cat. Data augmentation artificially expands your training set by applying random transformations at training time. At inference time, no augmentation is applied — only normalisation.

from torchvision.transforms import v2

# Modern torchvision v2 augmentation pipeline
train_transform = v2.Compose([
    v2.RandomResizedCrop(224, scale=(0.6, 1.0)),
    v2.RandomHorizontalFlip(p=0.5),
    v2.ColorJitter(brightness=0.3, contrast=0.3,
                   saturation=0.3, hue=0.1),
    v2.RandomRotation(degrees=15),
    v2.RandomErasing(p=0.25, scale=(0.02, 0.20)),
    v2.ToImage(),
    v2.ToDtype(torch.float32, scale=True),
    v2.Normalize(mean=[0.485, 0.456, 0.406],
                  std=[0.229, 0.224, 0.225])
])

# Validation: NO augmentation, only resize + normalise
val_transform = v2.Compose([
    v2.Resize((224, 224)),
    v2.ToImage(),
    v2.ToDtype(torch.float32, scale=True),
    v2.Normalize(mean=[0.485, 0.456, 0.406],
                  std=[0.229, 0.224, 0.225])
])

Object Detection — YOLO Architecture Deep Dive

from ultralytics import YOLO

# Load pretrained YOLOv8 nano (fastest model)
model = YOLO('yolov8n.pt')

# ── Inference on a single image ────────────────────────────────
results = model.predict(
    source='street.jpg',
    conf=0.45,      # confidence threshold
    iou=0.45,       # NMS IoU threshold
    device='cuda'
)

for r in results:
    for box in r.boxes:
        cls   = r.names[int(box.cls)]
        conf  = float(box.conf)
        xyxy  = box.xyxy[0].tolist()     # [x1,y1,x2,y2]
        print(f"{cls}: {conf:.2f} @ {[int(v) for v in xyxy]}")

# ── Fine-tune on custom dataset ────────────────────────────────
# Requires dataset.yaml defining: path, train, val, nc, names
model.train(
    data='dataset.yaml',
    epochs=50,
    imgsz=640,
    batch=16,
    lr0=0.01,
    patience=10,    # early stopping
    device='0'
)

# ── Evaluate — prints mAP50, mAP50-95 ─────────────────────────
metrics = model.val()
print(f"mAP50: {metrics.box.map50:.3f}")

Evaluation Metrics for Vision Models

Vision Transformers (ViT) — Attention Meets Images

Real-World CV Pipeline — End to End

Model Interpretability — GradCAM Explained

One common criticism of deep learning is the "black box" problem. GradCAM (Gradient-weighted Class Activation Mapping) solves this for CNNs by visualising which spatial regions the model attended to when making a prediction.

import torch
import torch.nn.functional as F
import numpy as np
import cv2

class GradCAM:
    def __init__(self, model, target_layer):
        self.model = model
        self.target_layer = target_layer
        self.gradients = None
        self.activations = None

        target_layer.register_forward_hook(self._save_activation)
        target_layer.register_backward_hook(self._save_gradient)

    def _save_activation(self, module, input, output):
        self.activations = output.detach()

    def _save_gradient(self, module, grad_input, grad_output):
        self.gradients = grad_output[0].detach()

    def generate(self, input_tensor, class_idx=None):
        logits = self.model(input_tensor)
        target = logits[0, class_idx if class_idx else logits.argmax()]
        self.model.zero_grad()
        target.backward()

        # Global average pooling of gradients → channel weights
        weights = self.gradients.mean(dim=(2, 3), keepdim=True)

        # Weighted combination of activation maps
        cam = (weights * self.activations).sum(dim=1, keepdim=True)
        cam = F.relu(cam)  # only positive contributions
        cam = F.interpolate(cam, input_tensor.shape[2:], mode='bilinear')
        cam = cam.squeeze().numpy()

        # Normalise to 0–1
        cam = (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)
        return cam

# Usage on a ResNet50
model = models.resnet50(weights='IMAGENET1K_V2').eval()
gcam = GradCAM(model, model.layer4[2].conv3)
heatmap = gcam.generate(img_tensor, class_idx=281)  # 281=tabby cat

# Overlay as colour map
heatmap_bgr = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
overlay = cv2.addWeighted(original_bgr, 0.6, heatmap_bgr, 0.4, 0)
cv2.imwrite('gradcam_output.jpg', overlay)

Golden Rules for Computer Vision Projects