Item-Based Collaborative Filtering

The Story That Explains Item-Based Filtering

Item-Based CF was pioneered at Amazon in 2003 and remains the backbone of how the world's largest retailers, streaming services, and marketplaces serve personalised recommendations at billion-user scale. Understanding it deeply is one of the most directly employable skills in applied machine learning.

Why Industry Overwhelmingly Prefers Item-Item

The 2003 Amazon paper "Item-to-Item Collaborative Filtering" by Linden, Smith & York changed the industry permanently. They showed three concrete reasons why item-based approaches dominate production systems — and those reasons are even more true today.

Scalability Advantages — A Deep Dive

The Mechanics — Adjusted Cosine Item Similarity

IBCF is built on one core computation: how similar is item A to item B? The standard choice for explicit ratings is Adjusted Cosine Similarity — cosine similarity with each user's mean rating subtracted first, correcting for the fact that different users use the rating scale differently.

Building IBCF from Scratch — Pure Python + NumPy

We build every component by hand: the rating matrix, adjusted cosine similarity, the item-item similarity matrix, and the weighted prediction engine. No shortcuts — understanding the mechanics from scratch is what separates practitioners from API callers.

Step 1 & 2 — Build and Mean-Centre the Rating Matrix

import numpy as np
import pandas as pd
from itertools import combinations

# ── Raw ratings: users × products ────────────────────────────
data = {
    'Wireless Earbuds':            {'Alice':5, 'Bob':4, 'David':5, 'Eve':5},
    'Noise-Cancelling Headphones': {'Alice':4, 'Bob':5, 'Carol':3, 'David':4},
    'Bluetooth Speaker':           {'Bob':4, 'Carol':2, 'David':3, 'Eve':4},
    'Gaming Keyboard':             {'Alice':2, 'Carol':4},
    'Webcam':                      {'Alice':1, 'Bob':2, 'Carol':3},
}

# users × items
ratings = pd.DataFrame(data)
print("=== Rating Matrix ===")
print(ratings)

# ── Mean-centre by USER means ─────────────────────────────────
user_means = ratings.mean(axis=1)   # one mean per user (row)
ratings_centered = ratings.subtract(user_means, axis=0)

print("\n=== User Means ===")
print(user_means.round(2))
print("\n=== Mean-Centred Matrix ===")
print(ratings_centered.round(2))

Step 3 — Compute Adjusted Cosine Item-Item Similarity

def adjusted_cosine(item_i: str, item_j: str,
                     ratings_c: pd.DataFrame) -> float:
    """
    Adjusted cosine similarity between two items.
    Uses the mean-centred rating matrix (ratings_c).
    Only users who rated BOTH items contribute.
    """
    col_i = ratings_c[item_i]
    col_j = ratings_c[item_j]

    # Only rows (users) with both ratings present
    mask = col_i.notna() & col_j.notna()

    if mask.sum() < 2:
        return np.nan

    vi = col_i[mask].values
    vj = col_j[mask].values

    numerator   = np.dot(vi, vj)
    denominator = np.sqrt(np.dot(vi, vi)) * np.sqrt(np.dot(vj, vj))

    if denominator == 0:
        return 0.0

    return float(np.clip(numerator / denominator, -1, 1))


# ── Build full item × item similarity matrix ──────────────────
items = ratings.columns.tolist()
sim_df = pd.DataFrame(np.eye(len(items)),
                        index=items, columns=items)

for i, j in combinations(items, 2):
    s = adjusted_cosine(i, j, ratings_centered)
    sim_df.loc[i, j] = s
    sim_df.loc[j, i] = s   # symmetric

print(sim_df.round(3))

Step 4 & 5 — Predict Ratings and Generate Recommendations

def predict_ibcf(user: str, target_item: str,
                  ratings: pd.DataFrame,
                  sim_df: pd.DataFrame,
                  k: int = 3) -> float:
    """
    Predicts user's rating for target_item using IBCF.
    Uses k most similar items that the user HAS already rated.

    Formula:
        pred(u, i) = Σ[sim(i,j) * r_uj] / Σ|sim(i,j)|
        where j ∈ {items rated by u, j ≠ i}
    """
    # Items user has already rated (excluding target)
    rated_items = [it for it in ratings.columns
                   if it != target_item
                   and pd.notna(ratings.loc[user, it])]

    if not rated_items:
        return ratings[target_item].mean()  # global fallback

    # Similarities between target_item and each rated item
    sims = sim_df.loc[target_item, rated_items].dropna()

    # Keep only positive similarities (items in same cluster)
    sims = sims[sims > 0]

    if sims.empty:
        return ratings[target_item].mean()

    # Top-k most similar rated items
    top_sims = sims.nlargest(k)

    numerator   = 0.0
    denominator = 0.0
    for item, s in top_sims.items():
        numerator   += s * ratings.loc[user, item]
        denominator += abs(s)

    if denominator == 0:
        return ratings[target_item].mean()

    return float(np.clip(numerator / denominator, 1, 5))


def recommend_ibcf(user: str, ratings: pd.DataFrame,
                    sim_df: pd.DataFrame,
                    k: int = 3, top_n: int = 5) -> pd.DataFrame:
    """Returns top-N unrated item recommendations for a user."""
    unrated = [it for it in ratings.columns
               if pd.isna(ratings.loc[user, it])]

    results = []
    for item in unrated:
        pred = predict_ibcf(user, item, ratings, sim_df, k)
        top_neighbours = sim_df.loc[item, :].dropna()
        top_neighbours = top_neighbours[top_neighbours > 0].nlargest(k)
        results.append({
            'Item':                 item,
            'Predicted Rating':     round(pred, 2),
            'Most Similar To':      top_neighbours.index[0] if len(top_neighbours) > 0 else '—',
            'Top Similarity Score': round(top_neighbours.max(), 3) if not top_neighbours.empty else np.nan
        })

    df_out = pd.DataFrame(results).sort_values('Predicted Rating', ascending=False)
    return df_out.head(top_n).reset_index(drop=True)


# ── Recommendations for Eve ───────────────────────────────────
print("=== Eve's Rated Items ===")
print(ratings.loc['Eve'].dropna())

print("\n=== Top Recommendations for Eve ===")
recs = recommend_ibcf('Eve', ratings, sim_df, k=3, top_n=5)
print(recs.to_string())

Visual Architecture — IBCF End-to-End

Visualising Item Clusters — What the Matrix Learns

🛒 Full Project — Amazon-Style Product Recommender

Install Dependencies

pip install scikit-surprise pandas numpy scipy matplotlib

Step 1 — Load & Prepare Amazon Electronics Data

import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.linalg import norm as sparse_norm

# ── Load Amazon Electronics reviews ──────────────────────────
# Download: https://nijianmo.github.io/amazon/index.html
# Small version: 'Electronics_5.json.gz' (5-core, ~1.7M reviews)

df = pd.read_json('Electronics_5.json.gz', lines=True)
df = df[['reviewerID', 'asin', 'overall']]  # keep only needed cols
df.columns = ['user_id', 'item_id', 'rating']

print(f"Raw rows: {len(df):,}")

# ── Filter: keep users with ≥5 reviews, items with ≥10 reviews ─
item_counts = df['item_id'].value_counts()
user_counts = df['user_id'].value_counts()

df = df[df['item_id'].isin(item_counts[item_counts >= 10].index)]
df = df[df['user_id'].isin(user_counts[user_counts >= 5].index)]
df = df.drop_duplicates(['user_id', 'item_id'])  # one rating per user-item

n_users = df['user_id'].nunique()
n_items = df['item_id'].nunique()
sparsity = 1 - len(df) / (n_users * n_items)

print(f"Filtered rows: {len(df):,}")
print(f"Users: {n_users:,}    Items: {n_items:,}")
print(f"Sparsity: {sparsity:.3%}")
print(df['rating'].value_counts().sort_index())

# ── Encode user/item IDs to integer indices ───────────────────
user_enc = {u: i for i, u in enumerate(df['user_id'].unique())}
item_enc = {it: i for i, it in enumerate(df['item_id'].unique())}
item_dec = {v: k for k, v in item_enc.items()}  # reverse map

df['u_idx'] = df['user_id'].map(user_enc)
df['i_idx'] = df['item_id'].map(item_enc)

Step 2 — Efficient Adjusted Cosine via Sparse Matrix

def build_item_similarity_sparse(df: pd.DataFrame,
                                   n_users: int, n_items: int,
                                   top_k: int = 50) -> dict:
    """
    Builds a sparse item-item adjusted cosine similarity store.
    Returns: {item_idx: [(neighbour_idx, similarity), ...]} — top-K only.

    Steps:
      1. Build CSR matrix (users × items)
      2. Mean-centre each user's ratings
      3. Compute column-normalised dot products (= adjusted cosine)
      4. Keep top-K per item
    """
    # ── Step A: Sparse user × item matrix ────────────────────────
    R = csr_matrix(
        (df['rating'].values,
         (df['u_idx'].values, df['i_idx'].values)),
        shape=(n_users, n_items),
        dtype=np.float32
    )

    # ── Step B: Mean-centre by user (row) means ───────────────────
    # Only subtract mean from non-zero (rated) entries
    R_coo = R.tocoo()
    user_means = np.array(R.sum(axis=1)).flatten() / np.array((R != 0).sum(axis=1)).flatten()
    centered_data = R_coo.data - user_means[R_coo.row]
    R_centered = csr_matrix(
        (centered_data, (R_coo.row, R_coo.col)),
        shape=(n_users, n_items), dtype=np.float32
    )

    # ── Step C: Column norms (L2) for cosine normalisation ────────
    col_norms = np.sqrt(np.array(R_centered.power(2).sum(axis=0))).flatten()
    col_norms[col_norms == 0] = 1e-10   # avoid divide by zero

    # Normalise columns
    R_norm = R_centered.copy()
    R_norm = R_norm.multiply(1 / col_norms)   # broadcast across rows

    # ── Step D: Item-item similarity = Rᵀ @ R ────────────────────
    # Shape: (n_items × n_items) — but we compute in chunks to save RAM
    CHUNK = 500
    item_sim_store = {}

    for start in range(0, n_items, CHUNK):
        end   = min(start + CHUNK, n_items)
        chunk = R_norm.T[start:end]            # (chunk × n_users)
        sims  = (chunk @ R_norm).toarray()   # (chunk × n_items)

        for local_idx in range(len(sims)):
            global_idx = start + local_idx
            row = sims[local_idx]
            row[global_idx] = 0              # exclude self-similarity

            # Top-K neighbours
            top_indices = np.argsort(row)[-top_k:][::-1]
            item_sim_store[global_idx] = [
                (idx, float(row[idx]))
                for idx in top_indices
                if row[idx] > 0
            ]

    return item_sim_store


print("Building item similarity matrix (chunked sparse)...")
item_sim_store = build_item_similarity_sparse(
    df, n_users, n_items, top_k=50
)
print(f"Similarity store built. Items with neighbours: {len(item_sim_store):,}")

# Inspect one item
sample_item_idx = item_enc[list(item_enc.keys())[0]]
print(f"\nTop-5 neighbours for item {item_dec[sample_item_idx]}:")
for nb_idx, sim in item_sim_store[sample_item_idx][:5]:
    print(f"  {item_dec[nb_idx]:30s}  sim={sim:.4f}")

Step 3 — Benchmark Against Surprise (Validation)

from surprise import Dataset, Reader, KNNWithMeans
from surprise.model_selection import cross_validate

# Use a 50K sample for manageable CV time
df_sample = df.sample(50_000, random_state=42)[['user_id','item_id','rating']]

reader  = Reader(rating_scale=(1, 5))
dataset = Dataset.load_from_df(df_sample, reader)

# Item-based KNN (IBCF) with adjusted cosine
algo_item = KNNWithMeans(
    k=50,
    sim_options={
        'name':       'cosine',   # Surprise uses adjusted cosine internally
        'user_based': False,      # ← IBCF flag
        'min_support':3
    }
)

cv_results = cross_validate(
    algo_item, dataset,
    measures=['MAE', 'RMSE'],
    cv=5, verbose=True
)

print(f"\nIBCF (item-based) — Mean MAE:  {cv_results['test_mae'].mean():.4f}")
print(f"IBCF (item-based) — Mean RMSE: {cv_results['test_rmse'].mean():.4f}")

# Compare with UBCF on same data
algo_user = KNNWithMeans(
    k=50,
    sim_options={'name':'pearson', 'user_based':True, 'min_support':3}
)
cv_user = cross_validate(algo_user, dataset, measures=['MAE','RMSE'], cv=5, verbose=False)
print(f"UBCF (user-based) — Mean MAE:  {cv_user['test_mae'].mean():.4f}")
print(f"UBCF (user-based) — Mean RMSE: {cv_user['test_rmse'].mean():.4f}")

Step 4 — "Customers Also Bought" Widget (Item-Level)

def customers_also_bought(asin: str,
                           item_enc: dict,
                           item_dec: dict,
                           item_sim_store: dict,
                           top_n: int = 10) -> pd.DataFrame:
    """
    Given a product ASIN, returns the top-N most similar products —
    the 'Customers Also Bought' widget on every Amazon product page.
    """
    if asin not in item_enc:
        raise ValueError(f"ASIN {asin} not in catalogue.")

    idx = item_enc[asin]
    neighbours = item_sim_store.get(idx, [])

    rows = []
    for nb_idx, sim in neighbours[:top_n]:
        nb_asin = item_dec[nb_idx]
        rows.append({
            'ASIN':       nb_asin,
            'Similarity': round(sim, 4),
            'Strength':   'Strong' if sim > 0.7 else
                          ('Medium' if sim > 0.4 else 'Weak')
        })

    return pd.DataFrame(rows)

widget = customers_also_bought('B00004TBKZ', item_enc, item_dec, item_sim_store)
print(widget.to_string(index=False))

Step 5 — Personalised Top-N for a Specific User

def personalised_recs(user_id: str, df: pd.DataFrame,
                       item_enc: dict, item_dec: dict,
                       item_sim_store: dict,
                       k: int = 20, top_n: int = 10) -> pd.DataFrame:
    """Personalised IBCF: aggregate item similarities across purchase history."""
    user_df      = df[df['user_id'] == user_id]
    rated_items  = set(user_df['item_id'])
    user_ratings = dict(zip(user_df['item_id'], user_df['rating']))

    scores, sim_totals = {}, {}

    for asin, user_rating in user_ratings.items():
        if asin not in item_enc:
            continue
        for nb_idx, sim in item_sim_store.get(item_enc[asin], [])[:k]:
            nb_asin = item_dec[nb_idx]
            if nb_asin in rated_items:
                continue
            scores[nb_asin]     = scores.get(nb_asin, 0)     + sim * user_rating
            sim_totals[nb_asin] = sim_totals.get(nb_asin, 0) + abs(sim)

    predicted = {a: scores[a]/sim_totals[a] for a in scores if sim_totals[a] > 0}
    top = sorted(predicted.items(), key=lambda x: -x[1])[:top_n]

    return pd.DataFrame([{'Rank':r+1,'ASIN':a,'Predicted Score':round(s,2)}
                          for r,(a,s) in enumerate(top)])

sample_user = df['user_id'].value_counts().index[10]
recs = personalised_recs(sample_user, df, item_enc, item_dec, item_sim_store)
print(recs.to_string(index=False))

Evaluation & Tuning Your IBCF System

Tuning k — The Neighbourhood Size

from surprise.model_selection import GridSearchCV

param_grid = {
    'k': [10, 20, 40, 60, 80],
    'sim_options': [{'name':'cosine', 'user_based':False, 'min_support':3}]
}

gs = GridSearchCV(KNNWithMeans, param_grid, measures=['rmse'], cv=3, n_jobs=-1)
gs.fit(dataset)

print(f"Best RMSE: {gs.best_score['rmse']:.4f}")
print(f"Best k:    {gs.best_params['rmse']['k']}")

results_df = pd.DataFrame(gs.cv_results)
print(results_df[['param_k', 'mean_test_rmse']].sort_values('param_k'))

Golden Rules for Production IBCF

IBCF in the Broader Recommender Landscape