M48: Sequence Modelling & Vanilla RNNs
CSCI 3151 — Foundations of Machine Learning
Unrolling the RNN
We can visualise the RNN “unrolled” through time — this makes it look like a very deep feedforward network:
Each [h_t] box applies the same weights \((W_{xh}, W_{hh}, W_{hy})\).
Key analogy:
Unrolling in time ↔︎ a very deep network in depth.
The depth equals the sequence length \(T\).
For \(T = 100\), you effectively have a 100-layer network with tied weights.
This is exactly why gradient problems (M33–M34) arise here too — but we’ll get there.
Initial state \(\mathbf{h}_0\): Often set to the zero vector. Sometimes learned.
Generate the data
Code
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import TensorDataset, DataLoader
rng = np.random.default_rng(3151)
T_SEQ = 30 # full sequence length seen by the RNN
W_MLP = 5 # short window seen by the MLP
SIGMA = 0.05 # per-step noise std
N_TRAIN = 4000
N_VAL = 1000
def make_walks(n, T, sigma, rng):
"""Generate n random walks of length T+1 starting from U(0,1)."""
x0 = rng.uniform(0, 1, size=(n, 1)) # seed: (n, 1)
noise = rng.normal(0, sigma, size=(n, T)) # increments: (n, T)
steps = np.cumsum(noise, axis=1) # cumulative drift
walks = np.concatenate([x0, x0 + steps], axis=1) # (n, T+1)
return walks.astype(np.float32)
walks_train = make_walks(N_TRAIN, T_SEQ, SIGMA, rng)
walks_val = make_walks(N_VAL, T_SEQ, SIGMA, rng)
# Target: x_0 (the seed, first column)
y_train = walks_train[:, 0:1] # (N_TRAIN, 1)
y_val = walks_val[:, 0:1]
# RNN input: full walk x_0 … x_{T-1} (T steps) — shape (N, T, 1)
X_rnn_train = walks_train[:, :T_SEQ, np.newaxis]
X_rnn_val = walks_val[:, :T_SEQ, np.newaxis]
# MLP input: last W_MLP steps only — shape (N, W_MLP)
X_mlp_train = walks_train[:, -W_MLP:]
X_mlp_val = walks_val[:, -W_MLP:]
# Plot a few example walks
fig, ax = plt.subplots(figsize=(9, 3))
for i in range(6):
ax.plot(walks_train[i], alpha=0.6, label=f"x₀={walks_train[i,0]:.2f}" if i < 3 else None)
ax.axvline(x=0, color="k", linestyle=":", linewidth=1.5, label="seed position (x₀)")
ax.set_xlabel("Time step")
ax.set_ylabel("Value")
ax.set_title(f"Six example random walks (σ={SIGMA}, T={T_SEQ})\nTarget = x₀ (leftmost point)")
ax.legend(fontsize=8)
plt.tight_layout()
Why the MLP window is blind to the seed
Code
fig, axes = plt.subplots(1, 2, figsize=(10, 3.5))
example = walks_train[0]
ax = axes[0]
ax.plot(np.arange(T_SEQ + 1), example, marker="o", markersize=4)
ax.axvspan(T_SEQ - W_MLP, T_SEQ, alpha=0.2, color="orange", label=f"MLP window (last {W_MLP})")
ax.axvline(0, color="red", linestyle="--", label=f"x₀ = {example[0]:.2f} (target)")
ax.set_xlabel("Time step"); ax.set_ylabel("Value")
ax.set_title("One walk: what the MLP sees"); ax.legend(fontsize=8)
# Theoretical MSE floor for MLP: variance of (x_0 - x_{T-k}) = k * sigma^2
steps_back = np.arange(1, T_SEQ + 1)
theoretical_mse = steps_back * SIGMA**2
ax2 = axes[1]
ax2.plot(steps_back, theoretical_mse, color="steelblue")
ax2.axvline(W_MLP, color="orange", linestyle="--",
label=f"MLP lookback = {W_MLP} steps\nfloor MSE ≈ {W_MLP * SIGMA**2:.4f}")
ax2.axhline(SIGMA**2, color="green", linestyle="--",
label=f"Per-step noise floor σ²={SIGMA**2:.4f}")
ax2.set_xlabel("Steps back from end of sequence")
ax2.set_ylabel("Theoretical MSE lower bound")
ax2.set_title("Information available from window position")
ax2.legend(fontsize=8); ax2.grid(True, alpha=0.3)
plt.tight_layout()
Learning curves: RNN vs MLP
Code
theoretical_floor = W_MLP * SIGMA**2
fig, ax = plt.subplots()
ax.plot(hist_rnn["epoch"], hist_rnn["train_mse"], label="RNN train")
ax.plot(hist_rnn["epoch"], hist_rnn["val_mse"], label="RNN val")
ax.plot(hist_mlp["epoch"], hist_mlp["train_mse"], label="MLP train", linestyle="--")
ax.plot(hist_mlp["epoch"], hist_mlp["val_mse"], label="MLP val", linestyle="--")
ax.axhline(theoretical_floor, color="orange", linestyle=":", linewidth=1.5,
label=f"MLP theoretical floor ({theoretical_floor:.4f})")
ax.axhline(SIGMA**2, color="green", linestyle=":", linewidth=1.5,
label=f"Noise floor σ²={SIGMA**2:.4f}")
ax.set_xlabel("Epoch"); ax.set_ylabel("MSE (log scale)")
ax.set_yscale("log")
ax.set_title("Memory retrieval: RNN can cross the MLP's hard floor")
ax.legend(fontsize=8)
plt.tight_layout()
Learning curves: RNN on AG News
Code
import matplotlib.pyplot as plt
epochs = range(1, len(history["train_acc"]) + 1)
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
# Accuracy
axes[0].plot(epochs, history["train_acc"], label="Train acc")
axes[0].plot(epochs, history["val_acc"], label="Val acc", linestyle="--")
axes[0].set_xlabel("Epoch")
axes[0].set_ylabel("Accuracy")
axes[0].set_title("Accuracy")
axes[0].legend()
# Loss
axes[1].plot(epochs, history["train_loss"], label="Train loss")
axes[1].plot(epochs, history["val_loss"], label="Val loss", linestyle="--")
axes[1].set_xlabel("Epoch")
axes[1].set_ylabel("Cross-entropy loss")
axes[1].set_title("Loss")
axes[1].legend()
plt.tight_layout()
plt.show()
- Train accuracy climbs close to 1.0
- Validation accuracy stabilizes well above chance (25%)
- Train–val gap shows overfitting, but overall the RNN is still learning a useful representation for news topics.
